{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.816471099853516, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.762155532836914, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.023956298828125, "KL/mean": 0.01683010905981064, "KL/rejected_KL_mean": 0.00971221923828125, "KL/std": 0.29489660263061523, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014244243502616882, "fcm_dpo/q_t": 0.49964389204978943, "grad_norm": 26.05791664123535, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42734113335609436, "logits/rejected": 0.6505087018013, "logps/chosen": -93.78703308105469, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.21978759765625, "loss": 1.3854, "margin_dpo/margin_mean": 0.014244511723518372, "margin_dpo/margin_std": 0.4581794738769531, "step": 3 }, { "KL/chosen_KL_mean": -0.036930084228515625, "KL/mean": -0.031013213098049164, "KL/rejected_KL_mean": -0.0251007080078125, "KL/std": 0.3202870190143585, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01182486116886139, "fcm_dpo/q_t": 0.5002953410148621, "grad_norm": 32.77095413208008, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6473371982574463, "logits/rejected": 0.7790506482124329, "logps/chosen": -87.32939910888672, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.84935760498047, "loss": 1.388, "margin_dpo/margin_mean": -0.011824682354927063, "margin_dpo/margin_std": 0.4406079053878784, "step": 4 }, { "KL/chosen_KL_mean": -0.01953887939453125, "KL/mean": -0.023597635328769684, "KL/rejected_KL_mean": -0.027660369873046875, "KL/std": 0.29826152324676514, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.008125528693199158, "fcm_dpo/q_t": 0.4997962713241577, "grad_norm": 35.24354553222656, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4454003572463989, "logits/rejected": 0.6567134261131287, "logps/chosen": -89.35629272460938, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.77549743652344, "loss": 1.3859, "margin_dpo/margin_mean": 0.008126139640808105, "margin_dpo/margin_std": 0.42889225482940674, "step": 5 }, { "KL/chosen_KL_mean": 0.049747467041015625, "KL/mean": 0.03715410828590393, "KL/rejected_KL_mean": 0.02455902099609375, "KL/std": 0.28524962067604065, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0251825749874115, "fcm_dpo/q_t": 0.49936991930007935, "grad_norm": 35.85267639160156, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.48449331521987915, "logits/rejected": 0.8011096715927124, "logps/chosen": -97.271728515625, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.85888671875, "loss": 1.3842, "margin_dpo/margin_mean": 0.025182008743286133, "margin_dpo/margin_std": 0.3964432179927826, "step": 6 }, { "KL/chosen_KL_mean": 0.014583587646484375, "KL/mean": -0.017501547932624817, "KL/rejected_KL_mean": -0.049594879150390625, "KL/std": 0.29236072301864624, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06417509913444519, "fcm_dpo/q_t": 0.49839699268341064, "grad_norm": 37.870521545410156, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4985416531562805, "logits/rejected": 0.5467824339866638, "logps/chosen": -86.63394927978516, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.665771484375, "loss": 1.3803, "margin_dpo/margin_mean": 0.06417447328567505, "margin_dpo/margin_std": 0.394360214471817, "step": 7 }, { "KL/chosen_KL_mean": -0.08135604858398438, "KL/mean": -0.02125602960586548, "KL/rejected_KL_mean": 0.038845062255859375, "KL/std": 0.32567089796066284, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12019762396812439, "fcm_dpo/q_t": 0.5030021667480469, "grad_norm": 32.98783493041992, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.41831934452056885, "logits/rejected": 0.7309536933898926, "logps/chosen": -90.00503540039062, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.18919372558594, "loss": 1.3989, "margin_dpo/margin_mean": -0.12019725143909454, "margin_dpo/margin_std": 0.447502076625824, "step": 8 }, { "KL/chosen_KL_mean": 0.03011322021484375, "KL/mean": -0.009677760303020477, "KL/rejected_KL_mean": -0.049465179443359375, "KL/std": 0.3352567255496979, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07956834137439728, "fcm_dpo/q_t": 0.49801239371299744, "grad_norm": 35.9222526550293, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.6572669744491577, "logits/rejected": 0.8368101119995117, "logps/chosen": -103.82701873779297, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.36878967285156, "loss": 1.3789, "margin_dpo/margin_mean": 0.07956790924072266, "margin_dpo/margin_std": 0.4475556015968323, "step": 9 }, { "KL/chosen_KL_mean": -0.020671844482421875, "KL/mean": 0.04531882703304291, "KL/rejected_KL_mean": 0.11130905151367188, "KL/std": 0.2782899737358093, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.1319890320301056, "fcm_dpo/q_t": 0.5032972693443298, "grad_norm": 33.726863861083984, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.4817013442516327, "logits/rejected": 0.6930489540100098, "logps/chosen": -76.22503662109375, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.0407943725586, "loss": 1.3999, "margin_dpo/margin_mean": -0.13198921084403992, "margin_dpo/margin_std": 0.37467730045318604, "step": 10 }, { "KL/chosen_KL_mean": 0.059215545654296875, "KL/mean": -0.0029697269201278687, "KL/rejected_KL_mean": -0.06515121459960938, "KL/std": 0.354614794254303, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12436237931251526, "fcm_dpo/q_t": 0.4968932569026947, "grad_norm": 36.549781799316406, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.3200716972351074, "logits/rejected": 0.5384095311164856, "logps/chosen": -82.30728149414062, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.32977294921875, "loss": 1.3744, "margin_dpo/margin_mean": 0.1243630051612854, "margin_dpo/margin_std": 0.43884149193763733, "step": 11 }, { "KL/chosen_KL_mean": 0.09013748168945312, "KL/mean": 0.015200257301330566, "KL/rejected_KL_mean": -0.059734344482421875, "KL/std": 0.3625867962837219, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1498645842075348, "fcm_dpo/q_t": 0.4962559938430786, "grad_norm": 39.76643753051758, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.3321595788002014, "logits/rejected": 0.4577009975910187, "logps/chosen": -99.01536560058594, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.33114624023438, "loss": 1.3721, "margin_dpo/margin_mean": 0.14986476302146912, "margin_dpo/margin_std": 0.5402764081954956, "step": 12 }, { "KL/chosen_KL_mean": 0.014560699462890625, "KL/mean": 0.012032613158226013, "KL/rejected_KL_mean": 0.009510040283203125, "KL/std": 0.31362950801849365, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005053430795669556, "fcm_dpo/q_t": 0.4998742938041687, "grad_norm": 41.79119873046875, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.4745793342590332, "logits/rejected": 0.72971510887146, "logps/chosen": -90.545166015625, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.68158721923828, "loss": 1.3863, "margin_dpo/margin_mean": 0.005053281784057617, "margin_dpo/margin_std": 0.4518609046936035, "step": 13 }, { "KL/chosen_KL_mean": -0.05303955078125, "KL/mean": -0.008498556911945343, "KL/rejected_KL_mean": 0.036041259765625, "KL/std": 0.29658782482147217, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0890849381685257, "fcm_dpo/q_t": 0.5022256970405579, "grad_norm": 35.894351959228516, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.9461093544960022, "logits/rejected": 1.0367159843444824, "logps/chosen": -99.88021850585938, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.90596008300781, "loss": 1.3956, "margin_dpo/margin_mean": -0.08908471465110779, "margin_dpo/margin_std": 0.3927665650844574, "step": 14 }, { "KL/chosen_KL_mean": 0.02091217041015625, "KL/mean": 0.009802624583244324, "KL/rejected_KL_mean": -0.001308441162109375, "KL/std": 0.3136335611343384, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.022223085165023804, "fcm_dpo/q_t": 0.4994439482688904, "grad_norm": 31.851465225219727, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.2689778208732605, "logits/rejected": 0.4296504855155945, "logps/chosen": -78.88905334472656, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.06365203857422, "loss": 1.3845, "margin_dpo/margin_mean": 0.02222272753715515, "margin_dpo/margin_std": 0.4053305685520172, "step": 15 }, { "KL/chosen_KL_mean": 0.014652252197265625, "KL/mean": 0.006631836295127869, "KL/rejected_KL_mean": -0.00139617919921875, "KL/std": 0.35333725810050964, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016044646501541138, "fcm_dpo/q_t": 0.4995981454849243, "grad_norm": 34.46835708618164, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.6591732501983643, "logits/rejected": 0.9268049001693726, "logps/chosen": -97.40863037109375, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.60086059570312, "loss": 1.3853, "margin_dpo/margin_mean": 0.016045063734054565, "margin_dpo/margin_std": 0.4730644226074219, "step": 16 }, { "KL/chosen_KL_mean": -0.0024261474609375, "KL/mean": 0.005953803658485413, "KL/rejected_KL_mean": 0.014331817626953125, "KL/std": 0.3177465796470642, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01675829291343689, "fcm_dpo/q_t": 0.5004173517227173, "grad_norm": 35.75648498535156, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5976489782333374, "logits/rejected": 0.8414093255996704, "logps/chosen": -104.36672973632812, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.4533920288086, "loss": 1.3885, "margin_dpo/margin_mean": -0.016758739948272705, "margin_dpo/margin_std": 0.46477240324020386, "step": 17 }, { "KL/chosen_KL_mean": 0.049022674560546875, "KL/mean": 0.05634636431932449, "KL/rejected_KL_mean": 0.06367111206054688, "KL/std": 0.34601879119873047, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014653608202934265, "fcm_dpo/q_t": 0.5003670454025269, "grad_norm": 41.81201934814453, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.2849951386451721, "logits/rejected": 0.617476224899292, "logps/chosen": -87.04293823242188, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.78704833984375, "loss": 1.3884, "margin_dpo/margin_mean": -0.01465316116809845, "margin_dpo/margin_std": 0.509171724319458, "step": 18 }, { "KL/chosen_KL_mean": -0.0611114501953125, "KL/mean": -0.06147870421409607, "KL/rejected_KL_mean": -0.0618438720703125, "KL/std": 0.326216459274292, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0007204413414001465, "fcm_dpo/q_t": 0.49998342990875244, "grad_norm": 31.24576759338379, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.5962898135185242, "logits/rejected": 0.8725696206092834, "logps/chosen": -105.93465423583984, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.99208068847656, "loss": 1.3868, "margin_dpo/margin_mean": 0.0007206201553344727, "margin_dpo/margin_std": 0.4606618583202362, "step": 19 }, { "KL/chosen_KL_mean": 0.07299041748046875, "KL/mean": 0.03186173737049103, "KL/rejected_KL_mean": -0.009273529052734375, "KL/std": 0.361385703086853, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08225861191749573, "fcm_dpo/q_t": 0.49794384837150574, "grad_norm": 32.69158172607422, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.44651734828948975, "logits/rejected": 0.7442775964736938, "logps/chosen": -90.68513488769531, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.92159271240234, "loss": 1.3787, "margin_dpo/margin_mean": 0.08225792646408081, "margin_dpo/margin_std": 0.49800044298171997, "step": 20 }, { "KL/chosen_KL_mean": 0.03143310546875, "KL/mean": 0.02778691053390503, "KL/rejected_KL_mean": 0.024139404296875, "KL/std": 0.27684861421585083, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0072969794273376465, "fcm_dpo/q_t": 0.4998171329498291, "grad_norm": 31.63650894165039, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.6309342384338379, "logits/rejected": 0.8168145418167114, "logps/chosen": -80.3020248413086, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.90959167480469, "loss": 1.3859, "margin_dpo/margin_mean": 0.007297217845916748, "margin_dpo/margin_std": 0.38107311725616455, "step": 21 }, { "KL/chosen_KL_mean": 0.035614013671875, "KL/mean": 0.029419660568237305, "KL/rejected_KL_mean": 0.023223876953125, "KL/std": 0.30232906341552734, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012379288673400879, "fcm_dpo/q_t": 0.4996913969516754, "grad_norm": 42.787086486816406, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.7734129428863525, "logits/rejected": 0.9002683758735657, "logps/chosen": -95.35968780517578, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.4502944946289, "loss": 1.3855, "margin_dpo/margin_mean": 0.012379348278045654, "margin_dpo/margin_std": 0.42307227849960327, "step": 22 }, { "KL/chosen_KL_mean": 0.07875823974609375, "KL/mean": 0.061934612691402435, "KL/rejected_KL_mean": 0.0451202392578125, "KL/std": 0.3263903260231018, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03363649547100067, "fcm_dpo/q_t": 0.49915897846221924, "grad_norm": 31.48447608947754, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 1.0141654014587402, "logits/rejected": 1.208448886871338, "logps/chosen": -90.55875396728516, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.54913330078125, "loss": 1.3834, "margin_dpo/margin_mean": 0.033637046813964844, "margin_dpo/margin_std": 0.4322824478149414, "step": 23 }, { "KL/chosen_KL_mean": 0.03389549255371094, "KL/mean": -0.024844534695148468, "KL/rejected_KL_mean": -0.08358383178710938, "KL/std": 0.30540019273757935, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11747793853282928, "fcm_dpo/q_t": 0.49706536531448364, "grad_norm": 44.137550354003906, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.4238646328449249, "logits/rejected": 0.4586022198200226, "logps/chosen": -69.88339233398438, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.6934814453125, "loss": 1.375, "margin_dpo/margin_mean": 0.11747895181179047, "margin_dpo/margin_std": 0.4316919147968292, "step": 24 }, { "KL/chosen_KL_mean": 0.004180908203125, "KL/mean": -0.03611262887716293, "KL/rejected_KL_mean": -0.07640838623046875, "KL/std": 0.29988306760787964, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08058185875415802, "fcm_dpo/q_t": 0.49798738956451416, "grad_norm": 36.75385284423828, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.20623520016670227, "logits/rejected": 0.36125850677490234, "logps/chosen": -80.82130432128906, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.03351593017578, "loss": 1.3787, "margin_dpo/margin_mean": 0.08058205246925354, "margin_dpo/margin_std": 0.4336202144622803, "step": 25 }, { "KL/chosen_KL_mean": 0.0511322021484375, "KL/mean": -0.003944575786590576, "KL/rejected_KL_mean": -0.05902099609375, "KL/std": 0.3815140128135681, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11015695333480835, "fcm_dpo/q_t": 0.49724942445755005, "grad_norm": 40.68088912963867, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.3260531425476074, "logits/rejected": 0.5353327393531799, "logps/chosen": -88.85003662109375, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.88721466064453, "loss": 1.3761, "margin_dpo/margin_mean": 0.110157310962677, "margin_dpo/margin_std": 0.5539075136184692, "step": 26 }, { "KL/chosen_KL_mean": 0.0164947509765625, "KL/mean": -0.0690927505493164, "KL/rejected_KL_mean": -0.15468978881835938, "KL/std": 0.34798353910446167, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17118597030639648, "fcm_dpo/q_t": 0.49572277069091797, "grad_norm": 44.13279724121094, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.10160086303949356, "logits/rejected": 0.2893596887588501, "logps/chosen": -77.57951354980469, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.09320068359375, "loss": 1.3698, "margin_dpo/margin_mean": 0.1711861789226532, "margin_dpo/margin_std": 0.4790102243423462, "step": 27 }, { "KL/chosen_KL_mean": 0.029026031494140625, "KL/mean": -0.04597693681716919, "KL/rejected_KL_mean": -0.12097549438476562, "KL/std": 0.3828321695327759, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15000391006469727, "fcm_dpo/q_t": 0.4962540864944458, "grad_norm": 36.23957443237305, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5922123789787292, "logits/rejected": 0.8133383989334106, "logps/chosen": -102.19953918457031, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.08047485351562, "loss": 1.3722, "margin_dpo/margin_mean": 0.15000486373901367, "margin_dpo/margin_std": 0.5746721625328064, "step": 28 }, { "KL/chosen_KL_mean": 0.054515838623046875, "KL/mean": -0.030326135456562042, "KL/rejected_KL_mean": -0.11516952514648438, "KL/std": 0.3723677694797516, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16969476640224457, "fcm_dpo/q_t": 0.4957624673843384, "grad_norm": 42.077754974365234, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.3846359848976135, "logits/rejected": 0.6488187313079834, "logps/chosen": -88.59253692626953, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.07528686523438, "loss": 1.3701, "margin_dpo/margin_mean": 0.16969551146030426, "margin_dpo/margin_std": 0.5162564516067505, "step": 29 }, { "KL/chosen_KL_mean": 0.005817413330078125, "KL/mean": -0.04626373201608658, "KL/rejected_KL_mean": -0.09835433959960938, "KL/std": 0.3184727430343628, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10417263209819794, "fcm_dpo/q_t": 0.4973965883255005, "grad_norm": 39.406333923339844, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.42230910062789917, "logits/rejected": 0.530748188495636, "logps/chosen": -88.382568359375, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.41725158691406, "loss": 1.3762, "margin_dpo/margin_mean": 0.10417288541793823, "margin_dpo/margin_std": 0.36842113733291626, "step": 30 }, { "KL/chosen_KL_mean": -0.08332443237304688, "KL/mean": -0.10066639631986618, "KL/rejected_KL_mean": -0.11800384521484375, "KL/std": 0.3287389278411865, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03467629849910736, "fcm_dpo/q_t": 0.49913397431373596, "grad_norm": 31.765897750854492, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.834971010684967, "logits/rejected": 1.0930125713348389, "logps/chosen": -101.208984375, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.97642517089844, "loss": 1.3833, "margin_dpo/margin_mean": 0.03467628359794617, "margin_dpo/margin_std": 0.42142367362976074, "step": 31 }, { "KL/chosen_KL_mean": -0.11858749389648438, "KL/mean": -0.12307121604681015, "KL/rejected_KL_mean": -0.12755203247070312, "KL/std": 0.2951400876045227, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00896655023097992, "fcm_dpo/q_t": 0.49977412819862366, "grad_norm": 35.40614318847656, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.4510706067085266, "logits/rejected": 0.5933880805969238, "logps/chosen": -96.73561096191406, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.35191345214844, "loss": 1.3859, "margin_dpo/margin_mean": 0.008967205882072449, "margin_dpo/margin_std": 0.4403691291809082, "step": 32 }, { "KL/chosen_KL_mean": 0.015445709228515625, "KL/mean": -0.022996827960014343, "KL/rejected_KL_mean": -0.0614471435546875, "KL/std": 0.3343868851661682, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07689568400382996, "fcm_dpo/q_t": 0.49807852506637573, "grad_norm": 33.84413146972656, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5546838045120239, "logits/rejected": 0.7345223426818848, "logps/chosen": -81.50556945800781, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.86740112304688, "loss": 1.3792, "margin_dpo/margin_mean": 0.07689633965492249, "margin_dpo/margin_std": 0.45086145401000977, "step": 33 }, { "KL/chosen_KL_mean": -0.009105682373046875, "KL/mean": -0.05371619015932083, "KL/rejected_KL_mean": -0.09832000732421875, "KL/std": 0.3600292503833771, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08922134339809418, "fcm_dpo/q_t": 0.49778062105178833, "grad_norm": 41.00299835205078, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5889803171157837, "logits/rejected": 0.7611876130104065, "logps/chosen": -77.21114349365234, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.81708526611328, "loss": 1.3782, "margin_dpo/margin_mean": 0.08922100067138672, "margin_dpo/margin_std": 0.5424289107322693, "step": 34 }, { "KL/chosen_KL_mean": -0.00128173828125, "KL/mean": -0.11901651322841644, "KL/rejected_KL_mean": -0.23675918579101562, "KL/std": 0.3551691174507141, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23547878861427307, "fcm_dpo/q_t": 0.4941176176071167, "grad_norm": 42.542884826660156, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.36765557527542114, "logits/rejected": 0.5232113599777222, "logps/chosen": -77.5716323852539, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.42530822753906, "loss": 1.3634, "margin_dpo/margin_mean": 0.2354794442653656, "margin_dpo/margin_std": 0.45697638392448425, "step": 35 }, { "KL/chosen_KL_mean": 0.00437164306640625, "KL/mean": -0.07553590089082718, "KL/rejected_KL_mean": -0.15544891357421875, "KL/std": 0.3380998969078064, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15981484949588776, "fcm_dpo/q_t": 0.4960077404975891, "grad_norm": 32.67436218261719, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.5308000445365906, "logits/rejected": 0.7261749505996704, "logps/chosen": -83.28387451171875, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.9128646850586, "loss": 1.3709, "margin_dpo/margin_mean": 0.15981517732143402, "margin_dpo/margin_std": 0.43543359637260437, "step": 36 }, { "KL/chosen_KL_mean": -0.040554046630859375, "KL/mean": -0.1503819078207016, "KL/rejected_KL_mean": -0.2602081298828125, "KL/std": 0.356597900390625, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21965128183364868, "fcm_dpo/q_t": 0.4945143163204193, "grad_norm": 36.286834716796875, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.6976536512374878, "logits/rejected": 0.8593810200691223, "logps/chosen": -94.8116455078125, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -85.97747802734375, "loss": 1.365, "margin_dpo/margin_mean": 0.21965140104293823, "margin_dpo/margin_std": 0.4810381531715393, "step": 37 }, { "KL/chosen_KL_mean": 0.08419418334960938, "KL/mean": -0.04857628792524338, "KL/rejected_KL_mean": -0.18134689331054688, "KL/std": 0.4369877278804779, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.265545129776001, "fcm_dpo/q_t": 0.4933720827102661, "grad_norm": 36.828086853027344, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.518415629863739, "logits/rejected": 0.6572738289833069, "logps/chosen": -75.8943099975586, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.28536987304688, "loss": 1.3609, "margin_dpo/margin_mean": 0.26554563641548157, "margin_dpo/margin_std": 0.6099990606307983, "step": 38 }, { "KL/chosen_KL_mean": -0.02442169189453125, "KL/mean": -0.18029499053955078, "KL/rejected_KL_mean": -0.3361663818359375, "KL/std": 0.39840167760849, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.31174010038375854, "fcm_dpo/q_t": 0.49222108721733093, "grad_norm": 36.06353759765625, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.43547242879867554, "logits/rejected": 0.6204301118850708, "logps/chosen": -81.210205078125, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.53208923339844, "loss": 1.3562, "margin_dpo/margin_mean": 0.3117403984069824, "margin_dpo/margin_std": 0.557281494140625, "step": 39 }, { "KL/chosen_KL_mean": -0.004962921142578125, "KL/mean": -0.1012190729379654, "KL/rejected_KL_mean": -0.19747543334960938, "KL/std": 0.34504544734954834, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19251349568367004, "fcm_dpo/q_t": 0.49518975615501404, "grad_norm": 29.203916549682617, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.6042391061782837, "logits/rejected": 0.7995736598968506, "logps/chosen": -83.33753967285156, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.45338439941406, "loss": 1.3676, "margin_dpo/margin_mean": 0.19251284003257751, "margin_dpo/margin_std": 0.4426041841506958, "step": 40 }, { "KL/chosen_KL_mean": -0.041439056396484375, "KL/mean": -0.15384149551391602, "KL/rejected_KL_mean": -0.2662467956542969, "KL/std": 0.40542417764663696, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.224808931350708, "fcm_dpo/q_t": 0.4943850040435791, "grad_norm": 33.55887985229492, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.7481341361999512, "logits/rejected": 0.9423681497573853, "logps/chosen": -93.19010162353516, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.345458984375, "loss": 1.3647, "margin_dpo/margin_mean": 0.22480908036231995, "margin_dpo/margin_std": 0.5345613360404968, "step": 41 }, { "KL/chosen_KL_mean": 0.05681610107421875, "KL/mean": -0.16962608695030212, "KL/rejected_KL_mean": -0.39606475830078125, "KL/std": 0.5022512674331665, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4528765082359314, "fcm_dpo/q_t": 0.48869359493255615, "grad_norm": 40.74055480957031, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.480833500623703, "logits/rejected": 0.6227739453315735, "logps/chosen": -90.64480590820312, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.26071166992188, "loss": 1.3425, "margin_dpo/margin_mean": 0.45287686586380005, "margin_dpo/margin_std": 0.6316689848899841, "step": 42 }, { "KL/chosen_KL_mean": -0.026905059814453125, "KL/mean": -0.21896962821483612, "KL/rejected_KL_mean": -0.4110450744628906, "KL/std": 0.4944770038127899, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3841424286365509, "fcm_dpo/q_t": 0.4904171824455261, "grad_norm": 38.02309036254883, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.6374416351318359, "logits/rejected": 0.830112099647522, "logps/chosen": -89.67093658447266, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.36289978027344, "loss": 1.3496, "margin_dpo/margin_mean": 0.38414281606674194, "margin_dpo/margin_std": 0.7231905460357666, "step": 43 }, { "KL/chosen_KL_mean": -0.05538177490234375, "KL/mean": -0.2489972710609436, "KL/rejected_KL_mean": -0.4426116943359375, "KL/std": 0.5009016990661621, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.38723504543304443, "fcm_dpo/q_t": 0.4903343915939331, "grad_norm": 34.3946533203125, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.7974671125411987, "logits/rejected": 0.9198058843612671, "logps/chosen": -81.93370056152344, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.86424255371094, "loss": 1.3489, "margin_dpo/margin_mean": 0.3872356414794922, "margin_dpo/margin_std": 0.6130525469779968, "step": 44 }, { "KL/chosen_KL_mean": -0.027496337890625, "KL/mean": -0.14699363708496094, "KL/rejected_KL_mean": -0.26648712158203125, "KL/std": 0.4084077477455139, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23899514973163605, "fcm_dpo/q_t": 0.49403226375579834, "grad_norm": 24.506587982177734, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.5265241861343384, "logits/rejected": 0.6354086399078369, "logps/chosen": -77.3720932006836, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.51423645019531, "loss": 1.3633, "margin_dpo/margin_mean": 0.2389957159757614, "margin_dpo/margin_std": 0.5532289743423462, "step": 45 }, { "KL/chosen_KL_mean": -0.057544708251953125, "KL/mean": -0.23640096187591553, "KL/rejected_KL_mean": -0.415252685546875, "KL/std": 0.5307238101959229, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3577152490615845, "fcm_dpo/q_t": 0.49106886982917786, "grad_norm": 33.535221099853516, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.6970125436782837, "logits/rejected": 0.9494351148605347, "logps/chosen": -90.39836883544922, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.09176635742188, "loss": 1.3517, "margin_dpo/margin_mean": 0.35771578550338745, "margin_dpo/margin_std": 0.5735461115837097, "step": 46 }, { "KL/chosen_KL_mean": -0.01141357421875, "KL/mean": -0.22173254191875458, "KL/rejected_KL_mean": -0.4320487976074219, "KL/std": 0.5231642723083496, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.42063960433006287, "fcm_dpo/q_t": 0.4895017147064209, "grad_norm": 33.9151496887207, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.2650867700576782, "logits/rejected": 0.4353049397468567, "logps/chosen": -104.42271423339844, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.68695068359375, "loss": 1.3458, "margin_dpo/margin_mean": 0.42063969373703003, "margin_dpo/margin_std": 0.6537460088729858, "step": 47 }, { "KL/chosen_KL_mean": -0.01540374755859375, "KL/mean": -0.29151830077171326, "KL/rejected_KL_mean": -0.5676345825195312, "KL/std": 0.6568649411201477, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5522333979606628, "fcm_dpo/q_t": 0.48623013496398926, "grad_norm": 38.80614471435547, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.3442534804344177, "logits/rejected": 0.55370032787323, "logps/chosen": -91.11567687988281, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.64820861816406, "loss": 1.3337, "margin_dpo/margin_mean": 0.5522340536117554, "margin_dpo/margin_std": 0.8551031351089478, "step": 48 }, { "KL/chosen_KL_mean": 0.062091827392578125, "KL/mean": -0.3264499306678772, "KL/rejected_KL_mean": -0.714996337890625, "KL/std": 0.8029529452323914, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7770804166793823, "fcm_dpo/q_t": 0.48064514994621277, "grad_norm": 39.6201171875, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.08297931402921677, "logits/rejected": 0.5518176555633545, "logps/chosen": -92.94158935546875, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.4639892578125, "loss": 1.3124, "margin_dpo/margin_mean": 0.7770806550979614, "margin_dpo/margin_std": 0.9510141611099243, "step": 49 }, { "KL/chosen_KL_mean": -0.1368408203125, "KL/mean": -0.4520326256752014, "KL/rejected_KL_mean": -0.7672157287597656, "KL/std": 0.7675960063934326, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6303742527961731, "fcm_dpo/q_t": 0.4842909872531891, "grad_norm": 33.27421188354492, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.6695551872253418, "logits/rejected": 0.8451331853866577, "logps/chosen": -94.76365661621094, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.3415756225586, "loss": 1.3264, "margin_dpo/margin_mean": 0.6303740739822388, "margin_dpo/margin_std": 0.9290628433227539, "step": 50 }, { "KL/chosen_KL_mean": -0.0882720947265625, "KL/mean": -0.389836847782135, "KL/rejected_KL_mean": -0.6914024353027344, "KL/std": 0.8204355239868164, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6031308174133301, "fcm_dpo/q_t": 0.48499026894569397, "grad_norm": 30.04303550720215, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5496084094047546, "logits/rejected": 0.6869431138038635, "logps/chosen": -87.59555053710938, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.16375732421875, "loss": 1.3297, "margin_dpo/margin_mean": 0.6031309962272644, "margin_dpo/margin_std": 1.0491979122161865, "step": 51 }, { "KL/chosen_KL_mean": 0.01444244384765625, "KL/mean": -0.4322391450405121, "KL/rejected_KL_mean": -0.8789253234863281, "KL/std": 0.7673888206481934, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.893365204334259, "fcm_dpo/q_t": 0.477752685546875, "grad_norm": 31.96318817138672, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.35411036014556885, "logits/rejected": 0.561944305896759, "logps/chosen": -90.6158218383789, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.91798400878906, "loss": 1.3011, "margin_dpo/margin_mean": 0.8933653831481934, "margin_dpo/margin_std": 0.9279263019561768, "step": 52 }, { "KL/chosen_KL_mean": -0.049350738525390625, "KL/mean": -0.5298468470573425, "KL/rejected_KL_mean": -1.0103416442871094, "KL/std": 1.060207486152649, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9609969854354858, "fcm_dpo/q_t": 0.4762192964553833, "grad_norm": 39.51531219482422, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.05912104249000549, "logits/rejected": 0.17086248099803925, "logps/chosen": -81.63243103027344, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.671875, "loss": 1.2976, "margin_dpo/margin_mean": 0.960996687412262, "margin_dpo/margin_std": 1.368351936340332, "step": 53 }, { "KL/chosen_KL_mean": 0.018436431884765625, "KL/mean": -0.5521670579910278, "KL/rejected_KL_mean": -1.122772216796875, "KL/std": 1.2533080577850342, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1412081718444824, "fcm_dpo/q_t": 0.47166353464126587, "grad_norm": 39.88654708862305, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.41032275557518005, "logits/rejected": 0.5890465974807739, "logps/chosen": -88.89173126220703, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.24032592773438, "loss": 1.2794, "margin_dpo/margin_mean": 1.1412081718444824, "margin_dpo/margin_std": 1.2643764019012451, "step": 54 }, { "KL/chosen_KL_mean": 0.08905792236328125, "KL/mean": -0.4935726821422577, "KL/rejected_KL_mean": -1.0761985778808594, "KL/std": 1.2417843341827393, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1652543544769287, "fcm_dpo/q_t": 0.47120052576065063, "grad_norm": 35.27983474731445, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.11833828687667847, "logits/rejected": 0.1704709231853485, "logps/chosen": -92.36686706542969, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.04713439941406, "loss": 1.2799, "margin_dpo/margin_mean": 1.1652547121047974, "margin_dpo/margin_std": 1.6067180633544922, "step": 55 }, { "KL/chosen_KL_mean": -0.06926345825195312, "KL/mean": -0.6079235076904297, "KL/rejected_KL_mean": -1.1465797424316406, "KL/std": 1.1910362243652344, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0773160457611084, "fcm_dpo/q_t": 0.47341078519821167, "grad_norm": 32.99076843261719, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.23610873520374298, "logits/rejected": 0.5750013589859009, "logps/chosen": -87.30592346191406, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.79212188720703, "loss": 1.2888, "margin_dpo/margin_mean": 1.077315092086792, "margin_dpo/margin_std": 1.706512689590454, "step": 56 }, { "KL/chosen_KL_mean": -0.06531906127929688, "KL/mean": -0.6360455751419067, "KL/rejected_KL_mean": -1.2067756652832031, "KL/std": 1.3176164627075195, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.141467809677124, "fcm_dpo/q_t": 0.47181951999664307, "grad_norm": 33.57544708251953, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.2027873694896698, "logits/rejected": 0.4157256484031677, "logps/chosen": -98.216064453125, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.25961303710938, "loss": 1.2827, "margin_dpo/margin_mean": 1.1414680480957031, "margin_dpo/margin_std": 1.7143833637237549, "step": 57 }, { "KL/chosen_KL_mean": 0.2312774658203125, "KL/mean": -0.506363034248352, "KL/rejected_KL_mean": -1.2439994812011719, "KL/std": 1.3707640171051025, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4752788543701172, "fcm_dpo/q_t": 0.46371233463287354, "grad_norm": 38.20697021484375, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.10058627277612686, "logits/rejected": 0.41236135363578796, "logps/chosen": -99.34969329833984, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.46627807617188, "loss": 1.2528, "margin_dpo/margin_mean": 1.4752788543701172, "margin_dpo/margin_std": 1.8025479316711426, "step": 58 }, { "KL/chosen_KL_mean": -0.00600433349609375, "KL/mean": -0.61235111951828, "KL/rejected_KL_mean": -1.2186965942382812, "KL/std": 1.206531047821045, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2126848697662354, "fcm_dpo/q_t": 0.46994543075561523, "grad_norm": 34.26017379760742, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.03608077019453049, "logits/rejected": 0.22423508763313293, "logps/chosen": -89.80833435058594, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.49537658691406, "loss": 1.274, "margin_dpo/margin_mean": 1.2126843929290771, "margin_dpo/margin_std": 1.4489891529083252, "step": 59 }, { "KL/chosen_KL_mean": 0.01334381103515625, "KL/mean": -0.557938277721405, "KL/rejected_KL_mean": -1.1292266845703125, "KL/std": 1.2249584197998047, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1425749063491821, "fcm_dpo/q_t": 0.4716295003890991, "grad_norm": 28.773168563842773, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.3138789236545563, "logits/rejected": 0.5801891684532166, "logps/chosen": -95.14236450195312, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.50166320800781, "loss": 1.2799, "margin_dpo/margin_mean": 1.1425745487213135, "margin_dpo/margin_std": 1.3638784885406494, "step": 60 }, { "KL/chosen_KL_mean": -0.25988006591796875, "KL/mean": -0.8018027544021606, "KL/rejected_KL_mean": -1.3437271118164062, "KL/std": 1.322080373764038, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.083847165107727, "fcm_dpo/q_t": 0.4733179807662964, "grad_norm": 28.59319305419922, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.16929276287555695, "logits/rejected": 0.3096129298210144, "logps/chosen": -85.9921875, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.30948638916016, "loss": 1.2884, "margin_dpo/margin_mean": 1.0838478803634644, "margin_dpo/margin_std": 1.718015193939209, "step": 61 }, { "KL/chosen_KL_mean": -0.14051437377929688, "KL/mean": -0.7414339184761047, "KL/rejected_KL_mean": -1.3423500061035156, "KL/std": 1.4131072759628296, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2018373012542725, "fcm_dpo/q_t": 0.4702993631362915, "grad_norm": 28.47686767578125, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.3492276966571808, "logits/rejected": 0.5544486045837402, "logps/chosen": -81.77589416503906, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.38066101074219, "loss": 1.279, "margin_dpo/margin_mean": 1.2018367052078247, "margin_dpo/margin_std": 1.8115359544754028, "step": 62 }, { "KL/chosen_KL_mean": -0.030788421630859375, "KL/mean": -0.7764010429382324, "KL/rejected_KL_mean": -1.5220184326171875, "KL/std": 1.341199278831482, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.491230845451355, "fcm_dpo/q_t": 0.46307259798049927, "grad_norm": 35.11778259277344, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.428670734167099, "logits/rejected": 0.6436530351638794, "logps/chosen": -103.65484619140625, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.925048828125, "loss": 1.2491, "margin_dpo/margin_mean": 1.4912304878234863, "margin_dpo/margin_std": 1.5769506692886353, "step": 63 }, { "KL/chosen_KL_mean": -0.10185623168945312, "KL/mean": -0.9925527572631836, "KL/rejected_KL_mean": -1.8832588195800781, "KL/std": 1.8783800601959229, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.2899116277694702, "fcm_dpo/delta": 0.11999016255140305, "fcm_dpo/margin": 1.781402826309204, "fcm_dpo/q_t": 0.43315911293029785, "grad_norm": 65.60782623291016, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.47705525159835815, "logits/rejected": 0.6941580176353455, "logps/chosen": -87.10343933105469, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.46873474121094, "loss": 1.0305, "margin_dpo/margin_mean": 1.7814027070999146, "margin_dpo/margin_std": 2.1789302825927734, "step": 64 }, { "KL/chosen_KL_mean": -0.13920211791992188, "KL/mean": -0.9870806932449341, "KL/rejected_KL_mean": -1.8349609375, "KL/std": 1.619195580482483, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.2169416844844818, "fcm_dpo/delta": -0.029760468751192093, "fcm_dpo/margin": 1.6957579851150513, "fcm_dpo/q_t": 0.4012138545513153, "grad_norm": 64.99551391601562, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.010010235011577606, "logits/rejected": 0.1342170685529709, "logps/chosen": -91.36111450195312, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.17049407958984, "loss": 1.0966, "margin_dpo/margin_mean": 1.69575834274292, "margin_dpo/margin_std": 1.9860968589782715, "step": 65 }, { "KL/chosen_KL_mean": -0.31231689453125, "KL/mean": -0.9466223120689392, "KL/rejected_KL_mean": -1.5809249877929688, "KL/std": 1.6071903705596924, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.1957913190126419, "fcm_dpo/delta": -0.023041775450110435, "fcm_dpo/margin": 1.268608570098877, "fcm_dpo/q_t": 0.43200498819351196, "grad_norm": 43.137229919433594, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.001981348730623722, "logits/rejected": 0.196447491645813, "logps/chosen": -84.09654235839844, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.33302307128906, "loss": 1.1883, "margin_dpo/margin_mean": 1.2686089277267456, "margin_dpo/margin_std": 1.8867642879486084, "step": 66 }, { "KL/chosen_KL_mean": -0.8941574096679688, "KL/mean": -1.472898006439209, "KL/rejected_KL_mean": -2.0516319274902344, "KL/std": 2.067988395690918, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.35979416966438293, "fcm_dpo/delta": 0.03292868658900261, "fcm_dpo/margin": 1.1574702262878418, "fcm_dpo/q_t": 0.43413203954696655, "grad_norm": 89.7978744506836, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.5418111681938171, "logits/rejected": 0.837544322013855, "logps/chosen": -88.56712341308594, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.13838195800781, "loss": 1.2103, "margin_dpo/margin_mean": 1.1574699878692627, "margin_dpo/margin_std": 2.794504165649414, "step": 67 }, { "KL/chosen_KL_mean": -0.2670097351074219, "KL/mean": -1.2747807502746582, "KL/rejected_KL_mean": -2.2825393676757812, "KL/std": 2.147158145904541, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.19313547015190125, "fcm_dpo/delta": -0.007606826722621918, "fcm_dpo/margin": 2.0155303478240967, "fcm_dpo/q_t": 0.40916287899017334, "grad_norm": 43.49911117553711, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.09655466675758362, "logits/rejected": 0.18649354577064514, "logps/chosen": -97.34585571289062, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.4012680053711, "loss": 1.104, "margin_dpo/margin_mean": 2.015530824661255, "margin_dpo/margin_std": 2.661637306213379, "step": 68 }, { "KL/chosen_KL_mean": -0.19721603393554688, "KL/mean": -1.3244526386260986, "KL/rejected_KL_mean": -2.4516868591308594, "KL/std": 2.434475898742676, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.16623175144195557, "fcm_dpo/delta": -0.02176831290125847, "fcm_dpo/margin": 2.2544713020324707, "fcm_dpo/q_t": 0.4042189419269562, "grad_norm": 41.069671630859375, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.5196509957313538, "logits/rejected": 0.6236632466316223, "logps/chosen": -85.91693115234375, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.93189239501953, "loss": 1.1084, "margin_dpo/margin_mean": 2.254472255706787, "margin_dpo/margin_std": 3.0831375122070312, "step": 69 }, { "KL/chosen_KL_mean": -0.6268424987792969, "KL/mean": -1.8375498056411743, "KL/rejected_KL_mean": -3.048259735107422, "KL/std": 2.64456844329834, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.16262836754322052, "fcm_dpo/delta": -0.0052594877779483795, "fcm_dpo/margin": 2.4214158058166504, "fcm_dpo/q_t": 0.40794846415519714, "grad_norm": 39.359798431396484, "learning_rate": 5e-07, "logits/chosen": 0.3382246792316437, "logits/rejected": 0.5159094929695129, "logps/chosen": -95.63679504394531, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.260986328125, "loss": 1.1053, "margin_dpo/margin_mean": 2.421415328979492, "margin_dpo/margin_std": 3.2600908279418945, "step": 70 }, { "KL/chosen_KL_mean": 0.025638580322265625, "KL/mean": -1.7682576179504395, "KL/rejected_KL_mean": -3.562152862548828, "KL/std": 3.3900156021118164, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": -0.19697315990924835, "fcm_dpo/margin": 3.5877926349639893, "fcm_dpo/q_t": 0.40759119391441345, "grad_norm": 12.852182388305664, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.0033807819709181786, "logits/rejected": 0.3664165735244751, "logps/chosen": -90.95171356201172, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.15318298339844, "loss": 1.295, "margin_dpo/margin_mean": 3.58779239654541, "margin_dpo/margin_std": 3.88043212890625, "step": 71 }, { "KL/chosen_KL_mean": -0.4679679870605469, "KL/mean": -1.8547499179840088, "KL/rejected_KL_mean": -3.2415390014648438, "KL/std": 2.812838554382324, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.7735695838928223, "fcm_dpo/q_t": 0.48144084215164185, "grad_norm": 10.203835487365723, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.2300165593624115, "logits/rejected": 0.07958346605300903, "logps/chosen": -98.3594970703125, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.43325805664062, "loss": 1.3152, "margin_dpo/margin_mean": 2.773569107055664, "margin_dpo/margin_std": 3.2940104007720947, "step": 72 }, { "KL/chosen_KL_mean": -0.7196731567382812, "KL/mean": -2.206515312194824, "KL/rejected_KL_mean": -3.6933555603027344, "KL/std": 3.661653995513916, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9736855030059814, "fcm_dpo/q_t": 0.4801853895187378, "grad_norm": 10.998189926147461, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.37059852480888367, "logits/rejected": -0.07341644912958145, "logps/chosen": -96.48869323730469, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.62579345703125, "loss": 1.3121, "margin_dpo/margin_mean": 2.9736852645874023, "margin_dpo/margin_std": 4.740559101104736, "step": 73 }, { "KL/chosen_KL_mean": -0.9126091003417969, "KL/mean": -2.6155929565429688, "KL/rejected_KL_mean": -4.318576812744141, "KL/std": 3.6316914558410645, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4059605598449707, "fcm_dpo/q_t": 0.47725576162338257, "grad_norm": 12.19267463684082, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.2862359881401062, "logits/rejected": -0.1447485387325287, "logps/chosen": -79.72100830078125, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -105.96534729003906, "loss": 1.3002, "margin_dpo/margin_mean": 3.4059600830078125, "margin_dpo/margin_std": 4.253868103027344, "step": 74 }, { "KL/chosen_KL_mean": -0.9842529296875, "KL/mean": -3.1885385513305664, "KL/rejected_KL_mean": -5.392822265625, "KL/std": 4.431934356689453, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.408570289611816, "fcm_dpo/q_t": 0.47067025303840637, "grad_norm": 13.740071296691895, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.2818906307220459, "logits/rejected": -0.08002206683158875, "logps/chosen": -79.26610565185547, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -120.79593658447266, "loss": 1.2765, "margin_dpo/margin_mean": 4.408570289611816, "margin_dpo/margin_std": 5.263299942016602, "step": 75 }, { "KL/chosen_KL_mean": -1.2972373962402344, "KL/mean": -2.8007211685180664, "KL/rejected_KL_mean": -4.304210662841797, "KL/std": 3.660342216491699, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.0069775581359863, "fcm_dpo/q_t": 0.47994327545166016, "grad_norm": 7.791311264038086, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.5875853300094604, "logits/rejected": -0.21344146132469177, "logps/chosen": -89.17437744140625, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.02389526367188, "loss": 1.3108, "margin_dpo/margin_mean": 3.0069773197174072, "margin_dpo/margin_std": 4.433416366577148, "step": 76 }, { "KL/chosen_KL_mean": -1.5638465881347656, "KL/mean": -3.186514377593994, "KL/rejected_KL_mean": -4.8091888427734375, "KL/std": 4.263186931610107, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.245338201522827, "fcm_dpo/q_t": 0.47834035754203796, "grad_norm": 10.043089866638184, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.24897505342960358, "logits/rejected": 0.11017333716154099, "logps/chosen": -101.26419067382812, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.01495361328125, "loss": 1.3053, "margin_dpo/margin_mean": 3.245338201522827, "margin_dpo/margin_std": 4.854216575622559, "step": 77 }, { "KL/chosen_KL_mean": -0.6364631652832031, "KL/mean": -2.3863377571105957, "KL/rejected_KL_mean": -4.136207580566406, "KL/std": 3.578500747680664, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4997406005859375, "fcm_dpo/q_t": 0.4766370356082916, "grad_norm": 9.772367477416992, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.09509723633527756, "logits/rejected": 0.32817816734313965, "logps/chosen": -90.93317413330078, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.27393341064453, "loss": 1.2978, "margin_dpo/margin_mean": 3.4997410774230957, "margin_dpo/margin_std": 4.289996147155762, "step": 78 }, { "KL/chosen_KL_mean": -0.8145217895507812, "KL/mean": -3.197850465774536, "KL/rejected_KL_mean": -5.581184387207031, "KL/std": 5.2968244552612305, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.766660690307617, "fcm_dpo/q_t": 0.4683226943016052, "grad_norm": 12.005343437194824, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.5704345703125, "logits/rejected": -0.18759649991989136, "logps/chosen": -87.1928482055664, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.68894958496094, "loss": 1.2686, "margin_dpo/margin_mean": 4.766660690307617, "margin_dpo/margin_std": 5.902681827545166, "step": 79 }, { "KL/chosen_KL_mean": -1.4987678527832031, "KL/mean": -3.3575353622436523, "KL/rejected_KL_mean": -5.216300964355469, "KL/std": 4.6430816650390625, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.7175345420837402, "fcm_dpo/q_t": 0.4752568006515503, "grad_norm": 10.531587600708008, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.8498926758766174, "logits/rejected": -0.5098190903663635, "logps/chosen": -95.46908569335938, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.79071807861328, "loss": 1.2937, "margin_dpo/margin_mean": 3.7175345420837402, "margin_dpo/margin_std": 5.0693254470825195, "step": 80 }, { "KL/chosen_KL_mean": -1.0690956115722656, "KL/mean": -3.6927976608276367, "KL/rejected_KL_mean": -6.316501617431641, "KL/std": 5.8687968254089355, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.247396469116211, "fcm_dpo/q_t": 0.4653970003128052, "grad_norm": 13.076894760131836, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5759814381599426, "logits/rejected": -0.15155600011348724, "logps/chosen": -100.89921569824219, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -139.9889678955078, "loss": 1.2595, "margin_dpo/margin_mean": 5.247397422790527, "margin_dpo/margin_std": 7.016992092132568, "step": 81 }, { "KL/chosen_KL_mean": -1.979583740234375, "KL/mean": -3.773425579071045, "KL/rejected_KL_mean": -5.567264556884766, "KL/std": 4.495739459991455, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.587681531906128, "fcm_dpo/q_t": 0.4760867655277252, "grad_norm": 9.986774444580078, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.29466521739959717, "logits/rejected": -0.0805768370628357, "logps/chosen": -85.02556610107422, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.09321594238281, "loss": 1.2969, "margin_dpo/margin_mean": 3.587681293487549, "margin_dpo/margin_std": 4.903266906738281, "step": 82 }, { "KL/chosen_KL_mean": -2.3032913208007812, "KL/mean": -4.400589942932129, "KL/rejected_KL_mean": -6.497886657714844, "KL/std": 5.237455368041992, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.194599151611328, "fcm_dpo/q_t": 0.4721631407737732, "grad_norm": 10.017520904541016, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.5145091414451599, "logits/rejected": -0.09129942208528519, "logps/chosen": -94.47950744628906, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.2425308227539, "loss": 1.2845, "margin_dpo/margin_mean": 4.194599151611328, "margin_dpo/margin_std": 6.56648063659668, "step": 83 }, { "KL/chosen_KL_mean": -2.9902420043945312, "KL/mean": -4.29372501373291, "KL/rejected_KL_mean": -5.5972137451171875, "KL/std": 4.738142967224121, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.6069717407226562, "fcm_dpo/q_t": 0.4826567769050598, "grad_norm": 8.190064430236816, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.5621448755264282, "logits/rejected": -0.17918795347213745, "logps/chosen": -104.51673889160156, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.02330017089844, "loss": 1.3226, "margin_dpo/margin_mean": 2.6069722175598145, "margin_dpo/margin_std": 5.041725158691406, "step": 84 }, { "KL/chosen_KL_mean": -2.654682159423828, "KL/mean": -4.414559364318848, "KL/rejected_KL_mean": -6.1744384765625, "KL/std": 5.240026950836182, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5197579860687256, "fcm_dpo/q_t": 0.4766012728214264, "grad_norm": 7.745766639709473, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.37055906653404236, "logits/rejected": -0.03429099917411804, "logps/chosen": -98.78205871582031, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.22962951660156, "loss": 1.3013, "margin_dpo/margin_mean": 3.5197582244873047, "margin_dpo/margin_std": 6.349936008453369, "step": 85 }, { "KL/chosen_KL_mean": -2.127002716064453, "KL/mean": -4.4175801277160645, "KL/rejected_KL_mean": -6.708164215087891, "KL/std": 6.120536804199219, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.5811614990234375, "fcm_dpo/q_t": 0.4698545038700104, "grad_norm": 9.386637687683105, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.465280681848526, "logits/rejected": -0.21400773525238037, "logps/chosen": -83.83126831054688, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.36366271972656, "loss": 1.2775, "margin_dpo/margin_mean": 4.581161022186279, "margin_dpo/margin_std": 7.631292819976807, "step": 86 }, { "KL/chosen_KL_mean": -2.496784210205078, "KL/mean": -4.9431657791137695, "KL/rejected_KL_mean": -7.3895416259765625, "KL/std": 6.742612838745117, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.89276123046875, "fcm_dpo/q_t": 0.46753138303756714, "grad_norm": 9.791629791259766, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.911993145942688, "logits/rejected": -0.5388570427894592, "logps/chosen": -95.18275451660156, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.30772399902344, "loss": 1.271, "margin_dpo/margin_mean": 4.892762184143066, "margin_dpo/margin_std": 7.999984264373779, "step": 87 }, { "KL/chosen_KL_mean": -3.1674423217773438, "KL/mean": -5.111578941345215, "KL/rejected_KL_mean": -7.055713653564453, "KL/std": 6.611425399780273, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.88826322555542, "fcm_dpo/q_t": 0.4742400646209717, "grad_norm": 9.02611255645752, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.15880034863948822, "logits/rejected": 0.037292104214429855, "logps/chosen": -103.930419921875, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -120.20608520507812, "loss": 1.2935, "margin_dpo/margin_mean": 3.88826322555542, "margin_dpo/margin_std": 6.930730819702148, "step": 88 }, { "KL/chosen_KL_mean": -2.6511611938476562, "KL/mean": -4.757400989532471, "KL/rejected_KL_mean": -6.863643646240234, "KL/std": 5.618893146514893, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.212480068206787, "fcm_dpo/q_t": 0.4723007082939148, "grad_norm": 9.82164192199707, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.6959099173545837, "logits/rejected": -0.3489062488079071, "logps/chosen": -92.34223937988281, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.43197631835938, "loss": 1.2863, "margin_dpo/margin_mean": 4.212479591369629, "margin_dpo/margin_std": 7.457820892333984, "step": 89 }, { "KL/chosen_KL_mean": -2.461589813232422, "KL/mean": -5.383166313171387, "KL/rejected_KL_mean": -8.304740905761719, "KL/std": 6.9413251876831055, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.843151092529297, "fcm_dpo/q_t": 0.4615885615348816, "grad_norm": 13.454115867614746, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6775976419448853, "logits/rejected": -0.5495983362197876, "logps/chosen": -83.84414672851562, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -126.08187866210938, "loss": 1.2484, "margin_dpo/margin_mean": 5.843151092529297, "margin_dpo/margin_std": 8.416637420654297, "step": 90 }, { "KL/chosen_KL_mean": -2.9930152893066406, "KL/mean": -5.380769729614258, "KL/rejected_KL_mean": -7.768524169921875, "KL/std": 6.850108623504639, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.7755126953125, "fcm_dpo/q_t": 0.46861329674720764, "grad_norm": 8.811869621276855, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.42839449644088745, "logits/rejected": 0.010534651577472687, "logps/chosen": -100.21489715576172, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.74526977539062, "loss": 1.2741, "margin_dpo/margin_mean": 4.7755126953125, "margin_dpo/margin_std": 8.200583457946777, "step": 91 }, { "KL/chosen_KL_mean": -3.1302566528320312, "KL/mean": -5.274267196655273, "KL/rejected_KL_mean": -7.418270111083984, "KL/std": 6.758179664611816, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.288013458251953, "fcm_dpo/q_t": 0.4716838598251343, "grad_norm": 9.424541473388672, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.9399533271789551, "logits/rejected": -0.5639652013778687, "logps/chosen": -87.66053771972656, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.91958618164062, "loss": 1.284, "margin_dpo/margin_mean": 4.288012504577637, "margin_dpo/margin_std": 7.254549980163574, "step": 92 }, { "KL/chosen_KL_mean": -3.572887420654297, "KL/mean": -5.562145233154297, "KL/rejected_KL_mean": -7.5514068603515625, "KL/std": 6.539409160614014, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.02684980072081089, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.9785189628601074, "fcm_dpo/q_t": 0.4737132489681244, "grad_norm": 8.088454246520996, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.9244620203971863, "logits/rejected": -0.3181036710739136, "logps/chosen": -93.82331848144531, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -91.64562225341797, "loss": 1.2937, "margin_dpo/margin_mean": 3.9785189628601074, "margin_dpo/margin_std": 7.918880462646484, "step": 93 }, { "KL/chosen_KL_mean": -3.9812850952148438, "KL/mean": -7.084136009216309, "KL/rejected_KL_mean": -10.186988830566406, "KL/std": 7.927455902099609, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.08922934532165527, "fcm_dpo/delta": 0.14126655459403992, "fcm_dpo/margin": 6.205711364746094, "fcm_dpo/q_t": 0.43943023681640625, "grad_norm": 28.482498168945312, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.8034812808036804, "logits/rejected": -0.48460859060287476, "logps/chosen": -96.43550109863281, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -115.43428039550781, "loss": 1.087, "margin_dpo/margin_mean": 6.205710411071777, "margin_dpo/margin_std": 9.258569717407227, "step": 94 }, { "KL/chosen_KL_mean": -3.1143417358398438, "KL/mean": -5.587283134460449, "KL/rejected_KL_mean": -8.060218811035156, "KL/std": 6.550907135009766, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.06798820942640305, "fcm_dpo/delta": -0.09557369351387024, "fcm_dpo/margin": 4.945874214172363, "fcm_dpo/q_t": 0.398811399936676, "grad_norm": 19.387113571166992, "learning_rate": 4.979441529392784e-07, "logits/chosen": -1.077991008758545, "logits/rejected": -0.730659008026123, "logps/chosen": -81.98805236816406, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -91.65142822265625, "loss": 1.1652, "margin_dpo/margin_mean": 4.945874214172363, "margin_dpo/margin_std": 8.124834060668945, "step": 95 }, { "KL/chosen_KL_mean": -3.0191726684570312, "KL/mean": -6.114728927612305, "KL/rejected_KL_mean": -9.210289001464844, "KL/std": 7.959561824798584, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.07466340065002441, "fcm_dpo/delta": 0.06181112676858902, "fcm_dpo/margin": 6.191125869750977, "fcm_dpo/q_t": 0.4224807024002075, "grad_norm": 19.158327102661133, "learning_rate": 4.977766449015534e-07, "logits/chosen": -1.0653278827667236, "logits/rejected": -0.6107670068740845, "logps/chosen": -109.61133575439453, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -110.97831726074219, "loss": 1.076, "margin_dpo/margin_mean": 6.191125869750977, "margin_dpo/margin_std": 8.967657089233398, "step": 96 }, { "KL/chosen_KL_mean": -4.30352783203125, "KL/mean": -6.982109069824219, "KL/rejected_KL_mean": -9.660690307617188, "KL/std": 7.105029582977295, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.07914964854717255, "fcm_dpo/delta": -0.29934895038604736, "fcm_dpo/margin": 5.357160568237305, "fcm_dpo/q_t": 0.36751437187194824, "grad_norm": 49.182891845703125, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.7157871723175049, "logits/rejected": -0.30353012681007385, "logps/chosen": -92.98341369628906, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -94.47298431396484, "loss": 1.2817, "margin_dpo/margin_mean": 5.357159614562988, "margin_dpo/margin_std": 7.681626319885254, "step": 97 }, { "KL/chosen_KL_mean": -3.354320526123047, "KL/mean": -6.616291046142578, "KL/rejected_KL_mean": -9.878257751464844, "KL/std": 6.736590385437012, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.523929595947266, "fcm_dpo/q_t": 0.49357253313064575, "grad_norm": 1.8291617631912231, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.971349835395813, "logits/rejected": -0.6960892081260681, "logps/chosen": -95.59681701660156, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -111.39774322509766, "loss": 1.361, "margin_dpo/margin_mean": 6.523929595947266, "margin_dpo/margin_std": 7.5871663093566895, "step": 98 }, { "KL/chosen_KL_mean": -2.650562286376953, "KL/mean": -6.3315582275390625, "KL/rejected_KL_mean": -10.012557983398438, "KL/std": 8.302359580993652, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.361995220184326, "fcm_dpo/q_t": 0.49275022745132446, "grad_norm": 2.3475043773651123, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.739467442035675, "logits/rejected": -0.5279399156570435, "logps/chosen": -74.83521270751953, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -101.89387512207031, "loss": 1.3579, "margin_dpo/margin_mean": 7.361995220184326, "margin_dpo/margin_std": 10.69651985168457, "step": 99 }, { "KL/chosen_KL_mean": -3.303607940673828, "KL/mean": -5.747438430786133, "KL/rejected_KL_mean": -8.191272735595703, "KL/std": 6.2550177574157715, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.8876633644104, "fcm_dpo/q_t": 0.4951840937137604, "grad_norm": 1.2957541942596436, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.9040879011154175, "logits/rejected": -0.39069873094558716, "logps/chosen": -92.81743621826172, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -89.40840148925781, "loss": 1.3673, "margin_dpo/margin_mean": 4.887662887573242, "margin_dpo/margin_std": 6.744594573974609, "step": 100 }, { "KL/chosen_KL_mean": -4.970386505126953, "KL/mean": -7.274693489074707, "KL/rejected_KL_mean": -9.578998565673828, "KL/std": 8.564298629760742, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.608610153198242, "fcm_dpo/q_t": 0.49545907974243164, "grad_norm": 1.5817487239837646, "learning_rate": 4.968412618365215e-07, "logits/chosen": -1.048797607421875, "logits/rejected": -0.7618355751037598, "logps/chosen": -99.37358093261719, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -99.489013671875, "loss": 1.3685, "margin_dpo/margin_mean": 4.608610153198242, "margin_dpo/margin_std": 9.184547424316406, "step": 101 }, { "KL/chosen_KL_mean": -6.0095062255859375, "KL/mean": -7.967520236968994, "KL/rejected_KL_mean": -9.925533294677734, "KL/std": 8.757617950439453, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.9160349369049072, "fcm_dpo/q_t": 0.4961429238319397, "grad_norm": 1.3767644166946411, "learning_rate": 4.966346511559149e-07, "logits/chosen": -1.0467605590820312, "logits/rejected": -0.6352392435073853, "logps/chosen": -97.61892700195312, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -85.6944580078125, "loss": 1.3713, "margin_dpo/margin_mean": 3.916034698486328, "margin_dpo/margin_std": 10.163015365600586, "step": 102 }, { "KL/chosen_KL_mean": -5.156093597412109, "KL/mean": -8.714576721191406, "KL/rejected_KL_mean": -12.273063659667969, "KL/std": 9.425952911376953, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.11696720123291, "fcm_dpo/q_t": 0.49298959970474243, "grad_norm": 1.790459394454956, "learning_rate": 4.964215414228785e-07, "logits/chosen": -1.1928706169128418, "logits/rejected": -0.7721550464630127, "logps/chosen": -102.55847930908203, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -111.25474548339844, "loss": 1.3589, "margin_dpo/margin_mean": 7.11696720123291, "margin_dpo/margin_std": 10.748941421508789, "step": 103 }, { "KL/chosen_KL_mean": -4.559867858886719, "KL/mean": -8.352380752563477, "KL/rejected_KL_mean": -12.144889831542969, "KL/std": 9.972511291503906, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.585015296936035, "fcm_dpo/q_t": 0.49253207445144653, "grad_norm": 1.7629283666610718, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.3680508136749268, "logits/rejected": -0.9734266996383667, "logps/chosen": -99.41622924804688, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -109.17158508300781, "loss": 1.3572, "margin_dpo/margin_mean": 7.585015296936035, "margin_dpo/margin_std": 12.237903594970703, "step": 104 }, { "KL/chosen_KL_mean": -5.988334655761719, "KL/mean": -9.985481262207031, "KL/rejected_KL_mean": -13.98263168334961, "KL/std": 9.38239860534668, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.994292736053467, "fcm_dpo/q_t": 0.49212610721588135, "grad_norm": 2.058182716369629, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.4645403623580933, "logits/rejected": -0.9535760879516602, "logps/chosen": -102.74623107910156, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -115.56011962890625, "loss": 1.3556, "margin_dpo/margin_mean": 7.994292736053467, "margin_dpo/margin_std": 11.662306785583496, "step": 105 }, { "KL/chosen_KL_mean": -6.178928375244141, "KL/mean": -9.716201782226562, "KL/rejected_KL_mean": -13.253471374511719, "KL/std": 9.75356674194336, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.074536323547363, "fcm_dpo/q_t": 0.4930341839790344, "grad_norm": 1.6064693927764893, "learning_rate": 4.957432749209755e-07, "logits/chosen": -1.214765191078186, "logits/rejected": -0.7557087540626526, "logps/chosen": -98.61683654785156, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -104.26316833496094, "loss": 1.3592, "margin_dpo/margin_mean": 7.074536323547363, "margin_dpo/margin_std": 12.676677703857422, "step": 106 }, { "KL/chosen_KL_mean": -5.604583740234375, "KL/mean": -9.965259552001953, "KL/rejected_KL_mean": -14.325939178466797, "KL/std": 12.738728523254395, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.721357345581055, "fcm_dpo/q_t": 0.4914305806159973, "grad_norm": 1.8672988414764404, "learning_rate": 4.955042268449307e-07, "logits/chosen": -1.2397130727767944, "logits/rejected": -0.6523370742797852, "logps/chosen": -110.7295913696289, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -115.50843811035156, "loss": 1.3533, "margin_dpo/margin_mean": 8.721357345581055, "margin_dpo/margin_std": 15.56640911102295, "step": 107 }, { "KL/chosen_KL_mean": -6.963321685791016, "KL/mean": -11.544520378112793, "KL/rejected_KL_mean": -16.125713348388672, "KL/std": 13.545372009277344, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.162386894226074, "fcm_dpo/q_t": 0.4909858703613281, "grad_norm": 2.371248483657837, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.5895195007324219, "logits/rejected": -1.1092265844345093, "logps/chosen": -96.17268371582031, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -117.96356201171875, "loss": 1.3515, "margin_dpo/margin_mean": 9.16238784790039, "margin_dpo/margin_std": 15.416194915771484, "step": 108 }, { "KL/chosen_KL_mean": -6.225807189941406, "KL/mean": -11.49411392211914, "KL/rejected_KL_mean": -16.76242446899414, "KL/std": 12.210805892944336, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.53661060333252, "fcm_dpo/q_t": 0.48963087797164917, "grad_norm": 2.0009870529174805, "learning_rate": 4.95006729368358e-07, "logits/chosen": -1.0352593660354614, "logits/rejected": -0.7654536962509155, "logps/chosen": -106.41289520263672, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -120.29981994628906, "loss": 1.3462, "margin_dpo/margin_mean": 10.53661060333252, "margin_dpo/margin_std": 15.981732368469238, "step": 109 }, { "KL/chosen_KL_mean": -8.133296966552734, "KL/mean": -10.793672561645508, "KL/rejected_KL_mean": -13.454055786132812, "KL/std": 11.326553344726562, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.320757865905762, "fcm_dpo/q_t": 0.49476101994514465, "grad_norm": 1.5833492279052734, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.7530525326728821, "logits/rejected": -0.2550654709339142, "logps/chosen": -115.02901458740234, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -99.2652587890625, "loss": 1.3661, "margin_dpo/margin_mean": 5.3207573890686035, "margin_dpo/margin_std": 12.921648025512695, "step": 110 }, { "KL/chosen_KL_mean": -7.288822174072266, "KL/mean": -13.54676628112793, "KL/rejected_KL_mean": -19.804710388183594, "KL/std": 14.473100662231445, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.515886306762695, "fcm_dpo/q_t": 0.4876821041107178, "grad_norm": 2.095782995223999, "learning_rate": 4.944834074412042e-07, "logits/chosen": -1.5955216884613037, "logits/rejected": -1.2875690460205078, "logps/chosen": -100.61819458007812, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -123.98219299316406, "loss": 1.3388, "margin_dpo/margin_mean": 12.515885353088379, "margin_dpo/margin_std": 17.623064041137695, "step": 111 }, { "KL/chosen_KL_mean": -7.8140716552734375, "KL/mean": -10.702799797058105, "KL/rejected_KL_mean": -13.59152603149414, "KL/std": 8.969317436218262, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.7774577140808105, "fcm_dpo/q_t": 0.49431025981903076, "grad_norm": 1.441429853439331, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.8942629098892212, "logits/rejected": -1.3193554878234863, "logps/chosen": -90.56111145019531, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -85.14997863769531, "loss": 1.3643, "margin_dpo/margin_mean": 5.777458190917969, "margin_dpo/margin_std": 12.472896575927734, "step": 112 }, { "KL/chosen_KL_mean": -9.836822509765625, "KL/mean": -13.043176651000977, "KL/rejected_KL_mean": -16.249526977539062, "KL/std": 12.523353576660156, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.412707328796387, "fcm_dpo/q_t": 0.4936884641647339, "grad_norm": 1.7968049049377441, "learning_rate": 4.939343162231841e-07, "logits/chosen": -1.6148682832717896, "logits/rejected": -1.023512601852417, "logps/chosen": -107.73631286621094, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -100.66912841796875, "loss": 1.3619, "margin_dpo/margin_mean": 6.412707328796387, "margin_dpo/margin_std": 13.005274772644043, "step": 113 }, { "KL/chosen_KL_mean": -9.579078674316406, "KL/mean": -14.345686912536621, "KL/rejected_KL_mean": -19.112293243408203, "KL/std": 13.567691802978516, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.533212661743164, "fcm_dpo/q_t": 0.49062275886535645, "grad_norm": 1.9230034351348877, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.7460644245147705, "logits/rejected": -1.139358639717102, "logps/chosen": -106.0741958618164, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -111.12765502929688, "loss": 1.3504, "margin_dpo/margin_mean": 9.533212661743164, "margin_dpo/margin_std": 18.38678550720215, "step": 114 }, { "KL/chosen_KL_mean": -10.882068634033203, "KL/mean": -15.692159652709961, "KL/rejected_KL_mean": -20.502243041992188, "KL/std": 15.815770149230957, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.620177268981934, "fcm_dpo/q_t": 0.4905366897583008, "grad_norm": 1.9674190282821655, "learning_rate": 4.933595135901732e-07, "logits/chosen": -1.5452475547790527, "logits/rejected": -1.0950497388839722, "logps/chosen": -114.06219482421875, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -126.21820068359375, "loss": 1.3501, "margin_dpo/margin_mean": 9.6201753616333, "margin_dpo/margin_std": 18.11780548095703, "step": 115 }, { "KL/chosen_KL_mean": -8.899002075195312, "KL/mean": -13.218119621276855, "KL/rejected_KL_mean": -17.537235260009766, "KL/std": 12.484780311584473, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.63823127746582, "fcm_dpo/q_t": 0.4914973974227905, "grad_norm": 1.8663828372955322, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.720348834991455, "logits/rejected": -1.300096035003662, "logps/chosen": -90.057373046875, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -103.84309387207031, "loss": 1.3533, "margin_dpo/margin_mean": 8.63823127746582, "margin_dpo/margin_std": 13.769462585449219, "step": 116 }, { "KL/chosen_KL_mean": -11.259132385253906, "KL/mean": -13.59769058227539, "KL/rejected_KL_mean": -15.936256408691406, "KL/std": 12.77949333190918, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.677123546600342, "fcm_dpo/q_t": 0.4953954219818115, "grad_norm": 1.7530142068862915, "learning_rate": 4.927590601281083e-07, "logits/chosen": -1.4592324495315552, "logits/rejected": -0.9577888250350952, "logps/chosen": -108.340087890625, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -91.076171875, "loss": 1.3691, "margin_dpo/margin_mean": 4.6771240234375, "margin_dpo/margin_std": 17.432037353515625, "step": 117 }, { "KL/chosen_KL_mean": -8.619972229003906, "KL/mean": -13.368680953979492, "KL/rejected_KL_mean": -18.117382049560547, "KL/std": 13.58875846862793, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.497408866882324, "fcm_dpo/q_t": 0.49065154790878296, "grad_norm": 1.7460366487503052, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.8650840520858765, "logits/rejected": -1.515581488609314, "logps/chosen": -95.15620422363281, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -101.05302429199219, "loss": 1.3501, "margin_dpo/margin_mean": 9.497407913208008, "margin_dpo/margin_std": 15.009272575378418, "step": 118 }, { "KL/chosen_KL_mean": -12.024051666259766, "KL/mean": -17.345012664794922, "KL/rejected_KL_mean": -22.665966033935547, "KL/std": 16.26428985595703, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.641914367675781, "fcm_dpo/q_t": 0.48952963948249817, "grad_norm": 2.106955051422119, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.8704497814178467, "logits/rejected": -1.5009846687316895, "logps/chosen": -116.63313293457031, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -131.8237762451172, "loss": 1.3465, "margin_dpo/margin_mean": 10.641915321350098, "margin_dpo/margin_std": 20.992694854736328, "step": 119 }, { "KL/chosen_KL_mean": -12.3184814453125, "KL/mean": -18.373321533203125, "KL/rejected_KL_mean": -24.42816162109375, "KL/std": 18.914430618286133, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.1096830368042, "fcm_dpo/q_t": 0.48810136318206787, "grad_norm": 1.971928596496582, "learning_rate": 4.918104238142103e-07, "logits/chosen": -2.0691912174224854, "logits/rejected": -1.4720251560211182, "logps/chosen": -127.00590515136719, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -135.61402893066406, "loss": 1.3415, "margin_dpo/margin_mean": 12.1096830368042, "margin_dpo/margin_std": 24.588058471679688, "step": 120 }, { "KL/chosen_KL_mean": -10.267005920410156, "KL/mean": -16.96421241760254, "KL/rejected_KL_mean": -23.661415100097656, "KL/std": 15.638692855834961, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.39441967010498, "fcm_dpo/q_t": 0.48681768774986267, "grad_norm": 2.482743501663208, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.825352430343628, "logits/rejected": -1.4946660995483398, "logps/chosen": -102.3836669921875, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -138.63430786132812, "loss": 1.3354, "margin_dpo/margin_mean": 13.394418716430664, "margin_dpo/margin_std": 17.572250366210938, "step": 121 }, { "KL/chosen_KL_mean": -9.972522735595703, "KL/mean": -17.743377685546875, "KL/rejected_KL_mean": -25.51422882080078, "KL/std": 16.18366813659668, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.541709899902344, "fcm_dpo/q_t": 0.48471248149871826, "grad_norm": 2.4590680599212646, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.7944577932357788, "logits/rejected": -1.7713662385940552, "logps/chosen": -94.85009765625, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -136.5972137451172, "loss": 1.3275, "margin_dpo/margin_mean": 15.541708946228027, "margin_dpo/margin_std": 19.758499145507812, "step": 122 }, { "KL/chosen_KL_mean": -11.918022155761719, "KL/mean": -17.037946701049805, "KL/rejected_KL_mean": -22.15787124633789, "KL/std": 15.060951232910156, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.239845275878906, "fcm_dpo/q_t": 0.48992547392845154, "grad_norm": 1.8092713356018066, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.9298293590545654, "logits/rejected": -1.5948786735534668, "logps/chosen": -110.39677429199219, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -116.01204681396484, "loss": 1.3478, "margin_dpo/margin_mean": 10.239845275878906, "margin_dpo/margin_std": 18.941972732543945, "step": 123 }, { "KL/chosen_KL_mean": -12.795635223388672, "KL/mean": -19.518369674682617, "KL/rejected_KL_mean": -26.241104125976562, "KL/std": 19.489463806152344, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.44546890258789, "fcm_dpo/q_t": 0.4868032932281494, "grad_norm": 2.221646785736084, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.7650707960128784, "logits/rejected": -1.5195913314819336, "logps/chosen": -115.55853271484375, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -149.24671936035156, "loss": 1.3366, "margin_dpo/margin_mean": 13.445467948913574, "margin_dpo/margin_std": 25.682212829589844, "step": 124 }, { "KL/chosen_KL_mean": -12.746185302734375, "KL/mean": -16.97201156616211, "KL/rejected_KL_mean": -21.197845458984375, "KL/std": 15.874967575073242, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.451654434204102, "fcm_dpo/q_t": 0.4916868209838867, "grad_norm": 1.7263103723526, "learning_rate": 4.90102044194588e-07, "logits/chosen": -1.4998936653137207, "logits/rejected": -1.2371689081192017, "logps/chosen": -103.1724624633789, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -110.44686126708984, "loss": 1.3549, "margin_dpo/margin_mean": 8.451654434204102, "margin_dpo/margin_std": 20.155799865722656, "step": 125 }, { "KL/chosen_KL_mean": -12.49234390258789, "KL/mean": -19.045732498168945, "KL/rejected_KL_mean": -25.599124908447266, "KL/std": 18.871963500976562, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.106780052185059, "fcm_dpo/q_t": 0.48711156845092773, "grad_norm": 1.9026752710342407, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.5265045166015625, "logits/rejected": -1.2378771305084229, "logps/chosen": -106.91522216796875, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -129.48068237304688, "loss": 1.3371, "margin_dpo/margin_mean": 13.106779098510742, "margin_dpo/margin_std": 21.688739776611328, "step": 126 }, { "KL/chosen_KL_mean": -12.38589096069336, "KL/mean": -16.578889846801758, "KL/rejected_KL_mean": -20.77188491821289, "KL/std": 15.507426261901855, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.385990142822266, "fcm_dpo/q_t": 0.491754412651062, "grad_norm": 1.755142092704773, "learning_rate": 4.89374339765481e-07, "logits/chosen": -1.5169181823730469, "logits/rejected": -1.12302827835083, "logps/chosen": -97.13694763183594, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -104.31712341308594, "loss": 1.355, "margin_dpo/margin_mean": 8.385990142822266, "margin_dpo/margin_std": 19.524280548095703, "step": 127 }, { "KL/chosen_KL_mean": -12.665786743164062, "KL/mean": -17.327199935913086, "KL/rejected_KL_mean": -21.988611221313477, "KL/std": 19.077045440673828, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.322829246520996, "fcm_dpo/q_t": 0.49084585905075073, "grad_norm": 1.8369626998901367, "learning_rate": 4.890010211106795e-07, "logits/chosen": -1.7185537815093994, "logits/rejected": -1.1907461881637573, "logps/chosen": -103.28543090820312, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -105.09208679199219, "loss": 1.3524, "margin_dpo/margin_mean": 9.322829246520996, "margin_dpo/margin_std": 25.408164978027344, "step": 128 }, { "KL/chosen_KL_mean": -13.75457763671875, "KL/mean": -19.102842330932617, "KL/rejected_KL_mean": -24.45111846923828, "KL/std": 18.285858154296875, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.696538925170898, "fcm_dpo/q_t": 0.48952019214630127, "grad_norm": 1.8121739625930786, "learning_rate": 4.88621404556699e-07, "logits/chosen": -1.675736427307129, "logits/rejected": -1.3551983833312988, "logps/chosen": -112.67328643798828, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -127.73531341552734, "loss": 1.3473, "margin_dpo/margin_mean": 10.696538925170898, "margin_dpo/margin_std": 26.051410675048828, "step": 129 }, { "KL/chosen_KL_mean": -10.79001235961914, "KL/mean": -17.570369720458984, "KL/rejected_KL_mean": -24.350727081298828, "KL/std": 16.611814498901367, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.56071949005127, "fcm_dpo/q_t": 0.48666661977767944, "grad_norm": 1.943054437637329, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.9714510440826416, "logits/rejected": -1.773998737335205, "logps/chosen": -83.97370147705078, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -112.37226867675781, "loss": 1.3354, "margin_dpo/margin_mean": 13.560718536376953, "margin_dpo/margin_std": 21.191560745239258, "step": 130 }, { "KL/chosen_KL_mean": -10.031551361083984, "KL/mean": -17.580753326416016, "KL/rejected_KL_mean": -25.129966735839844, "KL/std": 18.671642303466797, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.098409652709961, "fcm_dpo/q_t": 0.4851546883583069, "grad_norm": 2.4563302993774414, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.9276299476623535, "logits/rejected": -1.7717530727386475, "logps/chosen": -83.98799133300781, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -119.5857925415039, "loss": 1.3296, "margin_dpo/margin_mean": 15.098409652709961, "margin_dpo/margin_std": 21.681339263916016, "step": 131 }, { "KL/chosen_KL_mean": -13.750251770019531, "KL/mean": -19.72500228881836, "KL/rejected_KL_mean": -25.699748992919922, "KL/std": 19.65793228149414, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.94949722290039, "fcm_dpo/q_t": 0.4882674217224121, "grad_norm": 2.0281028747558594, "learning_rate": 4.874448683603694e-07, "logits/chosen": -2.1728646755218506, "logits/rejected": -1.7506976127624512, "logps/chosen": -106.18973541259766, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -119.83226776123047, "loss": 1.3424, "margin_dpo/margin_mean": 11.94949722290039, "margin_dpo/margin_std": 26.210208892822266, "step": 132 }, { "KL/chosen_KL_mean": -13.485633850097656, "KL/mean": -19.977392196655273, "KL/rejected_KL_mean": -26.46915054321289, "KL/std": 18.46392059326172, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.983511924743652, "fcm_dpo/q_t": 0.48723775148391724, "grad_norm": 1.9585597515106201, "learning_rate": 4.870401618977415e-07, "logits/chosen": -2.3112425804138184, "logits/rejected": -1.9153029918670654, "logps/chosen": -106.602294921875, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -128.56405639648438, "loss": 1.3375, "margin_dpo/margin_mean": 12.983511924743652, "margin_dpo/margin_std": 21.228500366210938, "step": 133 }, { "KL/chosen_KL_mean": -13.026458740234375, "KL/mean": -20.040264129638672, "KL/rejected_KL_mean": -27.054061889648438, "KL/std": 17.41130828857422, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.027605056762695, "fcm_dpo/q_t": 0.4862053394317627, "grad_norm": 2.0930588245391846, "learning_rate": 4.866292092063986e-07, "logits/chosen": -2.4215569496154785, "logits/rejected": -2.1038706302642822, "logps/chosen": -96.57658386230469, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -120.93563079833984, "loss": 1.3334, "margin_dpo/margin_mean": 14.027604103088379, "margin_dpo/margin_std": 19.67420196533203, "step": 134 }, { "KL/chosen_KL_mean": -11.901660919189453, "KL/mean": -20.908950805664062, "KL/rejected_KL_mean": -29.916244506835938, "KL/std": 21.206796646118164, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.014572143554688, "fcm_dpo/q_t": 0.48234063386917114, "grad_norm": 2.970766067504883, "learning_rate": 4.862120211153265e-07, "logits/chosen": -2.422943592071533, "logits/rejected": -2.410861015319824, "logps/chosen": -89.48428344726562, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -150.97271728515625, "loss": 1.3196, "margin_dpo/margin_mean": 18.014572143554688, "margin_dpo/margin_std": 26.540422439575195, "step": 135 }, { "KL/chosen_KL_mean": -15.761180877685547, "KL/mean": -22.522605895996094, "KL/rejected_KL_mean": -29.284034729003906, "KL/std": 23.943511962890625, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.522860527038574, "fcm_dpo/q_t": 0.4867728352546692, "grad_norm": 2.1683151721954346, "learning_rate": 4.857886086178193e-07, "logits/chosen": -2.211361885070801, "logits/rejected": -1.8358337879180908, "logps/chosen": -109.93391418457031, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -132.54832458496094, "loss": 1.3369, "margin_dpo/margin_mean": 13.522859573364258, "margin_dpo/margin_std": 28.928546905517578, "step": 136 }, { "KL/chosen_KL_mean": -11.953262329101562, "KL/mean": -24.288393020629883, "KL/rejected_KL_mean": -36.62351989746094, "KL/std": 27.769271850585938, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.670257568359375, "fcm_dpo/q_t": 0.475881963968277, "grad_norm": 2.8174755573272705, "learning_rate": 4.853589828711902e-07, "logits/chosen": -2.2476491928100586, "logits/rejected": -2.1286709308624268, "logps/chosen": -90.07200622558594, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -154.5458984375, "loss": 1.2958, "margin_dpo/margin_mean": 24.67026138305664, "margin_dpo/margin_std": 33.03590774536133, "step": 137 }, { "KL/chosen_KL_mean": -16.220993041992188, "KL/mean": -22.97465705871582, "KL/rejected_KL_mean": -29.728321075439453, "KL/std": 20.102527618408203, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.507326126098633, "fcm_dpo/q_t": 0.48672208189964294, "grad_norm": 2.04052996635437, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.8360660076141357, "logits/rejected": -1.5419467687606812, "logps/chosen": -109.05093383789062, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -128.37672424316406, "loss": 1.3359, "margin_dpo/margin_mean": 13.507326126098633, "margin_dpo/margin_std": 23.270442962646484, "step": 138 }, { "KL/chosen_KL_mean": -11.911380767822266, "KL/mean": -19.67681121826172, "KL/rejected_KL_mean": -27.442241668701172, "KL/std": 18.929052352905273, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.530868530273438, "fcm_dpo/q_t": 0.4847281873226166, "grad_norm": 2.018105983734131, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.750136137008667, "logits/rejected": -1.4226644039154053, "logps/chosen": -90.91439819335938, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -113.26957702636719, "loss": 1.3284, "margin_dpo/margin_mean": 15.530868530273438, "margin_dpo/margin_std": 24.44626235961914, "step": 139 }, { "KL/chosen_KL_mean": -14.42239761352539, "KL/mean": -21.190736770629883, "KL/rejected_KL_mean": -27.95907974243164, "KL/std": 18.942726135253906, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.536685943603516, "fcm_dpo/q_t": 0.48668986558914185, "grad_norm": 1.9829339981079102, "learning_rate": 4.840329401637809e-07, "logits/chosen": -2.483273506164551, "logits/rejected": -2.090954303741455, "logps/chosen": -102.77484130859375, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -118.53385162353516, "loss": 1.3357, "margin_dpo/margin_mean": 13.536685943603516, "margin_dpo/margin_std": 22.84204864501953, "step": 140 }, { "KL/chosen_KL_mean": -20.067184448242188, "KL/mean": -26.720977783203125, "KL/rejected_KL_mean": -33.37477111816406, "KL/std": 28.104387283325195, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.30759048461914, "fcm_dpo/q_t": 0.48689892888069153, "grad_norm": 2.142012357711792, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.95183265209198, "logits/rejected": -1.484389305114746, "logps/chosen": -127.28734588623047, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -138.08700561523438, "loss": 1.338, "margin_dpo/margin_mean": 13.307589530944824, "margin_dpo/margin_std": 29.758960723876953, "step": 141 }, { "KL/chosen_KL_mean": -16.411205291748047, "KL/mean": -23.70339584350586, "KL/rejected_KL_mean": -30.995590209960938, "KL/std": 23.255191802978516, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.584381103515625, "fcm_dpo/q_t": 0.4856886565685272, "grad_norm": 2.0269546508789062, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.8836333751678467, "logits/rejected": -1.6355130672454834, "logps/chosen": -105.14015197753906, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -130.72598266601562, "loss": 1.3322, "margin_dpo/margin_mean": 14.584380149841309, "margin_dpo/margin_std": 25.214208602905273, "step": 142 }, { "KL/chosen_KL_mean": -16.24040985107422, "KL/mean": -21.275131225585938, "KL/rejected_KL_mean": -26.309844970703125, "KL/std": 18.963544845581055, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.069433212280273, "fcm_dpo/q_t": 0.49009907245635986, "grad_norm": 2.1272330284118652, "learning_rate": 4.826513955607734e-07, "logits/chosen": -2.436145782470703, "logits/rejected": -2.0172533988952637, "logps/chosen": -101.74498748779297, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -112.11550903320312, "loss": 1.3487, "margin_dpo/margin_mean": 10.069433212280273, "margin_dpo/margin_std": 20.3162841796875, "step": 143 }, { "KL/chosen_KL_mean": -20.508296966552734, "KL/mean": -24.51533317565918, "KL/rejected_KL_mean": -28.522369384765625, "KL/std": 21.260501861572266, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.014068603515625, "fcm_dpo/q_t": 0.49211233854293823, "grad_norm": 2.5289130210876465, "learning_rate": 4.821786031898176e-07, "logits/chosen": -2.193833827972412, "logits/rejected": -1.5854945182800293, "logps/chosen": -125.19538879394531, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -116.241455078125, "loss": 1.3577, "margin_dpo/margin_mean": 8.014068603515625, "margin_dpo/margin_std": 26.302814483642578, "step": 144 }, { "KL/chosen_KL_mean": -15.285694122314453, "KL/mean": -21.622055053710938, "KL/rejected_KL_mean": -27.958415985107422, "KL/std": 19.135927200317383, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.672712326049805, "fcm_dpo/q_t": 0.4875432252883911, "grad_norm": 2.224977731704712, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.809239387512207, "logits/rejected": -1.2684640884399414, "logps/chosen": -103.80558776855469, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -111.72355651855469, "loss": 1.3389, "margin_dpo/margin_mean": 12.672713279724121, "margin_dpo/margin_std": 21.98320770263672, "step": 145 }, { "KL/chosen_KL_mean": -17.408668518066406, "KL/mean": -23.627788543701172, "KL/rejected_KL_mean": -29.846900939941406, "KL/std": 20.392807006835938, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.438226699829102, "fcm_dpo/q_t": 0.4877690076828003, "grad_norm": 2.474684238433838, "learning_rate": 4.812146767012779e-07, "logits/chosen": -2.3225269317626953, "logits/rejected": -1.615492582321167, "logps/chosen": -120.31452941894531, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -118.7962646484375, "loss": 1.3404, "margin_dpo/margin_mean": 12.438226699829102, "margin_dpo/margin_std": 25.214767456054688, "step": 146 }, { "KL/chosen_KL_mean": -16.11016845703125, "KL/mean": -22.63544464111328, "KL/rejected_KL_mean": -29.160720825195312, "KL/std": 18.536537170410156, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.050557136535645, "fcm_dpo/q_t": 0.4871768355369568, "grad_norm": 2.2620561122894287, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.9613345861434937, "logits/rejected": -1.3589062690734863, "logps/chosen": -105.37236022949219, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -105.66083526611328, "loss": 1.3373, "margin_dpo/margin_mean": 13.050557136535645, "margin_dpo/margin_std": 21.501733779907227, "step": 147 }, { "KL/chosen_KL_mean": -14.356269836425781, "KL/mean": -22.399616241455078, "KL/rejected_KL_mean": -30.44296646118164, "KL/std": 22.815998077392578, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.08669662475586, "fcm_dpo/q_t": 0.4842432141304016, "grad_norm": 2.1645374298095703, "learning_rate": 4.802263794862384e-07, "logits/chosen": -2.305239200592041, "logits/rejected": -1.9455174207687378, "logps/chosen": -106.11915588378906, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -140.8284912109375, "loss": 1.3272, "margin_dpo/margin_mean": 16.086694717407227, "margin_dpo/margin_std": 26.695873260498047, "step": 148 }, { "KL/chosen_KL_mean": -18.23978042602539, "KL/mean": -25.192874908447266, "KL/rejected_KL_mean": -32.145957946777344, "KL/std": 24.190887451171875, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.90617561340332, "fcm_dpo/q_t": 0.48640912771224976, "grad_norm": 2.0881261825561523, "learning_rate": 4.797231243092118e-07, "logits/chosen": -2.003110885620117, "logits/rejected": -1.6203808784484863, "logps/chosen": -111.72380065917969, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -137.78965759277344, "loss": 1.3367, "margin_dpo/margin_mean": 13.90617561340332, "margin_dpo/margin_std": 32.61283874511719, "step": 149 }, { "KL/chosen_KL_mean": -14.877021789550781, "KL/mean": -22.304553985595703, "KL/rejected_KL_mean": -29.732086181640625, "KL/std": 23.959932327270508, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.855060577392578, "fcm_dpo/q_t": 0.48541879653930664, "grad_norm": 2.0779125690460205, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.8175764083862305, "logits/rejected": -1.6101596355438232, "logps/chosen": -90.65570831298828, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -119.75885009765625, "loss": 1.3314, "margin_dpo/margin_mean": 14.855060577392578, "margin_dpo/margin_std": 26.653568267822266, "step": 150 }, { "KL/chosen_KL_mean": -19.089908599853516, "KL/mean": -26.241573333740234, "KL/rejected_KL_mean": -33.393226623535156, "KL/std": 23.946325302124023, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.303313255310059, "fcm_dpo/q_t": 0.48597365617752075, "grad_norm": 2.332406520843506, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.5951154232025146, "logits/rejected": -2.011615514755249, "logps/chosen": -121.82412719726562, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -139.4310302734375, "loss": 1.3341, "margin_dpo/margin_mean": 14.303314208984375, "margin_dpo/margin_std": 29.224233627319336, "step": 151 }, { "KL/chosen_KL_mean": -16.097530364990234, "KL/mean": -25.271503448486328, "KL/rejected_KL_mean": -34.445472717285156, "KL/std": 23.761032104492188, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.347936630249023, "fcm_dpo/q_t": 0.4819948077201843, "grad_norm": 2.2451746463775635, "learning_rate": 4.78177092112495e-07, "logits/chosen": -2.268007516860962, "logits/rejected": -1.8705179691314697, "logps/chosen": -113.68827819824219, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -144.94461059570312, "loss": 1.319, "margin_dpo/margin_mean": 18.347938537597656, "margin_dpo/margin_std": 30.36518669128418, "step": 152 }, { "KL/chosen_KL_mean": -16.796897888183594, "KL/mean": -22.94899559020996, "KL/rejected_KL_mean": -29.101097106933594, "KL/std": 22.983842849731445, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.304205894470215, "fcm_dpo/q_t": 0.4879310131072998, "grad_norm": 2.1937999725341797, "learning_rate": 4.776497044244016e-07, "logits/chosen": -2.3755455017089844, "logits/rejected": -2.062321186065674, "logps/chosen": -103.82249450683594, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -120.16963195800781, "loss": 1.3418, "margin_dpo/margin_mean": 12.304206848144531, "margin_dpo/margin_std": 29.279294967651367, "step": 153 }, { "KL/chosen_KL_mean": -16.420574188232422, "KL/mean": -26.677425384521484, "KL/rejected_KL_mean": -36.93428039550781, "KL/std": 24.10045623779297, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.51371192932129, "fcm_dpo/q_t": 0.4798734784126282, "grad_norm": 2.3687610626220703, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.9934017658233643, "logits/rejected": -1.7078893184661865, "logps/chosen": -107.5262451171875, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -148.30667114257812, "loss": 1.3108, "margin_dpo/margin_mean": 20.513713836669922, "margin_dpo/margin_std": 30.87636947631836, "step": 154 }, { "KL/chosen_KL_mean": -18.551990509033203, "KL/mean": -24.816028594970703, "KL/rejected_KL_mean": -31.080055236816406, "KL/std": 23.88965606689453, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.52806282043457, "fcm_dpo/q_t": 0.48765215277671814, "grad_norm": 2.206148386001587, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -2.1548995971679688, "logits/rejected": -1.7116855382919312, "logps/chosen": -113.11296081542969, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -115.85364532470703, "loss": 1.3416, "margin_dpo/margin_mean": 12.52806282043457, "margin_dpo/margin_std": 32.16478729248047, "step": 155 }, { "KL/chosen_KL_mean": -21.328575134277344, "KL/mean": -26.921852111816406, "KL/rejected_KL_mean": -32.5151252746582, "KL/std": 22.982667922973633, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.18655014038086, "fcm_dpo/q_t": 0.4890629053115845, "grad_norm": 2.250460624694824, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -2.2679522037506104, "logits/rejected": -1.7857239246368408, "logps/chosen": -115.98396301269531, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -116.93446350097656, "loss": 1.3467, "margin_dpo/margin_mean": 11.186551094055176, "margin_dpo/margin_std": 31.006500244140625, "step": 156 }, { "KL/chosen_KL_mean": -19.422882080078125, "KL/mean": -30.326126098632812, "KL/rejected_KL_mean": -41.229366302490234, "KL/std": 26.89075469970703, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.806480407714844, "fcm_dpo/q_t": 0.4786023795604706, "grad_norm": 2.440798282623291, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.598083257675171, "logits/rejected": -2.196641683578491, "logps/chosen": -116.43527221679688, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -152.9307098388672, "loss": 1.3063, "margin_dpo/margin_mean": 21.806480407714844, "margin_dpo/margin_std": 31.745500564575195, "step": 157 }, { "KL/chosen_KL_mean": -18.585174560546875, "KL/mean": -29.107662200927734, "KL/rejected_KL_mean": -39.630149841308594, "KL/std": 28.736848831176758, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.044979095458984, "fcm_dpo/q_t": 0.4794214069843292, "grad_norm": 2.470993757247925, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.8310312032699585, "logits/rejected": -1.631330966949463, "logps/chosen": -113.29814147949219, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -160.56748962402344, "loss": 1.3097, "margin_dpo/margin_mean": 21.044979095458984, "margin_dpo/margin_std": 34.67277526855469, "step": 158 }, { "KL/chosen_KL_mean": -20.59093475341797, "KL/mean": -28.2563419342041, "KL/rejected_KL_mean": -35.921756744384766, "KL/std": 26.909860610961914, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.330831527709961, "fcm_dpo/q_t": 0.4849628210067749, "grad_norm": 2.4251906871795654, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.493447780609131, "logits/rejected": -1.9657599925994873, "logps/chosen": -128.80917358398438, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -130.76077270507812, "loss": 1.3305, "margin_dpo/margin_mean": 15.330833435058594, "margin_dpo/margin_std": 30.907394409179688, "step": 159 }, { "KL/chosen_KL_mean": -17.27471160888672, "KL/mean": -25.730287551879883, "KL/rejected_KL_mean": -34.18586730957031, "KL/std": 21.565738677978516, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.911142349243164, "fcm_dpo/q_t": 0.4833832383155823, "grad_norm": 2.2291905879974365, "learning_rate": 4.737908228387656e-07, "logits/chosen": -2.3083481788635254, "logits/rejected": -1.8286737203598022, "logps/chosen": -115.43812561035156, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -136.4668426513672, "loss": 1.3232, "margin_dpo/margin_mean": 16.911144256591797, "margin_dpo/margin_std": 24.921266555786133, "step": 160 }, { "KL/chosen_KL_mean": -17.761592864990234, "KL/mean": -24.87548828125, "KL/rejected_KL_mean": -31.98938751220703, "KL/std": 20.47555160522461, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.227804183959961, "fcm_dpo/q_t": 0.48603445291519165, "grad_norm": 2.1814329624176025, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -2.528625011444092, "logits/rejected": -2.0782361030578613, "logps/chosen": -104.35346221923828, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -121.00582122802734, "loss": 1.3344, "margin_dpo/margin_mean": 14.227803230285645, "margin_dpo/margin_std": 28.934669494628906, "step": 161 }, { "KL/chosen_KL_mean": -17.810882568359375, "KL/mean": -26.036602020263672, "KL/rejected_KL_mean": -34.26231384277344, "KL/std": 25.157089233398438, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.451438903808594, "fcm_dpo/q_t": 0.4838644862174988, "grad_norm": 2.3289473056793213, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.8429043292999268, "logits/rejected": -1.4287272691726685, "logps/chosen": -115.2137680053711, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -130.1315460205078, "loss": 1.3261, "margin_dpo/margin_mean": 16.451438903808594, "margin_dpo/margin_std": 29.797039031982422, "step": 162 }, { "KL/chosen_KL_mean": -15.792465209960938, "KL/mean": -25.016803741455078, "KL/rejected_KL_mean": -34.241153717041016, "KL/std": 24.935949325561523, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.44869041442871, "fcm_dpo/q_t": 0.4819217026233673, "grad_norm": 2.2428908348083496, "learning_rate": 4.720482655449212e-07, "logits/chosen": -2.205623149871826, "logits/rejected": -1.8201701641082764, "logps/chosen": -97.96785736083984, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -117.10589599609375, "loss": 1.3183, "margin_dpo/margin_mean": 18.44869041442871, "margin_dpo/margin_std": 29.241764068603516, "step": 163 }, { "KL/chosen_KL_mean": -20.068470001220703, "KL/mean": -27.03874969482422, "KL/rejected_KL_mean": -34.00902557373047, "KL/std": 20.435285568237305, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.940549850463867, "fcm_dpo/q_t": 0.4863065779209137, "grad_norm": 2.307478427886963, "learning_rate": 4.714556901942599e-07, "logits/chosen": -2.023374080657959, "logits/rejected": -1.5521650314331055, "logps/chosen": -110.54273986816406, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -121.03711700439453, "loss": 1.335, "margin_dpo/margin_mean": 13.940549850463867, "margin_dpo/margin_std": 26.846755981445312, "step": 164 }, { "KL/chosen_KL_mean": -15.19522476196289, "KL/mean": -24.038387298583984, "KL/rejected_KL_mean": -32.88154602050781, "KL/std": 21.42752456665039, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.686321258544922, "fcm_dpo/q_t": 0.48266953229904175, "grad_norm": 2.1229000091552734, "learning_rate": 4.708572792802069e-07, "logits/chosen": -2.329153060913086, "logits/rejected": -1.8646923303604126, "logps/chosen": -102.50845336914062, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -114.78413391113281, "loss": 1.3208, "margin_dpo/margin_mean": 17.686321258544922, "margin_dpo/margin_std": 27.178512573242188, "step": 165 }, { "KL/chosen_KL_mean": -19.30014419555664, "KL/mean": -30.64197540283203, "KL/rejected_KL_mean": -41.98381042480469, "KL/std": 29.514850616455078, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.68366241455078, "fcm_dpo/q_t": 0.47791221737861633, "grad_norm": 2.66757869720459, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.9907479286193848, "logits/rejected": -1.828920841217041, "logps/chosen": -100.88621520996094, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -145.14898681640625, "loss": 1.3046, "margin_dpo/margin_mean": 22.683666229248047, "margin_dpo/margin_std": 38.20446014404297, "step": 166 }, { "KL/chosen_KL_mean": -18.67438507080078, "KL/mean": -29.651823043823242, "KL/rejected_KL_mean": -40.629268646240234, "KL/std": 25.506364822387695, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.954875946044922, "fcm_dpo/q_t": 0.4784806966781616, "grad_norm": 2.505065441131592, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.6930651664733887, "logits/rejected": -1.5580589771270752, "logps/chosen": -102.23605346679688, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -139.68231201171875, "loss": 1.3052, "margin_dpo/margin_mean": 21.954875946044922, "margin_dpo/margin_std": 30.218002319335938, "step": 167 }, { "KL/chosen_KL_mean": -17.395973205566406, "KL/mean": -26.433486938476562, "KL/rejected_KL_mean": -35.471004486083984, "KL/std": 22.67333984375, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.075031280517578, "fcm_dpo/q_t": 0.48226022720336914, "grad_norm": 2.2234251499176025, "learning_rate": 4.690271916109034e-07, "logits/chosen": -2.237651824951172, "logits/rejected": -1.9032844305038452, "logps/chosen": -97.32720947265625, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -115.57316589355469, "loss": 1.3191, "margin_dpo/margin_mean": 18.075031280517578, "margin_dpo/margin_std": 26.771896362304688, "step": 168 }, { "KL/chosen_KL_mean": -15.447311401367188, "KL/mean": -26.49721908569336, "KL/rejected_KL_mean": -37.54712677001953, "KL/std": 26.142513275146484, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.099811553955078, "fcm_dpo/q_t": 0.4783303737640381, "grad_norm": 2.170121431350708, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -2.2626466751098633, "logits/rejected": -2.0637898445129395, "logps/chosen": -98.35101318359375, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -127.66903686523438, "loss": 1.3045, "margin_dpo/margin_mean": 22.099811553955078, "margin_dpo/margin_std": 28.92761993408203, "step": 169 }, { "KL/chosen_KL_mean": -16.94698715209961, "KL/mean": -25.5399169921875, "KL/rejected_KL_mean": -34.132850646972656, "KL/std": 21.02767562866211, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.18586540222168, "fcm_dpo/q_t": 0.48310935497283936, "grad_norm": 2.4395320415496826, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.7925678491592407, "logits/rejected": -1.6052627563476562, "logps/chosen": -91.928955078125, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -116.29845428466797, "loss": 1.3219, "margin_dpo/margin_mean": 17.18586540222168, "margin_dpo/margin_std": 23.293609619140625, "step": 170 }, { "KL/chosen_KL_mean": -26.638866424560547, "KL/mean": -35.35610580444336, "KL/rejected_KL_mean": -44.0733528137207, "KL/std": 35.927249908447266, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.434494018554688, "fcm_dpo/q_t": 0.48308736085891724, "grad_norm": 2.705294132232666, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -2.337517261505127, "logits/rejected": -1.7665531635284424, "logps/chosen": -145.61740112304688, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -159.8426513671875, "loss": 1.3276, "margin_dpo/margin_mean": 17.434494018554688, "margin_dpo/margin_std": 47.77015686035156, "step": 171 }, { "KL/chosen_KL_mean": -25.95379638671875, "KL/mean": -32.8676872253418, "KL/rejected_KL_mean": -39.78157424926758, "KL/std": 24.107616424560547, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.827778816223145, "fcm_dpo/q_t": 0.48641008138656616, "grad_norm": 2.6508285999298096, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.9749245643615723, "logits/rejected": -2.1650679111480713, "logps/chosen": -127.10758972167969, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -130.78726196289062, "loss": 1.3345, "margin_dpo/margin_mean": 13.827777862548828, "margin_dpo/margin_std": 22.47052001953125, "step": 172 }, { "KL/chosen_KL_mean": -20.121109008789062, "KL/mean": -27.29183578491211, "KL/rejected_KL_mean": -34.462562561035156, "KL/std": 22.449979782104492, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.341460227966309, "fcm_dpo/q_t": 0.48590683937072754, "grad_norm": 2.2465462684631348, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -2.4041528701782227, "logits/rejected": -1.8870983123779297, "logps/chosen": -119.17779541015625, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -118.31329345703125, "loss": 1.3328, "margin_dpo/margin_mean": 14.341460227966309, "margin_dpo/margin_std": 23.557044982910156, "step": 173 }, { "KL/chosen_KL_mean": -20.460721969604492, "KL/mean": -32.66575622558594, "KL/rejected_KL_mean": -44.87078857421875, "KL/std": 32.8420295715332, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.410062789916992, "fcm_dpo/q_t": 0.4761740267276764, "grad_norm": 2.6189169883728027, "learning_rate": 4.652116329460919e-07, "logits/chosen": -2.3972954750061035, "logits/rejected": -2.414523124694824, "logps/chosen": -100.86962890625, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -153.072509765625, "loss": 1.298, "margin_dpo/margin_mean": 24.41006088256836, "margin_dpo/margin_std": 38.19987487792969, "step": 174 }, { "KL/chosen_KL_mean": -23.737075805664062, "KL/mean": -34.45552062988281, "KL/rejected_KL_mean": -45.17396545410156, "KL/std": 27.62860107421875, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.4368896484375, "fcm_dpo/q_t": 0.47894227504730225, "grad_norm": 2.70660400390625, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.748844861984253, "logits/rejected": -1.4731838703155518, "logps/chosen": -106.45356750488281, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -141.84109497070312, "loss": 1.3071, "margin_dpo/margin_mean": 21.4368896484375, "margin_dpo/margin_std": 29.92498207092285, "step": 175 }, { "KL/chosen_KL_mean": -19.755828857421875, "KL/mean": -32.56519317626953, "KL/rejected_KL_mean": -45.37455368041992, "KL/std": 29.2282772064209, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.618715286254883, "fcm_dpo/q_t": 0.47492530941963196, "grad_norm": 3.1132380962371826, "learning_rate": 4.638942309888058e-07, "logits/chosen": -2.18644642829895, "logits/rejected": -2.0900259017944336, "logps/chosen": -99.3851318359375, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -148.06314086914062, "loss": 1.2927, "margin_dpo/margin_mean": 25.618715286254883, "margin_dpo/margin_std": 35.60376739501953, "step": 176 }, { "KL/chosen_KL_mean": -24.30795669555664, "KL/mean": -37.46440505981445, "KL/rejected_KL_mean": -50.620853424072266, "KL/std": 33.79766845703125, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.31289291381836, "fcm_dpo/q_t": 0.474303662776947, "grad_norm": 2.839789390563965, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -2.300055503845215, "logits/rejected": -1.9878208637237549, "logps/chosen": -116.86905670166016, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -153.95924377441406, "loss": 1.2906, "margin_dpo/margin_mean": 26.31289291381836, "margin_dpo/margin_std": 37.134971618652344, "step": 177 }, { "KL/chosen_KL_mean": -22.46865463256836, "KL/mean": -38.696285247802734, "KL/rejected_KL_mean": -54.92390441894531, "KL/std": 33.955718994140625, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.45525360107422, "fcm_dpo/q_t": 0.4682769775390625, "grad_norm": 3.163456678390503, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -2.0517773628234863, "logits/rejected": -1.9020154476165771, "logps/chosen": -106.73445129394531, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -165.72122192382812, "loss": 1.2677, "margin_dpo/margin_mean": 32.45525360107422, "margin_dpo/margin_std": 37.07232666015625, "step": 178 }, { "KL/chosen_KL_mean": -29.51805877685547, "KL/mean": -37.62396240234375, "KL/rejected_KL_mean": -45.729862213134766, "KL/std": 31.43106460571289, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.211795806884766, "fcm_dpo/q_t": 0.4841870665550232, "grad_norm": 2.80216908454895, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.92525315284729, "logits/rejected": -1.522099256515503, "logps/chosen": -122.96603393554688, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -139.9852752685547, "loss": 1.3298, "margin_dpo/margin_mean": 16.2117977142334, "margin_dpo/margin_std": 40.396270751953125, "step": 179 }, { "KL/chosen_KL_mean": -22.782543182373047, "KL/mean": -36.95042037963867, "KL/rejected_KL_mean": -51.1182861328125, "KL/std": 29.197961807250977, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.335742950439453, "fcm_dpo/q_t": 0.47231632471084595, "grad_norm": 3.1997897624969482, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.75786292552948, "logits/rejected": -1.567931890487671, "logps/chosen": -111.61808776855469, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -155.02525329589844, "loss": 1.282, "margin_dpo/margin_mean": 28.335742950439453, "margin_dpo/margin_std": 32.893497467041016, "step": 180 }, { "KL/chosen_KL_mean": -21.673599243164062, "KL/mean": -29.447795867919922, "KL/rejected_KL_mean": -37.22199249267578, "KL/std": 27.200061798095703, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.548398971557617, "fcm_dpo/q_t": 0.4847892224788666, "grad_norm": 2.4382197856903076, "learning_rate": 4.605024008834863e-07, "logits/chosen": -2.496183156967163, "logits/rejected": -2.0607752799987793, "logps/chosen": -100.90756225585938, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -106.39641571044922, "loss": 1.3299, "margin_dpo/margin_mean": 15.548399925231934, "margin_dpo/margin_std": 32.0458984375, "step": 181 }, { "KL/chosen_KL_mean": -24.855987548828125, "KL/mean": -38.51593780517578, "KL/rejected_KL_mean": -52.17588806152344, "KL/std": 33.03102111816406, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.319900512695312, "fcm_dpo/q_t": 0.47330808639526367, "grad_norm": 2.975815773010254, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.618466854095459, "logits/rejected": -1.4646153450012207, "logps/chosen": -100.93817138671875, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -147.14450073242188, "loss": 1.2871, "margin_dpo/margin_mean": 27.319900512695312, "margin_dpo/margin_std": 37.59684753417969, "step": 182 }, { "KL/chosen_KL_mean": -32.134517669677734, "KL/mean": -41.24468994140625, "KL/rejected_KL_mean": -50.3548583984375, "KL/std": 28.784761428833008, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.22032928466797, "fcm_dpo/q_t": 0.4821339249610901, "grad_norm": 3.098867893218994, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -2.1643433570861816, "logits/rejected": -1.5747535228729248, "logps/chosen": -126.04241943359375, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -133.91546630859375, "loss": 1.3196, "margin_dpo/margin_mean": 18.22032928466797, "margin_dpo/margin_std": 30.255268096923828, "step": 183 }, { "KL/chosen_KL_mean": -27.548324584960938, "KL/mean": -34.44281005859375, "KL/rejected_KL_mean": -41.33728790283203, "KL/std": 29.312541961669922, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.788966178894043, "fcm_dpo/q_t": 0.4864732027053833, "grad_norm": 2.5663468837738037, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -2.276223659515381, "logits/rejected": -1.9761215448379517, "logps/chosen": -114.41622924804688, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -125.62184143066406, "loss": 1.3373, "margin_dpo/margin_mean": 13.788966178894043, "margin_dpo/margin_std": 34.25421142578125, "step": 184 }, { "KL/chosen_KL_mean": -28.556865692138672, "KL/mean": -35.7197265625, "KL/rejected_KL_mean": -42.88258361816406, "KL/std": 28.106971740722656, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.32571792602539, "fcm_dpo/q_t": 0.4859681725502014, "grad_norm": 2.816969871520996, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.6802477836608887, "logits/rejected": -2.0671162605285645, "logps/chosen": -122.57829284667969, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -124.33926391601562, "loss": 1.3348, "margin_dpo/margin_mean": 14.32571792602539, "margin_dpo/margin_std": 32.54096603393555, "step": 185 }, { "KL/chosen_KL_mean": -38.90485763549805, "KL/mean": -46.655517578125, "KL/rejected_KL_mean": -54.40618133544922, "KL/std": 30.275718688964844, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.501333236694336, "fcm_dpo/q_t": 0.48479369282722473, "grad_norm": 3.509957790374756, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.458968162536621, "logits/rejected": -1.4537272453308105, "logps/chosen": -153.09654235839844, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -147.74530029296875, "loss": 1.3313, "margin_dpo/margin_mean": 15.501333236694336, "margin_dpo/margin_std": 36.141563415527344, "step": 186 }, { "KL/chosen_KL_mean": -27.39602279663086, "KL/mean": -38.040008544921875, "KL/rejected_KL_mean": -48.68399429321289, "KL/std": 31.226436614990234, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.28797149658203, "fcm_dpo/q_t": 0.4791731834411621, "grad_norm": 2.637464761734009, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -2.3198585510253906, "logits/rejected": -2.1157209873199463, "logps/chosen": -104.26557159423828, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -136.84893798828125, "loss": 1.3091, "margin_dpo/margin_mean": 21.28797149658203, "margin_dpo/margin_std": 35.693016052246094, "step": 187 }, { "KL/chosen_KL_mean": -24.23682403564453, "KL/mean": -35.946895599365234, "KL/rejected_KL_mean": -47.65697479248047, "KL/std": 30.8372802734375, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.420143127441406, "fcm_dpo/q_t": 0.477014422416687, "grad_norm": 2.843477725982666, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.3912947177886963, "logits/rejected": -2.167414426803589, "logps/chosen": -103.43162536621094, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -144.48587036132812, "loss": 1.3003, "margin_dpo/margin_mean": 23.420143127441406, "margin_dpo/margin_std": 31.88623809814453, "step": 188 }, { "KL/chosen_KL_mean": -30.069568634033203, "KL/mean": -39.4371223449707, "KL/rejected_KL_mean": -48.80467224121094, "KL/std": 29.185684204101562, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.735107421875, "fcm_dpo/q_t": 0.4816925525665283, "grad_norm": 2.8568553924560547, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -2.1543917655944824, "logits/rejected": -1.4970344305038452, "logps/chosen": -113.96237182617188, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -128.27056884765625, "loss": 1.3186, "margin_dpo/margin_mean": 18.735107421875, "margin_dpo/margin_std": 34.98820877075195, "step": 189 }, { "KL/chosen_KL_mean": -27.554359436035156, "KL/mean": -41.00345993041992, "KL/rejected_KL_mean": -54.45256042480469, "KL/std": 34.461097717285156, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.898193359375, "fcm_dpo/q_t": 0.4737001061439514, "grad_norm": 2.926095724105835, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -2.3579511642456055, "logits/rejected": -1.7853758335113525, "logps/chosen": -130.99581909179688, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -163.66758728027344, "loss": 1.2885, "margin_dpo/margin_mean": 26.898193359375, "margin_dpo/margin_std": 37.25028991699219, "step": 190 }, { "KL/chosen_KL_mean": -26.10947036743164, "KL/mean": -44.21515655517578, "KL/rejected_KL_mean": -62.32084655761719, "KL/std": 39.56842041015625, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.21138000488281, "fcm_dpo/q_t": 0.4647136628627777, "grad_norm": 3.4432170391082764, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -2.0007400512695312, "logits/rejected": -1.711686611175537, "logps/chosen": -125.89498901367188, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -178.32113647460938, "loss": 1.2558, "margin_dpo/margin_mean": 36.21138000488281, "margin_dpo/margin_std": 43.17432403564453, "step": 191 }, { "KL/chosen_KL_mean": -34.9302978515625, "KL/mean": -46.89610290527344, "KL/rejected_KL_mean": -58.86189651489258, "KL/std": 31.04035186767578, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.931610107421875, "fcm_dpo/q_t": 0.47661298513412476, "grad_norm": 2.943469524383545, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -2.028528928756714, "logits/rejected": -1.8008787631988525, "logps/chosen": -135.79061889648438, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -168.30690002441406, "loss": 1.2997, "margin_dpo/margin_mean": 23.931610107421875, "margin_dpo/margin_std": 37.685142517089844, "step": 192 }, { "KL/chosen_KL_mean": -31.441795349121094, "KL/mean": -38.88845443725586, "KL/rejected_KL_mean": -46.33510971069336, "KL/std": 34.23279571533203, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.893312454223633, "fcm_dpo/q_t": 0.4854867160320282, "grad_norm": 2.9098246097564697, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.675116539001465, "logits/rejected": -2.108222007751465, "logps/chosen": -124.38011169433594, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -129.4442901611328, "loss": 1.3358, "margin_dpo/margin_mean": 14.893312454223633, "margin_dpo/margin_std": 42.886695861816406, "step": 193 }, { "KL/chosen_KL_mean": -24.895896911621094, "KL/mean": -35.334625244140625, "KL/rejected_KL_mean": -45.773353576660156, "KL/std": 28.17508316040039, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.87746238708496, "fcm_dpo/q_t": 0.479539692401886, "grad_norm": 2.734489679336548, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.719881534576416, "logits/rejected": -2.333237886428833, "logps/chosen": -88.68746948242188, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -114.50000762939453, "loss": 1.3093, "margin_dpo/margin_mean": 20.877460479736328, "margin_dpo/margin_std": 30.344284057617188, "step": 194 }, { "KL/chosen_KL_mean": -28.677204132080078, "KL/mean": -43.13445281982422, "KL/rejected_KL_mean": -57.59169387817383, "KL/std": 32.23984909057617, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.91449737548828, "fcm_dpo/q_t": 0.47169792652130127, "grad_norm": 2.8423221111297607, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.862783670425415, "logits/rejected": -1.7948139905929565, "logps/chosen": -112.56266784667969, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -161.59295654296875, "loss": 1.2807, "margin_dpo/margin_mean": 28.91449737548828, "margin_dpo/margin_std": 36.444297790527344, "step": 195 }, { "KL/chosen_KL_mean": -32.754638671875, "KL/mean": -45.229148864746094, "KL/rejected_KL_mean": -57.70365524291992, "KL/std": 34.0529670715332, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.949026107788086, "fcm_dpo/q_t": 0.4755308926105499, "grad_norm": 2.9268593788146973, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.6001067161560059, "logits/rejected": -1.1427620649337769, "logps/chosen": -122.156005859375, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -141.05099487304688, "loss": 1.2952, "margin_dpo/margin_mean": 24.949026107788086, "margin_dpo/margin_std": 35.32540512084961, "step": 196 }, { "KL/chosen_KL_mean": -31.860488891601562, "KL/mean": -38.67365646362305, "KL/rejected_KL_mean": -45.486820220947266, "KL/std": 25.852073669433594, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.626327514648438, "fcm_dpo/q_t": 0.4866221845149994, "grad_norm": 2.6637537479400635, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.7146830558776855, "logits/rejected": -1.3727842569351196, "logps/chosen": -118.7011947631836, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -128.27972412109375, "loss": 1.3367, "margin_dpo/margin_mean": 13.626327514648438, "margin_dpo/margin_std": 29.484539031982422, "step": 197 }, { "KL/chosen_KL_mean": -35.80447769165039, "KL/mean": -50.05065155029297, "KL/rejected_KL_mean": -64.29682922363281, "KL/std": 34.69794464111328, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.49234390258789, "fcm_dpo/q_t": 0.47211170196533203, "grad_norm": 3.045300245285034, "learning_rate": 4.479470611971645e-07, "logits/chosen": -2.1825413703918457, "logits/rejected": -2.062140464782715, "logps/chosen": -120.85272216796875, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -168.40048217773438, "loss": 1.2814, "margin_dpo/margin_mean": 28.492341995239258, "margin_dpo/margin_std": 33.316619873046875, "step": 198 }, { "KL/chosen_KL_mean": -33.55036163330078, "KL/mean": -45.13542175292969, "KL/rejected_KL_mean": -56.72047805786133, "KL/std": 31.565311431884766, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.17011260986328, "fcm_dpo/q_t": 0.4773210287094116, "grad_norm": 2.859048843383789, "learning_rate": 4.471606039587695e-07, "logits/chosen": -2.1535563468933105, "logits/rejected": -1.9127907752990723, "logps/chosen": -115.3370590209961, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -147.51126098632812, "loss": 1.3023, "margin_dpo/margin_mean": 23.17011260986328, "margin_dpo/margin_std": 36.928253173828125, "step": 199 }, { "KL/chosen_KL_mean": -32.8321647644043, "KL/mean": -45.55905532836914, "KL/rejected_KL_mean": -58.285945892333984, "KL/std": 32.778743743896484, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.453777313232422, "fcm_dpo/q_t": 0.4751233160495758, "grad_norm": 3.0321691036224365, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.9948928356170654, "logits/rejected": -1.4228618144989014, "logps/chosen": -117.10025787353516, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -145.25381469726562, "loss": 1.2951, "margin_dpo/margin_mean": 25.453777313232422, "margin_dpo/margin_std": 41.048484802246094, "step": 200 }, { "KL/chosen_KL_mean": -39.96355056762695, "KL/mean": -52.20111083984375, "KL/rejected_KL_mean": -64.43865966796875, "KL/std": 40.19508743286133, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.475112915039062, "fcm_dpo/q_t": 0.47616374492645264, "grad_norm": 3.0089099407196045, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.8119118213653564, "logits/rejected": -1.469679355621338, "logps/chosen": -147.73605346679688, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -184.23114013671875, "loss": 1.2999, "margin_dpo/margin_mean": 24.475114822387695, "margin_dpo/margin_std": 43.237831115722656, "step": 201 }, { "KL/chosen_KL_mean": -31.179424285888672, "KL/mean": -43.042606353759766, "KL/rejected_KL_mean": -54.905792236328125, "KL/std": 30.903419494628906, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.72636604309082, "fcm_dpo/q_t": 0.47683677077293396, "grad_norm": 2.906609296798706, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -2.073345184326172, "logits/rejected": -1.971888780593872, "logps/chosen": -107.1518783569336, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -149.36578369140625, "loss": 1.301, "margin_dpo/margin_mean": 23.72636604309082, "margin_dpo/margin_std": 38.153133392333984, "step": 202 }, { "KL/chosen_KL_mean": -32.3636360168457, "KL/mean": -43.33467102050781, "KL/rejected_KL_mean": -54.305702209472656, "KL/std": 30.808349609375, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.942066192626953, "fcm_dpo/q_t": 0.4785350263118744, "grad_norm": 2.8555636405944824, "learning_rate": 4.439630306414758e-07, "logits/chosen": -2.3963828086853027, "logits/rejected": -1.9650273323059082, "logps/chosen": -127.33079528808594, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -147.19337463378906, "loss": 1.3069, "margin_dpo/margin_mean": 21.942066192626953, "margin_dpo/margin_std": 36.761714935302734, "step": 203 }, { "KL/chosen_KL_mean": -39.9107666015625, "KL/mean": -48.989898681640625, "KL/rejected_KL_mean": -58.06903076171875, "KL/std": 34.40058135986328, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.15825843811035, "fcm_dpo/q_t": 0.4822615087032318, "grad_norm": 3.2055132389068604, "learning_rate": 4.431508065452897e-07, "logits/chosen": -2.3677725791931152, "logits/rejected": -1.71311616897583, "logps/chosen": -156.26795959472656, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -151.46661376953125, "loss": 1.3231, "margin_dpo/margin_mean": 18.158260345458984, "margin_dpo/margin_std": 40.9262809753418, "step": 204 }, { "KL/chosen_KL_mean": -35.680389404296875, "KL/mean": -49.78004455566406, "KL/rejected_KL_mean": -63.87969970703125, "KL/std": 35.868194580078125, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.199316024780273, "fcm_dpo/q_t": 0.47236043214797974, "grad_norm": 3.097104072570801, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.506355047225952, "logits/rejected": -2.0495710372924805, "logps/chosen": -124.53973388671875, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -155.73419189453125, "loss": 1.2822, "margin_dpo/margin_mean": 28.199317932128906, "margin_dpo/margin_std": 31.482276916503906, "step": 205 }, { "KL/chosen_KL_mean": -33.008487701416016, "KL/mean": -46.80091857910156, "KL/rejected_KL_mean": -60.59335708618164, "KL/std": 31.938980102539062, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.58487319946289, "fcm_dpo/q_t": 0.47291675209999084, "grad_norm": 2.928156614303589, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.878208875656128, "logits/rejected": -1.7781707048416138, "logps/chosen": -109.55484008789062, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -163.54649353027344, "loss": 1.2838, "margin_dpo/margin_mean": 27.58487319946289, "margin_dpo/margin_std": 28.706565856933594, "step": 206 }, { "KL/chosen_KL_mean": -37.874786376953125, "KL/mean": -52.75365447998047, "KL/rejected_KL_mean": -67.63252258300781, "KL/std": 43.94690704345703, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.757732391357422, "fcm_dpo/q_t": 0.4711493253707886, "grad_norm": 3.291799306869507, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.391835927963257, "logits/rejected": -1.9787577390670776, "logps/chosen": -124.10643005371094, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -158.2876434326172, "loss": 1.2827, "margin_dpo/margin_mean": 29.757732391357422, "margin_dpo/margin_std": 51.80152893066406, "step": 207 }, { "KL/chosen_KL_mean": -40.450050354003906, "KL/mean": -55.36357116699219, "KL/rejected_KL_mean": -70.27708435058594, "KL/std": 35.62689208984375, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.8270263671875, "fcm_dpo/q_t": 0.4707724153995514, "grad_norm": 3.3417177200317383, "learning_rate": 4.398512291636768e-07, "logits/chosen": -2.167989730834961, "logits/rejected": -1.8495962619781494, "logps/chosen": -134.60958862304688, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -171.23941040039062, "loss": 1.2767, "margin_dpo/margin_mean": 29.8270263671875, "margin_dpo/margin_std": 33.676544189453125, "step": 208 }, { "KL/chosen_KL_mean": -35.5634880065918, "KL/mean": -46.98072052001953, "KL/rejected_KL_mean": -58.397945404052734, "KL/std": 33.18840026855469, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.83446502685547, "fcm_dpo/q_t": 0.4776771068572998, "grad_norm": 3.1201300621032715, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.3986451625823975, "logits/rejected": -1.9226100444793701, "logps/chosen": -119.73405456542969, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -146.01748657226562, "loss": 1.3048, "margin_dpo/margin_mean": 22.83446502685547, "margin_dpo/margin_std": 40.24559783935547, "step": 209 }, { "KL/chosen_KL_mean": -36.966678619384766, "KL/mean": -49.4761848449707, "KL/rejected_KL_mean": -61.98569107055664, "KL/std": 33.77473449707031, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.019004821777344, "fcm_dpo/q_t": 0.4755290150642395, "grad_norm": 2.9408082962036133, "learning_rate": 4.381713366536311e-07, "logits/chosen": -2.207655191421509, "logits/rejected": -1.8308310508728027, "logps/chosen": -118.13784790039062, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -146.1604766845703, "loss": 1.2956, "margin_dpo/margin_mean": 25.019004821777344, "margin_dpo/margin_std": 37.50033187866211, "step": 210 }, { "KL/chosen_KL_mean": -48.0158576965332, "KL/mean": -57.91759490966797, "KL/rejected_KL_mean": -67.8193359375, "KL/std": 37.2100830078125, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.8034725189209, "fcm_dpo/q_t": 0.4806859791278839, "grad_norm": 3.5875132083892822, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.315256118774414, "logits/rejected": -1.6579055786132812, "logps/chosen": -156.72857666015625, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -161.37498474121094, "loss": 1.3159, "margin_dpo/margin_mean": 19.803470611572266, "margin_dpo/margin_std": 39.22863006591797, "step": 211 }, { "KL/chosen_KL_mean": -34.778770446777344, "KL/mean": -54.1689453125, "KL/rejected_KL_mean": -73.55912780761719, "KL/std": 45.90531921386719, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.78035354614258, "fcm_dpo/q_t": 0.46234989166259766, "grad_norm": 3.2839736938476562, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -2.1497392654418945, "logits/rejected": -1.7433969974517822, "logps/chosen": -133.14071655273438, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -183.44912719726562, "loss": 1.2496, "margin_dpo/margin_mean": 38.78035354614258, "margin_dpo/margin_std": 51.40272521972656, "step": 212 }, { "KL/chosen_KL_mean": -38.59524917602539, "KL/mean": -51.67693328857422, "KL/rejected_KL_mean": -64.75862121582031, "KL/std": 38.6029052734375, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.16337013244629, "fcm_dpo/q_t": 0.47440946102142334, "grad_norm": 3.525412082672119, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -2.1493072509765625, "logits/rejected": -1.3168166875839233, "logps/chosen": -146.6505584716797, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -164.90277099609375, "loss": 1.2939, "margin_dpo/margin_mean": 26.163368225097656, "margin_dpo/margin_std": 45.669315338134766, "step": 213 }, { "KL/chosen_KL_mean": -37.44565200805664, "KL/mean": -50.31850814819336, "KL/rejected_KL_mean": -63.191368103027344, "KL/std": 39.4818000793457, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.745716094970703, "fcm_dpo/q_t": 0.4749181866645813, "grad_norm": 2.9864683151245117, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -2.008094310760498, "logits/rejected": -1.8850256204605103, "logps/chosen": -115.2503890991211, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -147.4671630859375, "loss": 1.2951, "margin_dpo/margin_mean": 25.745716094970703, "margin_dpo/margin_std": 44.849849700927734, "step": 214 }, { "KL/chosen_KL_mean": -34.48417663574219, "KL/mean": -52.575889587402344, "KL/rejected_KL_mean": -70.6676025390625, "KL/std": 40.429718017578125, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.18342208862305, "fcm_dpo/q_t": 0.4647441506385803, "grad_norm": 3.256943941116333, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -2.0765020847320557, "logits/rejected": -1.628333330154419, "logps/chosen": -119.59806823730469, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -167.52911376953125, "loss": 1.2559, "margin_dpo/margin_mean": 36.18342208862305, "margin_dpo/margin_std": 43.13408660888672, "step": 215 }, { "KL/chosen_KL_mean": -37.166908264160156, "KL/mean": -51.09469223022461, "KL/rejected_KL_mean": -65.0224838256836, "KL/std": 33.28211212158203, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.855571746826172, "fcm_dpo/q_t": 0.47274652123451233, "grad_norm": 2.9497435092926025, "learning_rate": 4.330133748510036e-07, "logits/chosen": -2.3853635787963867, "logits/rejected": -1.9950261116027832, "logps/chosen": -117.75921630859375, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -146.44232177734375, "loss": 1.2854, "margin_dpo/margin_mean": 27.855571746826172, "margin_dpo/margin_std": 38.13648223876953, "step": 216 }, { "KL/chosen_KL_mean": -46.56953430175781, "KL/mean": -58.504364013671875, "KL/rejected_KL_mean": -70.43919372558594, "KL/std": 41.2020149230957, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.869651794433594, "fcm_dpo/q_t": 0.47683387994766235, "grad_norm": 3.464966058731079, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.158262252807617, "logits/rejected": -1.860333800315857, "logps/chosen": -140.0421142578125, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -173.92771911621094, "loss": 1.3058, "margin_dpo/margin_mean": 23.86964988708496, "margin_dpo/margin_std": 54.20922088623047, "step": 217 }, { "KL/chosen_KL_mean": -44.744728088378906, "KL/mean": -58.84639358520508, "KL/rejected_KL_mean": -72.94805908203125, "KL/std": 41.33476257324219, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.203327178955078, "fcm_dpo/q_t": 0.4727003872394562, "grad_norm": 3.26340913772583, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.825913429260254, "logits/rejected": -1.4752275943756104, "logps/chosen": -133.80355834960938, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -167.25486755371094, "loss": 1.2896, "margin_dpo/margin_mean": 28.203327178955078, "margin_dpo/margin_std": 53.922950744628906, "step": 218 }, { "KL/chosen_KL_mean": -42.79061508178711, "KL/mean": -54.33694839477539, "KL/rejected_KL_mean": -65.88328552246094, "KL/std": 46.10359191894531, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.092683792114258, "fcm_dpo/q_t": 0.4774833917617798, "grad_norm": 3.472902774810791, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.9319207668304443, "logits/rejected": -1.4453465938568115, "logps/chosen": -143.79795837402344, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -158.35122680664062, "loss": 1.3095, "margin_dpo/margin_mean": 23.092681884765625, "margin_dpo/margin_std": 54.0699462890625, "step": 219 }, { "KL/chosen_KL_mean": -41.33209228515625, "KL/mean": -53.158958435058594, "KL/rejected_KL_mean": -64.98582458496094, "KL/std": 35.063018798828125, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.65373420715332, "fcm_dpo/q_t": 0.47683560848236084, "grad_norm": 3.2416741847991943, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.8220744132995605, "logits/rejected": -1.491058111190796, "logps/chosen": -136.8693084716797, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -159.29286193847656, "loss": 1.3003, "margin_dpo/margin_mean": 23.65373420715332, "margin_dpo/margin_std": 36.15251159667969, "step": 220 }, { "KL/chosen_KL_mean": -37.68191909790039, "KL/mean": -59.50053024291992, "KL/rejected_KL_mean": -81.31913757324219, "KL/std": 46.101707458496094, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.637237548828125, "fcm_dpo/q_t": 0.4575921893119812, "grad_norm": 3.7014410495758057, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.4037028551101685, "logits/rejected": -1.2951120138168335, "logps/chosen": -120.52677917480469, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -192.13095092773438, "loss": 1.2305, "margin_dpo/margin_mean": 43.637237548828125, "margin_dpo/margin_std": 47.726112365722656, "step": 221 }, { "KL/chosen_KL_mean": -47.94321823120117, "KL/mean": -62.967010498046875, "KL/rejected_KL_mean": -77.99079132080078, "KL/std": 44.54785919189453, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.047563552856445, "fcm_dpo/q_t": 0.4707600474357605, "grad_norm": 3.3761861324310303, "learning_rate": 4.276818137766118e-07, "logits/chosen": -2.3395192623138428, "logits/rejected": -1.928642749786377, "logps/chosen": -143.085205078125, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -184.79519653320312, "loss": 1.2837, "margin_dpo/margin_mean": 30.047563552856445, "margin_dpo/margin_std": 56.0097541809082, "step": 222 }, { "KL/chosen_KL_mean": -45.519325256347656, "KL/mean": -56.63220977783203, "KL/rejected_KL_mean": -67.74510192871094, "KL/std": 37.266822814941406, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.225772857666016, "fcm_dpo/q_t": 0.47838348150253296, "grad_norm": 3.445647716522217, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.294569253921509, "logits/rejected": -1.9133875370025635, "logps/chosen": -131.0944366455078, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -154.19747924804688, "loss": 1.3085, "margin_dpo/margin_mean": 22.225772857666016, "margin_dpo/margin_std": 44.91571807861328, "step": 223 }, { "KL/chosen_KL_mean": -42.53110122680664, "KL/mean": -57.44699478149414, "KL/rejected_KL_mean": -72.3628921508789, "KL/std": 41.82231140136719, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.83179473876953, "fcm_dpo/q_t": 0.4710521996021271, "grad_norm": 3.3922059535980225, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -2.012925863265991, "logits/rejected": -1.5501737594604492, "logps/chosen": -125.2549057006836, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -154.95828247070312, "loss": 1.2815, "margin_dpo/margin_mean": 29.83179473876953, "margin_dpo/margin_std": 49.0733528137207, "step": 224 }, { "KL/chosen_KL_mean": -49.4376335144043, "KL/mean": -66.04546356201172, "KL/rejected_KL_mean": -82.6532974243164, "KL/std": 44.56757354736328, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.215675354003906, "fcm_dpo/q_t": 0.46802201867103577, "grad_norm": 3.8752338886260986, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.155578374862671, "logits/rejected": -1.6785998344421387, "logps/chosen": -145.11532592773438, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -187.75018310546875, "loss": 1.2728, "margin_dpo/margin_mean": 33.215675354003906, "margin_dpo/margin_std": 57.83137512207031, "step": 225 }, { "KL/chosen_KL_mean": -41.00728988647461, "KL/mean": -56.1566276550293, "KL/rejected_KL_mean": -71.30596160888672, "KL/std": 42.267852783203125, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.298660278320312, "fcm_dpo/q_t": 0.4704384207725525, "grad_norm": 3.359588623046875, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -2.0742321014404297, "logits/rejected": -1.5207934379577637, "logps/chosen": -134.4682159423828, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -158.00772094726562, "loss": 1.277, "margin_dpo/margin_mean": 30.298660278320312, "margin_dpo/margin_std": 39.041873931884766, "step": 226 }, { "KL/chosen_KL_mean": -51.651065826416016, "KL/mean": -68.78773498535156, "KL/rejected_KL_mean": -85.92439270019531, "KL/std": 45.077964782714844, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.27333068847656, "fcm_dpo/q_t": 0.4667772650718689, "grad_norm": 3.6794869899749756, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.9908442497253418, "logits/rejected": -1.65433931350708, "logps/chosen": -144.46533203125, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -190.66131591796875, "loss": 1.2671, "margin_dpo/margin_mean": 34.27333068847656, "margin_dpo/margin_std": 54.271888732910156, "step": 227 }, { "KL/chosen_KL_mean": -50.06334686279297, "KL/mean": -61.491363525390625, "KL/rejected_KL_mean": -72.91937255859375, "KL/std": 42.04429626464844, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.856021881103516, "fcm_dpo/q_t": 0.47772690653800964, "grad_norm": 3.4955241680145264, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -2.2086195945739746, "logits/rejected": -1.7940622568130493, "logps/chosen": -144.1004638671875, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -168.94088745117188, "loss": 1.3074, "margin_dpo/margin_mean": 22.856021881103516, "margin_dpo/margin_std": 48.51336669921875, "step": 228 }, { "KL/chosen_KL_mean": -47.684043884277344, "KL/mean": -65.58924865722656, "KL/rejected_KL_mean": -83.49446105957031, "KL/std": 47.740272521972656, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.81040954589844, "fcm_dpo/q_t": 0.4654579162597656, "grad_norm": 3.865912914276123, "learning_rate": 4.212490049118951e-07, "logits/chosen": -2.2352817058563232, "logits/rejected": -1.6269776821136475, "logps/chosen": -143.26171875, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -172.66824340820312, "loss": 1.2626, "margin_dpo/margin_mean": 35.81040954589844, "margin_dpo/margin_std": 56.99607467651367, "step": 229 }, { "KL/chosen_KL_mean": -40.72993087768555, "KL/mean": -63.83673858642578, "KL/rejected_KL_mean": -86.94354248046875, "KL/std": 45.238983154296875, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 46.21360397338867, "fcm_dpo/q_t": 0.4550536870956421, "grad_norm": 4.004880905151367, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8988010883331299, "logits/rejected": -1.8736968040466309, "logps/chosen": -104.44355773925781, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -178.8522491455078, "loss": 1.2208, "margin_dpo/margin_mean": 46.213600158691406, "margin_dpo/margin_std": 46.77851104736328, "step": 230 }, { "KL/chosen_KL_mean": -48.99016571044922, "KL/mean": -64.01972198486328, "KL/rejected_KL_mean": -79.04927825927734, "KL/std": 41.10475158691406, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.05911636352539, "fcm_dpo/q_t": 0.47063589096069336, "grad_norm": 3.4561145305633545, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -2.1500701904296875, "logits/rejected": -1.5681706666946411, "logps/chosen": -144.44583129882812, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -160.0049591064453, "loss": 1.2776, "margin_dpo/margin_mean": 30.059120178222656, "margin_dpo/margin_std": 40.43292999267578, "step": 231 }, { "KL/chosen_KL_mean": -52.23115539550781, "KL/mean": -65.32601165771484, "KL/rejected_KL_mean": -78.42085266113281, "KL/std": 41.296302795410156, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.189701080322266, "fcm_dpo/q_t": 0.47443145513534546, "grad_norm": 3.6677427291870117, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.703056812286377, "logits/rejected": -1.2184672355651855, "logps/chosen": -149.12841796875, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -168.18548583984375, "loss": 1.2922, "margin_dpo/margin_mean": 26.189701080322266, "margin_dpo/margin_std": 40.76703643798828, "step": 232 }, { "KL/chosen_KL_mean": -52.21504211425781, "KL/mean": -74.2791748046875, "KL/rejected_KL_mean": -96.34329986572266, "KL/std": 53.35480499267578, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.003942476119846106, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.12826156616211, "fcm_dpo/q_t": 0.45725512504577637, "grad_norm": 3.882728099822998, "learning_rate": 4.174733034541245e-07, "logits/chosen": -2.07017183303833, "logits/rejected": -1.9266438484191895, "logps/chosen": -141.26536560058594, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -209.10247802734375, "loss": 1.2328, "margin_dpo/margin_mean": 44.128257751464844, "margin_dpo/margin_std": 57.60142517089844, "step": 233 }, { "KL/chosen_KL_mean": -49.443233489990234, "KL/mean": -72.97052764892578, "KL/rejected_KL_mean": -96.4978256225586, "KL/std": 51.87455749511719, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.012918921187520027, "fcm_dpo/delta": 0.14951452612876892, "fcm_dpo/margin": 47.054588317871094, "fcm_dpo/q_t": 0.43992647528648376, "grad_norm": 10.90788459777832, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.4887762069702148, "logits/rejected": -1.459015130996704, "logps/chosen": -123.7615966796875, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -202.88540649414062, "loss": 1.0491, "margin_dpo/margin_mean": 47.054588317871094, "margin_dpo/margin_std": 61.331626892089844, "step": 234 }, { "KL/chosen_KL_mean": -61.7285270690918, "KL/mean": -76.63621520996094, "KL/rejected_KL_mean": -91.54389190673828, "KL/std": 45.83916473388672, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.01294376514852047, "fcm_dpo/delta": -0.011987816542387009, "fcm_dpo/margin": 29.81536102294922, "fcm_dpo/q_t": 0.4134935140609741, "grad_norm": 12.427983283996582, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.83735990524292, "logits/rejected": -1.284834623336792, "logps/chosen": -159.94557189941406, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -188.79066467285156, "loss": 1.1351, "margin_dpo/margin_mean": 29.81536293029785, "margin_dpo/margin_std": 50.3234748840332, "step": 235 }, { "KL/chosen_KL_mean": -45.484893798828125, "KL/mean": -71.12178039550781, "KL/rejected_KL_mean": -96.7586669921875, "KL/std": 60.65960693359375, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.0064881606958806515, "fcm_dpo/delta": -0.052223168313503265, "fcm_dpo/margin": 51.273773193359375, "fcm_dpo/q_t": 0.4006412625312805, "grad_norm": 5.868508338928223, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.751079797744751, "logits/rejected": -1.7836753129959106, "logps/chosen": -124.32262420654297, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -205.82211303710938, "loss": 1.1359, "margin_dpo/margin_mean": 51.273773193359375, "margin_dpo/margin_std": 67.23406982421875, "step": 236 }, { "KL/chosen_KL_mean": -59.35281753540039, "KL/mean": -74.46095275878906, "KL/rejected_KL_mean": -89.56908416748047, "KL/std": 48.16114807128906, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.012492372654378414, "fcm_dpo/delta": -0.06435012072324753, "fcm_dpo/margin": 30.216259002685547, "fcm_dpo/q_t": 0.416096955537796, "grad_norm": 12.642706871032715, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.5874991416931152, "logits/rejected": -1.2785063982009888, "logps/chosen": -144.56411743164062, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -189.4790802001953, "loss": 1.2382, "margin_dpo/margin_mean": 30.216259002685547, "margin_dpo/margin_std": 63.123191833496094, "step": 237 }, { "KL/chosen_KL_mean": -60.389278411865234, "KL/mean": -78.39015197753906, "KL/rejected_KL_mean": -96.39102935791016, "KL/std": 49.731727600097656, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.00486148102208972, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.00175094604492, "fcm_dpo/q_t": 0.4575156569480896, "grad_norm": 4.369856834411621, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.5797834396362305, "logits/rejected": -1.455578327178955, "logps/chosen": -139.12051391601562, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -191.80943298339844, "loss": 1.2361, "margin_dpo/margin_mean": 36.00175094604492, "margin_dpo/margin_std": 54.050819396972656, "step": 238 }, { "KL/chosen_KL_mean": -57.54976272583008, "KL/mean": -73.69355773925781, "KL/rejected_KL_mean": -89.83735656738281, "KL/std": 42.3623046875, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.00486148102208972, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.2875862121582, "fcm_dpo/q_t": 0.46155476570129395, "grad_norm": 4.500835418701172, "learning_rate": 4.116778689174514e-07, "logits/chosen": -2.2787864208221436, "logits/rejected": -1.828361988067627, "logps/chosen": -150.15069580078125, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -190.35504150390625, "loss": 1.2512, "margin_dpo/margin_mean": 32.28758239746094, "margin_dpo/margin_std": 50.53156280517578, "step": 239 }, { "KL/chosen_KL_mean": -58.807369232177734, "KL/mean": -71.83892822265625, "KL/rejected_KL_mean": -84.87049865722656, "KL/std": 42.137718200683594, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.00486148102208972, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.063140869140625, "fcm_dpo/q_t": 0.46887078881263733, "grad_norm": 4.9921488761901855, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.519554615020752, "logits/rejected": -2.151667594909668, "logps/chosen": -144.96713256835938, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -165.326171875, "loss": 1.2771, "margin_dpo/margin_mean": 26.063140869140625, "margin_dpo/margin_std": 47.363189697265625, "step": 240 }, { "KL/chosen_KL_mean": -56.09170150756836, "KL/mean": -80.79498291015625, "KL/rejected_KL_mean": -105.49827575683594, "KL/std": 49.216278076171875, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.006828675977885723, "fcm_dpo/delta": 0.05929393693804741, "fcm_dpo/margin": 49.40656280517578, "fcm_dpo/q_t": 0.4416468143463135, "grad_norm": 6.402169227600098, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.8975108861923218, "logits/rejected": -1.1852669715881348, "logps/chosen": -157.13763427734375, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -199.54762268066406, "loss": 1.1125, "margin_dpo/margin_mean": 49.40656280517578, "margin_dpo/margin_std": 54.83492660522461, "step": 241 }, { "KL/chosen_KL_mean": -63.43190383911133, "KL/mean": -81.91851806640625, "KL/rejected_KL_mean": -100.4051284790039, "KL/std": 47.86243438720703, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.011302069760859013, "fcm_dpo/delta": -0.2462189793586731, "fcm_dpo/margin": 36.973228454589844, "fcm_dpo/q_t": 0.37354418635368347, "grad_norm": 18.26535415649414, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.952657699584961, "logits/rejected": -1.2550981044769287, "logps/chosen": -159.1045684814453, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -191.05967712402344, "loss": 1.2896, "margin_dpo/margin_mean": 36.973228454589844, "margin_dpo/margin_std": 55.809898376464844, "step": 242 }, { "KL/chosen_KL_mean": -62.26166534423828, "KL/mean": -79.39157104492188, "KL/rejected_KL_mean": -96.52149200439453, "KL/std": 46.99899673461914, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.25983428955078, "fcm_dpo/q_t": 0.4935813248157501, "grad_norm": 0.8495025634765625, "learning_rate": 4.07728699811968e-07, "logits/chosen": -2.207411766052246, "logits/rejected": -1.3814442157745361, "logps/chosen": -160.29306030273438, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -179.70956420898438, "loss": 1.3612, "margin_dpo/margin_mean": 34.25983428955078, "margin_dpo/margin_std": 55.682003021240234, "step": 243 }, { "KL/chosen_KL_mean": -59.65989685058594, "KL/mean": -80.84715270996094, "KL/rejected_KL_mean": -102.03440856933594, "KL/std": 53.58961486816406, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.374507904052734, "fcm_dpo/q_t": 0.4920603930950165, "grad_norm": 0.7875893115997314, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.4628987312316895, "logits/rejected": -2.166200637817383, "logps/chosen": -148.55381774902344, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -204.607177734375, "loss": 1.3551, "margin_dpo/margin_mean": 42.374507904052734, "margin_dpo/margin_std": 47.34454345703125, "step": 244 }, { "KL/chosen_KL_mean": -56.21487045288086, "KL/mean": -73.89393615722656, "KL/rejected_KL_mean": -91.57300567626953, "KL/std": 44.180572509765625, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.358131408691406, "fcm_dpo/q_t": 0.4933754801750183, "grad_norm": 0.7549958229064941, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.566220283508301, "logits/rejected": -1.7162425518035889, "logps/chosen": -160.40887451171875, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -184.22946166992188, "loss": 1.3603, "margin_dpo/margin_mean": 35.358131408691406, "margin_dpo/margin_std": 49.148277282714844, "step": 245 }, { "KL/chosen_KL_mean": -66.1896743774414, "KL/mean": -82.22379302978516, "KL/rejected_KL_mean": -98.25790405273438, "KL/std": 47.31822204589844, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.0682373046875, "fcm_dpo/q_t": 0.4939919114112854, "grad_norm": 0.7453804612159729, "learning_rate": 4.047230911780736e-07, "logits/chosen": -2.0505242347717285, "logits/rejected": -1.4882137775421143, "logps/chosen": -169.40872192382812, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -189.2501220703125, "loss": 1.3628, "margin_dpo/margin_mean": 32.0682373046875, "margin_dpo/margin_std": 52.00699996948242, "step": 246 }, { "KL/chosen_KL_mean": -59.84515380859375, "KL/mean": -88.25391387939453, "KL/rejected_KL_mean": -116.66265869140625, "KL/std": 55.422569274902344, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.8175163269043, "fcm_dpo/q_t": 0.48936009407043457, "grad_norm": 0.9287959933280945, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.3642892837524414, "logits/rejected": -2.2371389865875244, "logps/chosen": -146.83953857421875, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -228.0006866455078, "loss": 1.3447, "margin_dpo/margin_mean": 56.81752014160156, "margin_dpo/margin_std": 61.945343017578125, "step": 247 }, { "KL/chosen_KL_mean": -63.277400970458984, "KL/mean": -82.63103485107422, "KL/rejected_KL_mean": -101.98465728759766, "KL/std": 44.84254837036133, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.70726013183594, "fcm_dpo/q_t": 0.49274715781211853, "grad_norm": 0.8046744465827942, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -2.0113143920898438, "logits/rejected": -2.0222272872924805, "logps/chosen": -138.06292724609375, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -200.2615509033203, "loss": 1.3578, "margin_dpo/margin_mean": 38.70726013183594, "margin_dpo/margin_std": 45.104469299316406, "step": 248 }, { "KL/chosen_KL_mean": -75.53176879882812, "KL/mean": -88.68106079101562, "KL/rejected_KL_mean": -101.83036804199219, "KL/std": 51.320777893066406, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.29861831665039, "fcm_dpo/q_t": 0.4950742721557617, "grad_norm": 0.7752062082290649, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.592108726501465, "logits/rejected": -2.0535848140716553, "logps/chosen": -171.23556518554688, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -188.97683715820312, "loss": 1.3672, "margin_dpo/margin_mean": 26.29861831665039, "margin_dpo/margin_std": 60.86722183227539, "step": 249 }, { "KL/chosen_KL_mean": -68.23214721679688, "KL/mean": -83.01423645019531, "KL/rejected_KL_mean": -97.79632568359375, "KL/std": 45.04075622558594, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.564184188842773, "fcm_dpo/q_t": 0.49445968866348267, "grad_norm": 0.7982592582702637, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.2814507484436035, "logits/rejected": -1.324074387550354, "logps/chosen": -182.28436279296875, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -178.88400268554688, "loss": 1.3646, "margin_dpo/margin_mean": 29.564186096191406, "margin_dpo/margin_std": 51.03428268432617, "step": 250 }, { "KL/chosen_KL_mean": -68.74237060546875, "KL/mean": -81.55670166015625, "KL/rejected_KL_mean": -94.37101745605469, "KL/std": 43.43121337890625, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.628643035888672, "fcm_dpo/q_t": 0.4951983392238617, "grad_norm": 0.846337080001831, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -2.2280702590942383, "logits/rejected": -1.5407836437225342, "logps/chosen": -166.4536590576172, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -177.8984375, "loss": 1.3676, "margin_dpo/margin_mean": 25.628644943237305, "margin_dpo/margin_std": 53.748939514160156, "step": 251 }, { "KL/chosen_KL_mean": -67.75337982177734, "KL/mean": -83.41988372802734, "KL/rejected_KL_mean": -99.08638000488281, "KL/std": 45.190467834472656, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.332996368408203, "fcm_dpo/q_t": 0.49412912130355835, "grad_norm": 0.8380826115608215, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.332660436630249, "logits/rejected": -1.7593233585357666, "logps/chosen": -176.73948669433594, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -190.650634765625, "loss": 1.3633, "margin_dpo/margin_mean": 31.332996368408203, "margin_dpo/margin_std": 50.06891632080078, "step": 252 }, { "KL/chosen_KL_mean": -72.7241439819336, "KL/mean": -88.5946273803711, "KL/rejected_KL_mean": -104.4651107788086, "KL/std": 47.35783386230469, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.740957260131836, "fcm_dpo/q_t": 0.4940521717071533, "grad_norm": 0.8278841972351074, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.1876282691955566, "logits/rejected": -1.988731861114502, "logps/chosen": -172.94044494628906, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -210.14181518554688, "loss": 1.3629, "margin_dpo/margin_mean": 31.740957260131836, "margin_dpo/margin_std": 45.19974136352539, "step": 253 }, { "KL/chosen_KL_mean": -72.06045532226562, "KL/mean": -84.73892211914062, "KL/rejected_KL_mean": -97.41737365722656, "KL/std": 48.263092041015625, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.35691261291504, "fcm_dpo/q_t": 0.4952482581138611, "grad_norm": 0.8024190664291382, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.275954484939575, "logits/rejected": -1.7677415609359741, "logps/chosen": -170.795654296875, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -191.15562438964844, "loss": 1.3677, "margin_dpo/margin_mean": 25.35691261291504, "margin_dpo/margin_std": 49.918148040771484, "step": 254 }, { "KL/chosen_KL_mean": -73.66374206542969, "KL/mean": -95.91549682617188, "KL/rejected_KL_mean": -118.16725158691406, "KL/std": 51.4613037109375, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.503509521484375, "fcm_dpo/q_t": 0.4916617274284363, "grad_norm": 0.9485985636711121, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.2522196769714355, "logits/rejected": -1.9258869886398315, "logps/chosen": -164.04632568359375, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -215.24349975585938, "loss": 1.3536, "margin_dpo/margin_mean": 44.503509521484375, "margin_dpo/margin_std": 54.452919006347656, "step": 255 }, { "KL/chosen_KL_mean": -68.78157043457031, "KL/mean": -88.72191619873047, "KL/rejected_KL_mean": -108.66226196289062, "KL/std": 53.56037902832031, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.88068389892578, "fcm_dpo/q_t": 0.4925292730331421, "grad_norm": 0.8354307413101196, "learning_rate": 3.944434578520628e-07, "logits/chosen": -2.168358325958252, "logits/rejected": -1.8572057485580444, "logps/chosen": -157.5343780517578, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -207.15609741210938, "loss": 1.3572, "margin_dpo/margin_mean": 39.88068389892578, "margin_dpo/margin_std": 63.00605010986328, "step": 256 }, { "KL/chosen_KL_mean": -70.8582992553711, "KL/mean": -91.40898895263672, "KL/rejected_KL_mean": -111.95968627929688, "KL/std": 55.498435974121094, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.10139846801758, "fcm_dpo/q_t": 0.49230098724365234, "grad_norm": 0.848185658454895, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.244913101196289, "logits/rejected": -1.8422893285751343, "logps/chosen": -153.6618194580078, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -197.82748413085938, "loss": 1.3563, "margin_dpo/margin_mean": 41.10139465332031, "margin_dpo/margin_std": 59.82185363769531, "step": 257 }, { "KL/chosen_KL_mean": -74.63509368896484, "KL/mean": -94.08375549316406, "KL/rejected_KL_mean": -113.53243255615234, "KL/std": 52.17498016357422, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.89734649658203, "fcm_dpo/q_t": 0.49271178245544434, "grad_norm": 0.8838553428649902, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.4037833213806152, "logits/rejected": -2.063913345336914, "logps/chosen": -164.82284545898438, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -217.00311279296875, "loss": 1.3578, "margin_dpo/margin_mean": 38.89734649658203, "margin_dpo/margin_std": 55.182472229003906, "step": 258 }, { "KL/chosen_KL_mean": -80.02920532226562, "KL/mean": -97.01299285888672, "KL/rejected_KL_mean": -113.99678039550781, "KL/std": 55.039485931396484, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.96757888793945, "fcm_dpo/q_t": 0.49363553524017334, "grad_norm": 0.8999978303909302, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.5181450843811035, "logits/rejected": -1.2585175037384033, "logps/chosen": -170.8017578125, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -208.58493041992188, "loss": 1.3615, "margin_dpo/margin_mean": 33.96757888793945, "margin_dpo/margin_std": 60.77301025390625, "step": 259 }, { "KL/chosen_KL_mean": -78.8233413696289, "KL/mean": -95.49571228027344, "KL/rejected_KL_mean": -112.16808319091797, "KL/std": 53.555450439453125, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.34474182128906, "fcm_dpo/q_t": 0.4937536120414734, "grad_norm": 0.907230019569397, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.8908485174179077, "logits/rejected": -1.9493842124938965, "logps/chosen": -154.41604614257812, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -213.0136260986328, "loss": 1.362, "margin_dpo/margin_mean": 33.34474563598633, "margin_dpo/margin_std": 62.46881866455078, "step": 260 }, { "KL/chosen_KL_mean": -69.0299072265625, "KL/mean": -91.35369873046875, "KL/rejected_KL_mean": -113.677490234375, "KL/std": 54.75978088378906, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.64759063720703, "fcm_dpo/q_t": 0.49163782596588135, "grad_norm": 0.8872746229171753, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.20039701461792, "logits/rejected": -1.8364017009735107, "logps/chosen": -163.28482055664062, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -207.33447265625, "loss": 1.3536, "margin_dpo/margin_mean": 44.64759063720703, "margin_dpo/margin_std": 57.86028289794922, "step": 261 }, { "KL/chosen_KL_mean": -72.75495910644531, "KL/mean": -94.6919174194336, "KL/rejected_KL_mean": -116.62887573242188, "KL/std": 55.59748077392578, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.87392044067383, "fcm_dpo/q_t": 0.49178093671798706, "grad_norm": 0.9073288440704346, "learning_rate": 3.880912432401264e-07, "logits/chosen": -2.1244940757751465, "logits/rejected": -1.6193374395370483, "logps/chosen": -158.0222625732422, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -207.45497131347656, "loss": 1.3541, "margin_dpo/margin_mean": 43.87392044067383, "margin_dpo/margin_std": 56.576560974121094, "step": 262 }, { "KL/chosen_KL_mean": -61.220680236816406, "KL/mean": -91.83334350585938, "KL/rejected_KL_mean": -122.44602966308594, "KL/std": 67.23827362060547, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.225341796875, "fcm_dpo/q_t": 0.48853549361228943, "grad_norm": 0.9414438009262085, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.3123207092285156, "logits/rejected": -1.7596337795257568, "logps/chosen": -156.97857666015625, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -224.27978515625, "loss": 1.3417, "margin_dpo/margin_mean": 61.225341796875, "margin_dpo/margin_std": 74.45008087158203, "step": 263 }, { "KL/chosen_KL_mean": -69.8609390258789, "KL/mean": -95.89552307128906, "KL/rejected_KL_mean": -121.93009948730469, "KL/std": 58.214111328125, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.06915283203125, "fcm_dpo/q_t": 0.4902456998825073, "grad_norm": 0.9757410883903503, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.4392552375793457, "logits/rejected": -2.177278995513916, "logps/chosen": -160.4835968017578, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -222.25564575195312, "loss": 1.3482, "margin_dpo/margin_mean": 52.06915283203125, "margin_dpo/margin_std": 61.96233367919922, "step": 264 }, { "KL/chosen_KL_mean": -74.7341537475586, "KL/mean": -96.87394714355469, "KL/rejected_KL_mean": -119.01374816894531, "KL/std": 59.320220947265625, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.27960205078125, "fcm_dpo/q_t": 0.49170637130737305, "grad_norm": 0.8949093818664551, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.3012285232543945, "logits/rejected": -1.8868340253829956, "logps/chosen": -167.10647583007812, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -213.64132690429688, "loss": 1.354, "margin_dpo/margin_mean": 44.27960205078125, "margin_dpo/margin_std": 66.13623046875, "step": 265 }, { "KL/chosen_KL_mean": -88.75712585449219, "KL/mean": -107.22935485839844, "KL/rejected_KL_mean": -125.70156860351562, "KL/std": 63.525474548339844, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.94444274902344, "fcm_dpo/q_t": 0.493081271648407, "grad_norm": 0.9999639987945557, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.4654886722564697, "logits/rejected": -1.6056798696517944, "logps/chosen": -190.9571533203125, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -200.0679931640625, "loss": 1.3595, "margin_dpo/margin_mean": 36.94444274902344, "margin_dpo/margin_std": 68.77296447753906, "step": 266 }, { "KL/chosen_KL_mean": -76.43965148925781, "KL/mean": -100.05720520019531, "KL/rejected_KL_mean": -123.67475891113281, "KL/std": 58.44633865356445, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 47.23509979248047, "fcm_dpo/q_t": 0.49115538597106934, "grad_norm": 1.0421093702316284, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -2.0589094161987305, "logits/rejected": -1.4213811159133911, "logps/chosen": -180.72564697265625, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -219.66195678710938, "loss": 1.3518, "margin_dpo/margin_mean": 47.23509979248047, "margin_dpo/margin_std": 66.11241149902344, "step": 267 }, { "KL/chosen_KL_mean": -75.33077239990234, "KL/mean": -103.068115234375, "KL/rejected_KL_mean": -130.8054656982422, "KL/std": 60.109397888183594, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.474693298339844, "fcm_dpo/q_t": 0.48960769176483154, "grad_norm": 1.130700945854187, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.8107173442840576, "logits/rejected": -1.6880958080291748, "logps/chosen": -162.02699279785156, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -239.997314453125, "loss": 1.3457, "margin_dpo/margin_mean": 55.474693298339844, "margin_dpo/margin_std": 65.44062805175781, "step": 268 }, { "KL/chosen_KL_mean": -85.94793701171875, "KL/mean": -113.86947631835938, "KL/rejected_KL_mean": -141.791015625, "KL/std": 67.0492935180664, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.84309387207031, "fcm_dpo/q_t": 0.48954328894615173, "grad_norm": 1.0637060403823853, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -2.097294569015503, "logits/rejected": -1.671635627746582, "logps/chosen": -190.9197540283203, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -254.26748657226562, "loss": 1.3455, "margin_dpo/margin_mean": 55.84309005737305, "margin_dpo/margin_std": 68.40371704101562, "step": 269 }, { "KL/chosen_KL_mean": -91.16667175292969, "KL/mean": -115.88008117675781, "KL/rejected_KL_mean": -140.59347534179688, "KL/std": 63.01862335205078, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.42679977416992, "fcm_dpo/q_t": 0.49074384570121765, "grad_norm": 1.0858466625213623, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.6901166439056396, "logits/rejected": -2.292452573776245, "logps/chosen": -192.24050903320312, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -258.34637451171875, "loss": 1.3504, "margin_dpo/margin_mean": 49.42680358886719, "margin_dpo/margin_std": 77.08574676513672, "step": 270 }, { "KL/chosen_KL_mean": -72.82658386230469, "KL/mean": -103.07574462890625, "KL/rejected_KL_mean": -133.32492065429688, "KL/std": 59.35320281982422, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.49834060668945, "fcm_dpo/q_t": 0.4886665642261505, "grad_norm": 1.1509487628936768, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.8020192384719849, "logits/rejected": -1.3601629734039307, "logps/chosen": -169.55117797851562, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -231.84933471679688, "loss": 1.3421, "margin_dpo/margin_mean": 60.49834060668945, "margin_dpo/margin_std": 65.2620620727539, "step": 271 }, { "KL/chosen_KL_mean": -65.21510314941406, "KL/mean": -95.32598876953125, "KL/rejected_KL_mean": -125.4368667602539, "KL/std": 67.95477294921875, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.221763610839844, "fcm_dpo/q_t": 0.4887227416038513, "grad_norm": 1.291757345199585, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.7158925533294678, "logits/rejected": -1.6411662101745605, "logps/chosen": -146.19232177734375, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -233.99221801757812, "loss": 1.3425, "margin_dpo/margin_mean": 60.221763610839844, "margin_dpo/margin_std": 75.51313781738281, "step": 272 }, { "KL/chosen_KL_mean": -87.780517578125, "KL/mean": -116.88139343261719, "KL/rejected_KL_mean": -145.98226928710938, "KL/std": 67.35155487060547, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.20175552368164, "fcm_dpo/q_t": 0.48910194635391235, "grad_norm": 1.1766048669815063, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.8135274648666382, "logits/rejected": -1.404639720916748, "logps/chosen": -180.005126953125, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -243.3453369140625, "loss": 1.3441, "margin_dpo/margin_mean": 58.20175552368164, "margin_dpo/margin_std": 78.68760681152344, "step": 273 }, { "KL/chosen_KL_mean": -89.35673522949219, "KL/mean": -112.33531188964844, "KL/rejected_KL_mean": -135.31387329101562, "KL/std": 66.47171783447266, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.95716094970703, "fcm_dpo/q_t": 0.49139082431793213, "grad_norm": 1.215266227722168, "learning_rate": 3.75e-07, "logits/chosen": -2.0976455211639404, "logits/rejected": -1.7162853479385376, "logps/chosen": -171.61761474609375, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -219.19088745117188, "loss": 1.3526, "margin_dpo/margin_mean": 45.957157135009766, "margin_dpo/margin_std": 59.3822021484375, "step": 274 }, { "KL/chosen_KL_mean": -78.82083129882812, "KL/mean": -104.2398681640625, "KL/rejected_KL_mean": -129.65890502929688, "KL/std": 65.93486785888672, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.83807373046875, "fcm_dpo/q_t": 0.4904789924621582, "grad_norm": 1.1010849475860596, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.304715633392334, "logits/rejected": -2.0019185543060303, "logps/chosen": -158.50779724121094, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -228.40988159179688, "loss": 1.3494, "margin_dpo/margin_mean": 50.83807373046875, "margin_dpo/margin_std": 74.62875366210938, "step": 275 }, { "KL/chosen_KL_mean": -90.76828002929688, "KL/mean": -118.34226989746094, "KL/rejected_KL_mean": -145.916259765625, "KL/std": 69.41792297363281, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.147979736328125, "fcm_dpo/q_t": 0.4896783232688904, "grad_norm": 1.1864073276519775, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.3094675540924072, "logits/rejected": -1.7294023036956787, "logps/chosen": -177.3079833984375, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -223.77020263671875, "loss": 1.3464, "margin_dpo/margin_mean": 55.147979736328125, "margin_dpo/margin_std": 86.2347640991211, "step": 276 }, { "KL/chosen_KL_mean": -100.0003662109375, "KL/mean": -119.78439331054688, "KL/rejected_KL_mean": -139.56842041015625, "KL/std": 65.22980499267578, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.56805419921875, "fcm_dpo/q_t": 0.4925883710384369, "grad_norm": 1.104003667831421, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.2019262313842773, "logits/rejected": -1.7979118824005127, "logps/chosen": -184.24447631835938, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -238.4026336669922, "loss": 1.3574, "margin_dpo/margin_mean": 39.56805419921875, "margin_dpo/margin_std": 63.80610656738281, "step": 277 }, { "KL/chosen_KL_mean": -93.67678833007812, "KL/mean": -121.53582763671875, "KL/rejected_KL_mean": -149.39486694335938, "KL/std": 69.25630187988281, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.71807098388672, "fcm_dpo/q_t": 0.4895629286766052, "grad_norm": 1.2485783100128174, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.3215789794921875, "logits/rejected": -2.1038029193878174, "logps/chosen": -176.10867309570312, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -247.25177001953125, "loss": 1.3457, "margin_dpo/margin_mean": 55.71807098388672, "margin_dpo/margin_std": 71.31480407714844, "step": 278 }, { "KL/chosen_KL_mean": -97.21517944335938, "KL/mean": -132.58399963378906, "KL/rejected_KL_mean": -167.9528045654297, "KL/std": 80.48412322998047, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.73764038085938, "fcm_dpo/q_t": 0.4867614507675171, "grad_norm": 1.7440743446350098, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.961909532546997, "logits/rejected": -1.7642630338668823, "logps/chosen": -182.3831787109375, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -262.0794677734375, "loss": 1.335, "margin_dpo/margin_mean": 70.73764038085938, "margin_dpo/margin_std": 84.1757583618164, "step": 279 }, { "KL/chosen_KL_mean": -99.2138671875, "KL/mean": -133.7057342529297, "KL/rejected_KL_mean": -168.1975860595703, "KL/std": 71.25033569335938, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.98371124267578, "fcm_dpo/q_t": 0.4870806932449341, "grad_norm": 1.2254314422607422, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -2.1313228607177734, "logits/rejected": -1.9547700881958008, "logps/chosen": -185.07028198242188, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -272.316162109375, "loss": 1.336, "margin_dpo/margin_mean": 68.98370361328125, "margin_dpo/margin_std": 74.4264144897461, "step": 280 }, { "KL/chosen_KL_mean": -107.82875061035156, "KL/mean": -141.58766174316406, "KL/rejected_KL_mean": -175.34658813476562, "KL/std": 79.8138427734375, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.517822265625, "fcm_dpo/q_t": 0.4873591661453247, "grad_norm": 1.1049168109893799, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.258763313293457, "logits/rejected": -1.5298941135406494, "logps/chosen": -194.37957763671875, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -263.9752502441406, "loss": 1.3373, "margin_dpo/margin_mean": 67.517822265625, "margin_dpo/margin_std": 81.44256591796875, "step": 281 }, { "KL/chosen_KL_mean": -105.86959075927734, "KL/mean": -150.191162109375, "KL/rejected_KL_mean": -194.51272583007812, "KL/std": 85.31196594238281, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.64315032958984, "fcm_dpo/q_t": 0.48341357707977295, "grad_norm": 1.329802393913269, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.8767670392990112, "logits/rejected": -1.494884967803955, "logps/chosen": -210.97470092773438, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -297.3660888671875, "loss": 1.3221, "margin_dpo/margin_mean": 88.64315032958984, "margin_dpo/margin_std": 90.69805908203125, "step": 282 }, { "KL/chosen_KL_mean": -127.11941528320312, "KL/mean": -157.56692504882812, "KL/rejected_KL_mean": -188.01446533203125, "KL/std": 81.84292602539062, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.895050048828125, "fcm_dpo/q_t": 0.4886007308959961, "grad_norm": 1.571425437927246, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.4978103637695312, "logits/rejected": -2.18035888671875, "logps/chosen": -222.17201232910156, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -291.55902099609375, "loss": 1.3423, "margin_dpo/margin_mean": 60.895050048828125, "margin_dpo/margin_std": 89.89836120605469, "step": 283 }, { "KL/chosen_KL_mean": -117.34901428222656, "KL/mean": -154.20004272460938, "KL/rejected_KL_mean": -191.0510711669922, "KL/std": 81.07917785644531, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.70205688476562, "fcm_dpo/q_t": 0.4862029552459717, "grad_norm": 1.6180373430252075, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.991154432296753, "logits/rejected": -1.6400357484817505, "logps/chosen": -205.01548767089844, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -289.8021240234375, "loss": 1.3327, "margin_dpo/margin_mean": 73.70205688476562, "margin_dpo/margin_std": 80.55685424804688, "step": 284 }, { "KL/chosen_KL_mean": -133.3653106689453, "KL/mean": -159.60784912109375, "KL/rejected_KL_mean": -185.8504180908203, "KL/std": 72.79084777832031, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.48508834838867, "fcm_dpo/q_t": 0.4901697635650635, "grad_norm": 1.569909691810608, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.586627721786499, "logits/rejected": -2.3078832626342773, "logps/chosen": -227.79833984375, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -289.92236328125, "loss": 1.3481, "margin_dpo/margin_mean": 52.48509216308594, "margin_dpo/margin_std": 76.03995513916016, "step": 285 }, { "KL/chosen_KL_mean": -143.09263610839844, "KL/mean": -179.37562561035156, "KL/rejected_KL_mean": -215.65859985351562, "KL/std": 84.29704284667969, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.56597137451172, "fcm_dpo/q_t": 0.4864155650138855, "grad_norm": 1.2915749549865723, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.193326711654663, "logits/rejected": -1.7382621765136719, "logps/chosen": -246.81304931640625, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -311.9163818359375, "loss": 1.3337, "margin_dpo/margin_mean": 72.56597137451172, "margin_dpo/margin_std": 87.52272033691406, "step": 286 }, { "KL/chosen_KL_mean": -158.85638427734375, "KL/mean": -177.94107055664062, "KL/rejected_KL_mean": -197.0257568359375, "KL/std": 66.99856567382812, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.169395446777344, "fcm_dpo/q_t": 0.4928576350212097, "grad_norm": 1.126685619354248, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.374863624572754, "logits/rejected": -1.768652319908142, "logps/chosen": -252.7462615966797, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -280.35943603515625, "loss": 1.3589, "margin_dpo/margin_mean": 38.169395446777344, "margin_dpo/margin_std": 84.71039581298828, "step": 287 }, { "KL/chosen_KL_mean": -153.23001098632812, "KL/mean": -191.44480895996094, "KL/rejected_KL_mean": -229.6595916748047, "KL/std": 78.3684310913086, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.42959594726562, "fcm_dpo/q_t": 0.48569637537002563, "grad_norm": 1.4093616008758545, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -2.0080392360687256, "logits/rejected": -1.5937647819519043, "logps/chosen": -241.38601684570312, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -322.9415283203125, "loss": 1.3309, "margin_dpo/margin_mean": 76.42959594726562, "margin_dpo/margin_std": 87.50643920898438, "step": 288 }, { "KL/chosen_KL_mean": -149.0620880126953, "KL/mean": -186.6866455078125, "KL/rejected_KL_mean": -224.3112030029297, "KL/std": 82.03190612792969, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.24910736083984, "fcm_dpo/q_t": 0.48592057824134827, "grad_norm": 1.5603917837142944, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.059061050415039, "logits/rejected": -1.8530421257019043, "logps/chosen": -224.45501708984375, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -317.4654846191406, "loss": 1.3319, "margin_dpo/margin_mean": 75.24911499023438, "margin_dpo/margin_std": 92.9822769165039, "step": 289 }, { "KL/chosen_KL_mean": -159.272705078125, "KL/mean": -200.63455200195312, "KL/rejected_KL_mean": -241.99642944335938, "KL/std": 93.33849334716797, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.72373962402344, "fcm_dpo/q_t": 0.4845364987850189, "grad_norm": 1.5876280069351196, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.303538203239441, "logits/rejected": -1.3814420700073242, "logps/chosen": -247.31466674804688, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -365.2085876464844, "loss": 1.3269, "margin_dpo/margin_mean": 82.72373962402344, "margin_dpo/margin_std": 103.92225646972656, "step": 290 }, { "KL/chosen_KL_mean": -154.41363525390625, "KL/mean": -186.93032836914062, "KL/rejected_KL_mean": -219.447021484375, "KL/std": 87.13306427001953, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.03338623046875, "fcm_dpo/q_t": 0.4878275394439697, "grad_norm": 1.8723284006118774, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.4107770919799805, "logits/rejected": -1.944793939590454, "logps/chosen": -248.50888061523438, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -315.4970703125, "loss": 1.3393, "margin_dpo/margin_mean": 65.03338623046875, "margin_dpo/margin_std": 92.52753448486328, "step": 291 }, { "KL/chosen_KL_mean": -155.26254272460938, "KL/mean": -191.09225463867188, "KL/rejected_KL_mean": -226.92193603515625, "KL/std": 85.65648651123047, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.6594009399414, "fcm_dpo/q_t": 0.4865839183330536, "grad_norm": 1.3005856275558472, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.156482696533203, "logits/rejected": -1.7326322793960571, "logps/chosen": -243.51296997070312, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -323.339599609375, "loss": 1.3342, "margin_dpo/margin_mean": 71.65939331054688, "margin_dpo/margin_std": 80.3996353149414, "step": 292 }, { "KL/chosen_KL_mean": -148.08865356445312, "KL/mean": -178.36181640625, "KL/rejected_KL_mean": -208.63497924804688, "KL/std": 93.88009643554688, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.54632568359375, "fcm_dpo/q_t": 0.4886786937713623, "grad_norm": 1.3504995107650757, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.423794746398926, "logits/rejected": -1.8463950157165527, "logps/chosen": -235.46519470214844, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -294.3907775878906, "loss": 1.3429, "margin_dpo/margin_mean": 60.54632568359375, "margin_dpo/margin_std": 102.08932495117188, "step": 293 }, { "KL/chosen_KL_mean": -148.74151611328125, "KL/mean": -172.98501586914062, "KL/rejected_KL_mean": -197.22853088378906, "KL/std": 82.56764221191406, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.48701858520508, "fcm_dpo/q_t": 0.4909246861934662, "grad_norm": 1.3076379299163818, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.767993688583374, "logits/rejected": -1.6282618045806885, "logps/chosen": -222.2494659423828, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -285.31732177734375, "loss": 1.3516, "margin_dpo/margin_mean": 48.487022399902344, "margin_dpo/margin_std": 95.79048156738281, "step": 294 }, { "KL/chosen_KL_mean": -137.94984436035156, "KL/mean": -175.755126953125, "KL/rejected_KL_mean": -213.56045532226562, "KL/std": 92.97337341308594, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.61060333251953, "fcm_dpo/q_t": 0.4858587086200714, "grad_norm": 1.4190195798873901, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.869284987449646, "logits/rejected": -1.70334792137146, "logps/chosen": -215.71533203125, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -307.8077087402344, "loss": 1.3319, "margin_dpo/margin_mean": 75.610595703125, "margin_dpo/margin_std": 102.93743896484375, "step": 295 }, { "KL/chosen_KL_mean": -156.04061889648438, "KL/mean": -193.81100463867188, "KL/rejected_KL_mean": -231.58139038085938, "KL/std": 88.27693176269531, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.54075622558594, "fcm_dpo/q_t": 0.48586204648017883, "grad_norm": 1.6333175897598267, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.6863853931427, "logits/rejected": -2.1925864219665527, "logps/chosen": -253.89703369140625, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -332.397705078125, "loss": 1.3317, "margin_dpo/margin_mean": 75.54075622558594, "margin_dpo/margin_std": 93.19606018066406, "step": 296 }, { "KL/chosen_KL_mean": -153.86599731445312, "KL/mean": -187.58914184570312, "KL/rejected_KL_mean": -221.31228637695312, "KL/std": 91.74603271484375, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.4462890625, "fcm_dpo/q_t": 0.4873807430267334, "grad_norm": 1.3923535346984863, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.3055821657180786, "logits/rejected": -1.0134344100952148, "logps/chosen": -242.4318389892578, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -322.86883544921875, "loss": 1.3376, "margin_dpo/margin_mean": 67.44628143310547, "margin_dpo/margin_std": 93.68558502197266, "step": 297 }, { "KL/chosen_KL_mean": -129.53509521484375, "KL/mean": -172.39068603515625, "KL/rejected_KL_mean": -215.24627685546875, "KL/std": 96.17247009277344, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.71119689941406, "fcm_dpo/q_t": 0.4839692711830139, "grad_norm": 1.5640844106674194, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.1976370811462402, "logits/rejected": -2.228729009628296, "logps/chosen": -224.41552734375, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -337.55731201171875, "loss": 1.3244, "margin_dpo/margin_mean": 85.71119689941406, "margin_dpo/margin_std": 97.04953002929688, "step": 298 }, { "KL/chosen_KL_mean": -136.6573486328125, "KL/mean": -174.93893432617188, "KL/rejected_KL_mean": -213.22052001953125, "KL/std": 91.65370178222656, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.56317138671875, "fcm_dpo/q_t": 0.48567402362823486, "grad_norm": 2.478612184524536, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.9176153540611267, "logits/rejected": -0.7064777612686157, "logps/chosen": -217.06570434570312, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -302.7576904296875, "loss": 1.3311, "margin_dpo/margin_mean": 76.56317138671875, "margin_dpo/margin_std": 98.42860412597656, "step": 299 }, { "KL/chosen_KL_mean": -145.34478759765625, "KL/mean": -181.54193115234375, "KL/rejected_KL_mean": -217.73907470703125, "KL/std": 95.27277374267578, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.39430236816406, "fcm_dpo/q_t": 0.4864570200443268, "grad_norm": 1.69949209690094, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.458991527557373, "logits/rejected": -1.25472092628479, "logps/chosen": -233.50369262695312, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -318.67828369140625, "loss": 1.3344, "margin_dpo/margin_mean": 72.39430236816406, "margin_dpo/margin_std": 108.0799560546875, "step": 300 }, { "KL/chosen_KL_mean": -137.0734100341797, "KL/mean": -181.09146118164062, "KL/rejected_KL_mean": -225.1094970703125, "KL/std": 93.66834259033203, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.03609466552734, "fcm_dpo/q_t": 0.4835318922996521, "grad_norm": 2.0061800479888916, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.943245768547058, "logits/rejected": -1.5396459102630615, "logps/chosen": -222.57830810546875, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -324.0263366699219, "loss": 1.323, "margin_dpo/margin_mean": 88.03609466552734, "margin_dpo/margin_std": 105.54109954833984, "step": 301 }, { "KL/chosen_KL_mean": -131.32778930664062, "KL/mean": -170.48629760742188, "KL/rejected_KL_mean": -209.6448211669922, "KL/std": 95.93925476074219, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.31702423095703, "fcm_dpo/q_t": 0.4853416383266449, "grad_norm": 1.402022361755371, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.500633955001831, "logits/rejected": -1.3473074436187744, "logps/chosen": -210.7781982421875, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -312.70391845703125, "loss": 1.3297, "margin_dpo/margin_mean": 78.31703186035156, "margin_dpo/margin_std": 96.01434326171875, "step": 302 }, { "KL/chosen_KL_mean": -133.18377685546875, "KL/mean": -173.2059326171875, "KL/rejected_KL_mean": -213.2281036376953, "KL/std": 92.25131225585938, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.0443344116211, "fcm_dpo/q_t": 0.48501861095428467, "grad_norm": 1.4939866065979004, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -2.0854170322418213, "logits/rejected": -1.627875566482544, "logps/chosen": -228.78207397460938, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -305.29266357421875, "loss": 1.3285, "margin_dpo/margin_mean": 80.04434204101562, "margin_dpo/margin_std": 96.65841674804688, "step": 303 }, { "KL/chosen_KL_mean": -135.19805908203125, "KL/mean": -175.7381591796875, "KL/rejected_KL_mean": -216.27828979492188, "KL/std": 92.19253540039062, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.08021545410156, "fcm_dpo/q_t": 0.48482638597488403, "grad_norm": 1.6548893451690674, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.2896907329559326, "logits/rejected": -1.8847473859786987, "logps/chosen": -234.65121459960938, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -314.36053466796875, "loss": 1.3277, "margin_dpo/margin_mean": 81.08021545410156, "margin_dpo/margin_std": 96.86013793945312, "step": 304 }, { "KL/chosen_KL_mean": -149.39247131347656, "KL/mean": -187.53536987304688, "KL/rejected_KL_mean": -225.67825317382812, "KL/std": 96.739501953125, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.28581237792969, "fcm_dpo/q_t": 0.4857264757156372, "grad_norm": 1.2528910636901855, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.0117123126983643, "logits/rejected": -1.5317509174346924, "logps/chosen": -252.65586853027344, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -323.5072326660156, "loss": 1.3318, "margin_dpo/margin_mean": 76.28581237792969, "margin_dpo/margin_std": 112.046875, "step": 305 }, { "KL/chosen_KL_mean": -135.45989990234375, "KL/mean": -179.7418212890625, "KL/rejected_KL_mean": -224.02371215820312, "KL/std": 88.54747772216797, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.56381225585938, "fcm_dpo/q_t": 0.48342210054397583, "grad_norm": 1.801374077796936, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.386655330657959, "logits/rejected": -1.843639850616455, "logps/chosen": -235.76373291015625, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -316.25457763671875, "loss": 1.3221, "margin_dpo/margin_mean": 88.56381225585938, "margin_dpo/margin_std": 87.718994140625, "step": 306 }, { "KL/chosen_KL_mean": -139.28236389160156, "KL/mean": -175.23760986328125, "KL/rejected_KL_mean": -211.1928253173828, "KL/std": 92.70634460449219, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.91045379638672, "fcm_dpo/q_t": 0.4865493178367615, "grad_norm": 1.6093735694885254, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.0579276084899902, "logits/rejected": -1.701674461364746, "logps/chosen": -235.93212890625, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -304.54144287109375, "loss": 1.335, "margin_dpo/margin_mean": 71.91046142578125, "margin_dpo/margin_std": 114.7800064086914, "step": 307 }, { "KL/chosen_KL_mean": -148.01919555664062, "KL/mean": -183.63253784179688, "KL/rejected_KL_mean": -219.24588012695312, "KL/std": 94.12076568603516, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.22666931152344, "fcm_dpo/q_t": 0.48667484521865845, "grad_norm": 1.5447392463684082, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7375619411468506, "logits/rejected": -1.4319477081298828, "logps/chosen": -233.74386596679688, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -305.6690673828125, "loss": 1.3354, "margin_dpo/margin_mean": 71.22666931152344, "margin_dpo/margin_std": 111.23309326171875, "step": 308 }, { "KL/chosen_KL_mean": -162.8796844482422, "KL/mean": -195.67568969726562, "KL/rejected_KL_mean": -228.47171020507812, "KL/std": 107.07435607910156, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.59203338623047, "fcm_dpo/q_t": 0.48772987723350525, "grad_norm": 1.852613925933838, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.6965781450271606, "logits/rejected": -1.263155460357666, "logps/chosen": -246.03114318847656, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -315.0577392578125, "loss": 1.3395, "margin_dpo/margin_mean": 65.592041015625, "margin_dpo/margin_std": 111.01177978515625, "step": 309 }, { "KL/chosen_KL_mean": -153.22161865234375, "KL/mean": -189.70895385742188, "KL/rejected_KL_mean": -226.1962890625, "KL/std": 89.24795532226562, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.97467041015625, "fcm_dpo/q_t": 0.48634210228919983, "grad_norm": 1.6017193794250488, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.298034191131592, "logits/rejected": -1.8882458209991455, "logps/chosen": -242.95962524414062, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -313.26708984375, "loss": 1.3336, "margin_dpo/margin_mean": 72.97466278076172, "margin_dpo/margin_std": 94.8542251586914, "step": 310 }, { "KL/chosen_KL_mean": -176.8011474609375, "KL/mean": -218.10595703125, "KL/rejected_KL_mean": -259.4107666015625, "KL/std": 92.58428955078125, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.60963439941406, "fcm_dpo/q_t": 0.48454737663269043, "grad_norm": 1.7805942296981812, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.5494651794433594, "logits/rejected": -0.8149286508560181, "logps/chosen": -265.32757568359375, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -341.6768493652344, "loss": 1.3268, "margin_dpo/margin_mean": 82.60963439941406, "margin_dpo/margin_std": 101.18122863769531, "step": 311 }, { "KL/chosen_KL_mean": -167.28433227539062, "KL/mean": -204.01658630371094, "KL/rejected_KL_mean": -240.74880981445312, "KL/std": 100.67498779296875, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.46446228027344, "fcm_dpo/q_t": 0.48625147342681885, "grad_norm": 1.6950063705444336, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.7287317514419556, "logits/rejected": -1.2230926752090454, "logps/chosen": -247.68502807617188, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -323.1959533691406, "loss": 1.3334, "margin_dpo/margin_mean": 73.46446228027344, "margin_dpo/margin_std": 101.20404052734375, "step": 312 }, { "KL/chosen_KL_mean": -169.1116943359375, "KL/mean": -204.22882080078125, "KL/rejected_KL_mean": -239.345947265625, "KL/std": 99.94741821289062, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.23424530029297, "fcm_dpo/q_t": 0.4868588149547577, "grad_norm": 1.7326157093048096, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.4047927856445312, "logits/rejected": -2.0162487030029297, "logps/chosen": -253.25360107421875, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -332.77276611328125, "loss": 1.3361, "margin_dpo/margin_mean": 70.23423767089844, "margin_dpo/margin_std": 109.85655212402344, "step": 313 }, { "KL/chosen_KL_mean": -171.88046264648438, "KL/mean": -205.6112060546875, "KL/rejected_KL_mean": -239.34193420410156, "KL/std": 106.45965576171875, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.46148681640625, "fcm_dpo/q_t": 0.4873883128166199, "grad_norm": 1.8477578163146973, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8601161241531372, "logits/rejected": -1.6590057611465454, "logps/chosen": -260.690185546875, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -343.5008544921875, "loss": 1.3383, "margin_dpo/margin_mean": 67.46148681640625, "margin_dpo/margin_std": 117.5742416381836, "step": 314 }, { "KL/chosen_KL_mean": -161.7198486328125, "KL/mean": -198.69338989257812, "KL/rejected_KL_mean": -235.66693115234375, "KL/std": 108.72269439697266, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.94706726074219, "fcm_dpo/q_t": 0.48617005348205566, "grad_norm": 1.8192142248153687, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.385285496711731, "logits/rejected": -1.234250545501709, "logps/chosen": -242.5135498046875, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -327.80401611328125, "loss": 1.3338, "margin_dpo/margin_mean": 73.94706726074219, "margin_dpo/margin_std": 124.23126220703125, "step": 315 }, { "KL/chosen_KL_mean": -164.91079711914062, "KL/mean": -208.87110900878906, "KL/rejected_KL_mean": -252.8314208984375, "KL/std": 109.53926086425781, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.92062377929688, "fcm_dpo/q_t": 0.4835713505744934, "grad_norm": 1.7363166809082031, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.7502096891403198, "logits/rejected": -1.7515931129455566, "logps/chosen": -249.0950469970703, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -358.522216796875, "loss": 1.3237, "margin_dpo/margin_mean": 87.92062377929688, "margin_dpo/margin_std": 125.98275756835938, "step": 316 }, { "KL/chosen_KL_mean": -169.54061889648438, "KL/mean": -220.13638305664062, "KL/rejected_KL_mean": -270.7321472167969, "KL/std": 116.44463348388672, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.19154357910156, "fcm_dpo/q_t": 0.48109036684036255, "grad_norm": 2.2765982151031494, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.8233227729797363, "logits/rejected": -1.3569589853286743, "logps/chosen": -257.4140930175781, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -378.9973449707031, "loss": 1.3142, "margin_dpo/margin_mean": 101.19153594970703, "margin_dpo/margin_std": 125.61209106445312, "step": 317 }, { "KL/chosen_KL_mean": -173.15655517578125, "KL/mean": -212.20391845703125, "KL/rejected_KL_mean": -251.25128173828125, "KL/std": 95.72006225585938, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.0947265625, "fcm_dpo/q_t": 0.485392302274704, "grad_norm": 1.958033561706543, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.2815892696380615, "logits/rejected": -1.5499211549758911, "logps/chosen": -271.3620910644531, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -339.3875732421875, "loss": 1.3303, "margin_dpo/margin_mean": 78.0947265625, "margin_dpo/margin_std": 109.33906555175781, "step": 318 }, { "KL/chosen_KL_mean": -143.1568603515625, "KL/mean": -195.76040649414062, "KL/rejected_KL_mean": -248.36392211914062, "KL/std": 106.71786499023438, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.20703887939453, "fcm_dpo/q_t": 0.48033732175827026, "grad_norm": 2.1304359436035156, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.8356809616088867, "logits/rejected": -1.803890347480774, "logps/chosen": -225.66024780273438, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -367.01458740234375, "loss": 1.3112, "margin_dpo/margin_mean": 105.20703887939453, "margin_dpo/margin_std": 126.10104370117188, "step": 319 }, { "KL/chosen_KL_mean": -147.56622314453125, "KL/mean": -187.6890869140625, "KL/rejected_KL_mean": -227.81195068359375, "KL/std": 101.06694030761719, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.24571990966797, "fcm_dpo/q_t": 0.48498809337615967, "grad_norm": 1.8001704216003418, "learning_rate": 3.208807785813777e-07, "logits/chosen": -1.8762221336364746, "logits/rejected": -1.6864088773727417, "logps/chosen": -234.7034454345703, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -331.03607177734375, "loss": 1.3286, "margin_dpo/margin_mean": 80.24571990966797, "margin_dpo/margin_std": 103.00386810302734, "step": 320 }, { "KL/chosen_KL_mean": -164.03793334960938, "KL/mean": -210.67388916015625, "KL/rejected_KL_mean": -257.309814453125, "KL/std": 106.82366943359375, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.27190399169922, "fcm_dpo/q_t": 0.48254668712615967, "grad_norm": 1.7484008073806763, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -2.0317392349243164, "logits/rejected": -1.6425573825836182, "logps/chosen": -257.1426086425781, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -354.7779541015625, "loss": 1.3191, "margin_dpo/margin_mean": 93.27189636230469, "margin_dpo/margin_std": 103.04649353027344, "step": 321 }, { "KL/chosen_KL_mean": -153.33889770507812, "KL/mean": -205.33584594726562, "KL/rejected_KL_mean": -257.33282470703125, "KL/std": 103.39689636230469, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.99391174316406, "fcm_dpo/q_t": 0.48055416345596313, "grad_norm": 2.3821053504943848, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.6860160827636719, "logits/rejected": -1.3285832405090332, "logps/chosen": -245.5318145751953, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -358.22601318359375, "loss": 1.3118, "margin_dpo/margin_mean": 103.9939193725586, "margin_dpo/margin_std": 114.73689270019531, "step": 322 }, { "KL/chosen_KL_mean": -177.37338256835938, "KL/mean": -215.71377563476562, "KL/rejected_KL_mean": -254.05416870117188, "KL/std": 106.259765625, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.6807861328125, "fcm_dpo/q_t": 0.4856625199317932, "grad_norm": 1.7359811067581177, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0475611686706543, "logits/rejected": -1.671327829360962, "logps/chosen": -257.6454772949219, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -334.3575134277344, "loss": 1.3315, "margin_dpo/margin_mean": 76.6807861328125, "margin_dpo/margin_std": 115.34764099121094, "step": 323 }, { "KL/chosen_KL_mean": -179.5297088623047, "KL/mean": -217.63865661621094, "KL/rejected_KL_mean": -255.74761962890625, "KL/std": 112.90201568603516, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.2179183959961, "fcm_dpo/q_t": 0.48574623465538025, "grad_norm": 1.8749815225601196, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.6392706632614136, "logits/rejected": -1.0110788345336914, "logps/chosen": -275.2590637207031, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -349.9168701171875, "loss": 1.3318, "margin_dpo/margin_mean": 76.21791076660156, "margin_dpo/margin_std": 113.80980682373047, "step": 324 }, { "KL/chosen_KL_mean": -163.34524536132812, "KL/mean": -204.83058166503906, "KL/rejected_KL_mean": -246.31588745117188, "KL/std": 97.76335906982422, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.97065734863281, "fcm_dpo/q_t": 0.48448073863983154, "grad_norm": 2.114514112472534, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.8505685329437256, "logits/rejected": -1.1071385145187378, "logps/chosen": -262.0521240234375, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -322.7239990234375, "loss": 1.3267, "margin_dpo/margin_mean": 82.97065734863281, "margin_dpo/margin_std": 105.86332702636719, "step": 325 }, { "KL/chosen_KL_mean": -174.87513732910156, "KL/mean": -215.03538513183594, "KL/rejected_KL_mean": -255.19561767578125, "KL/std": 97.26174926757812, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.32047271728516, "fcm_dpo/q_t": 0.4849695563316345, "grad_norm": 1.5879391431808472, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.610487461090088, "logits/rejected": -1.9356813430786133, "logps/chosen": -279.53643798828125, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -342.02099609375, "loss": 1.3285, "margin_dpo/margin_mean": 80.32047271728516, "margin_dpo/margin_std": 105.4498291015625, "step": 326 }, { "KL/chosen_KL_mean": -171.59085083007812, "KL/mean": -211.1751708984375, "KL/rejected_KL_mean": -250.75949096679688, "KL/std": 91.59095764160156, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.16862487792969, "fcm_dpo/q_t": 0.4851953685283661, "grad_norm": 1.7196553945541382, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.7971864938735962, "logits/rejected": -1.2378114461898804, "logps/chosen": -279.03106689453125, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -367.812744140625, "loss": 1.3295, "margin_dpo/margin_mean": 79.16863250732422, "margin_dpo/margin_std": 107.70355987548828, "step": 327 }, { "KL/chosen_KL_mean": -167.00955200195312, "KL/mean": -218.50228881835938, "KL/rejected_KL_mean": -269.9949951171875, "KL/std": 106.99576568603516, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.98544311523438, "fcm_dpo/q_t": 0.48074284195899963, "grad_norm": 1.763940453529358, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.7728610038757324, "logits/rejected": -1.476957082748413, "logps/chosen": -253.62025451660156, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -367.3968811035156, "loss": 1.3124, "margin_dpo/margin_mean": 102.98544311523438, "margin_dpo/margin_std": 113.4393081665039, "step": 328 }, { "KL/chosen_KL_mean": -177.99749755859375, "KL/mean": -218.81741333007812, "KL/rejected_KL_mean": -259.6373291015625, "KL/std": 107.40089416503906, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.63980102539062, "fcm_dpo/q_t": 0.48474419116973877, "grad_norm": 1.9768574237823486, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.518836498260498, "logits/rejected": -1.0219032764434814, "logps/chosen": -278.6105651855469, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -364.59100341796875, "loss": 1.3284, "margin_dpo/margin_mean": 81.63980102539062, "margin_dpo/margin_std": 127.27141571044922, "step": 329 }, { "KL/chosen_KL_mean": -142.2100372314453, "KL/mean": -187.89468383789062, "KL/rejected_KL_mean": -233.5792999267578, "KL/std": 100.66641235351562, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.3692626953125, "fcm_dpo/q_t": 0.48289719223976135, "grad_norm": 1.7705209255218506, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.9744439125061035, "logits/rejected": -1.904162883758545, "logps/chosen": -219.16847229003906, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -326.39422607421875, "loss": 1.3202, "margin_dpo/margin_mean": 91.3692626953125, "margin_dpo/margin_std": 94.27861022949219, "step": 330 }, { "KL/chosen_KL_mean": -158.49269104003906, "KL/mean": -205.53692626953125, "KL/rejected_KL_mean": -252.58116149902344, "KL/std": 96.6388931274414, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.08847045898438, "fcm_dpo/q_t": 0.48240286111831665, "grad_norm": 2.031865358352661, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8764307498931885, "logits/rejected": -1.5838327407836914, "logps/chosen": -241.55209350585938, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -346.7122802734375, "loss": 1.3186, "margin_dpo/margin_mean": 94.08847045898438, "margin_dpo/margin_std": 106.49085998535156, "step": 331 }, { "KL/chosen_KL_mean": -164.1211700439453, "KL/mean": -196.66876220703125, "KL/rejected_KL_mean": -229.21633911132812, "KL/std": 107.08634185791016, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.09516906738281, "fcm_dpo/q_t": 0.4878199100494385, "grad_norm": 1.9289159774780273, "learning_rate": 3.059876462596758e-07, "logits/chosen": -1.979122519493103, "logits/rejected": -1.5413931608200073, "logps/chosen": -246.05206298828125, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -312.062255859375, "loss": 1.3397, "margin_dpo/margin_mean": 65.09516906738281, "margin_dpo/margin_std": 107.55201721191406, "step": 332 }, { "KL/chosen_KL_mean": -186.67465209960938, "KL/mean": -233.2981414794922, "KL/rejected_KL_mean": -279.921630859375, "KL/std": 114.95674133300781, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.24699401855469, "fcm_dpo/q_t": 0.48257219791412354, "grad_norm": 1.8126145601272583, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.505399465560913, "logits/rejected": -1.1656548976898193, "logps/chosen": -275.20721435546875, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -382.38763427734375, "loss": 1.32, "margin_dpo/margin_mean": 93.24698638916016, "margin_dpo/margin_std": 127.58860778808594, "step": 333 }, { "KL/chosen_KL_mean": -170.8585968017578, "KL/mean": -212.82620239257812, "KL/rejected_KL_mean": -254.79379272460938, "KL/std": 111.58554077148438, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.93519592285156, "fcm_dpo/q_t": 0.48430150747299194, "grad_norm": 2.298536777496338, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.6401374340057373, "logits/rejected": -1.3456929922103882, "logps/chosen": -268.61639404296875, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -354.79962158203125, "loss": 1.3268, "margin_dpo/margin_mean": 83.93519592285156, "margin_dpo/margin_std": 127.92625427246094, "step": 334 }, { "KL/chosen_KL_mean": -171.27037048339844, "KL/mean": -206.06692504882812, "KL/rejected_KL_mean": -240.86346435546875, "KL/std": 107.12718200683594, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.59307861328125, "fcm_dpo/q_t": 0.48697859048843384, "grad_norm": 2.997488260269165, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.2017264366149902, "logits/rejected": -1.7558424472808838, "logps/chosen": -267.64642333984375, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -335.7874755859375, "loss": 1.3368, "margin_dpo/margin_mean": 69.59307861328125, "margin_dpo/margin_std": 119.34259033203125, "step": 335 }, { "KL/chosen_KL_mean": -173.12887573242188, "KL/mean": -221.12106323242188, "KL/rejected_KL_mean": -269.1132507324219, "KL/std": 115.34730529785156, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.98436737060547, "fcm_dpo/q_t": 0.4820683002471924, "grad_norm": 2.84354829788208, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.5848820209503174, "logits/rejected": -1.4520578384399414, "logps/chosen": -261.65838623046875, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -378.6690368652344, "loss": 1.3179, "margin_dpo/margin_mean": 95.98436737060547, "margin_dpo/margin_std": 126.19679260253906, "step": 336 }, { "KL/chosen_KL_mean": -175.91058349609375, "KL/mean": -218.54782104492188, "KL/rejected_KL_mean": -261.1850891113281, "KL/std": 111.39888000488281, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.27452850341797, "fcm_dpo/q_t": 0.48406457901000977, "grad_norm": 1.8968899250030518, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.5374832153320312, "logits/rejected": -2.2763664722442627, "logps/chosen": -273.22296142578125, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -371.3164367675781, "loss": 1.326, "margin_dpo/margin_mean": 85.27452087402344, "margin_dpo/margin_std": 137.1028594970703, "step": 337 }, { "KL/chosen_KL_mean": -172.55836486816406, "KL/mean": -215.05465698242188, "KL/rejected_KL_mean": -257.5509338378906, "KL/std": 111.29939270019531, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.99256896972656, "fcm_dpo/q_t": 0.484096497297287, "grad_norm": 1.9294191598892212, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.9294686317443848, "logits/rejected": -1.7994991540908813, "logps/chosen": -256.1884765625, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -351.2432861328125, "loss": 1.3253, "margin_dpo/margin_mean": 84.99256134033203, "margin_dpo/margin_std": 108.66987609863281, "step": 338 }, { "KL/chosen_KL_mean": -195.97569274902344, "KL/mean": -239.12396240234375, "KL/rejected_KL_mean": -282.2721862792969, "KL/std": 111.4539566040039, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.29649353027344, "fcm_dpo/q_t": 0.48386871814727783, "grad_norm": 1.8796781301498413, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -3.001610040664673, "logits/rejected": -2.5884108543395996, "logps/chosen": -295.8275146484375, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -391.3465576171875, "loss": 1.3251, "margin_dpo/margin_mean": 86.29649353027344, "margin_dpo/margin_std": 127.67636108398438, "step": 339 }, { "KL/chosen_KL_mean": -204.27601623535156, "KL/mean": -242.4334259033203, "KL/rejected_KL_mean": -280.5908203125, "KL/std": 109.43170928955078, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.31482696533203, "fcm_dpo/q_t": 0.48573023080825806, "grad_norm": 1.7793819904327393, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.1910557746887207, "logits/rejected": -0.675208568572998, "logps/chosen": -289.25, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -359.39117431640625, "loss": 1.3322, "margin_dpo/margin_mean": 76.3148193359375, "margin_dpo/margin_std": 128.40939331054688, "step": 340 }, { "KL/chosen_KL_mean": -213.48464965820312, "KL/mean": -249.03890991210938, "KL/rejected_KL_mean": -284.59320068359375, "KL/std": 111.91475677490234, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.10855102539062, "fcm_dpo/q_t": 0.4867076575756073, "grad_norm": 2.7591612339019775, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.3931262493133545, "logits/rejected": -1.1761419773101807, "logps/chosen": -298.7486572265625, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -390.67913818359375, "loss": 1.3363, "margin_dpo/margin_mean": 71.10855102539062, "margin_dpo/margin_std": 135.27841186523438, "step": 341 }, { "KL/chosen_KL_mean": -173.24964904785156, "KL/mean": -226.7955322265625, "KL/rejected_KL_mean": -280.3414001464844, "KL/std": 107.73362731933594, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.0917739868164, "fcm_dpo/q_t": 0.4799671471118927, "grad_norm": 1.839586615562439, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.381126880645752, "logits/rejected": -0.6057747602462769, "logps/chosen": -274.28826904296875, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -379.23992919921875, "loss": 1.3096, "margin_dpo/margin_mean": 107.0917739868164, "margin_dpo/margin_std": 117.62407684326172, "step": 342 }, { "KL/chosen_KL_mean": -187.06869506835938, "KL/mean": -233.98654174804688, "KL/rejected_KL_mean": -280.9043884277344, "KL/std": 115.51016235351562, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.83566284179688, "fcm_dpo/q_t": 0.4824519157409668, "grad_norm": 1.9299631118774414, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1900248527526855, "logits/rejected": -2.110774278640747, "logps/chosen": -270.8160705566406, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -397.48345947265625, "loss": 1.3192, "margin_dpo/margin_mean": 93.83566284179688, "margin_dpo/margin_std": 117.65565490722656, "step": 343 }, { "KL/chosen_KL_mean": -180.21946716308594, "KL/mean": -233.27694702148438, "KL/rejected_KL_mean": -286.3344421386719, "KL/std": 125.56156921386719, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.11498260498047, "fcm_dpo/q_t": 0.4801836907863617, "grad_norm": 2.609192371368408, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.4410922527313232, "logits/rejected": -1.199772834777832, "logps/chosen": -258.47344970703125, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -381.7631530761719, "loss": 1.3109, "margin_dpo/margin_mean": 106.11498260498047, "margin_dpo/margin_std": 131.50631713867188, "step": 344 }, { "KL/chosen_KL_mean": -196.92367553710938, "KL/mean": -248.86285400390625, "KL/rejected_KL_mean": -300.802001953125, "KL/std": 122.48582458496094, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.8783187866211, "fcm_dpo/q_t": 0.4805937707424164, "grad_norm": 2.5990960597991943, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.7193830013275146, "logits/rejected": -1.4424108266830444, "logps/chosen": -295.003173828125, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -404.8367919921875, "loss": 1.3124, "margin_dpo/margin_mean": 103.87832641601562, "margin_dpo/margin_std": 132.76129150390625, "step": 345 }, { "KL/chosen_KL_mean": -218.9744873046875, "KL/mean": -258.0343933105469, "KL/rejected_KL_mean": -297.09423828125, "KL/std": 113.39935302734375, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.11976623535156, "fcm_dpo/q_t": 0.48539960384368896, "grad_norm": 1.6109154224395752, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.8389122486114502, "logits/rejected": -1.541515827178955, "logps/chosen": -299.657958984375, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -382.9868469238281, "loss": 1.331, "margin_dpo/margin_mean": 78.11976623535156, "margin_dpo/margin_std": 131.702392578125, "step": 346 }, { "KL/chosen_KL_mean": -207.49905395507812, "KL/mean": -243.9525146484375, "KL/rejected_KL_mean": -280.40594482421875, "KL/std": 116.06382751464844, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.90690612792969, "fcm_dpo/q_t": 0.48637667298316956, "grad_norm": 1.7050715684890747, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.8481318950653076, "logits/rejected": -1.5454665422439575, "logps/chosen": -301.1784362792969, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -377.948486328125, "loss": 1.3348, "margin_dpo/margin_mean": 72.90689849853516, "margin_dpo/margin_std": 130.66159057617188, "step": 347 }, { "KL/chosen_KL_mean": -182.93736267089844, "KL/mean": -223.77056884765625, "KL/rejected_KL_mean": -264.603759765625, "KL/std": 109.15068054199219, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.66641235351562, "fcm_dpo/q_t": 0.4847267270088196, "grad_norm": 1.8869566917419434, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.1735219955444336, "logits/rejected": -1.917178750038147, "logps/chosen": -273.1896057128906, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -360.3740234375, "loss": 1.328, "margin_dpo/margin_mean": 81.66641235351562, "margin_dpo/margin_std": 109.67153930664062, "step": 348 }, { "KL/chosen_KL_mean": -203.35440063476562, "KL/mean": -238.85824584960938, "KL/rejected_KL_mean": -274.3620910644531, "KL/std": 121.47308349609375, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.00770568847656, "fcm_dpo/q_t": 0.486733078956604, "grad_norm": 2.120586633682251, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.934108853340149, "logits/rejected": -1.716012954711914, "logps/chosen": -295.4124755859375, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -378.3273620605469, "loss": 1.3365, "margin_dpo/margin_mean": 71.00770568847656, "margin_dpo/margin_std": 138.87158203125, "step": 349 }, { "KL/chosen_KL_mean": -205.0007781982422, "KL/mean": -239.32315063476562, "KL/rejected_KL_mean": -273.6455078125, "KL/std": 108.10479736328125, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.64474487304688, "fcm_dpo/q_t": 0.4871624708175659, "grad_norm": 2.1996688842773438, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.7905051708221436, "logits/rejected": -1.5643985271453857, "logps/chosen": -292.05499267578125, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -369.367431640625, "loss": 1.3378, "margin_dpo/margin_mean": 68.64474487304688, "margin_dpo/margin_std": 127.00064086914062, "step": 350 }, { "KL/chosen_KL_mean": -207.7464141845703, "KL/mean": -254.95458984375, "KL/rejected_KL_mean": -302.16278076171875, "KL/std": 122.4637451171875, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.41636657714844, "fcm_dpo/q_t": 0.4823535680770874, "grad_norm": 2.0570573806762695, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.2750723361968994, "logits/rejected": -0.9968114495277405, "logps/chosen": -296.91546630859375, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -410.4608154296875, "loss": 1.3193, "margin_dpo/margin_mean": 94.41636657714844, "margin_dpo/margin_std": 135.7715606689453, "step": 351 }, { "KL/chosen_KL_mean": -194.35650634765625, "KL/mean": -236.0874786376953, "KL/rejected_KL_mean": -277.8184509277344, "KL/std": 117.94984436035156, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.46194458007812, "fcm_dpo/q_t": 0.48441702127456665, "grad_norm": 2.2343902587890625, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.6724592447280884, "logits/rejected": -1.0964477062225342, "logps/chosen": -291.16796875, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -362.2232666015625, "loss": 1.3275, "margin_dpo/margin_mean": 83.46195220947266, "margin_dpo/margin_std": 138.1802215576172, "step": 352 }, { "KL/chosen_KL_mean": -214.028564453125, "KL/mean": -243.65545654296875, "KL/rejected_KL_mean": -273.2823486328125, "KL/std": 107.11181640625, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.25376892089844, "fcm_dpo/q_t": 0.48891353607177734, "grad_norm": 1.8328133821487427, "learning_rate": 2.794480701395219e-07, "logits/chosen": -1.9682350158691406, "logits/rejected": -1.5599172115325928, "logps/chosen": -299.39105224609375, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -360.649658203125, "loss": 1.3446, "margin_dpo/margin_mean": 59.25376892089844, "margin_dpo/margin_std": 123.41030883789062, "step": 353 }, { "KL/chosen_KL_mean": -190.698974609375, "KL/mean": -234.96786499023438, "KL/rejected_KL_mean": -279.2367858886719, "KL/std": 105.92338562011719, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.53778076171875, "fcm_dpo/q_t": 0.48344728350639343, "grad_norm": 1.949141025543213, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.9374115467071533, "logits/rejected": -1.4809579849243164, "logps/chosen": -286.5001525878906, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -374.1029052734375, "loss": 1.3227, "margin_dpo/margin_mean": 88.53778839111328, "margin_dpo/margin_std": 109.55422973632812, "step": 354 }, { "KL/chosen_KL_mean": -184.20248413085938, "KL/mean": -229.51220703125, "KL/rejected_KL_mean": -274.8218994140625, "KL/std": 106.01882934570312, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.619384765625, "fcm_dpo/q_t": 0.4830511808395386, "grad_norm": 2.4309730529785156, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.326524257659912, "logits/rejected": -1.8246828317642212, "logps/chosen": -270.3880615234375, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -363.283447265625, "loss": 1.3213, "margin_dpo/margin_mean": 90.619384765625, "margin_dpo/margin_std": 112.42620849609375, "step": 355 }, { "KL/chosen_KL_mean": -226.61892700195312, "KL/mean": -252.81829833984375, "KL/rejected_KL_mean": -279.0177001953125, "KL/std": 109.64193725585938, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.398780822753906, "fcm_dpo/q_t": 0.4902101457118988, "grad_norm": 1.9300626516342163, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8847599029541016, "logits/rejected": -1.4195287227630615, "logps/chosen": -317.43701171875, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -361.14501953125, "loss": 1.3499, "margin_dpo/margin_mean": 52.398780822753906, "margin_dpo/margin_std": 133.2839813232422, "step": 356 }, { "KL/chosen_KL_mean": -188.04754638671875, "KL/mean": -238.13047790527344, "KL/rejected_KL_mean": -288.2134094238281, "KL/std": 112.96527099609375, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.16590118408203, "fcm_dpo/q_t": 0.48126697540283203, "grad_norm": 1.9645127058029175, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.8223527669906616, "logits/rejected": -1.4570813179016113, "logps/chosen": -275.55230712890625, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -380.4627685546875, "loss": 1.3144, "margin_dpo/margin_mean": 100.1658935546875, "margin_dpo/margin_std": 110.48770141601562, "step": 357 }, { "KL/chosen_KL_mean": -196.13304138183594, "KL/mean": -235.9117431640625, "KL/rejected_KL_mean": -275.6904602050781, "KL/std": 111.81925964355469, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.55741882324219, "fcm_dpo/q_t": 0.48513180017471313, "grad_norm": 2.0048158168792725, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.6890497207641602, "logits/rejected": -1.2462100982666016, "logps/chosen": -291.1885070800781, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -372.16656494140625, "loss": 1.3294, "margin_dpo/margin_mean": 79.55741882324219, "margin_dpo/margin_std": 115.36127471923828, "step": 358 }, { "KL/chosen_KL_mean": -220.68499755859375, "KL/mean": -257.92401123046875, "KL/rejected_KL_mean": -295.16302490234375, "KL/std": 110.87359619140625, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.47798156738281, "fcm_dpo/q_t": 0.4860726594924927, "grad_norm": 2.4316370487213135, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4106663465499878, "logits/rejected": -0.810522198677063, "logps/chosen": -308.0583190917969, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -383.93487548828125, "loss": 1.3337, "margin_dpo/margin_mean": 74.47798156738281, "margin_dpo/margin_std": 131.02825927734375, "step": 359 }, { "KL/chosen_KL_mean": -202.25531005859375, "KL/mean": -246.13990783691406, "KL/rejected_KL_mean": -290.0245361328125, "KL/std": 103.00385284423828, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.76922607421875, "fcm_dpo/q_t": 0.48358258605003357, "grad_norm": 2.092036485671997, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.6504466533660889, "logits/rejected": -1.317915916442871, "logps/chosen": -277.478759765625, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -366.1185302734375, "loss": 1.3235, "margin_dpo/margin_mean": 87.76922607421875, "margin_dpo/margin_std": 116.0586929321289, "step": 360 }, { "KL/chosen_KL_mean": -207.54983520507812, "KL/mean": -249.42837524414062, "KL/rejected_KL_mean": -291.30694580078125, "KL/std": 117.9671630859375, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.75714111328125, "fcm_dpo/q_t": 0.4843447208404541, "grad_norm": 2.3063230514526367, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.5955922603607178, "logits/rejected": -1.3692234754562378, "logps/chosen": -289.503173828125, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -387.98614501953125, "loss": 1.3265, "margin_dpo/margin_mean": 83.75714111328125, "margin_dpo/margin_std": 119.43887329101562, "step": 361 }, { "KL/chosen_KL_mean": -211.30316162109375, "KL/mean": -259.80316162109375, "KL/rejected_KL_mean": -308.30316162109375, "KL/std": 119.29234313964844, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.99999237060547, "fcm_dpo/q_t": 0.4818745255470276, "grad_norm": 2.085251808166504, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.6707446575164795, "logits/rejected": -1.4764128923416138, "logps/chosen": -300.64288330078125, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -418.8051452636719, "loss": 1.3174, "margin_dpo/margin_mean": 96.99998474121094, "margin_dpo/margin_std": 132.0433349609375, "step": 362 }, { "KL/chosen_KL_mean": -198.04115295410156, "KL/mean": -255.04513549804688, "KL/rejected_KL_mean": -312.04913330078125, "KL/std": 120.0455322265625, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.00800323486328, "fcm_dpo/q_t": 0.4787050485610962, "grad_norm": 2.5454769134521484, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.8776438236236572, "logits/rejected": -1.465641975402832, "logps/chosen": -278.816162109375, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -391.4818115234375, "loss": 1.3051, "margin_dpo/margin_mean": 114.00800323486328, "margin_dpo/margin_std": 130.99520874023438, "step": 363 }, { "KL/chosen_KL_mean": -190.98196411132812, "KL/mean": -239.28634643554688, "KL/rejected_KL_mean": -287.5906982421875, "KL/std": 111.30044555664062, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.60877990722656, "fcm_dpo/q_t": 0.48193174600601196, "grad_norm": 2.6753499507904053, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.10219144821167, "logits/rejected": -1.986563801765442, "logps/chosen": -267.5115661621094, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -385.1524658203125, "loss": 1.3171, "margin_dpo/margin_mean": 96.60877990722656, "margin_dpo/margin_std": 114.6796646118164, "step": 364 }, { "KL/chosen_KL_mean": -197.79783630371094, "KL/mean": -245.99441528320312, "KL/rejected_KL_mean": -294.1910095214844, "KL/std": 120.54946899414062, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.3931655883789, "fcm_dpo/q_t": 0.48200884461402893, "grad_norm": 2.487633466720581, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.9868102073669434, "logits/rejected": -1.89994478225708, "logps/chosen": -278.34271240234375, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -393.51416015625, "loss": 1.318, "margin_dpo/margin_mean": 96.39315795898438, "margin_dpo/margin_std": 134.19805908203125, "step": 365 }, { "KL/chosen_KL_mean": -196.83172607421875, "KL/mean": -239.05426025390625, "KL/rejected_KL_mean": -281.27679443359375, "KL/std": 124.84422302246094, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.44503784179688, "fcm_dpo/q_t": 0.48423996567726135, "grad_norm": 1.967095136642456, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.8038487434387207, "logits/rejected": -1.8684226274490356, "logps/chosen": -269.0697937011719, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -372.9486083984375, "loss": 1.3266, "margin_dpo/margin_mean": 84.44503784179688, "margin_dpo/margin_std": 134.61257934570312, "step": 366 }, { "KL/chosen_KL_mean": -179.22076416015625, "KL/mean": -231.60177612304688, "KL/rejected_KL_mean": -283.9827575683594, "KL/std": 108.53471374511719, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.76200866699219, "fcm_dpo/q_t": 0.4804084300994873, "grad_norm": 2.1729555130004883, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.8216382265090942, "logits/rejected": -1.4519801139831543, "logps/chosen": -264.64276123046875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -367.50054931640625, "loss": 1.3109, "margin_dpo/margin_mean": 104.76200866699219, "margin_dpo/margin_std": 107.57371520996094, "step": 367 }, { "KL/chosen_KL_mean": -161.99427795410156, "KL/mean": -221.99395751953125, "KL/rejected_KL_mean": -281.99365234375, "KL/std": 118.74783325195312, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.99934387207031, "fcm_dpo/q_t": 0.47755587100982666, "grad_norm": 2.2534940242767334, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.443114995956421, "logits/rejected": -1.4341347217559814, "logps/chosen": -235.8354034423828, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -386.68890380859375, "loss": 1.3, "margin_dpo/margin_mean": 119.99934387207031, "margin_dpo/margin_std": 107.76819610595703, "step": 368 }, { "KL/chosen_KL_mean": -191.9196319580078, "KL/mean": -242.07676696777344, "KL/rejected_KL_mean": -292.2339172363281, "KL/std": 116.59150695800781, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.31428527832031, "fcm_dpo/q_t": 0.4812648892402649, "grad_norm": 2.2100255489349365, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.8020977973937988, "logits/rejected": -1.4606435298919678, "logps/chosen": -290.381591796875, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -391.390625, "loss": 1.3149, "margin_dpo/margin_mean": 100.31428527832031, "margin_dpo/margin_std": 130.4573211669922, "step": 369 }, { "KL/chosen_KL_mean": -213.18087768554688, "KL/mean": -247.20413208007812, "KL/rejected_KL_mean": -281.2274169921875, "KL/std": 109.03179168701172, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.0465087890625, "fcm_dpo/q_t": 0.487282395362854, "grad_norm": 2.259497880935669, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.959334135055542, "logits/rejected": -1.7763657569885254, "logps/chosen": -309.95660400390625, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -383.97149658203125, "loss": 1.3389, "margin_dpo/margin_mean": 68.0465087890625, "margin_dpo/margin_std": 143.55958557128906, "step": 370 }, { "KL/chosen_KL_mean": -202.30345153808594, "KL/mean": -239.2093505859375, "KL/rejected_KL_mean": -276.11529541015625, "KL/std": 109.3320541381836, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.81181335449219, "fcm_dpo/q_t": 0.4861924648284912, "grad_norm": 2.1594505310058594, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.9303300380706787, "logits/rejected": -1.5739960670471191, "logps/chosen": -305.29364013671875, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -381.46588134765625, "loss": 1.3342, "margin_dpo/margin_mean": 73.81180572509766, "margin_dpo/margin_std": 130.058837890625, "step": 371 }, { "KL/chosen_KL_mean": -175.415771484375, "KL/mean": -230.82057189941406, "KL/rejected_KL_mean": -286.225341796875, "KL/std": 121.67279052734375, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.80958557128906, "fcm_dpo/q_t": 0.479308545589447, "grad_norm": 2.3551340103149414, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.9448647499084473, "logits/rejected": -1.309738278388977, "logps/chosen": -266.4420166015625, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -371.8355712890625, "loss": 1.3075, "margin_dpo/margin_mean": 110.80958557128906, "margin_dpo/margin_std": 131.35858154296875, "step": 372 }, { "KL/chosen_KL_mean": -206.3367156982422, "KL/mean": -247.71420288085938, "KL/rejected_KL_mean": -289.0917053222656, "KL/std": 127.03855895996094, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.75497436523438, "fcm_dpo/q_t": 0.48453253507614136, "grad_norm": 2.988304615020752, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.1426215171813965, "logits/rejected": -1.6091229915618896, "logps/chosen": -306.4097900390625, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -380.2080383300781, "loss": 1.328, "margin_dpo/margin_mean": 82.75497436523438, "margin_dpo/margin_std": 141.1280517578125, "step": 373 }, { "KL/chosen_KL_mean": -177.1223907470703, "KL/mean": -225.25115966796875, "KL/rejected_KL_mean": -273.3798828125, "KL/std": 117.78491973876953, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.25750732421875, "fcm_dpo/q_t": 0.4820109009742737, "grad_norm": 2.0853121280670166, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.1371679306030273, "logits/rejected": -1.7520396709442139, "logps/chosen": -275.1347351074219, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -374.13214111328125, "loss": 1.3176, "margin_dpo/margin_mean": 96.25749206542969, "margin_dpo/margin_std": 123.01214599609375, "step": 374 }, { "KL/chosen_KL_mean": -175.10247802734375, "KL/mean": -231.9293212890625, "KL/rejected_KL_mean": -288.75616455078125, "KL/std": 111.16748046875, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.65367889404297, "fcm_dpo/q_t": 0.47875916957855225, "grad_norm": 1.983042597770691, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.7192519903182983, "logits/rejected": -1.4867249727249146, "logps/chosen": -259.498779296875, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -384.47503662109375, "loss": 1.3053, "margin_dpo/margin_mean": 113.65367889404297, "margin_dpo/margin_std": 128.7683868408203, "step": 375 }, { "KL/chosen_KL_mean": -180.15725708007812, "KL/mean": -240.06878662109375, "KL/rejected_KL_mean": -299.9803466796875, "KL/std": 134.05001831054688, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.82307434082031, "fcm_dpo/q_t": 0.4776192009449005, "grad_norm": 2.3758544921875, "learning_rate": 2.5e-07, "logits/chosen": -1.8710911273956299, "logits/rejected": -1.543565034866333, "logps/chosen": -275.89495849609375, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -411.9536437988281, "loss": 1.3014, "margin_dpo/margin_mean": 119.82307434082031, "margin_dpo/margin_std": 144.1407928466797, "step": 376 }, { "KL/chosen_KL_mean": -185.9791259765625, "KL/mean": -249.5006103515625, "KL/rejected_KL_mean": -313.02215576171875, "KL/std": 132.9515380859375, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.04302978515625, "fcm_dpo/q_t": 0.47625237703323364, "grad_norm": 2.077633857727051, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.58817458152771, "logits/rejected": -1.4919466972351074, "logps/chosen": -264.90606689453125, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -418.92108154296875, "loss": 1.2955, "margin_dpo/margin_mean": 127.04302978515625, "margin_dpo/margin_std": 125.34172821044922, "step": 377 }, { "KL/chosen_KL_mean": -175.15249633789062, "KL/mean": -226.288818359375, "KL/rejected_KL_mean": -277.4251708984375, "KL/std": 125.82097625732422, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.27265930175781, "fcm_dpo/q_t": 0.4808729887008667, "grad_norm": 2.2693285942077637, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.060354232788086, "logits/rejected": -1.8024005889892578, "logps/chosen": -250.52490234375, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -371.1292419433594, "loss": 1.3137, "margin_dpo/margin_mean": 102.27265930175781, "margin_dpo/margin_std": 134.22750854492188, "step": 378 }, { "KL/chosen_KL_mean": -207.5838623046875, "KL/mean": -253.81988525390625, "KL/rejected_KL_mean": -300.0559387207031, "KL/std": 129.25341796875, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.47209167480469, "fcm_dpo/q_t": 0.48273324966430664, "grad_norm": 2.3559682369232178, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.8973045349121094, "logits/rejected": -1.8792496919631958, "logps/chosen": -296.4395446777344, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -421.43707275390625, "loss": 1.3212, "margin_dpo/margin_mean": 92.47209167480469, "margin_dpo/margin_std": 146.677490234375, "step": 379 }, { "KL/chosen_KL_mean": -217.69314575195312, "KL/mean": -282.23980712890625, "KL/rejected_KL_mean": -346.78643798828125, "KL/std": 132.47064208984375, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.09329223632812, "fcm_dpo/q_t": 0.47590646147727966, "grad_norm": 2.44992995262146, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.429093599319458, "logits/rejected": -1.4150617122650146, "logps/chosen": -297.5263671875, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -457.53118896484375, "loss": 1.2949, "margin_dpo/margin_mean": 129.09329223632812, "margin_dpo/margin_std": 147.44207763671875, "step": 380 }, { "KL/chosen_KL_mean": -201.01144409179688, "KL/mean": -247.27537536621094, "KL/rejected_KL_mean": -293.539306640625, "KL/std": 130.00865173339844, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.52787780761719, "fcm_dpo/q_t": 0.48272275924682617, "grad_norm": 2.59283709526062, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.7739613056182861, "logits/rejected": -1.2501590251922607, "logps/chosen": -287.5782775878906, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -377.9560546875, "loss": 1.3217, "margin_dpo/margin_mean": 92.52787780761719, "margin_dpo/margin_std": 159.51612854003906, "step": 381 }, { "KL/chosen_KL_mean": -192.2664031982422, "KL/mean": -249.36886596679688, "KL/rejected_KL_mean": -306.4713134765625, "KL/std": 119.56370544433594, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.20491790771484, "fcm_dpo/q_t": 0.4786495864391327, "grad_norm": 2.2617204189300537, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -2.0662894248962402, "logits/rejected": -1.8332371711730957, "logps/chosen": -296.72821044921875, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -421.07354736328125, "loss": 1.3049, "margin_dpo/margin_mean": 114.20492553710938, "margin_dpo/margin_std": 129.36265563964844, "step": 382 }, { "KL/chosen_KL_mean": -210.74081420898438, "KL/mean": -258.48602294921875, "KL/rejected_KL_mean": -306.23126220703125, "KL/std": 128.7977294921875, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.4904556274414, "fcm_dpo/q_t": 0.4821586012840271, "grad_norm": 2.001458168029785, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.636322021484375, "logits/rejected": -1.5115256309509277, "logps/chosen": -292.85687255859375, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -405.0084228515625, "loss": 1.3189, "margin_dpo/margin_mean": 95.4904556274414, "margin_dpo/margin_std": 142.24583435058594, "step": 383 }, { "KL/chosen_KL_mean": -229.48770141601562, "KL/mean": -267.4931640625, "KL/rejected_KL_mean": -305.4986877441406, "KL/std": 128.28268432617188, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.0110092163086, "fcm_dpo/q_t": 0.48579728603363037, "grad_norm": 2.7901740074157715, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7477326393127441, "logits/rejected": -1.5417051315307617, "logps/chosen": -310.07293701171875, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -396.94659423828125, "loss": 1.3333, "margin_dpo/margin_mean": 76.0110092163086, "margin_dpo/margin_std": 148.33880615234375, "step": 384 }, { "KL/chosen_KL_mean": -199.04141235351562, "KL/mean": -248.03411865234375, "KL/rejected_KL_mean": -297.02685546875, "KL/std": 111.2469482421875, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.9854507446289, "fcm_dpo/q_t": 0.48168420791625977, "grad_norm": 4.3154191970825195, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.264610528945923, "logits/rejected": -1.6820578575134277, "logps/chosen": -291.80157470703125, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -381.48681640625, "loss": 1.317, "margin_dpo/margin_mean": 97.9854507446289, "margin_dpo/margin_std": 138.3977813720703, "step": 385 }, { "KL/chosen_KL_mean": -190.95977783203125, "KL/mean": -247.67291259765625, "KL/rejected_KL_mean": -304.3860778808594, "KL/std": 116.60881042480469, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.42628479003906, "fcm_dpo/q_t": 0.47878965735435486, "grad_norm": 2.27437424659729, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.7051974534988403, "logits/rejected": -1.4343218803405762, "logps/chosen": -276.3187255859375, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -401.376708984375, "loss": 1.3055, "margin_dpo/margin_mean": 113.42628479003906, "margin_dpo/margin_std": 130.1878662109375, "step": 386 }, { "KL/chosen_KL_mean": -234.921630859375, "KL/mean": -273.93304443359375, "KL/rejected_KL_mean": -312.9444580078125, "KL/std": 120.22225952148438, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.02284240722656, "fcm_dpo/q_t": 0.4854162335395813, "grad_norm": 2.49894118309021, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.487839460372925, "logits/rejected": -1.8270840644836426, "logps/chosen": -337.6097717285156, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -412.373046875, "loss": 1.3308, "margin_dpo/margin_mean": 78.02284240722656, "margin_dpo/margin_std": 124.25637817382812, "step": 387 }, { "KL/chosen_KL_mean": -219.42225646972656, "KL/mean": -276.08642578125, "KL/rejected_KL_mean": -332.75054931640625, "KL/std": 133.04432678222656, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.32830810546875, "fcm_dpo/q_t": 0.4788268208503723, "grad_norm": 2.3520195484161377, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.683074712753296, "logits/rejected": -1.6011897325515747, "logps/chosen": -303.00750732421875, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -442.8509216308594, "loss": 1.3058, "margin_dpo/margin_mean": 113.32829284667969, "margin_dpo/margin_std": 138.90773010253906, "step": 388 }, { "KL/chosen_KL_mean": -235.57354736328125, "KL/mean": -273.9796142578125, "KL/rejected_KL_mean": -312.3857421875, "KL/std": 132.59783935546875, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.81219482421875, "fcm_dpo/q_t": 0.4856501817703247, "grad_norm": 2.6768617630004883, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.460933208465576, "logits/rejected": -2.02463698387146, "logps/chosen": -325.39581298828125, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -397.239501953125, "loss": 1.3327, "margin_dpo/margin_mean": 76.81219482421875, "margin_dpo/margin_std": 149.66671752929688, "step": 389 }, { "KL/chosen_KL_mean": -217.5487060546875, "KL/mean": -267.90484619140625, "KL/rejected_KL_mean": -318.26104736328125, "KL/std": 131.77993774414062, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.71234130859375, "fcm_dpo/q_t": 0.4812266528606415, "grad_norm": 1.9427229166030884, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.4466137886047363, "logits/rejected": -1.245558261871338, "logps/chosen": -295.0811462402344, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -410.844970703125, "loss": 1.3158, "margin_dpo/margin_mean": 100.71234893798828, "margin_dpo/margin_std": 158.21273803710938, "step": 390 }, { "KL/chosen_KL_mean": -213.89271545410156, "KL/mean": -254.79710388183594, "KL/rejected_KL_mean": -295.70147705078125, "KL/std": 128.3821563720703, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.80879211425781, "fcm_dpo/q_t": 0.4846934974193573, "grad_norm": 2.5147948265075684, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.9462995529174805, "logits/rejected": -1.5259358882904053, "logps/chosen": -302.1256408691406, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -388.70904541015625, "loss": 1.3287, "margin_dpo/margin_mean": 81.80879211425781, "margin_dpo/margin_std": 136.8009033203125, "step": 391 }, { "KL/chosen_KL_mean": -198.14569091796875, "KL/mean": -264.30010986328125, "KL/rejected_KL_mean": -330.4545593261719, "KL/std": 145.79153442382812, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.30885314941406, "fcm_dpo/q_t": 0.47530385851860046, "grad_norm": 2.4054934978485107, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.5084149837493896, "logits/rejected": -1.4054985046386719, "logps/chosen": -285.8901062011719, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -439.0252685546875, "loss": 1.2931, "margin_dpo/margin_mean": 132.30886840820312, "margin_dpo/margin_std": 158.16635131835938, "step": 392 }, { "KL/chosen_KL_mean": -182.63487243652344, "KL/mean": -233.32662963867188, "KL/rejected_KL_mean": -284.01837158203125, "KL/std": 125.35281372070312, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.3835220336914, "fcm_dpo/q_t": 0.48105108737945557, "grad_norm": 1.7089182138442993, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.7333874702453613, "logits/rejected": -1.414301872253418, "logps/chosen": -259.1986999511719, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -363.74456787109375, "loss": 1.314, "margin_dpo/margin_mean": 101.3835220336914, "margin_dpo/margin_std": 127.85847473144531, "step": 393 }, { "KL/chosen_KL_mean": -198.5078125, "KL/mean": -254.21624755859375, "KL/rejected_KL_mean": -309.9246826171875, "KL/std": 124.21238708496094, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.41687774658203, "fcm_dpo/q_t": 0.4791812598705292, "grad_norm": 2.3547682762145996, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6734628677368164, "logits/rejected": -1.4628961086273193, "logps/chosen": -283.20648193359375, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -407.3824462890625, "loss": 1.3069, "margin_dpo/margin_mean": 111.4168701171875, "margin_dpo/margin_std": 130.08399963378906, "step": 394 }, { "KL/chosen_KL_mean": -205.06777954101562, "KL/mean": -250.15249633789062, "KL/rejected_KL_mean": -295.23724365234375, "KL/std": 126.99079132080078, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.16944885253906, "fcm_dpo/q_t": 0.4831814467906952, "grad_norm": 2.069394826889038, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5084080696105957, "logits/rejected": -2.1106679439544678, "logps/chosen": -293.9554443359375, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -386.28277587890625, "loss": 1.3241, "margin_dpo/margin_mean": 90.16944885253906, "margin_dpo/margin_std": 171.1063232421875, "step": 395 }, { "KL/chosen_KL_mean": -212.00624084472656, "KL/mean": -248.78155517578125, "KL/rejected_KL_mean": -285.556884765625, "KL/std": 116.52288055419922, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.55064392089844, "fcm_dpo/q_t": 0.4862578511238098, "grad_norm": 2.789499521255493, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.728729009628296, "logits/rejected": -1.2392408847808838, "logps/chosen": -314.4775695800781, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -378.79168701171875, "loss": 1.3347, "margin_dpo/margin_mean": 73.55064392089844, "margin_dpo/margin_std": 140.35275268554688, "step": 396 }, { "KL/chosen_KL_mean": -193.44876098632812, "KL/mean": -265.20953369140625, "KL/rejected_KL_mean": -336.9703063964844, "KL/std": 134.07769775390625, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 143.5215606689453, "fcm_dpo/q_t": 0.47321411967277527, "grad_norm": 2.28712797164917, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.139866828918457, "logits/rejected": -1.543352723121643, "logps/chosen": -294.4931640625, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -445.82147216796875, "loss": 1.2852, "margin_dpo/margin_mean": 143.52157592773438, "margin_dpo/margin_std": 160.88229370117188, "step": 397 }, { "KL/chosen_KL_mean": -206.92405700683594, "KL/mean": -257.6405029296875, "KL/rejected_KL_mean": -308.35693359375, "KL/std": 130.62677001953125, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.43287658691406, "fcm_dpo/q_t": 0.4810597896575928, "grad_norm": 2.5371594429016113, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.0150811672210693, "logits/rejected": -1.7552063465118408, "logps/chosen": -301.9732666015625, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -422.362548828125, "loss": 1.3153, "margin_dpo/margin_mean": 101.43287658691406, "margin_dpo/margin_std": 158.41050720214844, "step": 398 }, { "KL/chosen_KL_mean": -172.19081115722656, "KL/mean": -234.02032470703125, "KL/rejected_KL_mean": -295.8498229980469, "KL/std": 118.28659057617188, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.65900421142578, "fcm_dpo/q_t": 0.47690349817276, "grad_norm": 2.5326101779937744, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.0004758834838867, "logits/rejected": -1.169978380203247, "logps/chosen": -268.829345703125, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -379.38629150390625, "loss": 1.2986, "margin_dpo/margin_mean": 123.65900421142578, "margin_dpo/margin_std": 142.94349670410156, "step": 399 }, { "KL/chosen_KL_mean": -193.6880645751953, "KL/mean": -255.7099151611328, "KL/rejected_KL_mean": -317.73175048828125, "KL/std": 127.88251495361328, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.04368591308594, "fcm_dpo/q_t": 0.47684216499328613, "grad_norm": 2.980592966079712, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.3003973960876465, "logits/rejected": -1.8609031438827515, "logps/chosen": -298.9912109375, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -426.38726806640625, "loss": 1.2991, "margin_dpo/margin_mean": 124.04368591308594, "margin_dpo/margin_std": 156.67257690429688, "step": 400 }, { "KL/chosen_KL_mean": -207.02378845214844, "KL/mean": -243.39024353027344, "KL/rejected_KL_mean": -279.7567138671875, "KL/std": 138.27886962890625, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.73291015625, "fcm_dpo/q_t": 0.48646289110183716, "grad_norm": 2.7345128059387207, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.5098135471343994, "logits/rejected": -1.272717833518982, "logps/chosen": -291.3129577636719, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -370.6998291015625, "loss": 1.337, "margin_dpo/margin_mean": 72.73291015625, "margin_dpo/margin_std": 178.27313232421875, "step": 401 }, { "KL/chosen_KL_mean": -179.2232208251953, "KL/mean": -256.184814453125, "KL/rejected_KL_mean": -333.14642333984375, "KL/std": 146.7880096435547, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 153.9232177734375, "fcm_dpo/q_t": 0.47127941250801086, "grad_norm": 2.3253774642944336, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.8193799257278442, "logits/rejected": -1.3581407070159912, "logps/chosen": -262.81634521484375, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -420.95672607421875, "loss": 1.2777, "margin_dpo/margin_mean": 153.9232177734375, "margin_dpo/margin_std": 157.20892333984375, "step": 402 }, { "KL/chosen_KL_mean": -184.23416137695312, "KL/mean": -240.78480529785156, "KL/rejected_KL_mean": -297.33544921875, "KL/std": 133.58346557617188, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.10127258300781, "fcm_dpo/q_t": 0.47887909412384033, "grad_norm": 2.837219715118408, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.3437047004699707, "logits/rejected": -1.946314811706543, "logps/chosen": -257.98724365234375, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -381.25555419921875, "loss": 1.3064, "margin_dpo/margin_mean": 113.10127258300781, "margin_dpo/margin_std": 149.37289428710938, "step": 403 }, { "KL/chosen_KL_mean": -212.95523071289062, "KL/mean": -261.00250244140625, "KL/rejected_KL_mean": -309.0497741699219, "KL/std": 127.87992858886719, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.09453582763672, "fcm_dpo/q_t": 0.4820694923400879, "grad_norm": 3.0792086124420166, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.8549822568893433, "logits/rejected": -1.5171692371368408, "logps/chosen": -292.63140869140625, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -393.3302917480469, "loss": 1.3192, "margin_dpo/margin_mean": 96.09454345703125, "margin_dpo/margin_std": 160.61924743652344, "step": 404 }, { "KL/chosen_KL_mean": -220.28189086914062, "KL/mean": -266.0677490234375, "KL/rejected_KL_mean": -311.85357666015625, "KL/std": 127.51651000976562, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.57167053222656, "fcm_dpo/q_t": 0.48290836811065674, "grad_norm": 2.52311635017395, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.5847458839416504, "logits/rejected": -1.113439679145813, "logps/chosen": -316.74383544921875, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -404.72430419921875, "loss": 1.3219, "margin_dpo/margin_mean": 91.57167053222656, "margin_dpo/margin_std": 146.919189453125, "step": 405 }, { "KL/chosen_KL_mean": -208.19613647460938, "KL/mean": -254.60714721679688, "KL/rejected_KL_mean": -301.01812744140625, "KL/std": 129.20220947265625, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.82196807861328, "fcm_dpo/q_t": 0.48265495896339417, "grad_norm": 2.9394984245300293, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.7852003574371338, "logits/rejected": -1.6255347728729248, "logps/chosen": -286.3301086425781, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -399.3017272949219, "loss": 1.3208, "margin_dpo/margin_mean": 92.82196807861328, "margin_dpo/margin_std": 143.20538330078125, "step": 406 }, { "KL/chosen_KL_mean": -212.72459411621094, "KL/mean": -249.01718139648438, "KL/rejected_KL_mean": -285.3097839355469, "KL/std": 123.00564575195312, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.58518981933594, "fcm_dpo/q_t": 0.48642784357070923, "grad_norm": 2.147543430328369, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.1419026851654053, "logits/rejected": -1.629102110862732, "logps/chosen": -296.7374267578125, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -368.0908203125, "loss": 1.3351, "margin_dpo/margin_mean": 72.58518981933594, "margin_dpo/margin_std": 132.86550903320312, "step": 407 }, { "KL/chosen_KL_mean": -220.15966796875, "KL/mean": -252.8894805908203, "KL/rejected_KL_mean": -285.6192932128906, "KL/std": 124.65621948242188, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.45965576171875, "fcm_dpo/q_t": 0.4877682030200958, "grad_norm": 2.2763001918792725, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.3395206928253174, "logits/rejected": -1.7280678749084473, "logps/chosen": -324.62139892578125, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -381.9914855957031, "loss": 1.3412, "margin_dpo/margin_mean": 65.45965576171875, "margin_dpo/margin_std": 156.46835327148438, "step": 408 }, { "KL/chosen_KL_mean": -227.10003662109375, "KL/mean": -264.39813232421875, "KL/rejected_KL_mean": -301.6961975097656, "KL/std": 123.733642578125, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.59618377685547, "fcm_dpo/q_t": 0.48605042695999146, "grad_norm": 2.506699800491333, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5093998908996582, "logits/rejected": -1.0721710920333862, "logps/chosen": -324.7683410644531, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -391.7420349121094, "loss": 1.334, "margin_dpo/margin_mean": 74.59618377685547, "margin_dpo/margin_std": 141.0192413330078, "step": 409 }, { "KL/chosen_KL_mean": -213.66226196289062, "KL/mean": -264.12005615234375, "KL/rejected_KL_mean": -314.577880859375, "KL/std": 127.75316619873047, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.91563415527344, "fcm_dpo/q_t": 0.48112523555755615, "grad_norm": 2.628166913986206, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.6030099391937256, "logits/rejected": -1.4845497608184814, "logps/chosen": -290.1315002441406, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -403.2185363769531, "loss": 1.3144, "margin_dpo/margin_mean": 100.9156265258789, "margin_dpo/margin_std": 129.0129852294922, "step": 410 }, { "KL/chosen_KL_mean": -233.10989379882812, "KL/mean": -276.6300964355469, "KL/rejected_KL_mean": -320.1502990722656, "KL/std": 128.56146240234375, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.04042053222656, "fcm_dpo/q_t": 0.4837803244590759, "grad_norm": 2.5955636501312256, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.1193413734436035, "logits/rejected": -1.709669828414917, "logps/chosen": -320.27618408203125, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -407.246337890625, "loss": 1.3256, "margin_dpo/margin_mean": 87.04042053222656, "margin_dpo/margin_std": 154.06417846679688, "step": 411 }, { "KL/chosen_KL_mean": -214.4846954345703, "KL/mean": -265.4324951171875, "KL/rejected_KL_mean": -316.3802795410156, "KL/std": 128.7322235107422, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.89558410644531, "fcm_dpo/q_t": 0.48099127411842346, "grad_norm": 2.8030333518981934, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.411147117614746, "logits/rejected": -2.2294788360595703, "logps/chosen": -293.4320373535156, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -422.48583984375, "loss": 1.3152, "margin_dpo/margin_mean": 101.89557647705078, "margin_dpo/margin_std": 164.01687622070312, "step": 412 }, { "KL/chosen_KL_mean": -233.07749938964844, "KL/mean": -285.6881103515625, "KL/rejected_KL_mean": -338.2987060546875, "KL/std": 142.8193817138672, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.22122192382812, "fcm_dpo/q_t": 0.4803154468536377, "grad_norm": 2.0868327617645264, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.2890902757644653, "logits/rejected": -1.0358425378799438, "logps/chosen": -328.772216796875, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -446.25958251953125, "loss": 1.3134, "margin_dpo/margin_mean": 105.22122955322266, "margin_dpo/margin_std": 176.9695587158203, "step": 413 }, { "KL/chosen_KL_mean": -217.25250244140625, "KL/mean": -265.25286865234375, "KL/rejected_KL_mean": -313.2532653808594, "KL/std": 128.4976806640625, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.00076293945312, "fcm_dpo/q_t": 0.4820926785469055, "grad_norm": 3.5479159355163574, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.5779776573181152, "logits/rejected": -1.225874662399292, "logps/chosen": -305.5291748046875, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -406.123291015625, "loss": 1.3197, "margin_dpo/margin_mean": 96.00076293945312, "margin_dpo/margin_std": 170.79161071777344, "step": 414 }, { "KL/chosen_KL_mean": -243.37554931640625, "KL/mean": -279.4093017578125, "KL/rejected_KL_mean": -315.4430236816406, "KL/std": 121.48760986328125, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.06747436523438, "fcm_dpo/q_t": 0.4865492284297943, "grad_norm": 2.3449926376342773, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.7072596549987793, "logits/rejected": -1.3572174310684204, "logps/chosen": -352.29144287109375, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -422.91436767578125, "loss": 1.3366, "margin_dpo/margin_mean": 72.06747436523438, "margin_dpo/margin_std": 158.32510375976562, "step": 415 }, { "KL/chosen_KL_mean": -239.30447387695312, "KL/mean": -295.9998474121094, "KL/rejected_KL_mean": -352.6952209472656, "KL/std": 137.08724975585938, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.3907470703125, "fcm_dpo/q_t": 0.47882652282714844, "grad_norm": 2.171994209289551, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.7119426727294922, "logits/rejected": -1.2140345573425293, "logps/chosen": -332.703369140625, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -450.3681945800781, "loss": 1.3059, "margin_dpo/margin_mean": 113.3907470703125, "margin_dpo/margin_std": 141.44833374023438, "step": 416 }, { "KL/chosen_KL_mean": -226.33685302734375, "KL/mean": -269.6973876953125, "KL/rejected_KL_mean": -313.0579833984375, "KL/std": 119.21952056884766, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.72114562988281, "fcm_dpo/q_t": 0.4837896227836609, "grad_norm": 2.491236686706543, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.155160903930664, "logits/rejected": -1.2611862421035767, "logps/chosen": -309.8721923828125, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -387.49981689453125, "loss": 1.3245, "margin_dpo/margin_mean": 86.72113800048828, "margin_dpo/margin_std": 122.76805114746094, "step": 417 }, { "KL/chosen_KL_mean": -219.7808074951172, "KL/mean": -284.52349853515625, "KL/rejected_KL_mean": -349.2662353515625, "KL/std": 137.56683349609375, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.48541259765625, "fcm_dpo/q_t": 0.47582751512527466, "grad_norm": 3.1695291996002197, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.0720443725585938, "logits/rejected": -1.4238743782043457, "logps/chosen": -328.0023193359375, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -461.130859375, "loss": 1.295, "margin_dpo/margin_mean": 129.48541259765625, "margin_dpo/margin_std": 142.00296020507812, "step": 418 }, { "KL/chosen_KL_mean": -230.02926635742188, "KL/mean": -284.34576416015625, "KL/rejected_KL_mean": -338.66229248046875, "KL/std": 123.41677856445312, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.6330337524414, "fcm_dpo/q_t": 0.47971978783607483, "grad_norm": 3.100389242172241, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.582008719444275, "logits/rejected": -1.153680682182312, "logps/chosen": -327.2125549316406, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -436.84759521484375, "loss": 1.3099, "margin_dpo/margin_mean": 108.63304138183594, "margin_dpo/margin_std": 153.47459411621094, "step": 419 }, { "KL/chosen_KL_mean": -231.63824462890625, "KL/mean": -276.48309326171875, "KL/rejected_KL_mean": -321.3279113769531, "KL/std": 126.53128051757812, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.68964385986328, "fcm_dpo/q_t": 0.48325690627098083, "grad_norm": 3.9296462535858154, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.1818621158599854, "logits/rejected": -1.1011720895767212, "logps/chosen": -345.94671630859375, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -397.011474609375, "loss": 1.3235, "margin_dpo/margin_mean": 89.68965148925781, "margin_dpo/margin_std": 154.76513671875, "step": 420 }, { "KL/chosen_KL_mean": -209.14089965820312, "KL/mean": -267.628173828125, "KL/rejected_KL_mean": -326.115478515625, "KL/std": 128.77378845214844, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.97456359863281, "fcm_dpo/q_t": 0.47813981771469116, "grad_norm": 2.275118112564087, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.822321891784668, "logits/rejected": -1.4682667255401611, "logps/chosen": -295.020751953125, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -422.45196533203125, "loss": 1.3035, "margin_dpo/margin_mean": 116.97456359863281, "margin_dpo/margin_std": 145.6648712158203, "step": 421 }, { "KL/chosen_KL_mean": -219.56222534179688, "KL/mean": -272.2270812988281, "KL/rejected_KL_mean": -324.8919677734375, "KL/std": 117.08108520507812, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.32974243164062, "fcm_dpo/q_t": 0.4803217053413391, "grad_norm": 4.305285453796387, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.382503032684326, "logits/rejected": -2.042513847351074, "logps/chosen": -320.0428161621094, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -419.3001708984375, "loss": 1.3116, "margin_dpo/margin_mean": 105.32974243164062, "margin_dpo/margin_std": 140.40176391601562, "step": 422 }, { "KL/chosen_KL_mean": -191.89627075195312, "KL/mean": -254.83953857421875, "KL/rejected_KL_mean": -317.7828369140625, "KL/std": 145.78810119628906, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.88655853271484, "fcm_dpo/q_t": 0.47652626037597656, "grad_norm": 2.3042867183685303, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.8401424884796143, "logits/rejected": -1.7213852405548096, "logps/chosen": -270.34619140625, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -409.829345703125, "loss": 1.2982, "margin_dpo/margin_mean": 125.88655090332031, "margin_dpo/margin_std": 170.1717529296875, "step": 423 }, { "KL/chosen_KL_mean": -228.29205322265625, "KL/mean": -274.72271728515625, "KL/rejected_KL_mean": -321.15338134765625, "KL/std": 133.09481811523438, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.86128997802734, "fcm_dpo/q_t": 0.4826472997665405, "grad_norm": 3.0902650356292725, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.712888240814209, "logits/rejected": -1.1478955745697021, "logps/chosen": -315.9344177246094, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -393.51904296875, "loss": 1.321, "margin_dpo/margin_mean": 92.86128997802734, "margin_dpo/margin_std": 147.8531036376953, "step": 424 }, { "KL/chosen_KL_mean": -225.53810119628906, "KL/mean": -263.9624938964844, "KL/rejected_KL_mean": -302.38690185546875, "KL/std": 136.5972900390625, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.84878540039062, "fcm_dpo/q_t": 0.48567914962768555, "grad_norm": 3.7735304832458496, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.7254984378814697, "logits/rejected": -1.6840758323669434, "logps/chosen": -304.0578918457031, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -405.1355285644531, "loss": 1.3335, "margin_dpo/margin_mean": 76.84878540039062, "margin_dpo/margin_std": 169.38803100585938, "step": 425 }, { "KL/chosen_KL_mean": -206.81210327148438, "KL/mean": -255.70327758789062, "KL/rejected_KL_mean": -304.594482421875, "KL/std": 123.99125671386719, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.78237915039062, "fcm_dpo/q_t": 0.48173412680625916, "grad_norm": 2.434260368347168, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.7533936500549316, "logits/rejected": -1.0057125091552734, "logps/chosen": -315.31793212890625, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -392.89453125, "loss": 1.3177, "margin_dpo/margin_mean": 97.78237915039062, "margin_dpo/margin_std": 154.05340576171875, "step": 426 }, { "KL/chosen_KL_mean": -166.75863647460938, "KL/mean": -211.57293701171875, "KL/rejected_KL_mean": -256.3872375488281, "KL/std": 111.78739929199219, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.62860107421875, "fcm_dpo/q_t": 0.4832432270050049, "grad_norm": 3.172963857650757, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.4380829334259033, "logits/rejected": -1.8329308032989502, "logps/chosen": -265.87908935546875, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -342.11181640625, "loss": 1.3224, "margin_dpo/margin_mean": 89.62859344482422, "margin_dpo/margin_std": 123.31138610839844, "step": 427 }, { "KL/chosen_KL_mean": -235.9365997314453, "KL/mean": -278.49444580078125, "KL/rejected_KL_mean": -321.05224609375, "KL/std": 129.67874145507812, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.11565399169922, "fcm_dpo/q_t": 0.48411524295806885, "grad_norm": 2.4556148052215576, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.8440883159637451, "logits/rejected": -1.5966129302978516, "logps/chosen": -341.9058532714844, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -430.1544189453125, "loss": 1.327, "margin_dpo/margin_mean": 85.11565399169922, "margin_dpo/margin_std": 156.03152465820312, "step": 428 }, { "KL/chosen_KL_mean": -225.98056030273438, "KL/mean": -278.222900390625, "KL/rejected_KL_mean": -330.46527099609375, "KL/std": 137.8583221435547, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.48472595214844, "fcm_dpo/q_t": 0.48049256205558777, "grad_norm": 2.167076349258423, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9075427055358887, "logits/rejected": -1.2072385549545288, "logps/chosen": -339.5254211425781, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -428.707275390625, "loss": 1.3132, "margin_dpo/margin_mean": 104.48472595214844, "margin_dpo/margin_std": 158.04342651367188, "step": 429 }, { "KL/chosen_KL_mean": -204.56065368652344, "KL/mean": -248.5777587890625, "KL/rejected_KL_mean": -292.5948486328125, "KL/std": 131.38768005371094, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.03418731689453, "fcm_dpo/q_t": 0.4835924208164215, "grad_norm": 2.9233627319335938, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.6981136798858643, "logits/rejected": -1.6831955909729004, "logps/chosen": -295.8800048828125, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -402.7044982910156, "loss": 1.3252, "margin_dpo/margin_mean": 88.03419494628906, "margin_dpo/margin_std": 164.08734130859375, "step": 430 }, { "KL/chosen_KL_mean": -204.44683837890625, "KL/mean": -255.11138916015625, "KL/rejected_KL_mean": -305.77593994140625, "KL/std": 138.01947021484375, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.3291015625, "fcm_dpo/q_t": 0.48107534646987915, "grad_norm": 2.5565779209136963, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.836270809173584, "logits/rejected": -1.4731874465942383, "logps/chosen": -297.6280517578125, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -403.908203125, "loss": 1.3154, "margin_dpo/margin_mean": 101.3291015625, "margin_dpo/margin_std": 159.82142639160156, "step": 431 }, { "KL/chosen_KL_mean": -195.92144775390625, "KL/mean": -252.35499572753906, "KL/rejected_KL_mean": -308.78851318359375, "KL/std": 139.68263244628906, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.86708068847656, "fcm_dpo/q_t": 0.4789184629917145, "grad_norm": 2.1892361640930176, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.3142979145050049, "logits/rejected": -0.7676298022270203, "logps/chosen": -300.3562316894531, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -413.8780822753906, "loss": 1.3063, "margin_dpo/margin_mean": 112.86708068847656, "margin_dpo/margin_std": 141.7538299560547, "step": 432 }, { "KL/chosen_KL_mean": -220.94760131835938, "KL/mean": -267.4281921386719, "KL/rejected_KL_mean": -313.9088134765625, "KL/std": 127.52333068847656, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.96121978759766, "fcm_dpo/q_t": 0.4826398491859436, "grad_norm": 2.950871706008911, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.5267220735549927, "logits/rejected": -1.2338993549346924, "logps/chosen": -310.79083251953125, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -415.64227294921875, "loss": 1.3207, "margin_dpo/margin_mean": 92.96121978759766, "margin_dpo/margin_std": 143.0620574951172, "step": 433 }, { "KL/chosen_KL_mean": -190.644287109375, "KL/mean": -242.0233154296875, "KL/rejected_KL_mean": -293.40234375, "KL/std": 147.38214111328125, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.75804138183594, "fcm_dpo/q_t": 0.48090988397598267, "grad_norm": 2.989858865737915, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.371131658554077, "logits/rejected": -1.6903268098831177, "logps/chosen": -288.3367919921875, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -377.49365234375, "loss": 1.3154, "margin_dpo/margin_mean": 102.75804138183594, "margin_dpo/margin_std": 181.8870849609375, "step": 434 }, { "KL/chosen_KL_mean": -187.08688354492188, "KL/mean": -244.69830322265625, "KL/rejected_KL_mean": -302.30975341796875, "KL/std": 132.41065979003906, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.22286987304688, "fcm_dpo/q_t": 0.4785175025463104, "grad_norm": 2.321089267730713, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.384443759918213, "logits/rejected": -1.9534735679626465, "logps/chosen": -273.2588195800781, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -396.06146240234375, "loss": 1.3053, "margin_dpo/margin_mean": 115.22286987304688, "margin_dpo/margin_std": 157.01016235351562, "step": 435 }, { "KL/chosen_KL_mean": -164.3741455078125, "KL/mean": -218.8768310546875, "KL/rejected_KL_mean": -273.3795166015625, "KL/std": 112.46183776855469, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.00538635253906, "fcm_dpo/q_t": 0.4796183407306671, "grad_norm": 3.9240972995758057, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.6462223529815674, "logits/rejected": -0.8180787563323975, "logps/chosen": -270.1612548828125, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -362.0042419433594, "loss": 1.3084, "margin_dpo/margin_mean": 109.0053939819336, "margin_dpo/margin_std": 120.05644226074219, "step": 436 }, { "KL/chosen_KL_mean": -176.65310668945312, "KL/mean": -226.54296875, "KL/rejected_KL_mean": -276.4328308105469, "KL/std": 119.92024993896484, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.77970123291016, "fcm_dpo/q_t": 0.48134058713912964, "grad_norm": 4.617831230163574, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.84061598777771, "logits/rejected": -1.1808371543884277, "logps/chosen": -272.71514892578125, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -365.4450378417969, "loss": 1.3152, "margin_dpo/margin_mean": 99.77970123291016, "margin_dpo/margin_std": 127.61613464355469, "step": 437 }, { "KL/chosen_KL_mean": -192.2845458984375, "KL/mean": -237.13624572753906, "KL/rejected_KL_mean": -281.9879455566406, "KL/std": 119.74552917480469, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.70340728759766, "fcm_dpo/q_t": 0.4832267463207245, "grad_norm": 2.7240400314331055, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -1.9848185777664185, "logits/rejected": -1.1943778991699219, "logps/chosen": -294.13995361328125, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -371.43560791015625, "loss": 1.3228, "margin_dpo/margin_mean": 89.70340728759766, "margin_dpo/margin_std": 137.09054565429688, "step": 438 }, { "KL/chosen_KL_mean": -197.57278442382812, "KL/mean": -244.14227294921875, "KL/rejected_KL_mean": -290.7117614746094, "KL/std": 132.6284637451172, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.13896942138672, "fcm_dpo/q_t": 0.48261141777038574, "grad_norm": 2.9226772785186768, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.3724796772003174, "logits/rejected": -1.1545944213867188, "logps/chosen": -279.32843017578125, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -381.2981262207031, "loss": 1.3204, "margin_dpo/margin_mean": 93.13896942138672, "margin_dpo/margin_std": 134.04611206054688, "step": 439 }, { "KL/chosen_KL_mean": -186.47311401367188, "KL/mean": -243.037841796875, "KL/rejected_KL_mean": -299.60260009765625, "KL/std": 135.29049682617188, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.12948608398438, "fcm_dpo/q_t": 0.4788944721221924, "grad_norm": 2.1388111114501953, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.49202561378479, "logits/rejected": -2.0186967849731445, "logps/chosen": -292.11419677734375, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -403.0036315917969, "loss": 1.3067, "margin_dpo/margin_mean": 113.1294937133789, "margin_dpo/margin_std": 155.10800170898438, "step": 440 }, { "KL/chosen_KL_mean": -233.53546142578125, "KL/mean": -278.2824401855469, "KL/rejected_KL_mean": -323.0294189453125, "KL/std": 134.76431274414062, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.49397277832031, "fcm_dpo/q_t": 0.48333263397216797, "grad_norm": 2.2704029083251953, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.551321029663086, "logits/rejected": -1.2265297174453735, "logps/chosen": -325.06500244140625, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -426.6488037109375, "loss": 1.3242, "margin_dpo/margin_mean": 89.49397277832031, "margin_dpo/margin_std": 165.5093994140625, "step": 441 }, { "KL/chosen_KL_mean": -219.39376831054688, "KL/mean": -279.14093017578125, "KL/rejected_KL_mean": -338.8880310058594, "KL/std": 129.84979248046875, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.4942855834961, "fcm_dpo/q_t": 0.47768938541412354, "grad_norm": 2.9798710346221924, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.4814010858535767, "logits/rejected": -1.0086218118667603, "logps/chosen": -304.17132568359375, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -422.7121887207031, "loss": 1.3018, "margin_dpo/margin_mean": 119.4942855834961, "margin_dpo/margin_std": 147.96401977539062, "step": 442 }, { "KL/chosen_KL_mean": -222.1339569091797, "KL/mean": -254.53802490234375, "KL/rejected_KL_mean": -286.94207763671875, "KL/std": 133.91229248046875, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.80811309814453, "fcm_dpo/q_t": 0.4878942668437958, "grad_norm": 3.1072499752044678, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.7394073009490967, "logits/rejected": -1.171545147895813, "logps/chosen": -324.7832336425781, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -379.9801330566406, "loss": 1.3412, "margin_dpo/margin_mean": 64.80811309814453, "margin_dpo/margin_std": 142.51718139648438, "step": 443 }, { "KL/chosen_KL_mean": -222.07986450195312, "KL/mean": -270.88812255859375, "KL/rejected_KL_mean": -319.6964111328125, "KL/std": 130.78875732421875, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.61653137207031, "fcm_dpo/q_t": 0.4817734956741333, "grad_norm": 3.0346412658691406, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5253052711486816, "logits/rejected": -1.3643295764923096, "logps/chosen": -309.99957275390625, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -423.01983642578125, "loss": 1.3176, "margin_dpo/margin_mean": 97.61653137207031, "margin_dpo/margin_std": 149.9453125, "step": 444 }, { "KL/chosen_KL_mean": -200.41574096679688, "KL/mean": -254.0961456298828, "KL/rejected_KL_mean": -307.77655029296875, "KL/std": 126.97477722167969, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.36080932617188, "fcm_dpo/q_t": 0.4799421429634094, "grad_norm": 2.7677905559539795, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.7995564937591553, "logits/rejected": -1.393208622932434, "logps/chosen": -301.81658935546875, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -406.814453125, "loss": 1.3103, "margin_dpo/margin_mean": 107.36080169677734, "margin_dpo/margin_std": 140.34642028808594, "step": 445 }, { "KL/chosen_KL_mean": -192.586181640625, "KL/mean": -238.10118103027344, "KL/rejected_KL_mean": -283.61614990234375, "KL/std": 128.30526733398438, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.02998352050781, "fcm_dpo/q_t": 0.4829912483692169, "grad_norm": 3.122206926345825, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.68727445602417, "logits/rejected": -1.5667202472686768, "logps/chosen": -280.008544921875, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -390.3169250488281, "loss": 1.3222, "margin_dpo/margin_mean": 91.02998352050781, "margin_dpo/margin_std": 143.9282684326172, "step": 446 }, { "KL/chosen_KL_mean": -184.7958984375, "KL/mean": -249.05238342285156, "KL/rejected_KL_mean": -313.30889892578125, "KL/std": 140.1792449951172, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.51300048828125, "fcm_dpo/q_t": 0.47603508830070496, "grad_norm": 3.1798629760742188, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.009763240814209, "logits/rejected": -1.9724934101104736, "logps/chosen": -268.4111328125, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -418.2212829589844, "loss": 1.2963, "margin_dpo/margin_mean": 128.51300048828125, "margin_dpo/margin_std": 168.89508056640625, "step": 447 }, { "KL/chosen_KL_mean": -197.61639404296875, "KL/mean": -259.4129333496094, "KL/rejected_KL_mean": -321.2094421386719, "KL/std": 140.93443298339844, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.59304809570312, "fcm_dpo/q_t": 0.47693508863449097, "grad_norm": 2.8769078254699707, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.4738311767578125, "logits/rejected": -1.2040340900421143, "logps/chosen": -290.19219970703125, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -428.89923095703125, "loss": 1.2997, "margin_dpo/margin_mean": 123.59304809570312, "margin_dpo/margin_std": 162.56732177734375, "step": 448 }, { "KL/chosen_KL_mean": -192.94696044921875, "KL/mean": -257.8748474121094, "KL/rejected_KL_mean": -322.8027038574219, "KL/std": 136.18606567382812, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.8557586669922, "fcm_dpo/q_t": 0.47576647996902466, "grad_norm": 2.5559661388397217, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4509382247924805, "logits/rejected": -1.0107239484786987, "logps/chosen": -295.34588623046875, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -417.95159912109375, "loss": 1.295, "margin_dpo/margin_mean": 129.8557586669922, "margin_dpo/margin_std": 162.7983856201172, "step": 449 }, { "KL/chosen_KL_mean": -179.53587341308594, "KL/mean": -243.11111450195312, "KL/rejected_KL_mean": -306.68634033203125, "KL/std": 139.1621551513672, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.15047454833984, "fcm_dpo/q_t": 0.47625645995140076, "grad_norm": 2.31666898727417, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.1191439628601074, "logits/rejected": -1.927521824836731, "logps/chosen": -266.52874755859375, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -415.2183837890625, "loss": 1.2962, "margin_dpo/margin_mean": 127.15047454833984, "margin_dpo/margin_std": 146.05322265625, "step": 450 }, { "KL/chosen_KL_mean": -200.68321228027344, "KL/mean": -237.6129913330078, "KL/rejected_KL_mean": -274.54278564453125, "KL/std": 130.09423828125, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.85955047607422, "fcm_dpo/q_t": 0.48621025681495667, "grad_norm": 3.0506389141082764, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.1721272468566895, "logits/rejected": -1.6651277542114258, "logps/chosen": -287.4945068359375, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -354.39837646484375, "loss": 1.3359, "margin_dpo/margin_mean": 73.85955047607422, "margin_dpo/margin_std": 172.58737182617188, "step": 451 }, { "KL/chosen_KL_mean": -174.02749633789062, "KL/mean": -224.4292449951172, "KL/rejected_KL_mean": -274.83099365234375, "KL/std": 120.98470306396484, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.80349731445312, "fcm_dpo/q_t": 0.48116081953048706, "grad_norm": 2.6075327396392822, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7434451580047607, "logits/rejected": -1.7151966094970703, "logps/chosen": -253.40713500976562, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -378.54638671875, "loss": 1.3145, "margin_dpo/margin_mean": 100.80349731445312, "margin_dpo/margin_std": 127.45135498046875, "step": 452 }, { "KL/chosen_KL_mean": -203.81629943847656, "KL/mean": -270.99774169921875, "KL/rejected_KL_mean": -338.17919921875, "KL/std": 125.4898681640625, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.36288452148438, "fcm_dpo/q_t": 0.4749225676059723, "grad_norm": 3.2362561225891113, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.8851237297058105, "logits/rejected": -1.4285997152328491, "logps/chosen": -291.511474609375, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -428.2374267578125, "loss": 1.2912, "margin_dpo/margin_mean": 134.36289978027344, "margin_dpo/margin_std": 148.0814208984375, "step": 453 }, { "KL/chosen_KL_mean": -218.97509765625, "KL/mean": -265.7637023925781, "KL/rejected_KL_mean": -312.55230712890625, "KL/std": 127.52639770507812, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.57721710205078, "fcm_dpo/q_t": 0.4825332760810852, "grad_norm": 2.3103115558624268, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.8439137935638428, "logits/rejected": -1.3995423316955566, "logps/chosen": -308.54132080078125, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -405.47332763671875, "loss": 1.3215, "margin_dpo/margin_mean": 93.57721710205078, "margin_dpo/margin_std": 169.75875854492188, "step": 454 }, { "KL/chosen_KL_mean": -211.1138916015625, "KL/mean": -263.39129638671875, "KL/rejected_KL_mean": -315.6686706542969, "KL/std": 142.35183715820312, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.55479431152344, "fcm_dpo/q_t": 0.4805016815662384, "grad_norm": 2.638075590133667, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8904001712799072, "logits/rejected": -1.8741719722747803, "logps/chosen": -288.7438659667969, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -434.6466369628906, "loss": 1.3134, "margin_dpo/margin_mean": 104.55480194091797, "margin_dpo/margin_std": 165.65147399902344, "step": 455 }, { "KL/chosen_KL_mean": -200.10443115234375, "KL/mean": -258.47760009765625, "KL/rejected_KL_mean": -316.85076904296875, "KL/std": 135.27200317382812, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.74633026123047, "fcm_dpo/q_t": 0.47819411754608154, "grad_norm": 3.3313889503479004, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.0526340007781982, "logits/rejected": -1.9270200729370117, "logps/chosen": -289.7213134765625, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -426.41046142578125, "loss": 1.3033, "margin_dpo/margin_mean": 116.74632263183594, "margin_dpo/margin_std": 135.8311767578125, "step": 456 }, { "KL/chosen_KL_mean": -172.23208618164062, "KL/mean": -233.80929565429688, "KL/rejected_KL_mean": -295.386474609375, "KL/std": 129.04061889648438, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.15440368652344, "fcm_dpo/q_t": 0.47698354721069336, "grad_norm": 2.9794704914093018, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -1.9627540111541748, "logits/rejected": -1.6496200561523438, "logps/chosen": -259.55377197265625, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -386.153076171875, "loss": 1.2982, "margin_dpo/margin_mean": 123.15441131591797, "margin_dpo/margin_std": 113.814453125, "step": 457 }, { "KL/chosen_KL_mean": -197.53067016601562, "KL/mean": -243.77383422851562, "KL/rejected_KL_mean": -290.01702880859375, "KL/std": 131.42681884765625, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.4863510131836, "fcm_dpo/q_t": 0.48274898529052734, "grad_norm": 2.5871999263763428, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5915628671646118, "logits/rejected": -1.3662021160125732, "logps/chosen": -291.0511169433594, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -393.385986328125, "loss": 1.322, "margin_dpo/margin_mean": 92.48635864257812, "margin_dpo/margin_std": 166.12738037109375, "step": 458 }, { "KL/chosen_KL_mean": -185.57797241210938, "KL/mean": -246.20639038085938, "KL/rejected_KL_mean": -306.8348083496094, "KL/std": 132.03431701660156, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.2568359375, "fcm_dpo/q_t": 0.4773685932159424, "grad_norm": 2.1119604110717773, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6059105396270752, "logits/rejected": -1.3837541341781616, "logps/chosen": -261.26617431640625, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -399.0052795410156, "loss": 1.3007, "margin_dpo/margin_mean": 121.2568359375, "margin_dpo/margin_std": 149.27313232421875, "step": 459 }, { "KL/chosen_KL_mean": -192.89242553710938, "KL/mean": -247.4097442626953, "KL/rejected_KL_mean": -301.92706298828125, "KL/std": 134.7513427734375, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.03468322753906, "fcm_dpo/q_t": 0.479636013507843, "grad_norm": 3.448631763458252, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -1.8314173221588135, "logits/rejected": -1.8858757019042969, "logps/chosen": -274.01031494140625, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -412.2394714355469, "loss": 1.3097, "margin_dpo/margin_mean": 109.03468322753906, "margin_dpo/margin_std": 157.29644775390625, "step": 460 }, { "KL/chosen_KL_mean": -224.62220764160156, "KL/mean": -271.1238708496094, "KL/rejected_KL_mean": -317.62554931640625, "KL/std": 119.38990783691406, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.00333404541016, "fcm_dpo/q_t": 0.48263370990753174, "grad_norm": 2.951667308807373, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.3829995393753052, "logits/rejected": -1.1961634159088135, "logps/chosen": -306.20574951171875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -411.5026550292969, "loss": 1.3209, "margin_dpo/margin_mean": 93.00332641601562, "margin_dpo/margin_std": 148.73028564453125, "step": 461 }, { "KL/chosen_KL_mean": -181.29013061523438, "KL/mean": -237.28453063964844, "KL/rejected_KL_mean": -293.2789306640625, "KL/std": 123.73484802246094, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.98877716064453, "fcm_dpo/q_t": 0.4790824055671692, "grad_norm": 2.472874164581299, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.3736772537231445, "logits/rejected": -1.905167818069458, "logps/chosen": -275.4886779785156, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -378.9105529785156, "loss": 1.3066, "margin_dpo/margin_mean": 111.98878479003906, "margin_dpo/margin_std": 133.2924346923828, "step": 462 }, { "KL/chosen_KL_mean": -177.4019775390625, "KL/mean": -243.83999633789062, "KL/rejected_KL_mean": -310.27801513671875, "KL/std": 137.69427490234375, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.8760223388672, "fcm_dpo/q_t": 0.4751947522163391, "grad_norm": 3.6551504135131836, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.841729998588562, "logits/rejected": -1.4474916458129883, "logps/chosen": -263.32672119140625, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -407.17987060546875, "loss": 1.2927, "margin_dpo/margin_mean": 132.8760223388672, "margin_dpo/margin_std": 148.9471435546875, "step": 463 }, { "KL/chosen_KL_mean": -208.85791015625, "KL/mean": -269.6978759765625, "KL/rejected_KL_mean": -330.537841796875, "KL/std": 139.39895629882812, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.67992401123047, "fcm_dpo/q_t": 0.4773097038269043, "grad_norm": 2.8546268939971924, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.7100327014923096, "logits/rejected": -1.7970632314682007, "logps/chosen": -288.547119140625, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -437.83013916015625, "loss": 1.3009, "margin_dpo/margin_mean": 121.67992401123047, "margin_dpo/margin_std": 163.2327423095703, "step": 464 }, { "KL/chosen_KL_mean": -205.40951538085938, "KL/mean": -269.0036315917969, "KL/rejected_KL_mean": -332.5977783203125, "KL/std": 145.96728515625, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.18822479248047, "fcm_dpo/q_t": 0.4762471914291382, "grad_norm": 3.4777116775512695, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.5665435791015625, "logits/rejected": -1.2901265621185303, "logps/chosen": -297.2697448730469, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -451.3077697753906, "loss": 1.296, "margin_dpo/margin_mean": 127.18822479248047, "margin_dpo/margin_std": 141.31704711914062, "step": 465 }, { "KL/chosen_KL_mean": -179.81521606445312, "KL/mean": -246.32723999023438, "KL/rejected_KL_mean": -312.8392639160156, "KL/std": 138.1134033203125, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.0240478515625, "fcm_dpo/q_t": 0.47516506910324097, "grad_norm": 2.0875155925750732, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.585087776184082, "logits/rejected": -1.2858678102493286, "logps/chosen": -264.5166015625, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -408.8901062011719, "loss": 1.2925, "margin_dpo/margin_mean": 133.0240478515625, "margin_dpo/margin_std": 156.70968627929688, "step": 466 }, { "KL/chosen_KL_mean": -240.61111450195312, "KL/mean": -293.86322021484375, "KL/rejected_KL_mean": -347.1153259277344, "KL/std": 156.17855834960938, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.50419616699219, "fcm_dpo/q_t": 0.48012661933898926, "grad_norm": 2.5901663303375244, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.182767629623413, "logits/rejected": -1.6111836433410645, "logps/chosen": -349.909423828125, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -455.958984375, "loss": 1.3129, "margin_dpo/margin_mean": 106.50419616699219, "margin_dpo/margin_std": 181.83541870117188, "step": 467 }, { "KL/chosen_KL_mean": -184.54534912109375, "KL/mean": -235.17913818359375, "KL/rejected_KL_mean": -285.81292724609375, "KL/std": 143.1393585205078, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.26753997802734, "fcm_dpo/q_t": 0.48111215233802795, "grad_norm": 2.931168556213379, "learning_rate": 1.362737437810114e-07, "logits/chosen": -2.001765489578247, "logits/rejected": -1.7796076536178589, "logps/chosen": -282.86700439453125, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -392.4933776855469, "loss": 1.316, "margin_dpo/margin_mean": 101.26753997802734, "margin_dpo/margin_std": 173.38235473632812, "step": 468 }, { "KL/chosen_KL_mean": -225.48451232910156, "KL/mean": -286.72296142578125, "KL/rejected_KL_mean": -347.9613952636719, "KL/std": 143.04742431640625, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.47687530517578, "fcm_dpo/q_t": 0.47715169191360474, "grad_norm": 2.423753261566162, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.125657796859741, "logits/rejected": -1.7193701267242432, "logps/chosen": -322.24871826171875, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -457.556396484375, "loss": 1.3002, "margin_dpo/margin_mean": 122.47687530517578, "margin_dpo/margin_std": 156.31951904296875, "step": 469 }, { "KL/chosen_KL_mean": -197.51422119140625, "KL/mean": -257.70068359375, "KL/rejected_KL_mean": -317.88720703125, "KL/std": 129.17971801757812, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.37300109863281, "fcm_dpo/q_t": 0.47751057147979736, "grad_norm": 2.5448169708251953, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.541486382484436, "logits/rejected": -1.1399693489074707, "logps/chosen": -281.0108642578125, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -406.37298583984375, "loss": 1.3004, "margin_dpo/margin_mean": 120.37300872802734, "margin_dpo/margin_std": 126.85967254638672, "step": 470 }, { "KL/chosen_KL_mean": -231.31845092773438, "KL/mean": -283.90899658203125, "KL/rejected_KL_mean": -336.49951171875, "KL/std": 134.05325317382812, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.18106079101562, "fcm_dpo/q_t": 0.4803590178489685, "grad_norm": 2.3362152576446533, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.5472378730773926, "logits/rejected": -1.1089308261871338, "logps/chosen": -319.7927551269531, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -426.9812316894531, "loss": 1.3118, "margin_dpo/margin_mean": 105.18106079101562, "margin_dpo/margin_std": 141.64805603027344, "step": 471 }, { "KL/chosen_KL_mean": -227.3581085205078, "KL/mean": -289.81378173828125, "KL/rejected_KL_mean": -352.2695007324219, "KL/std": 137.46987915039062, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.911376953125, "fcm_dpo/q_t": 0.4767184853553772, "grad_norm": 2.572507858276367, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.0441436767578125, "logits/rejected": -1.34371018409729, "logps/chosen": -352.591796875, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -473.322998046875, "loss": 1.2987, "margin_dpo/margin_mean": 124.91138458251953, "margin_dpo/margin_std": 163.572509765625, "step": 472 }, { "KL/chosen_KL_mean": -253.68670654296875, "KL/mean": -288.7862548828125, "KL/rejected_KL_mean": -323.88580322265625, "KL/std": 143.443115234375, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.19908142089844, "fcm_dpo/q_t": 0.4869306683540344, "grad_norm": 2.5371081829071045, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8233697414398193, "logits/rejected": -1.3511872291564941, "logps/chosen": -349.298095703125, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -412.03692626953125, "loss": 1.3388, "margin_dpo/margin_mean": 70.19908142089844, "margin_dpo/margin_std": 176.6898193359375, "step": 473 }, { "KL/chosen_KL_mean": -232.5495147705078, "KL/mean": -275.2092590332031, "KL/rejected_KL_mean": -317.8690185546875, "KL/std": 138.1640625, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.31950378417969, "fcm_dpo/q_t": 0.48406845331192017, "grad_norm": 4.950549125671387, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.36810302734375, "logits/rejected": -1.1875860691070557, "logps/chosen": -314.0292663574219, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -414.33465576171875, "loss": 1.3281, "margin_dpo/margin_mean": 85.31949615478516, "margin_dpo/margin_std": 178.5922088623047, "step": 474 }, { "KL/chosen_KL_mean": -173.61073303222656, "KL/mean": -238.4129638671875, "KL/rejected_KL_mean": -303.2152099609375, "KL/std": 135.77024841308594, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.60446166992188, "fcm_dpo/q_t": 0.47584784030914307, "grad_norm": 2.6735544204711914, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.0357353687286377, "logits/rejected": -1.9991700649261475, "logps/chosen": -247.8067169189453, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -397.9076232910156, "loss": 1.2954, "margin_dpo/margin_mean": 129.6044464111328, "margin_dpo/margin_std": 167.6500244140625, "step": 475 }, { "KL/chosen_KL_mean": -201.90005493164062, "KL/mean": -254.14071655273438, "KL/rejected_KL_mean": -306.38140869140625, "KL/std": 138.9665069580078, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.4813461303711, "fcm_dpo/q_t": 0.4805123209953308, "grad_norm": 2.984710216522217, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.9619851112365723, "logits/rejected": -1.6193482875823975, "logps/chosen": -273.87115478515625, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -386.6436462402344, "loss": 1.313, "margin_dpo/margin_mean": 104.4813461303711, "margin_dpo/margin_std": 157.06617736816406, "step": 476 }, { "KL/chosen_KL_mean": -208.6039581298828, "KL/mean": -264.6419677734375, "KL/rejected_KL_mean": -320.6799621582031, "KL/std": 118.03338623046875, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.07597351074219, "fcm_dpo/q_t": 0.4790581464767456, "grad_norm": 2.4452149868011475, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.257868766784668, "logits/rejected": -1.345482349395752, "logps/chosen": -313.6094970703125, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -402.5583801269531, "loss": 1.3073, "margin_dpo/margin_mean": 112.07598876953125, "margin_dpo/margin_std": 149.7928466796875, "step": 477 }, { "KL/chosen_KL_mean": -203.43240356445312, "KL/mean": -258.9180603027344, "KL/rejected_KL_mean": -314.4036865234375, "KL/std": 132.0986328125, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.97127532958984, "fcm_dpo/q_t": 0.479291707277298, "grad_norm": 3.710623264312744, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.4226372241973877, "logits/rejected": -1.245201587677002, "logps/chosen": -280.2206115722656, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -404.8436279296875, "loss": 1.3084, "margin_dpo/margin_mean": 110.97127532958984, "margin_dpo/margin_std": 157.87594604492188, "step": 478 }, { "KL/chosen_KL_mean": -199.51406860351562, "KL/mean": -260.71630859375, "KL/rejected_KL_mean": -321.91851806640625, "KL/std": 139.95703125, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.40444946289062, "fcm_dpo/q_t": 0.47716224193573, "grad_norm": 4.358889102935791, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.4216914176940918, "logits/rejected": -1.2796061038970947, "logps/chosen": -280.87640380859375, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -423.0096740722656, "loss": 1.3005, "margin_dpo/margin_mean": 122.40444946289062, "margin_dpo/margin_std": 164.08511352539062, "step": 479 }, { "KL/chosen_KL_mean": -239.70700073242188, "KL/mean": -285.0233459472656, "KL/rejected_KL_mean": -330.33966064453125, "KL/std": 129.0963592529297, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.63264465332031, "fcm_dpo/q_t": 0.48306578397750854, "grad_norm": 2.61621356010437, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7424893379211426, "logits/rejected": -1.0999655723571777, "logps/chosen": -336.48089599609375, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -416.744384765625, "loss": 1.3223, "margin_dpo/margin_mean": 90.63264465332031, "margin_dpo/margin_std": 140.43528747558594, "step": 480 }, { "KL/chosen_KL_mean": -224.60638427734375, "KL/mean": -275.5166015625, "KL/rejected_KL_mean": -326.42681884765625, "KL/std": 134.68984985351562, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.8204345703125, "fcm_dpo/q_t": 0.48098719120025635, "grad_norm": 3.5046699047088623, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.7914048433303833, "logits/rejected": -1.3765017986297607, "logps/chosen": -316.27655029296875, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -425.1217041015625, "loss": 1.3145, "margin_dpo/margin_mean": 101.82042694091797, "margin_dpo/margin_std": 147.8446044921875, "step": 481 }, { "KL/chosen_KL_mean": -220.76934814453125, "KL/mean": -272.82049560546875, "KL/rejected_KL_mean": -324.87164306640625, "KL/std": 139.10650634765625, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.10231018066406, "fcm_dpo/q_t": 0.4805508852005005, "grad_norm": 2.3485841751098633, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.3395986557006836, "logits/rejected": -0.9926462173461914, "logps/chosen": -319.2894592285156, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -419.70111083984375, "loss": 1.3134, "margin_dpo/margin_mean": 104.10231018066406, "margin_dpo/margin_std": 160.00491333007812, "step": 482 }, { "KL/chosen_KL_mean": -235.88815307617188, "KL/mean": -288.361572265625, "KL/rejected_KL_mean": -340.8349609375, "KL/std": 134.47811889648438, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.94682312011719, "fcm_dpo/q_t": 0.4803844094276428, "grad_norm": 2.3078625202178955, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.9388898611068726, "logits/rejected": -1.3056683540344238, "logps/chosen": -343.00677490234375, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -441.949951171875, "loss": 1.312, "margin_dpo/margin_mean": 104.94683837890625, "margin_dpo/margin_std": 138.95883178710938, "step": 483 }, { "KL/chosen_KL_mean": -203.01473999023438, "KL/mean": -268.3775634765625, "KL/rejected_KL_mean": -333.74041748046875, "KL/std": 141.14572143554688, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.7256622314453, "fcm_dpo/q_t": 0.4756035804748535, "grad_norm": 2.485551595687866, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.186819314956665, "logits/rejected": -1.7204868793487549, "logps/chosen": -289.9946594238281, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -424.4640808105469, "loss": 1.2943, "margin_dpo/margin_mean": 130.7256622314453, "margin_dpo/margin_std": 158.65200805664062, "step": 484 }, { "KL/chosen_KL_mean": -205.00978088378906, "KL/mean": -269.60516357421875, "KL/rejected_KL_mean": -334.2005310058594, "KL/std": 151.53909301757812, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.19076538085938, "fcm_dpo/q_t": 0.4759119153022766, "grad_norm": 2.419785499572754, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.6419097185134888, "logits/rejected": -1.2625524997711182, "logps/chosen": -299.06854248046875, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -442.7635192871094, "loss": 1.2957, "margin_dpo/margin_mean": 129.19076538085938, "margin_dpo/margin_std": 167.09756469726562, "step": 485 }, { "KL/chosen_KL_mean": -230.72299194335938, "KL/mean": -293.6932373046875, "KL/rejected_KL_mean": -356.66351318359375, "KL/std": 146.785400390625, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.94051361083984, "fcm_dpo/q_t": 0.4764828681945801, "grad_norm": 2.1421806812286377, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.827906847000122, "logits/rejected": -1.1452302932739258, "logps/chosen": -324.4688720703125, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -454.73419189453125, "loss": 1.2979, "margin_dpo/margin_mean": 125.94052124023438, "margin_dpo/margin_std": 162.4217987060547, "step": 486 }, { "KL/chosen_KL_mean": -225.82882690429688, "KL/mean": -280.41046142578125, "KL/rejected_KL_mean": -334.9920654296875, "KL/std": 128.5418701171875, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.16323852539062, "fcm_dpo/q_t": 0.4795949161052704, "grad_norm": 2.5113344192504883, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.1101508140563965, "logits/rejected": -1.8077709674835205, "logps/chosen": -313.8623962402344, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -433.46417236328125, "loss": 1.3091, "margin_dpo/margin_mean": 109.16323852539062, "margin_dpo/margin_std": 144.17507934570312, "step": 487 }, { "KL/chosen_KL_mean": -188.63613891601562, "KL/mean": -248.41513061523438, "KL/rejected_KL_mean": -308.194091796875, "KL/std": 141.5706787109375, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.55796813964844, "fcm_dpo/q_t": 0.4776681959629059, "grad_norm": 2.663071870803833, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.617283821105957, "logits/rejected": -1.3627817630767822, "logps/chosen": -273.42578125, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -398.46759033203125, "loss": 1.3014, "margin_dpo/margin_mean": 119.5579605102539, "margin_dpo/margin_std": 139.0713653564453, "step": 488 }, { "KL/chosen_KL_mean": -234.0811004638672, "KL/mean": -281.5931091308594, "KL/rejected_KL_mean": -329.1051025390625, "KL/std": 132.08172607421875, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.02399444580078, "fcm_dpo/q_t": 0.48225080966949463, "grad_norm": 3.4114930629730225, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.3334327936172485, "logits/rejected": -0.8446152210235596, "logps/chosen": -324.5504150390625, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -415.5027160644531, "loss": 1.3193, "margin_dpo/margin_mean": 95.02400207519531, "margin_dpo/margin_std": 145.00865173339844, "step": 489 }, { "KL/chosen_KL_mean": -221.203857421875, "KL/mean": -279.44476318359375, "KL/rejected_KL_mean": -337.68560791015625, "KL/std": 147.33631896972656, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.48175048828125, "fcm_dpo/q_t": 0.47830355167388916, "grad_norm": 2.83760142326355, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.065483570098877, "logits/rejected": -2.1717522144317627, "logps/chosen": -306.52398681640625, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -453.679443359375, "loss": 1.3062, "margin_dpo/margin_mean": 116.48175048828125, "margin_dpo/margin_std": 191.59442138671875, "step": 490 }, { "KL/chosen_KL_mean": -225.7991943359375, "KL/mean": -268.99169921875, "KL/rejected_KL_mean": -312.1842041015625, "KL/std": 143.58282470703125, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.385009765625, "fcm_dpo/q_t": 0.48385560512542725, "grad_norm": 2.863807439804077, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8178702592849731, "logits/rejected": -1.3582427501678467, "logps/chosen": -311.9427185058594, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -392.8636474609375, "loss": 1.3256, "margin_dpo/margin_mean": 86.385009765625, "margin_dpo/margin_std": 146.44581604003906, "step": 491 }, { "KL/chosen_KL_mean": -218.5087890625, "KL/mean": -254.9041748046875, "KL/rejected_KL_mean": -291.29949951171875, "KL/std": 153.01087951660156, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.79073333740234, "fcm_dpo/q_t": 0.4864271581172943, "grad_norm": 2.795712471008301, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.7768990993499756, "logits/rejected": -1.4080734252929688, "logps/chosen": -299.61639404296875, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -372.051513671875, "loss": 1.3371, "margin_dpo/margin_mean": 72.79074096679688, "margin_dpo/margin_std": 180.40573120117188, "step": 492 }, { "KL/chosen_KL_mean": -219.41134643554688, "KL/mean": -259.8005676269531, "KL/rejected_KL_mean": -300.1898193359375, "KL/std": 129.7357940673828, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.77845764160156, "fcm_dpo/q_t": 0.48490965366363525, "grad_norm": 4.085540771484375, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.3330774307250977, "logits/rejected": -1.6510825157165527, "logps/chosen": -331.6186828613281, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -393.7969970703125, "loss": 1.3298, "margin_dpo/margin_mean": 80.77845764160156, "margin_dpo/margin_std": 147.03135681152344, "step": 493 }, { "KL/chosen_KL_mean": -201.76626586914062, "KL/mean": -269.3253479003906, "KL/rejected_KL_mean": -336.8844299316406, "KL/std": 137.19586181640625, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.1181640625, "fcm_dpo/q_t": 0.4747849106788635, "grad_norm": 2.852871894836426, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.3374569416046143, "logits/rejected": -1.2813893556594849, "logps/chosen": -274.88116455078125, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -429.04742431640625, "loss": 1.2912, "margin_dpo/margin_mean": 135.1181640625, "margin_dpo/margin_std": 157.82913208007812, "step": 494 }, { "KL/chosen_KL_mean": -220.84530639648438, "KL/mean": -274.27734375, "KL/rejected_KL_mean": -327.7093505859375, "KL/std": 132.48629760742188, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.86402130126953, "fcm_dpo/q_t": 0.48006874322891235, "grad_norm": 3.4120309352874756, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.232571601867676, "logits/rejected": -1.7715232372283936, "logps/chosen": -320.3656311035156, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -425.6402587890625, "loss": 1.3118, "margin_dpo/margin_mean": 106.864013671875, "margin_dpo/margin_std": 168.31631469726562, "step": 495 }, { "KL/chosen_KL_mean": -249.7236328125, "KL/mean": -294.7996826171875, "KL/rejected_KL_mean": -339.87567138671875, "KL/std": 130.43310546875, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.15203857421875, "fcm_dpo/q_t": 0.48314812779426575, "grad_norm": 2.8798770904541016, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.0033652782440186, "logits/rejected": -1.3672683238983154, "logps/chosen": -357.58038330078125, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -432.646240234375, "loss": 1.3231, "margin_dpo/margin_mean": 90.15203857421875, "margin_dpo/margin_std": 149.02516174316406, "step": 496 }, { "KL/chosen_KL_mean": -250.27682495117188, "KL/mean": -279.00408935546875, "KL/rejected_KL_mean": -307.7313232421875, "KL/std": 136.04237365722656, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.454471588134766, "fcm_dpo/q_t": 0.48928117752075195, "grad_norm": 2.1774070262908936, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.642315149307251, "logits/rejected": -1.2091455459594727, "logps/chosen": -342.36004638671875, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -389.5263366699219, "loss": 1.347, "margin_dpo/margin_mean": 57.4544677734375, "margin_dpo/margin_std": 149.70042419433594, "step": 497 }, { "KL/chosen_KL_mean": -222.25106811523438, "KL/mean": -273.79833984375, "KL/rejected_KL_mean": -325.34564208984375, "KL/std": 146.6027374267578, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.09456634521484, "fcm_dpo/q_t": 0.4807358384132385, "grad_norm": 2.746053695678711, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.662240743637085, "logits/rejected": -1.1621549129486084, "logps/chosen": -320.4454345703125, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -416.0330810546875, "loss": 1.3149, "margin_dpo/margin_mean": 103.09455871582031, "margin_dpo/margin_std": 176.64935302734375, "step": 498 }, { "KL/chosen_KL_mean": -220.58566284179688, "KL/mean": -279.2626953125, "KL/rejected_KL_mean": -337.93975830078125, "KL/std": 146.4928741455078, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.3541259765625, "fcm_dpo/q_t": 0.4781012535095215, "grad_norm": 3.033827066421509, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.8117303848266602, "logits/rejected": -1.7649179697036743, "logps/chosen": -301.62554931640625, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -444.8614501953125, "loss": 1.3043, "margin_dpo/margin_mean": 117.3541259765625, "margin_dpo/margin_std": 169.64990234375, "step": 499 }, { "KL/chosen_KL_mean": -247.7305908203125, "KL/mean": -298.7356872558594, "KL/rejected_KL_mean": -349.74078369140625, "KL/std": 139.93426513671875, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.01020812988281, "fcm_dpo/q_t": 0.4809700548648834, "grad_norm": 2.4893479347229004, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.3681640625, "logits/rejected": -1.1921244859695435, "logps/chosen": -336.9786376953125, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -450.1510009765625, "loss": 1.3149, "margin_dpo/margin_mean": 102.01020812988281, "margin_dpo/margin_std": 161.70640563964844, "step": 500 }, { "KL/chosen_KL_mean": -225.6497802734375, "KL/mean": -275.616943359375, "KL/rejected_KL_mean": -325.5841064453125, "KL/std": 155.71327209472656, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.9343032836914, "fcm_dpo/q_t": 0.4813776910305023, "grad_norm": 2.4970173835754395, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.3502709865570068, "logits/rejected": -0.9225509166717529, "logps/chosen": -316.2467041015625, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -410.88134765625, "loss": 1.3179, "margin_dpo/margin_mean": 99.93431091308594, "margin_dpo/margin_std": 186.3050994873047, "step": 501 }, { "KL/chosen_KL_mean": -223.92596435546875, "KL/mean": -268.0943603515625, "KL/rejected_KL_mean": -312.26275634765625, "KL/std": 121.43360900878906, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.3367691040039, "fcm_dpo/q_t": 0.4834824800491333, "grad_norm": 2.0512125492095947, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.775010108947754, "logits/rejected": -1.0697212219238281, "logps/chosen": -327.3966979980469, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -401.306884765625, "loss": 1.324, "margin_dpo/margin_mean": 88.3367691040039, "margin_dpo/margin_std": 139.7023162841797, "step": 502 }, { "KL/chosen_KL_mean": -219.4454345703125, "KL/mean": -271.7796630859375, "KL/rejected_KL_mean": -324.1139221191406, "KL/std": 131.18515014648438, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.66850280761719, "fcm_dpo/q_t": 0.4804626703262329, "grad_norm": 2.871922731399536, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.6447885036468506, "logits/rejected": -1.1650278568267822, "logps/chosen": -313.1754150390625, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -411.722900390625, "loss": 1.3127, "margin_dpo/margin_mean": 104.66850280761719, "margin_dpo/margin_std": 154.4967041015625, "step": 503 }, { "KL/chosen_KL_mean": -230.42111206054688, "KL/mean": -299.09173583984375, "KL/rejected_KL_mean": -367.76239013671875, "KL/std": 143.7586669921875, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.3412628173828, "fcm_dpo/q_t": 0.47437193989753723, "grad_norm": 2.7151482105255127, "learning_rate": 9.730107739932805e-08, "logits/chosen": -1.9214551448822021, "logits/rejected": -1.6032567024230957, "logps/chosen": -322.7129821777344, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -476.800048828125, "loss": 1.2897, "margin_dpo/margin_mean": 137.3412628173828, "margin_dpo/margin_std": 160.4349365234375, "step": 504 }, { "KL/chosen_KL_mean": -258.48931884765625, "KL/mean": -301.019775390625, "KL/rejected_KL_mean": -343.5502014160156, "KL/std": 144.49624633789062, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.06088256835938, "fcm_dpo/q_t": 0.48409539461135864, "grad_norm": 2.5015909671783447, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.188708782196045, "logits/rejected": -1.6548290252685547, "logps/chosen": -364.7662658691406, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -437.7563171386719, "loss": 1.3267, "margin_dpo/margin_mean": 85.06088256835938, "margin_dpo/margin_std": 147.93377685546875, "step": 505 }, { "KL/chosen_KL_mean": -230.0304412841797, "KL/mean": -285.6044921875, "KL/rejected_KL_mean": -341.1785888671875, "KL/std": 128.82492065429688, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.14815521240234, "fcm_dpo/q_t": 0.4792250990867615, "grad_norm": 3.0503876209259033, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9335882663726807, "logits/rejected": -1.5895378589630127, "logps/chosen": -311.4713134765625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -426.84295654296875, "loss": 1.3074, "margin_dpo/margin_mean": 111.14816284179688, "margin_dpo/margin_std": 137.26451110839844, "step": 506 }, { "KL/chosen_KL_mean": -249.32730102539062, "KL/mean": -298.1544189453125, "KL/rejected_KL_mean": -346.9815673828125, "KL/std": 141.3009796142578, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.65428161621094, "fcm_dpo/q_t": 0.4817865490913391, "grad_norm": 3.44724178314209, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.188180446624756, "logits/rejected": -1.7750639915466309, "logps/chosen": -348.7542419433594, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -449.23455810546875, "loss": 1.3186, "margin_dpo/margin_mean": 97.65428161621094, "margin_dpo/margin_std": 167.56637573242188, "step": 507 }, { "KL/chosen_KL_mean": -242.43008422851562, "KL/mean": -286.727294921875, "KL/rejected_KL_mean": -331.0244445800781, "KL/std": 147.21182250976562, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.59437561035156, "fcm_dpo/q_t": 0.48347893357276917, "grad_norm": 2.888066053390503, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.0570285320281982, "logits/rejected": -1.602394938468933, "logps/chosen": -340.3457336425781, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -434.822265625, "loss": 1.3257, "margin_dpo/margin_mean": 88.59437561035156, "margin_dpo/margin_std": 181.9033203125, "step": 508 }, { "KL/chosen_KL_mean": -225.02056884765625, "KL/mean": -263.1119079589844, "KL/rejected_KL_mean": -301.2032470703125, "KL/std": 122.82987976074219, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.18267822265625, "fcm_dpo/q_t": 0.48574739694595337, "grad_norm": 3.6827175617218018, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.1026625633239746, "logits/rejected": -1.4822242259979248, "logps/chosen": -325.75042724609375, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -388.1466064453125, "loss": 1.3341, "margin_dpo/margin_mean": 76.18267059326172, "margin_dpo/margin_std": 168.67529296875, "step": 509 }, { "KL/chosen_KL_mean": -260.73992919921875, "KL/mean": -312.5957336425781, "KL/rejected_KL_mean": -364.4515380859375, "KL/std": 135.18795776367188, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.71160888671875, "fcm_dpo/q_t": 0.48061394691467285, "grad_norm": 2.7668449878692627, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.3365200757980347, "logits/rejected": -1.1768691539764404, "logps/chosen": -349.9093017578125, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -472.37091064453125, "loss": 1.3127, "margin_dpo/margin_mean": 103.71160888671875, "margin_dpo/margin_std": 135.2669219970703, "step": 510 }, { "KL/chosen_KL_mean": -231.220703125, "KL/mean": -290.11944580078125, "KL/rejected_KL_mean": -349.0182189941406, "KL/std": 139.14443969726562, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.79753875732422, "fcm_dpo/q_t": 0.4780312180519104, "grad_norm": 3.162642478942871, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.041820764541626, "logits/rejected": -1.4883346557617188, "logps/chosen": -320.9704895019531, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -446.74951171875, "loss": 1.3038, "margin_dpo/margin_mean": 117.79753112792969, "margin_dpo/margin_std": 162.8411407470703, "step": 511 }, { "KL/chosen_KL_mean": -226.69754028320312, "KL/mean": -281.1231384277344, "KL/rejected_KL_mean": -335.5487060546875, "KL/std": 144.51596069335938, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.85115051269531, "fcm_dpo/q_t": 0.4796707034111023, "grad_norm": 3.7245802879333496, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.7443311214447021, "logits/rejected": -1.3200244903564453, "logps/chosen": -312.43267822265625, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -423.173583984375, "loss": 1.31, "margin_dpo/margin_mean": 108.85115051269531, "margin_dpo/margin_std": 160.43148803710938, "step": 512 }, { "KL/chosen_KL_mean": -240.21534729003906, "KL/mean": -284.7982177734375, "KL/rejected_KL_mean": -329.381103515625, "KL/std": 132.26751708984375, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.16575622558594, "fcm_dpo/q_t": 0.4833342730998993, "grad_norm": 2.701509475708008, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.2210681438446045, "logits/rejected": -1.5463500022888184, "logps/chosen": -336.4058532714844, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -412.298828125, "loss": 1.3242, "margin_dpo/margin_mean": 89.16575622558594, "margin_dpo/margin_std": 157.306884765625, "step": 513 }, { "KL/chosen_KL_mean": -240.42791748046875, "KL/mean": -284.64678955078125, "KL/rejected_KL_mean": -328.86566162109375, "KL/std": 139.82440185546875, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.43775939941406, "fcm_dpo/q_t": 0.4834962487220764, "grad_norm": 3.13967227935791, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.6610198020935059, "logits/rejected": -1.106550931930542, "logps/chosen": -338.2515869140625, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -414.2355041503906, "loss": 1.3249, "margin_dpo/margin_mean": 88.43775939941406, "margin_dpo/margin_std": 164.82388305664062, "step": 514 }, { "KL/chosen_KL_mean": -210.51864624023438, "KL/mean": -266.1867370605469, "KL/rejected_KL_mean": -321.85479736328125, "KL/std": 139.5894775390625, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.33617401123047, "fcm_dpo/q_t": 0.47921237349510193, "grad_norm": 2.7581024169921875, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.8042776584625244, "logits/rejected": -1.5885119438171387, "logps/chosen": -292.45220947265625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -415.431396484375, "loss": 1.3082, "margin_dpo/margin_mean": 111.336181640625, "margin_dpo/margin_std": 160.18417358398438, "step": 515 }, { "KL/chosen_KL_mean": -235.45989990234375, "KL/mean": -285.1554870605469, "KL/rejected_KL_mean": -334.85107421875, "KL/std": 168.2606201171875, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.39115905761719, "fcm_dpo/q_t": 0.4815008044242859, "grad_norm": 3.2672483921051025, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.114440441131592, "logits/rejected": -1.9969334602355957, "logps/chosen": -332.55157470703125, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -453.40771484375, "loss": 1.3189, "margin_dpo/margin_mean": 99.39115905761719, "margin_dpo/margin_std": 203.16189575195312, "step": 516 }, { "KL/chosen_KL_mean": -248.83160400390625, "KL/mean": -308.335205078125, "KL/rejected_KL_mean": -367.8388366699219, "KL/std": 140.98275756835938, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.0072250366211, "fcm_dpo/q_t": 0.4777877926826477, "grad_norm": 2.0641796588897705, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.863579273223877, "logits/rejected": -1.4619570970535278, "logps/chosen": -342.0181884765625, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -463.431640625, "loss": 1.3029, "margin_dpo/margin_mean": 119.0072250366211, "margin_dpo/margin_std": 158.8932647705078, "step": 517 }, { "KL/chosen_KL_mean": -254.31137084960938, "KL/mean": -293.679931640625, "KL/rejected_KL_mean": -333.0484313964844, "KL/std": 127.41337585449219, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.737060546875, "fcm_dpo/q_t": 0.48531025648117065, "grad_norm": 3.323793888092041, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.818824052810669, "logits/rejected": -1.5150971412658691, "logps/chosen": -341.332763671875, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -424.2929992675781, "loss": 1.3317, "margin_dpo/margin_mean": 78.737060546875, "margin_dpo/margin_std": 159.06838989257812, "step": 518 }, { "KL/chosen_KL_mean": -239.15664672851562, "KL/mean": -287.27056884765625, "KL/rejected_KL_mean": -335.384521484375, "KL/std": 123.94954681396484, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.22785949707031, "fcm_dpo/q_t": 0.48203787207603455, "grad_norm": 3.0840909481048584, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.081559658050537, "logits/rejected": -1.7444448471069336, "logps/chosen": -339.84417724609375, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -444.4149169921875, "loss": 1.318, "margin_dpo/margin_mean": 96.22785949707031, "margin_dpo/margin_std": 135.52943420410156, "step": 519 }, { "KL/chosen_KL_mean": -227.39144897460938, "KL/mean": -280.8648681640625, "KL/rejected_KL_mean": -334.33831787109375, "KL/std": 147.8077392578125, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.94685363769531, "fcm_dpo/q_t": 0.48001593351364136, "grad_norm": 2.495713472366333, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.3962624073028564, "logits/rejected": -0.9834322929382324, "logps/chosen": -326.1690673828125, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -441.28399658203125, "loss": 1.311, "margin_dpo/margin_mean": 106.94685363769531, "margin_dpo/margin_std": 151.57859802246094, "step": 520 }, { "KL/chosen_KL_mean": -217.18563842773438, "KL/mean": -273.59100341796875, "KL/rejected_KL_mean": -329.99639892578125, "KL/std": 125.0504379272461, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.81076049804688, "fcm_dpo/q_t": 0.4789217710494995, "grad_norm": 2.696787118911743, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9515177011489868, "logits/rejected": -1.628605842590332, "logps/chosen": -303.5367431640625, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -420.38958740234375, "loss": 1.3062, "margin_dpo/margin_mean": 112.81076049804688, "margin_dpo/margin_std": 136.91543579101562, "step": 521 }, { "KL/chosen_KL_mean": -226.6318359375, "KL/mean": -275.57452392578125, "KL/rejected_KL_mean": -324.51715087890625, "KL/std": 148.30645751953125, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.88531494140625, "fcm_dpo/q_t": 0.48171180486679077, "grad_norm": 2.7498366832733154, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.5271730422973633, "logits/rejected": -1.168497920036316, "logps/chosen": -302.639892578125, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -398.95843505859375, "loss": 1.3177, "margin_dpo/margin_mean": 97.88531494140625, "margin_dpo/margin_std": 156.40728759765625, "step": 522 }, { "KL/chosen_KL_mean": -256.62249755859375, "KL/mean": -291.5694885253906, "KL/rejected_KL_mean": -326.51641845703125, "KL/std": 146.20013427734375, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.89395141601562, "fcm_dpo/q_t": 0.48695120215415955, "grad_norm": 3.2422845363616943, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.8963487148284912, "logits/rejected": -1.2352466583251953, "logps/chosen": -353.1187744140625, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -417.53173828125, "loss": 1.3393, "margin_dpo/margin_mean": 69.89395141601562, "margin_dpo/margin_std": 183.20700073242188, "step": 523 }, { "KL/chosen_KL_mean": -242.25738525390625, "KL/mean": -290.04779052734375, "KL/rejected_KL_mean": -337.83819580078125, "KL/std": 138.75054931640625, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.580810546875, "fcm_dpo/q_t": 0.48215872049331665, "grad_norm": 2.9329564571380615, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.005033016204834, "logits/rejected": -1.2998918294906616, "logps/chosen": -345.66290283203125, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -425.74835205078125, "loss": 1.3196, "margin_dpo/margin_mean": 95.58080291748047, "margin_dpo/margin_std": 160.59947204589844, "step": 524 }, { "KL/chosen_KL_mean": -213.10797119140625, "KL/mean": -280.6202697753906, "KL/rejected_KL_mean": -348.132568359375, "KL/std": 146.9420623779297, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.02459716796875, "fcm_dpo/q_t": 0.4748413562774658, "grad_norm": 2.6640799045562744, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.4765006303787231, "logits/rejected": -1.3910722732543945, "logps/chosen": -297.314453125, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -454.3941650390625, "loss": 1.2925, "margin_dpo/margin_mean": 135.02459716796875, "margin_dpo/margin_std": 180.88243103027344, "step": 525 }, { "KL/chosen_KL_mean": -227.42359924316406, "KL/mean": -282.94415283203125, "KL/rejected_KL_mean": -338.4647216796875, "KL/std": 128.98049926757812, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.04109191894531, "fcm_dpo/q_t": 0.4792640507221222, "grad_norm": 2.7353060245513916, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7302055358886719, "logits/rejected": -1.111423134803772, "logps/chosen": -330.2777099609375, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -432.11614990234375, "loss": 1.3082, "margin_dpo/margin_mean": 111.04109954833984, "margin_dpo/margin_std": 156.33432006835938, "step": 526 }, { "KL/chosen_KL_mean": -224.2889404296875, "KL/mean": -279.9082336425781, "KL/rejected_KL_mean": -335.52752685546875, "KL/std": 132.20474243164062, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.23857879638672, "fcm_dpo/q_t": 0.47920048236846924, "grad_norm": 2.508111000061035, "learning_rate": 7.504749238082414e-08, "logits/chosen": -2.94213604927063, "logits/rejected": -2.1575708389282227, "logps/chosen": -321.05438232421875, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -420.9226989746094, "loss": 1.3065, "margin_dpo/margin_mean": 111.23857879638672, "margin_dpo/margin_std": 115.89666748046875, "step": 527 }, { "KL/chosen_KL_mean": -213.55999755859375, "KL/mean": -272.0622863769531, "KL/rejected_KL_mean": -330.5645751953125, "KL/std": 139.76571655273438, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.00456237792969, "fcm_dpo/q_t": 0.4781774878501892, "grad_norm": 3.2053911685943604, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.0348644256591797, "logits/rejected": -1.5180234909057617, "logps/chosen": -307.72491455078125, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -430.9368896484375, "loss": 1.3047, "margin_dpo/margin_mean": 117.00456237792969, "margin_dpo/margin_std": 170.54867553710938, "step": 528 }, { "KL/chosen_KL_mean": -234.4136199951172, "KL/mean": -292.0758972167969, "KL/rejected_KL_mean": -349.7381896972656, "KL/std": 143.36842346191406, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.3245849609375, "fcm_dpo/q_t": 0.47849607467651367, "grad_norm": 2.840238094329834, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8433358669281006, "logits/rejected": -1.6848459243774414, "logps/chosen": -315.1904296875, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -446.9293518066406, "loss": 1.3059, "margin_dpo/margin_mean": 115.3245849609375, "margin_dpo/margin_std": 169.78587341308594, "step": 529 }, { "KL/chosen_KL_mean": -227.28521728515625, "KL/mean": -276.7687072753906, "KL/rejected_KL_mean": -326.2522277832031, "KL/std": 136.855224609375, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.96701049804688, "fcm_dpo/q_t": 0.48154544830322266, "grad_norm": 2.2890827655792236, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.6973247528076172, "logits/rejected": -1.4122761487960815, "logps/chosen": -307.0609436035156, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -418.3061218261719, "loss": 1.3167, "margin_dpo/margin_mean": 98.96701049804688, "margin_dpo/margin_std": 151.55935668945312, "step": 530 }, { "KL/chosen_KL_mean": -213.584228515625, "KL/mean": -275.4671630859375, "KL/rejected_KL_mean": -337.3501281738281, "KL/std": 143.4992218017578, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.76590728759766, "fcm_dpo/q_t": 0.4768890142440796, "grad_norm": 3.923381805419922, "learning_rate": 7.141774982445147e-08, "logits/chosen": -2.0198397636413574, "logits/rejected": -1.464902639389038, "logps/chosen": -292.9034423828125, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -415.4584655761719, "loss": 1.2986, "margin_dpo/margin_mean": 123.76590728759766, "margin_dpo/margin_std": 145.36941528320312, "step": 531 }, { "KL/chosen_KL_mean": -240.36883544921875, "KL/mean": -306.0802917480469, "KL/rejected_KL_mean": -371.791748046875, "KL/std": 138.44107055664062, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.42291259765625, "fcm_dpo/q_t": 0.4754612147808075, "grad_norm": 3.2439699172973633, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.508775234222412, "logits/rejected": -0.8831478953361511, "logps/chosen": -342.0057373046875, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -465.13714599609375, "loss": 1.2938, "margin_dpo/margin_mean": 131.42291259765625, "margin_dpo/margin_std": 159.09173583984375, "step": 532 }, { "KL/chosen_KL_mean": -225.8460235595703, "KL/mean": -265.20428466796875, "KL/rejected_KL_mean": -304.5625305175781, "KL/std": 141.72572326660156, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.71651458740234, "fcm_dpo/q_t": 0.4853260815143585, "grad_norm": 2.739628553390503, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.536929130554199, "logits/rejected": -1.9522643089294434, "logps/chosen": -316.5926513671875, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -387.4510498046875, "loss": 1.3317, "margin_dpo/margin_mean": 78.71650695800781, "margin_dpo/margin_std": 157.9561309814453, "step": 533 }, { "KL/chosen_KL_mean": -192.76870727539062, "KL/mean": -251.14263916015625, "KL/rejected_KL_mean": -309.5166015625, "KL/std": 131.68356323242188, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.74790954589844, "fcm_dpo/q_t": 0.4781864583492279, "grad_norm": 2.2298476696014404, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.8520100116729736, "logits/rejected": -1.459557294845581, "logps/chosen": -279.3995361328125, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -394.3885803222656, "loss": 1.3033, "margin_dpo/margin_mean": 116.7479019165039, "margin_dpo/margin_std": 137.2545623779297, "step": 534 }, { "KL/chosen_KL_mean": -205.09136962890625, "KL/mean": -267.3884582519531, "KL/rejected_KL_mean": -329.685546875, "KL/std": 132.93528747558594, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.5941390991211, "fcm_dpo/q_t": 0.4767518639564514, "grad_norm": 2.241795063018799, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.7109732627868652, "logits/rejected": -1.5465950965881348, "logps/chosen": -295.436767578125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -430.4635009765625, "loss": 1.2983, "margin_dpo/margin_mean": 124.59414672851562, "margin_dpo/margin_std": 150.03736877441406, "step": 535 }, { "KL/chosen_KL_mean": -194.33282470703125, "KL/mean": -257.92816162109375, "KL/rejected_KL_mean": -321.5235290527344, "KL/std": 134.81883239746094, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.19068145751953, "fcm_dpo/q_t": 0.47625482082366943, "grad_norm": 3.5601696968078613, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.6216890811920166, "logits/rejected": -1.3779850006103516, "logps/chosen": -273.29241943359375, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -414.611328125, "loss": 1.2963, "margin_dpo/margin_mean": 127.190673828125, "margin_dpo/margin_std": 147.18434143066406, "step": 536 }, { "KL/chosen_KL_mean": -184.94509887695312, "KL/mean": -243.5758056640625, "KL/rejected_KL_mean": -302.20654296875, "KL/std": 141.54354858398438, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.26144409179688, "fcm_dpo/q_t": 0.4781017303466797, "grad_norm": 2.4173905849456787, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.5456569194793701, "logits/rejected": -0.9050878286361694, "logps/chosen": -271.65447998046875, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -381.1142272949219, "loss": 1.3038, "margin_dpo/margin_mean": 117.26144409179688, "margin_dpo/margin_std": 152.3922119140625, "step": 537 }, { "KL/chosen_KL_mean": -247.31048583984375, "KL/mean": -295.71673583984375, "KL/rejected_KL_mean": -344.1230163574219, "KL/std": 126.93928527832031, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.81250762939453, "fcm_dpo/q_t": 0.4819135069847107, "grad_norm": 2.0410172939300537, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.7219324111938477, "logits/rejected": -1.185032844543457, "logps/chosen": -345.75262451171875, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -435.2176513671875, "loss": 1.3177, "margin_dpo/margin_mean": 96.81251525878906, "margin_dpo/margin_std": 139.31597900390625, "step": 538 }, { "KL/chosen_KL_mean": -207.85159301757812, "KL/mean": -265.1238098144531, "KL/rejected_KL_mean": -322.39599609375, "KL/std": 133.97247314453125, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.54442596435547, "fcm_dpo/q_t": 0.47860628366470337, "grad_norm": 2.824282169342041, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.3833396434783936, "logits/rejected": -1.912332534790039, "logps/chosen": -298.78369140625, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -419.8414611816406, "loss": 1.3052, "margin_dpo/margin_mean": 114.54441833496094, "margin_dpo/margin_std": 144.195556640625, "step": 539 }, { "KL/chosen_KL_mean": -238.54757690429688, "KL/mean": -307.37298583984375, "KL/rejected_KL_mean": -376.1983947753906, "KL/std": 164.16156005859375, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.65081787109375, "fcm_dpo/q_t": 0.47441741824150085, "grad_norm": 3.663111925125122, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9062130451202393, "logits/rejected": -1.6877721548080444, "logps/chosen": -346.41748046875, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -497.5919189453125, "loss": 1.2919, "margin_dpo/margin_mean": 137.65081787109375, "margin_dpo/margin_std": 206.5270538330078, "step": 540 }, { "KL/chosen_KL_mean": -253.8505096435547, "KL/mean": -298.027099609375, "KL/rejected_KL_mean": -342.2036437988281, "KL/std": 128.92562866210938, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.3531494140625, "fcm_dpo/q_t": 0.4835139513015747, "grad_norm": 3.776151180267334, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.2674155235290527, "logits/rejected": -1.8375978469848633, "logps/chosen": -360.4179992675781, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -451.2699890136719, "loss": 1.3244, "margin_dpo/margin_mean": 88.3531494140625, "margin_dpo/margin_std": 139.0042724609375, "step": 541 }, { "KL/chosen_KL_mean": -203.76284790039062, "KL/mean": -271.67974853515625, "KL/rejected_KL_mean": -339.59661865234375, "KL/std": 141.50320434570312, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.83375549316406, "fcm_dpo/q_t": 0.4746397137641907, "grad_norm": 2.5607497692108154, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.262341022491455, "logits/rejected": -2.145379066467285, "logps/chosen": -291.13214111328125, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -442.4169616699219, "loss": 1.2905, "margin_dpo/margin_mean": 135.83377075195312, "margin_dpo/margin_std": 156.38327026367188, "step": 542 }, { "KL/chosen_KL_mean": -246.968994140625, "KL/mean": -283.1767578125, "KL/rejected_KL_mean": -319.384521484375, "KL/std": 154.32830810546875, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.41552734375, "fcm_dpo/q_t": 0.4865034222602844, "grad_norm": 3.3061554431915283, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.842086911201477, "logits/rejected": -1.9202001094818115, "logps/chosen": -334.12158203125, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -430.916259765625, "loss": 1.3382, "margin_dpo/margin_mean": 72.41553497314453, "margin_dpo/margin_std": 195.22171020507812, "step": 543 }, { "KL/chosen_KL_mean": -259.39422607421875, "KL/mean": -306.80792236328125, "KL/rejected_KL_mean": -354.22161865234375, "KL/std": 152.59002685546875, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.82740020751953, "fcm_dpo/q_t": 0.4824141561985016, "grad_norm": 2.8080837726593018, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.564019799232483, "logits/rejected": -1.4887635707855225, "logps/chosen": -345.5411376953125, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -460.61553955078125, "loss": 1.3215, "margin_dpo/margin_mean": 94.82740020751953, "margin_dpo/margin_std": 188.14691162109375, "step": 544 }, { "KL/chosen_KL_mean": -229.24819946289062, "KL/mean": -284.44146728515625, "KL/rejected_KL_mean": -339.634765625, "KL/std": 142.37359619140625, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.38655853271484, "fcm_dpo/q_t": 0.4793943464756012, "grad_norm": 2.821514129638672, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4887055158615112, "logits/rejected": -1.0736464262008667, "logps/chosen": -325.67242431640625, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -440.36883544921875, "loss": 1.3091, "margin_dpo/margin_mean": 110.38656616210938, "margin_dpo/margin_std": 166.3857879638672, "step": 545 }, { "KL/chosen_KL_mean": -243.70892333984375, "KL/mean": -285.1555480957031, "KL/rejected_KL_mean": -326.6022033691406, "KL/std": 137.81776428222656, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.89330291748047, "fcm_dpo/q_t": 0.4845101833343506, "grad_norm": 2.385624885559082, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.440197467803955, "logits/rejected": -1.1086068153381348, "logps/chosen": -329.73004150390625, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -413.974853515625, "loss": 1.3286, "margin_dpo/margin_mean": 82.89330291748047, "margin_dpo/margin_std": 157.81134033203125, "step": 546 }, { "KL/chosen_KL_mean": -230.33383178710938, "KL/mean": -288.0187072753906, "KL/rejected_KL_mean": -345.7035827636719, "KL/std": 141.52304077148438, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.3697509765625, "fcm_dpo/q_t": 0.47845107316970825, "grad_norm": 2.6981756687164307, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.7775790691375732, "logits/rejected": -1.322922706604004, "logps/chosen": -330.9848327636719, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -443.6153259277344, "loss": 1.3043, "margin_dpo/margin_mean": 115.36974334716797, "margin_dpo/margin_std": 135.03115844726562, "step": 547 }, { "KL/chosen_KL_mean": -226.72744750976562, "KL/mean": -270.318115234375, "KL/rejected_KL_mean": -313.90875244140625, "KL/std": 140.63821411132812, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.1812973022461, "fcm_dpo/q_t": 0.4837414622306824, "grad_norm": 3.8489596843719482, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.1934986114501953, "logits/rejected": -1.84761381149292, "logps/chosen": -322.757568359375, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -412.32196044921875, "loss": 1.3264, "margin_dpo/margin_mean": 87.1812973022461, "margin_dpo/margin_std": 176.93655395507812, "step": 548 }, { "KL/chosen_KL_mean": -216.90777587890625, "KL/mean": -257.9437561035156, "KL/rejected_KL_mean": -298.9797668457031, "KL/std": 131.54896545410156, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.07199096679688, "fcm_dpo/q_t": 0.4846654534339905, "grad_norm": 2.5178282260894775, "learning_rate": 5.603696935852426e-08, "logits/chosen": -2.0160152912139893, "logits/rejected": -1.6451306343078613, "logps/chosen": -300.3587341308594, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -380.3599853515625, "loss": 1.33, "margin_dpo/margin_mean": 82.07199096679688, "margin_dpo/margin_std": 169.788330078125, "step": 549 }, { "KL/chosen_KL_mean": -234.70523071289062, "KL/mean": -278.3212890625, "KL/rejected_KL_mean": -321.9372863769531, "KL/std": 128.1849365234375, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.23204040527344, "fcm_dpo/q_t": 0.4837181866168976, "grad_norm": 3.099229574203491, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.598528265953064, "logits/rejected": -1.4131920337677002, "logps/chosen": -324.299560546875, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -423.2528076171875, "loss": 1.3249, "margin_dpo/margin_mean": 87.23204040527344, "margin_dpo/margin_std": 145.12326049804688, "step": 550 }, { "KL/chosen_KL_mean": -206.50570678710938, "KL/mean": -273.36798095703125, "KL/rejected_KL_mean": -340.230224609375, "KL/std": 148.15065002441406, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.72451782226562, "fcm_dpo/q_t": 0.4750761389732361, "grad_norm": 2.5059351921081543, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.7352067232131958, "logits/rejected": -1.6128718852996826, "logps/chosen": -301.5878601074219, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -457.949462890625, "loss": 1.2924, "margin_dpo/margin_mean": 133.72450256347656, "margin_dpo/margin_std": 165.4195556640625, "step": 551 }, { "KL/chosen_KL_mean": -211.17556762695312, "KL/mean": -293.4539794921875, "KL/rejected_KL_mean": -375.7323913574219, "KL/std": 160.8712158203125, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.55682373046875, "fcm_dpo/q_t": 0.46932584047317505, "grad_norm": 5.347563743591309, "learning_rate": 5.363104864490034e-08, "logits/chosen": -1.9795013666152954, "logits/rejected": -1.6384661197662354, "logps/chosen": -312.5847473144531, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -486.5708312988281, "loss": 1.2711, "margin_dpo/margin_mean": 164.55682373046875, "margin_dpo/margin_std": 172.76641845703125, "step": 552 }, { "KL/chosen_KL_mean": -219.4801788330078, "KL/mean": -266.9822082519531, "KL/rejected_KL_mean": -314.4842529296875, "KL/std": 149.79664611816406, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.0040512084961, "fcm_dpo/q_t": 0.48226845264434814, "grad_norm": 3.0840790271759033, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.3532179594039917, "logits/rejected": -1.03049898147583, "logps/chosen": -317.4440612792969, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -418.0614318847656, "loss": 1.3202, "margin_dpo/margin_mean": 95.00404357910156, "margin_dpo/margin_std": 163.81768798828125, "step": 553 }, { "KL/chosen_KL_mean": -230.00704956054688, "KL/mean": -285.986083984375, "KL/rejected_KL_mean": -341.96514892578125, "KL/std": 143.4598846435547, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.95808410644531, "fcm_dpo/q_t": 0.47911733388900757, "grad_norm": 1.953932523727417, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.7792418003082275, "logits/rejected": -1.012389898300171, "logps/chosen": -333.69384765625, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -434.9572448730469, "loss": 1.309, "margin_dpo/margin_mean": 111.95808410644531, "margin_dpo/margin_std": 185.7437744140625, "step": 554 }, { "KL/chosen_KL_mean": -235.93524169921875, "KL/mean": -287.5007019042969, "KL/rejected_KL_mean": -339.066162109375, "KL/std": 151.07955932617188, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.13092041015625, "fcm_dpo/q_t": 0.48076900839805603, "grad_norm": 2.353976011276245, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.4626209735870361, "logits/rejected": -1.3825455904006958, "logps/chosen": -322.13885498046875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -453.6133728027344, "loss": 1.3147, "margin_dpo/margin_mean": 103.13092041015625, "margin_dpo/margin_std": 174.03384399414062, "step": 555 }, { "KL/chosen_KL_mean": -217.14129638671875, "KL/mean": -272.3252868652344, "KL/rejected_KL_mean": -327.50927734375, "KL/std": 126.7371826171875, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.36795806884766, "fcm_dpo/q_t": 0.47938498854637146, "grad_norm": 2.745551109313965, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.6635576486587524, "logits/rejected": -1.4132812023162842, "logps/chosen": -305.5447998046875, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -434.14837646484375, "loss": 1.3082, "margin_dpo/margin_mean": 110.36795806884766, "margin_dpo/margin_std": 145.40655517578125, "step": 556 }, { "KL/chosen_KL_mean": -234.36170959472656, "KL/mean": -296.340087890625, "KL/rejected_KL_mean": -358.3184509277344, "KL/std": 162.69204711914062, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.95672607421875, "fcm_dpo/q_t": 0.476902037858963, "grad_norm": 2.553267002105713, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.7139363288879395, "logits/rejected": -1.5506037473678589, "logps/chosen": -333.19696044921875, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -473.96734619140625, "loss": 1.3006, "margin_dpo/margin_mean": 123.95672607421875, "margin_dpo/margin_std": 187.78341674804688, "step": 557 }, { "KL/chosen_KL_mean": -234.14480590820312, "KL/mean": -299.5523681640625, "KL/rejected_KL_mean": -364.9599609375, "KL/std": 143.33338928222656, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.81517028808594, "fcm_dpo/q_t": 0.47557714581489563, "grad_norm": 3.2217812538146973, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.265990734100342, "logits/rejected": -1.6822986602783203, "logps/chosen": -340.1566162109375, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -473.3512878417969, "loss": 1.2935, "margin_dpo/margin_mean": 130.81517028808594, "margin_dpo/margin_std": 143.53675842285156, "step": 558 }, { "KL/chosen_KL_mean": -251.5071258544922, "KL/mean": -299.79644775390625, "KL/rejected_KL_mean": -348.08575439453125, "KL/std": 144.76040649414062, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.57862854003906, "fcm_dpo/q_t": 0.481955349445343, "grad_norm": 3.57910418510437, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.3775768280029297, "logits/rejected": -1.9365863800048828, "logps/chosen": -351.931884765625, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -448.49847412109375, "loss": 1.3194, "margin_dpo/margin_mean": 96.57862854003906, "margin_dpo/margin_std": 172.02413940429688, "step": 559 }, { "KL/chosen_KL_mean": -225.135009765625, "KL/mean": -278.10955810546875, "KL/rejected_KL_mean": -331.08404541015625, "KL/std": 135.88177490234375, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.94905853271484, "fcm_dpo/q_t": 0.48022356629371643, "grad_norm": 2.6301896572113037, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.9401184320449829, "logits/rejected": -0.4988725185394287, "logps/chosen": -312.7705383300781, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -418.46392822265625, "loss": 1.3121, "margin_dpo/margin_mean": 105.94905853271484, "margin_dpo/margin_std": 161.83114624023438, "step": 560 }, { "KL/chosen_KL_mean": -234.145751953125, "KL/mean": -290.10809326171875, "KL/rejected_KL_mean": -346.0704345703125, "KL/std": 134.6580047607422, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.92466735839844, "fcm_dpo/q_t": 0.47909796237945557, "grad_norm": 2.8854706287384033, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.76493501663208, "logits/rejected": -1.754115104675293, "logps/chosen": -309.02716064453125, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -452.118408203125, "loss": 1.3069, "margin_dpo/margin_mean": 111.92466735839844, "margin_dpo/margin_std": 137.0492401123047, "step": 561 }, { "KL/chosen_KL_mean": -217.61721801757812, "KL/mean": -265.86566162109375, "KL/rejected_KL_mean": -314.1141357421875, "KL/std": 136.08387756347656, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.49690246582031, "fcm_dpo/q_t": 0.48201045393943787, "grad_norm": 2.972957134246826, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.138160228729248, "logits/rejected": -1.7075974941253662, "logps/chosen": -318.28045654296875, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -416.35498046875, "loss": 1.3195, "margin_dpo/margin_mean": 96.49690246582031, "margin_dpo/margin_std": 172.51959228515625, "step": 562 }, { "KL/chosen_KL_mean": -227.67910766601562, "KL/mean": -265.709716796875, "KL/rejected_KL_mean": -303.7403564453125, "KL/std": 136.34646606445312, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.06124114990234, "fcm_dpo/q_t": 0.48578745126724243, "grad_norm": 3.1283719539642334, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.6521726846694946, "logits/rejected": -1.3449602127075195, "logps/chosen": -323.9383239746094, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -404.94537353515625, "loss": 1.3329, "margin_dpo/margin_mean": 76.06124114990234, "margin_dpo/margin_std": 138.85739135742188, "step": 563 }, { "KL/chosen_KL_mean": -226.86627197265625, "KL/mean": -272.26763916015625, "KL/rejected_KL_mean": -317.6690368652344, "KL/std": 131.53077697753906, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.80274963378906, "fcm_dpo/q_t": 0.48302334547042847, "grad_norm": 2.58054780960083, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.2342538833618164, "logits/rejected": -1.6622649431228638, "logps/chosen": -339.82666015625, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -417.19708251953125, "loss": 1.322, "margin_dpo/margin_mean": 90.80274963378906, "margin_dpo/margin_std": 137.28802490234375, "step": 564 }, { "KL/chosen_KL_mean": -233.16317749023438, "KL/mean": -284.23907470703125, "KL/rejected_KL_mean": -335.31500244140625, "KL/std": 152.0717010498047, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.15180969238281, "fcm_dpo/q_t": 0.48095667362213135, "grad_norm": 2.56844162940979, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.7068369388580322, "logits/rejected": -1.2357667684555054, "logps/chosen": -328.52874755859375, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -427.5323486328125, "loss": 1.3156, "margin_dpo/margin_mean": 102.15180969238281, "margin_dpo/margin_std": 177.01327514648438, "step": 565 }, { "KL/chosen_KL_mean": -242.52294921875, "KL/mean": -293.8273620605469, "KL/rejected_KL_mean": -345.13177490234375, "KL/std": 151.52069091796875, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.60881805419922, "fcm_dpo/q_t": 0.4808419346809387, "grad_norm": 2.5964386463165283, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.8553074598312378, "logits/rejected": -1.4466962814331055, "logps/chosen": -341.49615478515625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -455.9526672363281, "loss": 1.3142, "margin_dpo/margin_mean": 102.60881805419922, "margin_dpo/margin_std": 153.62527465820312, "step": 566 }, { "KL/chosen_KL_mean": -224.59475708007812, "KL/mean": -267.9803466796875, "KL/rejected_KL_mean": -311.36590576171875, "KL/std": 131.08297729492188, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.77113342285156, "fcm_dpo/q_t": 0.48377373814582825, "grad_norm": 2.830057144165039, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.0365662574768066, "logits/rejected": -1.36041259765625, "logps/chosen": -317.29833984375, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -402.5902099609375, "loss": 1.3255, "margin_dpo/margin_mean": 86.77113342285156, "margin_dpo/margin_std": 146.63174438476562, "step": 567 }, { "KL/chosen_KL_mean": -209.0989990234375, "KL/mean": -275.99395751953125, "KL/rejected_KL_mean": -342.888916015625, "KL/std": 140.37594604492188, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.78990173339844, "fcm_dpo/q_t": 0.4750339388847351, "grad_norm": 2.8402209281921387, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.106200098991394, "logits/rejected": -1.000565528869629, "logps/chosen": -282.23126220703125, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -437.841552734375, "loss": 1.2919, "margin_dpo/margin_mean": 133.78990173339844, "margin_dpo/margin_std": 154.15573120117188, "step": 568 }, { "KL/chosen_KL_mean": -225.69557189941406, "KL/mean": -278.95068359375, "KL/rejected_KL_mean": -332.205810546875, "KL/std": 146.8890380859375, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.51025390625, "fcm_dpo/q_t": 0.48011624813079834, "grad_norm": 2.4148361682891846, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.8295388221740723, "logits/rejected": -1.4515950679779053, "logps/chosen": -314.91265869140625, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -418.61688232421875, "loss": 1.3129, "margin_dpo/margin_mean": 106.51026153564453, "margin_dpo/margin_std": 183.4683074951172, "step": 569 }, { "KL/chosen_KL_mean": -202.05967712402344, "KL/mean": -270.5618896484375, "KL/rejected_KL_mean": -339.0640869140625, "KL/std": 138.13851928710938, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.00440979003906, "fcm_dpo/q_t": 0.4744052290916443, "grad_norm": 2.903806447982788, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.2158632278442383, "logits/rejected": -1.5605194568634033, "logps/chosen": -296.82244873046875, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -427.9110107421875, "loss": 1.2892, "margin_dpo/margin_mean": 137.00440979003906, "margin_dpo/margin_std": 146.96524047851562, "step": 570 }, { "KL/chosen_KL_mean": -205.18377685546875, "KL/mean": -270.6641845703125, "KL/rejected_KL_mean": -336.14459228515625, "KL/std": 134.00331115722656, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.96080017089844, "fcm_dpo/q_t": 0.47555845975875854, "grad_norm": 3.723468780517578, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.6530721187591553, "logits/rejected": -1.4993969202041626, "logps/chosen": -286.00506591796875, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -433.1598815917969, "loss": 1.2943, "margin_dpo/margin_mean": 130.96080017089844, "margin_dpo/margin_std": 163.29270935058594, "step": 571 }, { "KL/chosen_KL_mean": -212.09429931640625, "KL/mean": -276.9739685058594, "KL/rejected_KL_mean": -341.8536682128906, "KL/std": 139.84658813476562, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.75933837890625, "fcm_dpo/q_t": 0.47578930854797363, "grad_norm": 3.165109634399414, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.6178743839263916, "logits/rejected": -1.453852653503418, "logps/chosen": -289.4742431640625, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -432.4196472167969, "loss": 1.2946, "margin_dpo/margin_mean": 129.75933837890625, "margin_dpo/margin_std": 151.4270477294922, "step": 572 }, { "KL/chosen_KL_mean": -211.66497802734375, "KL/mean": -265.1546630859375, "KL/rejected_KL_mean": -318.6443176269531, "KL/std": 138.79562377929688, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.97933959960938, "fcm_dpo/q_t": 0.4800325930118561, "grad_norm": 3.1105995178222656, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.0926878452301025, "logits/rejected": -1.8707280158996582, "logps/chosen": -295.3710632324219, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -409.341796875, "loss": 1.3115, "margin_dpo/margin_mean": 106.97933197021484, "margin_dpo/margin_std": 160.90249633789062, "step": 573 }, { "KL/chosen_KL_mean": -227.5018310546875, "KL/mean": -288.3305358886719, "KL/rejected_KL_mean": -349.15924072265625, "KL/std": 140.05398559570312, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.65740966796875, "fcm_dpo/q_t": 0.4772789478302002, "grad_norm": 2.547578811645508, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.102932929992676, "logits/rejected": -1.8053398132324219, "logps/chosen": -312.6313171386719, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -448.2132873535156, "loss": 1.3004, "margin_dpo/margin_mean": 121.65740203857422, "margin_dpo/margin_std": 150.1446533203125, "step": 574 }, { "KL/chosen_KL_mean": -238.29165649414062, "KL/mean": -290.9857482910156, "KL/rejected_KL_mean": -343.67987060546875, "KL/std": 144.7447052001953, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.38819885253906, "fcm_dpo/q_t": 0.48034071922302246, "grad_norm": 3.3236470222473145, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.2946975231170654, "logits/rejected": -0.8284667730331421, "logps/chosen": -344.0625, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -444.6293029785156, "loss": 1.3132, "margin_dpo/margin_mean": 105.38819122314453, "margin_dpo/margin_std": 176.1435546875, "step": 575 }, { "KL/chosen_KL_mean": -223.13516235351562, "KL/mean": -280.13555908203125, "KL/rejected_KL_mean": -337.13592529296875, "KL/std": 137.86892700195312, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.00074768066406, "fcm_dpo/q_t": 0.4787105917930603, "grad_norm": 2.568302631378174, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.916085124015808, "logits/rejected": -1.8780686855316162, "logps/chosen": -304.3396911621094, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -442.5453186035156, "loss": 1.3064, "margin_dpo/margin_mean": 114.00074768066406, "margin_dpo/margin_std": 162.38050842285156, "step": 576 }, { "KL/chosen_KL_mean": -206.51889038085938, "KL/mean": -269.3373107910156, "KL/rejected_KL_mean": -332.1557312011719, "KL/std": 139.81484985351562, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.63679504394531, "fcm_dpo/q_t": 0.47657057642936707, "grad_norm": 3.279026746749878, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.8040984869003296, "logits/rejected": -1.129164457321167, "logps/chosen": -307.98309326171875, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -421.4820861816406, "loss": 1.2984, "margin_dpo/margin_mean": 125.63679504394531, "margin_dpo/margin_std": 168.97586059570312, "step": 577 }, { "KL/chosen_KL_mean": -238.64602661132812, "KL/mean": -288.0362548828125, "KL/rejected_KL_mean": -337.42645263671875, "KL/std": 134.03607177734375, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.7804183959961, "fcm_dpo/q_t": 0.48155421018600464, "grad_norm": 4.465363025665283, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.3204762935638428, "logits/rejected": -1.0889753103256226, "logps/chosen": -315.9384460449219, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -425.6734619140625, "loss": 1.317, "margin_dpo/margin_mean": 98.7804183959961, "margin_dpo/margin_std": 155.01918029785156, "step": 578 }, { "KL/chosen_KL_mean": -221.681884765625, "KL/mean": -271.17657470703125, "KL/rejected_KL_mean": -320.6712951660156, "KL/std": 137.55613708496094, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.98939514160156, "fcm_dpo/q_t": 0.48153698444366455, "grad_norm": 3.3012914657592773, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.6622679233551025, "logits/rejected": -1.4249765872955322, "logps/chosen": -309.0249328613281, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -422.79229736328125, "loss": 1.3168, "margin_dpo/margin_mean": 98.98939514160156, "margin_dpo/margin_std": 147.33534240722656, "step": 579 }, { "KL/chosen_KL_mean": -209.8216552734375, "KL/mean": -271.0985107421875, "KL/rejected_KL_mean": -332.3753967285156, "KL/std": 147.07659912109375, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.55374145507812, "fcm_dpo/q_t": 0.4771918058395386, "grad_norm": 3.1845366954803467, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.4725209474563599, "logits/rejected": -1.1649446487426758, "logps/chosen": -298.75042724609375, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -421.072021484375, "loss": 1.3014, "margin_dpo/margin_mean": 122.55374145507812, "margin_dpo/margin_std": 185.9619140625, "step": 580 }, { "KL/chosen_KL_mean": -228.62673950195312, "KL/mean": -280.120849609375, "KL/rejected_KL_mean": -331.61492919921875, "KL/std": 133.18528747558594, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.98818969726562, "fcm_dpo/q_t": 0.48077356815338135, "grad_norm": 3.1262059211730957, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.7708747386932373, "logits/rejected": -1.4373161792755127, "logps/chosen": -320.8499755859375, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -431.00439453125, "loss": 1.3143, "margin_dpo/margin_mean": 102.98818969726562, "margin_dpo/margin_std": 163.66651916503906, "step": 581 }, { "KL/chosen_KL_mean": -200.41949462890625, "KL/mean": -253.6851348876953, "KL/rejected_KL_mean": -306.9508056640625, "KL/std": 148.80413818359375, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.53130340576172, "fcm_dpo/q_t": 0.4801819324493408, "grad_norm": 2.395751476287842, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.9525909423828125, "logits/rejected": -1.9538204669952393, "logps/chosen": -287.755126953125, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -422.105224609375, "loss": 1.3128, "margin_dpo/margin_mean": 106.53130340576172, "margin_dpo/margin_std": 176.0354766845703, "step": 582 }, { "KL/chosen_KL_mean": -233.57626342773438, "KL/mean": -290.5015869140625, "KL/rejected_KL_mean": -347.4269104003906, "KL/std": 147.3800811767578, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.85064697265625, "fcm_dpo/q_t": 0.4787808954715729, "grad_norm": 3.5268752574920654, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.3595764636993408, "logits/rejected": -1.1263954639434814, "logps/chosen": -317.2933349609375, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -442.0115661621094, "loss": 1.3077, "margin_dpo/margin_mean": 113.85064697265625, "margin_dpo/margin_std": 186.25314331054688, "step": 583 }, { "KL/chosen_KL_mean": -212.98648071289062, "KL/mean": -276.23858642578125, "KL/rejected_KL_mean": -339.4906921386719, "KL/std": 143.2069091796875, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.50421142578125, "fcm_dpo/q_t": 0.47638875246047974, "grad_norm": 2.681764841079712, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.6702097654342651, "logits/rejected": -1.2272729873657227, "logps/chosen": -307.24603271484375, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -443.973388671875, "loss": 1.2978, "margin_dpo/margin_mean": 126.50422668457031, "margin_dpo/margin_std": 166.6014862060547, "step": 584 }, { "KL/chosen_KL_mean": -224.97561645507812, "KL/mean": -292.38421630859375, "KL/rejected_KL_mean": -359.79278564453125, "KL/std": 164.330322265625, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.81716918945312, "fcm_dpo/q_t": 0.4749140441417694, "grad_norm": 2.8456995487213135, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.8186898231506348, "logits/rejected": -1.5559121370315552, "logps/chosen": -310.61236572265625, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -450.0782470703125, "loss": 1.2937, "margin_dpo/margin_mean": 134.81716918945312, "margin_dpo/margin_std": 205.90191650390625, "step": 585 }, { "KL/chosen_KL_mean": -246.75259399414062, "KL/mean": -290.6927185058594, "KL/rejected_KL_mean": -334.63287353515625, "KL/std": 139.19235229492188, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.88029479980469, "fcm_dpo/q_t": 0.4835766851902008, "grad_norm": 4.567317008972168, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.5364530086517334, "logits/rejected": -1.2850267887115479, "logps/chosen": -330.4337158203125, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -432.9920349121094, "loss": 1.3244, "margin_dpo/margin_mean": 87.88028717041016, "margin_dpo/margin_std": 141.13095092773438, "step": 586 }, { "KL/chosen_KL_mean": -197.52883911132812, "KL/mean": -243.93435668945312, "KL/rejected_KL_mean": -290.3398742675781, "KL/std": 140.65615844726562, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.81100463867188, "fcm_dpo/q_t": 0.4826587438583374, "grad_norm": 3.6568593978881836, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.363705635070801, "logits/rejected": -2.1130459308624268, "logps/chosen": -275.1063537597656, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -379.66021728515625, "loss": 1.3211, "margin_dpo/margin_mean": 92.8110122680664, "margin_dpo/margin_std": 149.44886779785156, "step": 587 }, { "KL/chosen_KL_mean": -225.06765747070312, "KL/mean": -278.6062316894531, "KL/rejected_KL_mean": -332.14483642578125, "KL/std": 150.41317749023438, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.0771713256836, "fcm_dpo/q_t": 0.48007136583328247, "grad_norm": 2.427170515060425, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.6777353286743164, "logits/rejected": -1.662008285522461, "logps/chosen": -308.8719177246094, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -445.2642822265625, "loss": 1.3124, "margin_dpo/margin_mean": 107.07716369628906, "margin_dpo/margin_std": 183.16702270507812, "step": 588 }, { "KL/chosen_KL_mean": -238.58639526367188, "KL/mean": -298.85186767578125, "KL/rejected_KL_mean": -359.11724853515625, "KL/std": 136.771728515625, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.53087615966797, "fcm_dpo/q_t": 0.4774773120880127, "grad_norm": 2.3885083198547363, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.595583438873291, "logits/rejected": -1.237383246421814, "logps/chosen": -335.0514221191406, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -475.1544494628906, "loss": 1.3005, "margin_dpo/margin_mean": 120.53086853027344, "margin_dpo/margin_std": 135.0284423828125, "step": 589 }, { "KL/chosen_KL_mean": -221.94424438476562, "KL/mean": -281.45703125, "KL/rejected_KL_mean": -340.9698486328125, "KL/std": 127.40753173828125, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.02559661865234, "fcm_dpo/q_t": 0.4777853786945343, "grad_norm": 5.7808308601379395, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.7818667888641357, "logits/rejected": -1.683816909790039, "logps/chosen": -310.3721008300781, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -451.16864013671875, "loss": 1.3021, "margin_dpo/margin_mean": 119.02560424804688, "margin_dpo/margin_std": 145.03762817382812, "step": 590 }, { "KL/chosen_KL_mean": -205.62591552734375, "KL/mean": -273.180908203125, "KL/rejected_KL_mean": -340.73590087890625, "KL/std": 132.84939575195312, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.11001586914062, "fcm_dpo/q_t": 0.474764347076416, "grad_norm": 2.324849843978882, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.8741310834884644, "logits/rejected": -1.293553113937378, "logps/chosen": -293.4755859375, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -430.31512451171875, "loss": 1.2901, "margin_dpo/margin_mean": 135.11000061035156, "margin_dpo/margin_std": 134.28115844726562, "step": 591 }, { "KL/chosen_KL_mean": -244.51828002929688, "KL/mean": -295.3258361816406, "KL/rejected_KL_mean": -346.1333923339844, "KL/std": 133.82640075683594, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.61510467529297, "fcm_dpo/q_t": 0.4810141324996948, "grad_norm": 3.3346073627471924, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7603555917739868, "logits/rejected": -1.4445216655731201, "logps/chosen": -319.4940185546875, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -427.68603515625, "loss": 1.3157, "margin_dpo/margin_mean": 101.61509704589844, "margin_dpo/margin_std": 172.69859313964844, "step": 592 }, { "KL/chosen_KL_mean": -219.55612182617188, "KL/mean": -259.6684265136719, "KL/rejected_KL_mean": -299.78076171875, "KL/std": 137.32952880859375, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.22463989257812, "fcm_dpo/q_t": 0.4850119352340698, "grad_norm": 3.4564807415008545, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.7922645807266235, "logits/rejected": -0.4995706081390381, "logps/chosen": -291.795166015625, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -369.8772888183594, "loss": 1.3305, "margin_dpo/margin_mean": 80.22463989257812, "margin_dpo/margin_std": 157.02490234375, "step": 593 }, { "KL/chosen_KL_mean": -205.53065490722656, "KL/mean": -266.9476013183594, "KL/rejected_KL_mean": -328.36456298828125, "KL/std": 134.19216918945312, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.83389282226562, "fcm_dpo/q_t": 0.4770805537700653, "grad_norm": 4.158108234405518, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9517443180084229, "logits/rejected": -1.4566266536712646, "logps/chosen": -295.9683837890625, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -420.63214111328125, "loss": 1.2995, "margin_dpo/margin_mean": 122.83390045166016, "margin_dpo/margin_std": 146.39027404785156, "step": 594 }, { "KL/chosen_KL_mean": -218.60601806640625, "KL/mean": -271.45623779296875, "KL/rejected_KL_mean": -324.30645751953125, "KL/std": 135.58554077148438, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.70039367675781, "fcm_dpo/q_t": 0.4802814722061157, "grad_norm": 2.866910696029663, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.6972863674163818, "logits/rejected": -1.3268048763275146, "logps/chosen": -300.120849609375, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -410.826904296875, "loss": 1.3123, "margin_dpo/margin_mean": 105.70039367675781, "margin_dpo/margin_std": 161.84527587890625, "step": 595 }, { "KL/chosen_KL_mean": -222.6416015625, "KL/mean": -271.24237060546875, "KL/rejected_KL_mean": -319.8431396484375, "KL/std": 136.9354705810547, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.20152282714844, "fcm_dpo/q_t": 0.4818582534790039, "grad_norm": 3.2587060928344727, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.534786343574524, "logits/rejected": -0.9699172973632812, "logps/chosen": -310.84967041015625, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -405.9368896484375, "loss": 1.3188, "margin_dpo/margin_mean": 97.20152282714844, "margin_dpo/margin_std": 169.30215454101562, "step": 596 }, { "KL/chosen_KL_mean": -216.8161163330078, "KL/mean": -280.48687744140625, "KL/rejected_KL_mean": -344.1575927734375, "KL/std": 147.99200439453125, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.34150695800781, "fcm_dpo/q_t": 0.4762328863143921, "grad_norm": 2.747236490249634, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.2952357530593872, "logits/rejected": -1.1986753940582275, "logps/chosen": -300.4060974121094, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -451.10198974609375, "loss": 1.2965, "margin_dpo/margin_mean": 127.34149932861328, "margin_dpo/margin_std": 154.5725860595703, "step": 597 }, { "KL/chosen_KL_mean": -207.78652954101562, "KL/mean": -262.9382629394531, "KL/rejected_KL_mean": -318.0899658203125, "KL/std": 138.7903289794922, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.30345153808594, "fcm_dpo/q_t": 0.47938352823257446, "grad_norm": 2.6391210556030273, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.7546311616897583, "logits/rejected": -1.451805830001831, "logps/chosen": -297.96917724609375, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -412.8885498046875, "loss": 1.3087, "margin_dpo/margin_mean": 110.30345153808594, "margin_dpo/margin_std": 151.27230834960938, "step": 598 }, { "KL/chosen_KL_mean": -210.1345672607422, "KL/mean": -257.8169860839844, "KL/rejected_KL_mean": -305.4994201660156, "KL/std": 140.48385620117188, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.36487579345703, "fcm_dpo/q_t": 0.48217707872390747, "grad_norm": 3.0647988319396973, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.5942022800445557, "logits/rejected": -1.4911596775054932, "logps/chosen": -287.9768371582031, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -394.6970520019531, "loss": 1.3196, "margin_dpo/margin_mean": 95.3648681640625, "margin_dpo/margin_std": 153.74462890625, "step": 599 }, { "KL/chosen_KL_mean": -242.70452880859375, "KL/mean": -295.8871765136719, "KL/rejected_KL_mean": -349.06982421875, "KL/std": 138.20596313476562, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.36532592773438, "fcm_dpo/q_t": 0.4801507592201233, "grad_norm": 2.477203845977783, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.117032289505005, "logits/rejected": -1.7777535915374756, "logps/chosen": -334.892333984375, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -442.0241394042969, "loss": 1.3117, "margin_dpo/margin_mean": 106.36532592773438, "margin_dpo/margin_std": 158.6875, "step": 600 }, { "KL/chosen_KL_mean": -245.82174682617188, "KL/mean": -301.040771484375, "KL/rejected_KL_mean": -356.259765625, "KL/std": 137.65225219726562, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.43804931640625, "fcm_dpo/q_t": 0.4793722331523895, "grad_norm": 2.8573169708251953, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.637001395225525, "logits/rejected": -1.3280977010726929, "logps/chosen": -335.03790283203125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -456.4302978515625, "loss": 1.3078, "margin_dpo/margin_mean": 110.43804168701172, "margin_dpo/margin_std": 135.68576049804688, "step": 601 }, { "KL/chosen_KL_mean": -195.35964965820312, "KL/mean": -261.9444580078125, "KL/rejected_KL_mean": -328.529296875, "KL/std": 145.36526489257812, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.16966247558594, "fcm_dpo/q_t": 0.4751831591129303, "grad_norm": 3.677858829498291, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.9452719688415527, "logits/rejected": -1.6815335750579834, "logps/chosen": -275.417236328125, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -421.726806640625, "loss": 1.2931, "margin_dpo/margin_mean": 133.16966247558594, "margin_dpo/margin_std": 172.5902099609375, "step": 602 }, { "KL/chosen_KL_mean": -203.7841796875, "KL/mean": -262.05657958984375, "KL/rejected_KL_mean": -320.3289794921875, "KL/std": 132.73837280273438, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.5448226928711, "fcm_dpo/q_t": 0.47825661301612854, "grad_norm": 3.223254442214966, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.830800175666809, "logits/rejected": -1.2605353593826294, "logps/chosen": -306.09375, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -428.3978271484375, "loss": 1.3047, "margin_dpo/margin_mean": 116.54483032226562, "margin_dpo/margin_std": 167.35903930664062, "step": 603 }, { "KL/chosen_KL_mean": -209.63900756835938, "KL/mean": -274.07098388671875, "KL/rejected_KL_mean": -338.5029296875, "KL/std": 143.05001831054688, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.86392211914062, "fcm_dpo/q_t": 0.47593969106674194, "grad_norm": 2.8525619506835938, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.510689377784729, "logits/rejected": -1.3659579753875732, "logps/chosen": -287.8131103515625, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -437.9990234375, "loss": 1.2955, "margin_dpo/margin_mean": 128.86392211914062, "margin_dpo/margin_std": 156.9940185546875, "step": 604 }, { "KL/chosen_KL_mean": -233.50970458984375, "KL/mean": -287.5378112792969, "KL/rejected_KL_mean": -341.56591796875, "KL/std": 124.8137435913086, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.05622863769531, "fcm_dpo/q_t": 0.4798135459423065, "grad_norm": 2.893526792526245, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8962678909301758, "logits/rejected": -1.4393830299377441, "logps/chosen": -328.28302001953125, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -448.87078857421875, "loss": 1.3095, "margin_dpo/margin_mean": 108.05622863769531, "margin_dpo/margin_std": 135.7593231201172, "step": 605 }, { "KL/chosen_KL_mean": -226.79421997070312, "KL/mean": -283.5191650390625, "KL/rejected_KL_mean": -340.2441711425781, "KL/std": 143.7904052734375, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.44998168945312, "fcm_dpo/q_t": 0.4788230359554291, "grad_norm": 2.5587477684020996, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.638110876083374, "logits/rejected": -1.306382417678833, "logps/chosen": -314.547607421875, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -444.48638916015625, "loss": 1.3067, "margin_dpo/margin_mean": 113.44998168945312, "margin_dpo/margin_std": 159.24444580078125, "step": 606 }, { "KL/chosen_KL_mean": -212.788330078125, "KL/mean": -273.20166015625, "KL/rejected_KL_mean": -333.61505126953125, "KL/std": 145.17950439453125, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.82672119140625, "fcm_dpo/q_t": 0.4774804413318634, "grad_norm": 4.310309886932373, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.526931643486023, "logits/rejected": -1.0161640644073486, "logps/chosen": -301.11737060546875, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -420.38311767578125, "loss": 1.3028, "margin_dpo/margin_mean": 120.82672119140625, "margin_dpo/margin_std": 188.900390625, "step": 607 }, { "KL/chosen_KL_mean": -203.0638427734375, "KL/mean": -265.6777648925781, "KL/rejected_KL_mean": -328.29168701171875, "KL/std": 142.86758422851562, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.22786712646484, "fcm_dpo/q_t": 0.47661292552948, "grad_norm": 2.519831418991089, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.7311196327209473, "logits/rejected": -1.4500843286514282, "logps/chosen": -288.83319091796875, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -428.5245361328125, "loss": 1.298, "margin_dpo/margin_mean": 125.22785949707031, "margin_dpo/margin_std": 152.9814453125, "step": 608 }, { "KL/chosen_KL_mean": -226.29644775390625, "KL/mean": -280.4804992675781, "KL/rejected_KL_mean": -334.66455078125, "KL/std": 146.28749084472656, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.36811065673828, "fcm_dpo/q_t": 0.4797861576080322, "grad_norm": 2.719367265701294, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.151111602783203, "logits/rejected": -1.7583298683166504, "logps/chosen": -319.2630310058594, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -430.582763671875, "loss": 1.3107, "margin_dpo/margin_mean": 108.36810302734375, "margin_dpo/margin_std": 162.8912811279297, "step": 609 }, { "KL/chosen_KL_mean": -206.39218139648438, "KL/mean": -267.79962158203125, "KL/rejected_KL_mean": -329.2070617675781, "KL/std": 140.7444305419922, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.81486511230469, "fcm_dpo/q_t": 0.4770563244819641, "grad_norm": 3.2092933654785156, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.6336605548858643, "logits/rejected": -1.1144661903381348, "logps/chosen": -301.09246826171875, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -418.89447021484375, "loss": 1.2998, "margin_dpo/margin_mean": 122.81486511230469, "margin_dpo/margin_std": 156.54998779296875, "step": 610 }, { "KL/chosen_KL_mean": -218.11988830566406, "KL/mean": -280.755126953125, "KL/rejected_KL_mean": -343.390380859375, "KL/std": 141.8554229736328, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.2705078125, "fcm_dpo/q_t": 0.4766193628311157, "grad_norm": 4.261493682861328, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.9375190734863281, "logits/rejected": -1.6139534711837769, "logps/chosen": -304.9032287597656, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -433.23345947265625, "loss": 1.298, "margin_dpo/margin_mean": 125.27049255371094, "margin_dpo/margin_std": 153.66690063476562, "step": 611 }, { "KL/chosen_KL_mean": -242.83404541015625, "KL/mean": -305.0503234863281, "KL/rejected_KL_mean": -367.2666015625, "KL/std": 140.06939697265625, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.43258666992188, "fcm_dpo/q_t": 0.47677794098854065, "grad_norm": 2.9796133041381836, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.10237193107605, "logits/rejected": -1.3119711875915527, "logps/chosen": -343.85418701171875, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -461.04962158203125, "loss": 1.2987, "margin_dpo/margin_mean": 124.43258666992188, "margin_dpo/margin_std": 159.07992553710938, "step": 612 }, { "KL/chosen_KL_mean": -194.26910400390625, "KL/mean": -266.9717712402344, "KL/rejected_KL_mean": -339.6744384765625, "KL/std": 137.37423706054688, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.4053192138672, "fcm_dpo/q_t": 0.4728449285030365, "grad_norm": 2.3789477348327637, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.7169119119644165, "logits/rejected": -1.5314912796020508, "logps/chosen": -283.25775146484375, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -449.6699523925781, "loss": 1.2828, "margin_dpo/margin_mean": 145.40533447265625, "margin_dpo/margin_std": 135.75216674804688, "step": 613 }, { "KL/chosen_KL_mean": -233.25543212890625, "KL/mean": -294.3155212402344, "KL/rejected_KL_mean": -355.3755798339844, "KL/std": 132.63552856445312, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.12015533447266, "fcm_dpo/q_t": 0.47719377279281616, "grad_norm": 2.7950809001922607, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.296385407447815, "logits/rejected": -1.301984190940857, "logps/chosen": -313.45550537109375, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -465.23797607421875, "loss": 1.301, "margin_dpo/margin_mean": 122.12015533447266, "margin_dpo/margin_std": 172.07244873046875, "step": 614 }, { "KL/chosen_KL_mean": -222.57510375976562, "KL/mean": -269.38238525390625, "KL/rejected_KL_mean": -316.1896667480469, "KL/std": 135.27426147460938, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.61457824707031, "fcm_dpo/q_t": 0.4825076460838318, "grad_norm": 3.2616021633148193, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.6337181329727173, "logits/rejected": -1.2649847269058228, "logps/chosen": -323.0103759765625, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -417.3697509765625, "loss": 1.3202, "margin_dpo/margin_mean": 93.61457824707031, "margin_dpo/margin_std": 135.43016052246094, "step": 615 }, { "KL/chosen_KL_mean": -235.94863891601562, "KL/mean": -293.9284973144531, "KL/rejected_KL_mean": -351.9083251953125, "KL/std": 131.07591247558594, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.95966339111328, "fcm_dpo/q_t": 0.47836363315582275, "grad_norm": 3.0364432334899902, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.830662488937378, "logits/rejected": -1.3642970323562622, "logps/chosen": -328.44158935546875, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -433.968994140625, "loss": 1.3053, "margin_dpo/margin_mean": 115.95966339111328, "margin_dpo/margin_std": 168.0748748779297, "step": 616 }, { "KL/chosen_KL_mean": -237.478759765625, "KL/mean": -279.7459411621094, "KL/rejected_KL_mean": -322.0130920410156, "KL/std": 127.04332733154297, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.5343246459961, "fcm_dpo/q_t": 0.48420634865760803, "grad_norm": 2.923325777053833, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.3564269542694092, "logits/rejected": -0.7247529625892639, "logps/chosen": -328.57574462890625, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -392.42315673828125, "loss": 1.3269, "margin_dpo/margin_mean": 84.5343246459961, "margin_dpo/margin_std": 146.15206909179688, "step": 617 }, { "KL/chosen_KL_mean": -241.45611572265625, "KL/mean": -298.96685791015625, "KL/rejected_KL_mean": -356.4776306152344, "KL/std": 145.70919799804688, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.02149963378906, "fcm_dpo/q_t": 0.47850579023361206, "grad_norm": 3.076589345932007, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.061563730239868, "logits/rejected": -1.7056865692138672, "logps/chosen": -343.47674560546875, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -456.27880859375, "loss": 1.3056, "margin_dpo/margin_mean": 115.02149200439453, "margin_dpo/margin_std": 160.023193359375, "step": 618 }, { "KL/chosen_KL_mean": -232.05148315429688, "KL/mean": -283.2738952636719, "KL/rejected_KL_mean": -334.49627685546875, "KL/std": 142.17279052734375, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.44483947753906, "fcm_dpo/q_t": 0.4809134602546692, "grad_norm": 2.978163480758667, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.5172827243804932, "logits/rejected": -1.1808502674102783, "logps/chosen": -321.7928466796875, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -434.3976745605469, "loss": 1.315, "margin_dpo/margin_mean": 102.44483947753906, "margin_dpo/margin_std": 169.15625, "step": 619 }, { "KL/chosen_KL_mean": -228.1065673828125, "KL/mean": -280.0168151855469, "KL/rejected_KL_mean": -331.92706298828125, "KL/std": 150.201904296875, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.82046508789062, "fcm_dpo/q_t": 0.48062780499458313, "grad_norm": 2.4092557430267334, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.0634217262268066, "logits/rejected": -1.5564475059509277, "logps/chosen": -313.23089599609375, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -417.339599609375, "loss": 1.3142, "margin_dpo/margin_mean": 103.82046508789062, "margin_dpo/margin_std": 167.6929473876953, "step": 620 }, { "KL/chosen_KL_mean": -214.96063232421875, "KL/mean": -273.7718505859375, "KL/rejected_KL_mean": -332.5831298828125, "KL/std": 148.15817260742188, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.62246704101562, "fcm_dpo/q_t": 0.4780597984790802, "grad_norm": 2.8110575675964355, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.908807635307312, "logits/rejected": -1.6555733680725098, "logps/chosen": -304.20904541015625, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -428.0477294921875, "loss": 1.305, "margin_dpo/margin_mean": 117.62246704101562, "margin_dpo/margin_std": 177.79519653320312, "step": 621 }, { "KL/chosen_KL_mean": -227.42337036132812, "KL/mean": -283.957763671875, "KL/rejected_KL_mean": -340.4921569824219, "KL/std": 131.98194885253906, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.06877136230469, "fcm_dpo/q_t": 0.4788917899131775, "grad_norm": 2.935838222503662, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.441578984260559, "logits/rejected": -0.8279513120651245, "logps/chosen": -327.2174987792969, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -431.32037353515625, "loss": 1.3064, "margin_dpo/margin_mean": 113.06877136230469, "margin_dpo/margin_std": 148.11471557617188, "step": 622 }, { "KL/chosen_KL_mean": -233.2842254638672, "KL/mean": -270.21649169921875, "KL/rejected_KL_mean": -307.14874267578125, "KL/std": 150.03707885742188, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.8644790649414, "fcm_dpo/q_t": 0.48621851205825806, "grad_norm": 2.7917158603668213, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.0312962532043457, "logits/rejected": -1.6134121417999268, "logps/chosen": -323.73980712890625, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -398.47149658203125, "loss": 1.3364, "margin_dpo/margin_mean": 73.86447143554688, "margin_dpo/margin_std": 181.19915771484375, "step": 623 }, { "KL/chosen_KL_mean": -249.74301147460938, "KL/mean": -314.18389892578125, "KL/rejected_KL_mean": -378.62481689453125, "KL/std": 143.81045532226562, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.8817901611328, "fcm_dpo/q_t": 0.4759555757045746, "grad_norm": 2.938708543777466, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.3264384269714355, "logits/rejected": -1.4604101181030273, "logps/chosen": -359.61822509765625, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -483.39801025390625, "loss": 1.296, "margin_dpo/margin_mean": 128.8817901611328, "margin_dpo/margin_std": 169.51065063476562, "step": 624 }, { "KL/chosen_KL_mean": -247.3508758544922, "KL/mean": -296.3285827636719, "KL/rejected_KL_mean": -345.3062744140625, "KL/std": 135.01531982421875, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.95539093017578, "fcm_dpo/q_t": 0.4817218780517578, "grad_norm": 3.0797793865203857, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.4914311170578003, "logits/rejected": -1.1273784637451172, "logps/chosen": -334.51904296875, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -437.1677551269531, "loss": 1.3175, "margin_dpo/margin_mean": 97.95539093017578, "margin_dpo/margin_std": 153.63845825195312, "step": 625 }, { "KL/chosen_KL_mean": -234.2378387451172, "KL/mean": -279.7330627441406, "KL/rejected_KL_mean": -325.228271484375, "KL/std": 152.66761779785156, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.99042510986328, "fcm_dpo/q_t": 0.4830174148082733, "grad_norm": 2.7641239166259766, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.277144193649292, "logits/rejected": -0.8597748279571533, "logps/chosen": -338.4620361328125, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -429.4057312011719, "loss": 1.3232, "margin_dpo/margin_mean": 90.99042510986328, "margin_dpo/margin_std": 166.49215698242188, "step": 626 }, { "KL/chosen_KL_mean": -200.14598083496094, "KL/mean": -268.3508605957031, "KL/rejected_KL_mean": -336.55572509765625, "KL/std": 158.29049682617188, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 136.40972900390625, "fcm_dpo/q_t": 0.4746091961860657, "grad_norm": 2.1687510013580322, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.4831078052520752, "logits/rejected": -1.6413118839263916, "logps/chosen": -275.0805969238281, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -449.12860107421875, "loss": 1.2916, "margin_dpo/margin_mean": 136.40972900390625, "margin_dpo/margin_std": 189.23855590820312, "step": 627 }, { "KL/chosen_KL_mean": -223.91384887695312, "KL/mean": -289.42437744140625, "KL/rejected_KL_mean": -354.93487548828125, "KL/std": 149.38682556152344, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.02102661132812, "fcm_dpo/q_t": 0.47557002305984497, "grad_norm": 2.6899516582489014, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.9828526973724365, "logits/rejected": -1.7473845481872559, "logps/chosen": -317.60491943359375, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -462.27880859375, "loss": 1.2947, "margin_dpo/margin_mean": 131.02102661132812, "margin_dpo/margin_std": 173.58596801757812, "step": 628 }, { "KL/chosen_KL_mean": -241.31484985351562, "KL/mean": -296.77313232421875, "KL/rejected_KL_mean": -352.2314453125, "KL/std": 142.0338897705078, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.91655731201172, "fcm_dpo/q_t": 0.4792732298374176, "grad_norm": 2.7746167182922363, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.015077590942383, "logits/rejected": -1.5882477760314941, "logps/chosen": -344.54522705078125, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -449.39984130859375, "loss": 1.3083, "margin_dpo/margin_mean": 110.91656494140625, "margin_dpo/margin_std": 154.14022827148438, "step": 629 }, { "KL/chosen_KL_mean": -224.39895629882812, "KL/mean": -249.5462188720703, "KL/rejected_KL_mean": -274.6934814453125, "KL/std": 132.11199951171875, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.294532775878906, "fcm_dpo/q_t": 0.4905868172645569, "grad_norm": 2.64823317527771, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.8615837097167969, "logits/rejected": -1.383866548538208, "logps/chosen": -318.2965087890625, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -364.06787109375, "loss": 1.3522, "margin_dpo/margin_mean": 50.294532775878906, "margin_dpo/margin_std": 150.94970703125, "step": 630 }, { "KL/chosen_KL_mean": -205.47509765625, "KL/mean": -262.740478515625, "KL/rejected_KL_mean": -320.005859375, "KL/std": 133.64273071289062, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.53072357177734, "fcm_dpo/q_t": 0.47861772775650024, "grad_norm": 2.7346699237823486, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.2770087718963623, "logits/rejected": -1.860000729560852, "logps/chosen": -293.24591064453125, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -408.6882629394531, "loss": 1.3057, "margin_dpo/margin_mean": 114.53072357177734, "margin_dpo/margin_std": 156.97215270996094, "step": 631 }, { "KL/chosen_KL_mean": -206.00082397460938, "KL/mean": -262.8818359375, "KL/rejected_KL_mean": -319.76287841796875, "KL/std": 137.546142578125, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.76203155517578, "fcm_dpo/q_t": 0.4787876605987549, "grad_norm": 3.726548910140991, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.9574096202850342, "logits/rejected": -1.541906476020813, "logps/chosen": -294.6273498535156, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -411.21380615234375, "loss": 1.3075, "margin_dpo/margin_mean": 113.76202392578125, "margin_dpo/margin_std": 181.45750427246094, "step": 632 }, { "KL/chosen_KL_mean": -249.42298889160156, "KL/mean": -284.602783203125, "KL/rejected_KL_mean": -319.78253173828125, "KL/std": 140.1533966064453, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.35955810546875, "fcm_dpo/q_t": 0.4869043827056885, "grad_norm": 3.2025272846221924, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.280824661254883, "logits/rejected": -2.115692615509033, "logps/chosen": -330.79742431640625, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -418.40826416015625, "loss": 1.3396, "margin_dpo/margin_mean": 70.35955810546875, "margin_dpo/margin_std": 193.2945098876953, "step": 633 }, { "KL/chosen_KL_mean": -234.61459350585938, "KL/mean": -282.4373779296875, "KL/rejected_KL_mean": -330.26019287109375, "KL/std": 123.31548309326172, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.6456298828125, "fcm_dpo/q_t": 0.48213157057762146, "grad_norm": 3.531745433807373, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.3649067878723145, "logits/rejected": -1.2149584293365479, "logps/chosen": -330.7374267578125, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -443.1080017089844, "loss": 1.3185, "margin_dpo/margin_mean": 95.6456298828125, "margin_dpo/margin_std": 133.9752197265625, "step": 634 }, { "KL/chosen_KL_mean": -253.35321044921875, "KL/mean": -281.0209655761719, "KL/rejected_KL_mean": -308.688720703125, "KL/std": 146.91183471679688, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.33551788330078, "fcm_dpo/q_t": 0.4896904528141022, "grad_norm": 2.9642043113708496, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.6573972702026367, "logits/rejected": -1.1143567562103271, "logps/chosen": -352.0373229980469, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -398.5879211425781, "loss": 1.3497, "margin_dpo/margin_mean": 55.33551788330078, "margin_dpo/margin_std": 178.00881958007812, "step": 635 }, { "KL/chosen_KL_mean": -212.27993774414062, "KL/mean": -275.1605529785156, "KL/rejected_KL_mean": -338.0411376953125, "KL/std": 138.53482055664062, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.76123046875, "fcm_dpo/q_t": 0.47650930285453796, "grad_norm": 2.756849765777588, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.5323189496994019, "logits/rejected": -1.0715140104293823, "logps/chosen": -302.6979064941406, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -425.748046875, "loss": 1.2968, "margin_dpo/margin_mean": 125.76123809814453, "margin_dpo/margin_std": 135.817138671875, "step": 636 }, { "KL/chosen_KL_mean": -214.06515502929688, "KL/mean": -283.6993103027344, "KL/rejected_KL_mean": -353.33349609375, "KL/std": 155.1986541748047, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.268310546875, "fcm_dpo/q_t": 0.47399359941482544, "grad_norm": 2.697218656539917, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.614661693572998, "logits/rejected": -1.5092706680297852, "logps/chosen": -296.5148620605469, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -457.362060546875, "loss": 1.2875, "margin_dpo/margin_mean": 139.26832580566406, "margin_dpo/margin_std": 143.46875, "step": 637 }, { "KL/chosen_KL_mean": -212.17120361328125, "KL/mean": -269.2478942871094, "KL/rejected_KL_mean": -326.3245849609375, "KL/std": 142.13717651367188, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.15339660644531, "fcm_dpo/q_t": 0.47866755723953247, "grad_norm": 2.448617458343506, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.4919719696044922, "logits/rejected": -0.8508119583129883, "logps/chosen": -304.09619140625, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -412.61163330078125, "loss": 1.306, "margin_dpo/margin_mean": 114.15339660644531, "margin_dpo/margin_std": 153.63507080078125, "step": 638 }, { "KL/chosen_KL_mean": -278.0149230957031, "KL/mean": -299.6900634765625, "KL/rejected_KL_mean": -321.365234375, "KL/std": 125.88824462890625, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.35030746459961, "fcm_dpo/q_t": 0.49189069867134094, "grad_norm": 2.6335229873657227, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.215509295463562, "logits/rejected": -0.4676734209060669, "logps/chosen": -382.54248046875, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -402.8455810546875, "loss": 1.3579, "margin_dpo/margin_mean": 43.35030746459961, "margin_dpo/margin_std": 162.18515014648438, "step": 639 }, { "KL/chosen_KL_mean": -228.24986267089844, "KL/mean": -289.5536193847656, "KL/rejected_KL_mean": -350.8573303222656, "KL/std": 148.07235717773438, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.60749053955078, "fcm_dpo/q_t": 0.47712284326553345, "grad_norm": 3.3724520206451416, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8270769119262695, "logits/rejected": -1.341010332107544, "logps/chosen": -326.1351318359375, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -463.5623779296875, "loss": 1.3005, "margin_dpo/margin_mean": 122.60749053955078, "margin_dpo/margin_std": 156.3879852294922, "step": 640 }, { "KL/chosen_KL_mean": -217.90097045898438, "KL/mean": -273.86962890625, "KL/rejected_KL_mean": -329.8382263183594, "KL/std": 164.63864135742188, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.93727111816406, "fcm_dpo/q_t": 0.47925180196762085, "grad_norm": 2.2765235900878906, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.205932140350342, "logits/rejected": -1.6736149787902832, "logps/chosen": -314.3466491699219, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -424.97393798828125, "loss": 1.3095, "margin_dpo/margin_mean": 111.93727111816406, "margin_dpo/margin_std": 194.04501342773438, "step": 641 }, { "KL/chosen_KL_mean": -241.40005493164062, "KL/mean": -295.476806640625, "KL/rejected_KL_mean": -349.55352783203125, "KL/std": 149.46018981933594, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.15351104736328, "fcm_dpo/q_t": 0.4799058735370636, "grad_norm": 2.7814865112304688, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.3699712753295898, "logits/rejected": -1.0397937297821045, "logps/chosen": -342.159912109375, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -464.26116943359375, "loss": 1.3125, "margin_dpo/margin_mean": 108.15351104736328, "margin_dpo/margin_std": 198.84100341796875, "step": 642 }, { "KL/chosen_KL_mean": -219.92355346679688, "KL/mean": -273.14459228515625, "KL/rejected_KL_mean": -326.3656005859375, "KL/std": 138.46096801757812, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.4420166015625, "fcm_dpo/q_t": 0.48013022541999817, "grad_norm": 3.729930877685547, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.074728488922119, "logits/rejected": -1.7410621643066406, "logps/chosen": -304.667236328125, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -420.68402099609375, "loss": 1.3119, "margin_dpo/margin_mean": 106.4420166015625, "margin_dpo/margin_std": 161.14605712890625, "step": 643 }, { "KL/chosen_KL_mean": -230.06137084960938, "KL/mean": -272.560302734375, "KL/rejected_KL_mean": -315.0592041015625, "KL/std": 140.4134063720703, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.9978256225586, "fcm_dpo/q_t": 0.48412764072418213, "grad_norm": 3.1791374683380127, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.8308217525482178, "logits/rejected": -1.5965285301208496, "logps/chosen": -315.3841247558594, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -409.6678161621094, "loss": 1.3271, "margin_dpo/margin_mean": 84.99781799316406, "margin_dpo/margin_std": 157.36871337890625, "step": 644 }, { "KL/chosen_KL_mean": -244.36663818359375, "KL/mean": -272.5540771484375, "KL/rejected_KL_mean": -300.7415466308594, "KL/std": 144.05970764160156, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.374900817871094, "fcm_dpo/q_t": 0.48947393894195557, "grad_norm": 2.7095649242401123, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.4988312721252441, "logits/rejected": -1.367296814918518, "logps/chosen": -326.9568786621094, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -403.74530029296875, "loss": 1.3485, "margin_dpo/margin_mean": 56.374908447265625, "margin_dpo/margin_std": 167.49652099609375, "step": 645 }, { "KL/chosen_KL_mean": -216.6048126220703, "KL/mean": -271.05853271484375, "KL/rejected_KL_mean": -325.5122985839844, "KL/std": 126.53826904296875, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.90748596191406, "fcm_dpo/q_t": 0.47964218258857727, "grad_norm": 2.5405075550079346, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.624917984008789, "logits/rejected": -1.0488609075546265, "logps/chosen": -307.7968444824219, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -409.2906188964844, "loss": 1.3086, "margin_dpo/margin_mean": 108.90748596191406, "margin_dpo/margin_std": 125.0238265991211, "step": 646 }, { "KL/chosen_KL_mean": -227.20091247558594, "KL/mean": -277.7492370605469, "KL/rejected_KL_mean": -328.29754638671875, "KL/std": 148.75889587402344, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.09666442871094, "fcm_dpo/q_t": 0.4811328649520874, "grad_norm": 2.50854229927063, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.2378754615783691, "logits/rejected": -1.1080912351608276, "logps/chosen": -316.04534912109375, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -427.7958984375, "loss": 1.3158, "margin_dpo/margin_mean": 101.09666442871094, "margin_dpo/margin_std": 166.54736328125, "step": 647 }, { "KL/chosen_KL_mean": -218.10018920898438, "KL/mean": -287.427490234375, "KL/rejected_KL_mean": -356.7547607421875, "KL/std": 156.55841064453125, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.6545867919922, "fcm_dpo/q_t": 0.47412657737731934, "grad_norm": 2.796614170074463, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.8720585107803345, "logits/rejected": -1.8027348518371582, "logps/chosen": -305.94830322265625, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -461.4248352050781, "loss": 1.2891, "margin_dpo/margin_mean": 138.6545867919922, "margin_dpo/margin_std": 169.20953369140625, "step": 648 }, { "KL/chosen_KL_mean": -207.23248291015625, "KL/mean": -257.6666259765625, "KL/rejected_KL_mean": -308.10076904296875, "KL/std": 138.46714782714844, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.86827087402344, "fcm_dpo/q_t": 0.4811597764492035, "grad_norm": 2.479647636413574, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.2501227855682373, "logits/rejected": -1.8553112745285034, "logps/chosen": -296.92498779296875, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -396.80731201171875, "loss": 1.3153, "margin_dpo/margin_mean": 100.86827087402344, "margin_dpo/margin_std": 148.193359375, "step": 649 }, { "KL/chosen_KL_mean": -248.23092651367188, "KL/mean": -302.5274658203125, "KL/rejected_KL_mean": -356.82403564453125, "KL/std": 155.9969482421875, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.5931167602539, "fcm_dpo/q_t": 0.47978436946868896, "grad_norm": 2.799098491668701, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.1955901384353638, "logits/rejected": -0.9927818775177002, "logps/chosen": -338.1615295410156, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -459.4368896484375, "loss": 1.3121, "margin_dpo/margin_mean": 108.59310913085938, "margin_dpo/margin_std": 199.5858154296875, "step": 650 }, { "KL/chosen_KL_mean": -193.00372314453125, "KL/mean": -267.20477294921875, "KL/rejected_KL_mean": -341.4058532714844, "KL/std": 151.50503540039062, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.40216064453125, "fcm_dpo/q_t": 0.47231122851371765, "grad_norm": 4.07969856262207, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.1121726036071777, "logits/rejected": -1.9382414817810059, "logps/chosen": -272.1910400390625, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -447.33917236328125, "loss": 1.2815, "margin_dpo/margin_mean": 148.40216064453125, "margin_dpo/margin_std": 155.552490234375, "step": 651 }, { "KL/chosen_KL_mean": -211.31723022460938, "KL/mean": -263.3797302246094, "KL/rejected_KL_mean": -315.4422302246094, "KL/std": 131.8716278076172, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.1249771118164, "fcm_dpo/q_t": 0.4805474877357483, "grad_norm": 2.5738179683685303, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.4473280906677246, "logits/rejected": -2.19136905670166, "logps/chosen": -313.1074523925781, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -431.76678466796875, "loss": 1.3122, "margin_dpo/margin_mean": 104.12498474121094, "margin_dpo/margin_std": 131.46841430664062, "step": 652 }, { "KL/chosen_KL_mean": -242.0731658935547, "KL/mean": -297.6376953125, "KL/rejected_KL_mean": -353.20220947265625, "KL/std": 146.03692626953125, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.12904357910156, "fcm_dpo/q_t": 0.4792335629463196, "grad_norm": 2.4406802654266357, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.119443893432617, "logits/rejected": -1.8677217960357666, "logps/chosen": -335.4267578125, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -462.325439453125, "loss": 1.3081, "margin_dpo/margin_mean": 111.12904357910156, "margin_dpo/margin_std": 153.53993225097656, "step": 653 }, { "KL/chosen_KL_mean": -232.56832885742188, "KL/mean": -280.4932556152344, "KL/rejected_KL_mean": -328.41815185546875, "KL/std": 140.91104125976562, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.84984588623047, "fcm_dpo/q_t": 0.4821503460407257, "grad_norm": 2.5210328102111816, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.1840169429779053, "logits/rejected": -2.0149307250976562, "logps/chosen": -321.6838684082031, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -433.338134765625, "loss": 1.32, "margin_dpo/margin_mean": 95.84984588623047, "margin_dpo/margin_std": 174.12362670898438, "step": 654 }, { "KL/chosen_KL_mean": -216.65016174316406, "KL/mean": -274.90716552734375, "KL/rejected_KL_mean": -333.16412353515625, "KL/std": 158.21929931640625, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.51395416259766, "fcm_dpo/q_t": 0.47826701402664185, "grad_norm": 3.688742160797119, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.8090769052505493, "logits/rejected": -1.8249022960662842, "logps/chosen": -297.686279296875, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -435.9664611816406, "loss": 1.3051, "margin_dpo/margin_mean": 116.51394653320312, "margin_dpo/margin_std": 174.94566345214844, "step": 655 }, { "KL/chosen_KL_mean": -228.6700897216797, "KL/mean": -298.02056884765625, "KL/rejected_KL_mean": -367.37103271484375, "KL/std": 153.9044647216797, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.70095825195312, "fcm_dpo/q_t": 0.4741334319114685, "grad_norm": 3.4868154525756836, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.2732727527618408, "logits/rejected": -1.3434628248214722, "logps/chosen": -313.99542236328125, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -485.709716796875, "loss": 1.2887, "margin_dpo/margin_mean": 138.70095825195312, "margin_dpo/margin_std": 160.37698364257812, "step": 656 }, { "KL/chosen_KL_mean": -239.61798095703125, "KL/mean": -297.7270812988281, "KL/rejected_KL_mean": -355.836181640625, "KL/std": 150.00946044921875, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.2182388305664, "fcm_dpo/q_t": 0.4783613383769989, "grad_norm": 3.240980863571167, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9169104099273682, "logits/rejected": -1.7531216144561768, "logps/chosen": -319.8157043457031, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -458.4175109863281, "loss": 1.3058, "margin_dpo/margin_mean": 116.2182388305664, "margin_dpo/margin_std": 181.55105590820312, "step": 657 }, { "KL/chosen_KL_mean": -241.43600463867188, "KL/mean": -285.92547607421875, "KL/rejected_KL_mean": -330.4149169921875, "KL/std": 132.139892578125, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.97891235351562, "fcm_dpo/q_t": 0.48337817192077637, "grad_norm": 2.9665966033935547, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.8052482604980469, "logits/rejected": -1.3541498184204102, "logps/chosen": -336.0705261230469, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -432.0465087890625, "loss": 1.3235, "margin_dpo/margin_mean": 88.97891235351562, "margin_dpo/margin_std": 140.66143798828125, "step": 658 }, { "KL/chosen_KL_mean": -219.21136474609375, "KL/mean": -273.73492431640625, "KL/rejected_KL_mean": -328.2585144042969, "KL/std": 136.57138061523438, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.04716491699219, "fcm_dpo/q_t": 0.47965604066848755, "grad_norm": 2.6292343139648438, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.1363086700439453, "logits/rejected": -1.214653491973877, "logps/chosen": -331.7672119140625, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -422.18011474609375, "loss": 1.3104, "margin_dpo/margin_mean": 109.04716491699219, "margin_dpo/margin_std": 169.31570434570312, "step": 659 }, { "KL/chosen_KL_mean": -221.76962280273438, "KL/mean": -282.48175048828125, "KL/rejected_KL_mean": -343.1938781738281, "KL/std": 133.46400451660156, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.42427062988281, "fcm_dpo/q_t": 0.47733232378959656, "grad_norm": 3.7269186973571777, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.6664080619812012, "logits/rejected": -1.0522747039794922, "logps/chosen": -315.14703369140625, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -424.58868408203125, "loss": 1.3002, "margin_dpo/margin_mean": 121.42427062988281, "margin_dpo/margin_std": 144.5328369140625, "step": 660 }, { "KL/chosen_KL_mean": -218.50857543945312, "KL/mean": -288.33563232421875, "KL/rejected_KL_mean": -358.16265869140625, "KL/std": 166.63958740234375, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.65406799316406, "fcm_dpo/q_t": 0.4739971160888672, "grad_norm": 2.7362561225891113, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.7331154346466064, "logits/rejected": -1.2408311367034912, "logps/chosen": -306.3637390136719, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -450.56597900390625, "loss": 1.2895, "margin_dpo/margin_mean": 139.65406799316406, "margin_dpo/margin_std": 188.02781677246094, "step": 661 }, { "KL/chosen_KL_mean": -222.47463989257812, "KL/mean": -280.26025390625, "KL/rejected_KL_mean": -338.04583740234375, "KL/std": 131.5216064453125, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.57122039794922, "fcm_dpo/q_t": 0.4784170687198639, "grad_norm": 3.0945019721984863, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.8273260593414307, "logits/rejected": -1.2846848964691162, "logps/chosen": -325.2544250488281, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -433.2711486816406, "loss": 1.305, "margin_dpo/margin_mean": 115.57122039794922, "margin_dpo/margin_std": 156.78770446777344, "step": 662 }, { "KL/chosen_KL_mean": -212.2145233154297, "KL/mean": -274.2753601074219, "KL/rejected_KL_mean": -336.3362121582031, "KL/std": 151.01031494140625, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.1217041015625, "fcm_dpo/q_t": 0.47687897086143494, "grad_norm": 2.831491708755493, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.2985334396362305, "logits/rejected": -2.112649917602539, "logps/chosen": -308.56109619140625, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -456.8626708984375, "loss": 1.3001, "margin_dpo/margin_mean": 124.1217041015625, "margin_dpo/margin_std": 182.96783447265625, "step": 663 }, { "KL/chosen_KL_mean": -241.66387939453125, "KL/mean": -297.8927001953125, "KL/rejected_KL_mean": -354.12152099609375, "KL/std": 152.3071746826172, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.45767974853516, "fcm_dpo/q_t": 0.4790467619895935, "grad_norm": 2.821671962738037, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.4599525928497314, "logits/rejected": -1.305912733078003, "logps/chosen": -333.50628662109375, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -465.9582214355469, "loss": 1.3076, "margin_dpo/margin_mean": 112.45767211914062, "margin_dpo/margin_std": 162.72158813476562, "step": 664 }, { "KL/chosen_KL_mean": -201.75076293945312, "KL/mean": -258.6714172363281, "KL/rejected_KL_mean": -315.5920715332031, "KL/std": 135.3944091796875, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.84133911132812, "fcm_dpo/q_t": 0.47873276472091675, "grad_norm": 2.383733034133911, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.16715145111084, "logits/rejected": -1.7471940517425537, "logps/chosen": -289.93695068359375, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -407.50408935546875, "loss": 1.3056, "margin_dpo/margin_mean": 113.84133911132812, "margin_dpo/margin_std": 141.60125732421875, "step": 665 }, { "KL/chosen_KL_mean": -225.67323303222656, "KL/mean": -282.63232421875, "KL/rejected_KL_mean": -339.5914306640625, "KL/std": 137.11383056640625, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.91817474365234, "fcm_dpo/q_t": 0.4787454903125763, "grad_norm": 2.5642058849334717, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.647458791732788, "logits/rejected": -1.2770774364471436, "logps/chosen": -326.6478271484375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -440.8413391113281, "loss": 1.3063, "margin_dpo/margin_mean": 113.91818237304688, "margin_dpo/margin_std": 158.00228881835938, "step": 666 }, { "KL/chosen_KL_mean": -252.62477111816406, "KL/mean": -307.4694519042969, "KL/rejected_KL_mean": -362.31414794921875, "KL/std": 139.23544311523438, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.68937683105469, "fcm_dpo/q_t": 0.47950440645217896, "grad_norm": 2.5648889541625977, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.4194658994674683, "logits/rejected": -0.7977867126464844, "logps/chosen": -350.1959228515625, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -455.89892578125, "loss": 1.3085, "margin_dpo/margin_mean": 109.68937683105469, "margin_dpo/margin_std": 141.29977416992188, "step": 667 }, { "KL/chosen_KL_mean": -226.78192138671875, "KL/mean": -291.05596923828125, "KL/rejected_KL_mean": -355.33001708984375, "KL/std": 146.80776977539062, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.54806518554688, "fcm_dpo/q_t": 0.4760376214981079, "grad_norm": 2.74251651763916, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.081204414367676, "logits/rejected": -1.5766410827636719, "logps/chosen": -329.35699462890625, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -464.147705078125, "loss": 1.2964, "margin_dpo/margin_mean": 128.54806518554688, "margin_dpo/margin_std": 172.77294921875, "step": 668 }, { "KL/chosen_KL_mean": -202.57254028320312, "KL/mean": -266.5069885253906, "KL/rejected_KL_mean": -330.44146728515625, "KL/std": 144.8451385498047, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.86891174316406, "fcm_dpo/q_t": 0.47614002227783203, "grad_norm": 2.8896138668060303, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.4448649883270264, "logits/rejected": -1.0619765520095825, "logps/chosen": -292.06732177734375, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -412.9609375, "loss": 1.2962, "margin_dpo/margin_mean": 127.86891174316406, "margin_dpo/margin_std": 158.53366088867188, "step": 669 }, { "KL/chosen_KL_mean": -213.94186401367188, "KL/mean": -282.596435546875, "KL/rejected_KL_mean": -351.25103759765625, "KL/std": 153.62490844726562, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.30917358398438, "fcm_dpo/q_t": 0.474363774061203, "grad_norm": 2.752959966659546, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.2725176811218262, "logits/rejected": -0.7324990034103394, "logps/chosen": -309.396484375, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -452.78399658203125, "loss": 1.2896, "margin_dpo/margin_mean": 137.30917358398438, "margin_dpo/margin_std": 160.04266357421875, "step": 670 }, { "KL/chosen_KL_mean": -243.01788330078125, "KL/mean": -290.5675964355469, "KL/rejected_KL_mean": -338.1173095703125, "KL/std": 142.2958526611328, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.09942626953125, "fcm_dpo/q_t": 0.48225152492523193, "grad_norm": 2.903679132461548, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6856684684753418, "logits/rejected": -1.4807045459747314, "logps/chosen": -325.1409912109375, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -428.3370056152344, "loss": 1.3203, "margin_dpo/margin_mean": 95.09942626953125, "margin_dpo/margin_std": 168.96022033691406, "step": 671 }, { "KL/chosen_KL_mean": -249.73092651367188, "KL/mean": -289.1348876953125, "KL/rejected_KL_mean": -328.538818359375, "KL/std": 131.60208129882812, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.80785369873047, "fcm_dpo/q_t": 0.48529088497161865, "grad_norm": 3.580207347869873, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.928175926208496, "logits/rejected": -1.6369515657424927, "logps/chosen": -339.7928466796875, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -428.9920654296875, "loss": 1.3317, "margin_dpo/margin_mean": 78.807861328125, "margin_dpo/margin_std": 153.46914672851562, "step": 672 }, { "KL/chosen_KL_mean": -224.96038818359375, "KL/mean": -271.40606689453125, "KL/rejected_KL_mean": -317.8516845703125, "KL/std": 136.0777587890625, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.89131164550781, "fcm_dpo/q_t": 0.48265817761421204, "grad_norm": 2.399165153503418, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.907275676727295, "logits/rejected": -1.5743541717529297, "logps/chosen": -304.2235412597656, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -409.2009582519531, "loss": 1.3207, "margin_dpo/margin_mean": 92.89131164550781, "margin_dpo/margin_std": 141.99069213867188, "step": 673 }, { "KL/chosen_KL_mean": -194.6017608642578, "KL/mean": -267.942626953125, "KL/rejected_KL_mean": -341.28350830078125, "KL/std": 138.8739471435547, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 146.68173217773438, "fcm_dpo/q_t": 0.4726351499557495, "grad_norm": 2.1911773681640625, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.3993110656738281, "logits/rejected": -1.0007160902023315, "logps/chosen": -270.06005859375, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -417.48712158203125, "loss": 1.2825, "margin_dpo/margin_mean": 146.68173217773438, "margin_dpo/margin_std": 150.37503051757812, "step": 674 }, { "KL/chosen_KL_mean": -225.39395141601562, "KL/mean": -286.04083251953125, "KL/rejected_KL_mean": -346.68768310546875, "KL/std": 144.73532104492188, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.29374694824219, "fcm_dpo/q_t": 0.4773688018321991, "grad_norm": 2.3581955432891846, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.2584365606307983, "logits/rejected": -0.9119387269020081, "logps/chosen": -305.303466796875, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -427.90594482421875, "loss": 1.3021, "margin_dpo/margin_mean": 121.29375457763672, "margin_dpo/margin_std": 180.9144287109375, "step": 675 }, { "KL/chosen_KL_mean": -238.1473388671875, "KL/mean": -299.1657409667969, "KL/rejected_KL_mean": -360.18414306640625, "KL/std": 141.51023864746094, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.03678894042969, "fcm_dpo/q_t": 0.47723203897476196, "grad_norm": 2.552828073501587, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.7591078281402588, "logits/rejected": -1.1710798740386963, "logps/chosen": -336.3184509277344, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -455.20880126953125, "loss": 1.301, "margin_dpo/margin_mean": 122.03678894042969, "margin_dpo/margin_std": 170.703857421875, "step": 676 }, { "KL/chosen_KL_mean": -226.05081176757812, "KL/mean": -277.8521728515625, "KL/rejected_KL_mean": -329.6535949707031, "KL/std": 138.38217163085938, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.602783203125, "fcm_dpo/q_t": 0.4806548058986664, "grad_norm": 2.6232893466949463, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.477384328842163, "logits/rejected": -0.8311235904693604, "logps/chosen": -317.4300842285156, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -412.5313720703125, "loss": 1.3133, "margin_dpo/margin_mean": 103.602783203125, "margin_dpo/margin_std": 147.50140380859375, "step": 677 }, { "KL/chosen_KL_mean": -200.55703735351562, "KL/mean": -269.98046875, "KL/rejected_KL_mean": -339.4039001464844, "KL/std": 148.61474609375, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.84686279296875, "fcm_dpo/q_t": 0.47408929467201233, "grad_norm": 2.234687089920044, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.4711862802505493, "logits/rejected": -1.1561777591705322, "logps/chosen": -276.2029113769531, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -426.3699951171875, "loss": 1.2883, "margin_dpo/margin_mean": 138.84686279296875, "margin_dpo/margin_std": 154.29971313476562, "step": 678 }, { "KL/chosen_KL_mean": -236.57814025878906, "KL/mean": -291.4619445800781, "KL/rejected_KL_mean": -346.3457946777344, "KL/std": 121.90452575683594, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.76765441894531, "fcm_dpo/q_t": 0.47950834035873413, "grad_norm": 2.273527145385742, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.011234998703003, "logits/rejected": -1.5070514678955078, "logps/chosen": -317.3515625, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -429.22430419921875, "loss": 1.3085, "margin_dpo/margin_mean": 109.76763916015625, "margin_dpo/margin_std": 141.626953125, "step": 679 }, { "KL/chosen_KL_mean": -241.52853393554688, "KL/mean": -306.0274963378906, "KL/rejected_KL_mean": -370.5264892578125, "KL/std": 154.24136352539062, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.99798583984375, "fcm_dpo/q_t": 0.4759482741355896, "grad_norm": 3.1376230716705322, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.7654681205749512, "logits/rejected": -1.4120877981185913, "logps/chosen": -349.2114562988281, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -486.62139892578125, "loss": 1.2961, "margin_dpo/margin_mean": 128.99798583984375, "margin_dpo/margin_std": 171.2488250732422, "step": 680 }, { "KL/chosen_KL_mean": -218.44668579101562, "KL/mean": -271.7856140136719, "KL/rejected_KL_mean": -325.12451171875, "KL/std": 140.9796600341797, "epoch": 1.0, "fcm_dpo/beta": 0.000749831204302609, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.67784118652344, "fcm_dpo/q_t": 0.48010584712028503, "grad_norm": 2.5116751194000244, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.130545139312744, "logits/rejected": -1.8318228721618652, "logps/chosen": -311.457763671875, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -419.94671630859375, "loss": 1.3118, "margin_dpo/margin_mean": 106.67784118652344, "margin_dpo/margin_std": 162.38775634765625, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.3161681713686815, "train_runtime": 1969.3972, "train_samples_per_second": 22.138, "train_steps_per_second": 0.346 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }