{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10609889030456543, "fcm_dpo/delta": 0.3937801122665405, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984467327594757, "grad_norm": 41.251930236816406, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3804, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10819403827190399, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5033633708953857, "grad_norm": 32.21284866333008, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.4003, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.0274810791015625, "KL/mean": 0.022979460656642914, "KL/rejected_KL_mean": 0.018482208251953125, "KL/std": 0.32842397689819336, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.11038153618574142, "fcm_dpo/delta": 0.1982020139694214, "fcm_dpo/margin": 0.008995577692985535, "fcm_dpo/q_t": 0.49975651502609253, "grad_norm": 28.80890655517578, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4276501536369324, "logits/rejected": 0.650732159614563, "logps/chosen": -93.78350830078125, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.21102142333984, "loss": 1.386, "margin_dpo/margin_mean": 0.008995875716209412, "margin_dpo/margin_std": 0.5123732089996338, "step": 3 }, { "KL/chosen_KL_mean": -0.004695892333984375, "KL/mean": 0.00633549690246582, "KL/rejected_KL_mean": 0.017364501953125, "KL/std": 0.33055269718170166, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.11256903409957886, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.022053658962249756, "fcm_dpo/q_t": 0.5006164908409119, "grad_norm": 36.866031646728516, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6465466022491455, "logits/rejected": 0.7781882286071777, "logps/chosen": -87.29716491699219, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.80689239501953, "loss": 1.3896, "margin_dpo/margin_mean": -0.022053629159927368, "margin_dpo/margin_std": 0.499037504196167, "step": 4 }, { "KL/chosen_KL_mean": -0.0415802001953125, "KL/mean": -0.012181811034679413, "KL/rejected_KL_mean": 0.01721954345703125, "KL/std": 0.2892824411392212, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.11256903409957886, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.058802708983421326, "fcm_dpo/q_t": 0.5016533732414246, "grad_norm": 39.79860305786133, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4465774595737457, "logits/rejected": 0.6579518914222717, "logps/chosen": -89.37834167480469, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.7306137084961, "loss": 1.3935, "margin_dpo/margin_mean": -0.058802247047424316, "margin_dpo/margin_std": 0.40777578949928284, "step": 5 }, { "KL/chosen_KL_mean": -0.040332794189453125, "KL/mean": -0.017932526767253876, "KL/rejected_KL_mean": 0.004467010498046875, "KL/std": 0.306866854429245, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.11713411659002304, "fcm_dpo/delta": 0.19876445829868317, "fcm_dpo/margin": -0.04480220377445221, "fcm_dpo/q_t": 0.5013227462768555, "grad_norm": 42.343238830566406, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.7285869121551514, "logits/rejected": 1.0661801099777222, "logps/chosen": -97.36180877685547, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.87898254394531, "loss": 1.3922, "margin_dpo/margin_mean": -0.044802308082580566, "margin_dpo/margin_std": 0.4148600101470947, "step": 6 }, { "KL/chosen_KL_mean": 0.002674102783203125, "KL/mean": -0.040212854743003845, "KL/rejected_KL_mean": -0.08310317993164062, "KL/std": 0.31091660261154175, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.12422582507133484, "fcm_dpo/delta": 0.38970065116882324, "fcm_dpo/margin": 0.08577784895896912, "fcm_dpo/q_t": 0.4974261224269867, "grad_norm": 46.930755615234375, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.49828237295150757, "logits/rejected": 0.5457561016082764, "logps/chosen": -86.64585876464844, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.69927978515625, "loss": 1.3762, "margin_dpo/margin_mean": 0.0857776403427124, "margin_dpo/margin_std": 0.38655292987823486, "step": 7 }, { "KL/chosen_KL_mean": -0.018039703369140625, "KL/mean": 0.026049040257930756, "KL/rejected_KL_mean": 0.07014083862304688, "KL/std": 0.31119027733802795, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.12662875652313232, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08817459642887115, "fcm_dpo/q_t": 0.5027884840965271, "grad_norm": 41.915199279785156, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.3507783114910126, "logits/rejected": 0.6549879908561707, "logps/chosen": -89.94172668457031, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.15789794921875, "loss": 1.3985, "margin_dpo/margin_mean": -0.08817432820796967, "margin_dpo/margin_std": 0.4965929687023163, "step": 8 }, { "KL/chosen_KL_mean": 0.04393768310546875, "KL/mean": 0.0506223663687706, "KL/rejected_KL_mean": 0.057308197021484375, "KL/std": 0.32663288712501526, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.1317325234413147, "fcm_dpo/delta": 0.19756969809532166, "fcm_dpo/margin": -0.013376161456108093, "fcm_dpo/q_t": 0.5004613399505615, "grad_norm": 47.66386795043945, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.738162100315094, "logits/rejected": 0.9225601553916931, "logps/chosen": -103.81319427490234, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.26200866699219, "loss": 1.3891, "margin_dpo/margin_mean": -0.013375937938690186, "margin_dpo/margin_std": 0.4861551523208618, "step": 9 }, { "KL/chosen_KL_mean": 0.01479339599609375, "KL/mean": 0.05779782682657242, "KL/rejected_KL_mean": 0.10079574584960938, "KL/std": 0.25286445021629333, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.1317325234413147, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08600728213787079, "fcm_dpo/q_t": 0.502830445766449, "grad_norm": 44.372135162353516, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5548410415649414, "logits/rejected": 0.7682797312736511, "logps/chosen": -76.1895751953125, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.05130767822266, "loss": 1.3981, "margin_dpo/margin_mean": -0.08600735664367676, "margin_dpo/margin_std": 0.31241482496261597, "step": 10 }, { "KL/chosen_KL_mean": 0.02524566650390625, "KL/mean": -0.005339004099369049, "KL/rejected_KL_mean": -0.035923004150390625, "KL/std": 0.31739675998687744, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.13973329961299896, "fcm_dpo/delta": 0.39178359508514404, "fcm_dpo/margin": 0.061165615916252136, "fcm_dpo/q_t": 0.4979475736618042, "grad_norm": 51.3861198425293, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.2798558175563812, "logits/rejected": 0.49518418312072754, "logps/chosen": -82.34124755859375, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.30054473876953, "loss": 1.3785, "margin_dpo/margin_mean": 0.06116662919521332, "margin_dpo/margin_std": 0.3877822756767273, "step": 11 }, { "KL/chosen_KL_mean": -0.0330963134765625, "KL/mean": -0.0310685858130455, "KL/rejected_KL_mean": -0.029041290283203125, "KL/std": 0.37282195687294006, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.14537039399147034, "fcm_dpo/delta": 0.19955585896968842, "fcm_dpo/margin": -0.004062756896018982, "fcm_dpo/q_t": 0.5001388788223267, "grad_norm": 58.28773498535156, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.38442662358283997, "logits/rejected": 0.5129320025444031, "logps/chosen": -99.13859558105469, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.30045318603516, "loss": 1.3888, "margin_dpo/margin_mean": -0.004062682390213013, "margin_dpo/margin_std": 0.606316864490509, "step": 12 }, { "KL/chosen_KL_mean": -0.021022796630859375, "KL/mean": 0.013990692794322968, "KL/rejected_KL_mean": 0.04901123046875, "KL/std": 0.29282087087631226, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.14827094972133636, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07002590596675873, "fcm_dpo/q_t": 0.5025920867919922, "grad_norm": 62.25632858276367, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.19216829538345337, "logits/rejected": 0.4231150448322296, "logps/chosen": -90.58074951171875, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.64208984375, "loss": 1.3979, "margin_dpo/margin_mean": -0.07002612948417664, "margin_dpo/margin_std": 0.45393693447113037, "step": 13 }, { "KL/chosen_KL_mean": -0.01229095458984375, "KL/mean": 0.006553620100021362, "KL/rejected_KL_mean": 0.025402069091796875, "KL/std": 0.30935874581336975, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.15124410390853882, "fcm_dpo/delta": 0.1966051608324051, "fcm_dpo/margin": -0.037695348262786865, "fcm_dpo/q_t": 0.5013964772224426, "grad_norm": 54.50238037109375, "learning_rate": 9.420289855072464e-08, "logits/chosen": 1.0699594020843506, "logits/rejected": 1.1643542051315308, "logps/chosen": -99.83946228027344, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.91659545898438, "loss": 1.3927, "margin_dpo/margin_mean": -0.03769567608833313, "margin_dpo/margin_std": 0.39262664318084717, "step": 14 }, { "KL/chosen_KL_mean": 0.019443511962890625, "KL/mean": -0.010309271514415741, "KL/rejected_KL_mean": -0.04006195068359375, "KL/std": 0.33523422479629517, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.16352787613868713, "fcm_dpo/delta": 0.39071524143218994, "fcm_dpo/margin": 0.05950702726840973, "fcm_dpo/q_t": 0.4976808428764343, "grad_norm": 51.29329299926758, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.4237554371356964, "logits/rejected": 0.5942878723144531, "logps/chosen": -78.8905258178711, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.10240173339844, "loss": 1.378, "margin_dpo/margin_mean": 0.05950640141963959, "margin_dpo/margin_std": 0.44070136547088623, "step": 15 }, { "KL/chosen_KL_mean": -0.02983856201171875, "KL/mean": -0.0010115057229995728, "KL/rejected_KL_mean": 0.027812957763671875, "KL/std": 0.36071711778640747, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.16675162315368652, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0576571524143219, "fcm_dpo/q_t": 0.5023883581161499, "grad_norm": 57.59986877441406, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.7205063104629517, "logits/rejected": 0.9946610927581787, "logps/chosen": -97.45311737060547, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.57164764404297, "loss": 1.3976, "margin_dpo/margin_mean": -0.05765742063522339, "margin_dpo/margin_std": 0.4916079640388489, "step": 16 }, { "KL/chosen_KL_mean": 0.011501312255859375, "KL/mean": 0.0320146381855011, "KL/rejected_KL_mean": 0.052516937255859375, "KL/std": 0.35109764337539673, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.1733553558588028, "fcm_dpo/delta": 0.1941906213760376, "fcm_dpo/margin": -0.04101455211639404, "fcm_dpo/q_t": 0.5018252730369568, "grad_norm": 62.63567352294922, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5154228806495667, "logits/rejected": 0.7499574422836304, "logps/chosen": -104.3528060913086, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.41520690917969, "loss": 1.3955, "margin_dpo/margin_mean": -0.04101487994194031, "margin_dpo/margin_std": 0.5133906006813049, "step": 17 }, { "KL/chosen_KL_mean": 0.02033233642578125, "KL/mean": -0.01960884779691696, "KL/rejected_KL_mean": -0.059543609619140625, "KL/std": 0.2908901572227478, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.18379023671150208, "fcm_dpo/delta": 0.3857170045375824, "fcm_dpo/margin": 0.07987268269062042, "fcm_dpo/q_t": 0.4964374303817749, "grad_norm": 75.81092834472656, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.30209431052207947, "logits/rejected": 0.6371196508407593, "logps/chosen": -87.07162475585938, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.91026306152344, "loss": 1.3732, "margin_dpo/margin_mean": 0.07987302541732788, "margin_dpo/margin_std": 0.4489114284515381, "step": 18 }, { "KL/chosen_KL_mean": -0.0159454345703125, "KL/mean": -0.03483607620000839, "KL/rejected_KL_mean": -0.05373382568359375, "KL/std": 0.31353646516799927, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.19870582222938538, "fcm_dpo/delta": 0.39268553256988525, "fcm_dpo/margin": 0.03778375685214996, "fcm_dpo/q_t": 0.49817246198654175, "grad_norm": 62.638912200927734, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.7597043514251709, "logits/rejected": 1.0487170219421387, "logps/chosen": -105.88948822021484, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.98396301269531, "loss": 1.3805, "margin_dpo/margin_mean": 0.03778374195098877, "margin_dpo/margin_std": 0.4292582869529724, "step": 19 }, { "KL/chosen_KL_mean": -0.015350341796875, "KL/mean": -0.014215245842933655, "KL/rejected_KL_mean": -0.01308441162109375, "KL/std": 0.3167102336883545, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.2065596729516983, "fcm_dpo/delta": 0.19377554953098297, "fcm_dpo/margin": -0.002266496419906616, "fcm_dpo/q_t": 0.5001283884048462, "grad_norm": 66.9495849609375, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.4638875722885132, "logits/rejected": 0.7625367641448975, "logps/chosen": -90.77346801757812, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.92539978027344, "loss": 1.3886, "margin_dpo/margin_mean": -0.002266138792037964, "margin_dpo/margin_std": 0.4346674680709839, "step": 20 }, { "KL/chosen_KL_mean": 0.0288543701171875, "KL/mean": -0.004838965833187103, "KL/rejected_KL_mean": -0.038532257080078125, "KL/std": 0.26356515288352966, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.2230634093284607, "fcm_dpo/delta": 0.3856821656227112, "fcm_dpo/margin": 0.06738580763339996, "fcm_dpo/q_t": 0.49642497301101685, "grad_norm": 69.4036865234375, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5413563847541809, "logits/rejected": 0.7192718982696533, "logps/chosen": -80.30460357666016, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.9722671508789, "loss": 1.3729, "margin_dpo/margin_mean": 0.06738576292991638, "margin_dpo/margin_std": 0.3197932541370392, "step": 21 }, { "KL/chosen_KL_mean": 0.036098480224609375, "KL/mean": 0.029216498136520386, "KL/rejected_KL_mean": 0.022335052490234375, "KL/std": 0.3517240881919861, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.2318095862865448, "fcm_dpo/delta": 0.1882348358631134, "fcm_dpo/margin": 0.013759255409240723, "fcm_dpo/q_t": 0.4992424249649048, "grad_norm": 98.70708465576172, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.5662018656730652, "logits/rejected": 0.6827989816665649, "logps/chosen": -95.35920715332031, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.45118713378906, "loss": 1.3859, "margin_dpo/margin_mean": 0.013759046792984009, "margin_dpo/margin_std": 0.4810311198234558, "step": 22 }, { "KL/chosen_KL_mean": 0.050693511962890625, "KL/mean": 0.02165912836790085, "KL/rejected_KL_mean": -0.0073699951171875, "KL/std": 0.312529981136322, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.2501738369464874, "fcm_dpo/delta": 0.3862309157848358, "fcm_dpo/margin": 0.058066532015800476, "fcm_dpo/q_t": 0.49657338857650757, "grad_norm": 77.96316528320312, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.6677815914154053, "logits/rejected": 0.8405833840370178, "logps/chosen": -90.58682250976562, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.60162353515625, "loss": 1.3757, "margin_dpo/margin_mean": 0.05806627869606018, "margin_dpo/margin_std": 0.45669984817504883, "step": 23 }, { "KL/chosen_KL_mean": 0.051105499267578125, "KL/mean": 0.03489205241203308, "KL/rejected_KL_mean": 0.018680572509765625, "KL/std": 0.3133290112018585, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.26000264286994934, "fcm_dpo/delta": 0.18707986176013947, "fcm_dpo/margin": 0.03242594003677368, "fcm_dpo/q_t": 0.49796468019485474, "grad_norm": 115.18417358398438, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.4683370590209961, "logits/rejected": 0.5016816854476929, "logps/chosen": -69.8661880493164, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.59121704101562, "loss": 1.3809, "margin_dpo/margin_mean": 0.03242714703083038, "margin_dpo/margin_std": 0.42682135105133057, "step": 24 }, { "KL/chosen_KL_mean": 0.05570220947265625, "KL/mean": -0.009782277047634125, "KL/rejected_KL_mean": -0.0752716064453125, "KL/std": 0.29425162076950073, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.2798027992248535, "fcm_dpo/delta": 0.364682674407959, "fcm_dpo/margin": 0.1309651881456375, "fcm_dpo/q_t": 0.491201251745224, "grad_norm": 100.36483001708984, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.3833140730857849, "logits/rejected": 0.5460999011993408, "logps/chosen": -80.76978302001953, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.03237915039062, "loss": 1.3539, "margin_dpo/margin_mean": 0.13096454739570618, "margin_dpo/margin_std": 0.4431500732898712, "step": 25 }, { "KL/chosen_KL_mean": -0.020782470703125, "KL/mean": -0.05539718270301819, "KL/rejected_KL_mean": -0.09001541137695312, "KL/std": 0.32345426082611084, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.30148518085479736, "fcm_dpo/delta": 0.3801534175872803, "fcm_dpo/margin": 0.06923562288284302, "fcm_dpo/q_t": 0.4950401782989502, "grad_norm": 121.97190856933594, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.3393968343734741, "logits/rejected": 0.548342764377594, "logps/chosen": -88.92195129394531, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.918212890625, "loss": 1.3698, "margin_dpo/margin_mean": 0.06923580169677734, "margin_dpo/margin_std": 0.41606152057647705, "step": 26 }, { "KL/chosen_KL_mean": 0.03350067138671875, "KL/mean": -0.03292522579431534, "KL/rejected_KL_mean": -0.09935379028320312, "KL/std": 0.3495270013809204, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.3241084814071655, "fcm_dpo/delta": 0.3588481545448303, "fcm_dpo/margin": 0.13285885751247406, "fcm_dpo/q_t": 0.48977601528167725, "grad_norm": 138.99343872070312, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.1380520910024643, "logits/rejected": 0.3255367875099182, "logps/chosen": -77.56251525878906, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.03785705566406, "loss": 1.3495, "margin_dpo/margin_mean": 0.13285881280899048, "margin_dpo/margin_std": 0.445763498544693, "step": 27 }, { "KL/chosen_KL_mean": 0.0693359375, "KL/mean": 0.01536327600479126, "KL/rejected_KL_mean": -0.038608551025390625, "KL/std": 0.31252521276474, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.34867170453071594, "fcm_dpo/delta": 0.3639304041862488, "fcm_dpo/margin": 0.10794335603713989, "fcm_dpo/q_t": 0.49099764227867126, "grad_norm": 125.0436782836914, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5264641046524048, "logits/rejected": 0.7391597628593445, "logps/chosen": -102.15923309326172, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -96.99810791015625, "loss": 1.3561, "margin_dpo/margin_mean": 0.10794338583946228, "margin_dpo/margin_std": 0.47336459159851074, "step": 28 }, { "KL/chosen_KL_mean": 0.007293701171875, "KL/mean": -0.05067768692970276, "KL/rejected_KL_mean": -0.10864639282226562, "KL/std": 0.33801114559173584, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.3751586675643921, "fcm_dpo/delta": 0.3577496111392975, "fcm_dpo/margin": 0.11594089865684509, "fcm_dpo/q_t": 0.489596426486969, "grad_norm": 153.3795623779297, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.4081413745880127, "logits/rejected": 0.6730955839157104, "logps/chosen": -88.63975524902344, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.06875610351562, "loss": 1.3508, "margin_dpo/margin_mean": 0.11594228446483612, "margin_dpo/margin_std": 0.470255970954895, "step": 29 }, { "KL/chosen_KL_mean": -0.041561126708984375, "KL/mean": -0.06272705644369125, "KL/rejected_KL_mean": -0.08390045166015625, "KL/std": 0.3294770121574402, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.39581361413002014, "fcm_dpo/delta": 0.18322935700416565, "fcm_dpo/margin": 0.04233910143375397, "fcm_dpo/q_t": 0.49602073431015015, "grad_norm": 153.34808349609375, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.5828127861022949, "logits/rejected": 0.6982088088989258, "logps/chosen": -88.42994689941406, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.40279388427734, "loss": 1.375, "margin_dpo/margin_mean": 0.04233968257904053, "margin_dpo/margin_std": 0.35311275720596313, "step": 30 }, { "KL/chosen_KL_mean": 0.0334320068359375, "KL/mean": -0.04428648203611374, "KL/rejected_KL_mean": -0.12200546264648438, "KL/std": 0.31470417976379395, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.41656017303466797, "fcm_dpo/delta": 0.33729276061058044, "fcm_dpo/margin": 0.15543852746486664, "fcm_dpo/q_t": 0.48445427417755127, "grad_norm": 125.12789916992188, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.7943882942199707, "logits/rejected": 1.0489579439163208, "logps/chosen": -101.09222412109375, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.98042297363281, "loss": 1.3294, "margin_dpo/margin_mean": 0.15543913841247559, "margin_dpo/margin_std": 0.40076956152915955, "step": 31 }, { "KL/chosen_KL_mean": -0.07001113891601562, "KL/mean": -0.07657930999994278, "KL/rejected_KL_mean": -0.0831451416015625, "KL/std": 0.3476099967956543, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.4317340552806854, "fcm_dpo/delta": 0.19223041832447052, "fcm_dpo/margin": 0.013135209679603577, "fcm_dpo/q_t": 0.49855929613113403, "grad_norm": 150.8096923828125, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.4747728705406189, "logits/rejected": 0.6188691854476929, "logps/chosen": -96.68704223632812, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.30751037597656, "loss": 1.392, "margin_dpo/margin_mean": 0.013135567307472229, "margin_dpo/margin_std": 0.497408926486969, "step": 32 }, { "KL/chosen_KL_mean": -0.030864715576171875, "KL/mean": -0.07889585196971893, "KL/rejected_KL_mean": -0.12693023681640625, "KL/std": 0.3411206007003784, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.464775025844574, "fcm_dpo/delta": 0.356516569852829, "fcm_dpo/margin": 0.09606686234474182, "fcm_dpo/q_t": 0.489250123500824, "grad_norm": 153.46739196777344, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6412599086761475, "logits/rejected": 0.8231346011161804, "logps/chosen": -81.5518798828125, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.9328842163086, "loss": 1.3554, "margin_dpo/margin_mean": 0.09606766700744629, "margin_dpo/margin_std": 0.4980977773666382, "step": 33 }, { "KL/chosen_KL_mean": 0.07332611083984375, "KL/mean": -0.05434776842594147, "KL/rejected_KL_mean": -0.1820220947265625, "KL/std": 0.3962346017360687, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.49294236302375793, "fcm_dpo/delta": 0.27743732929229736, "fcm_dpo/margin": 0.25535640120506287, "fcm_dpo/q_t": 0.4703396260738373, "grad_norm": 176.7681121826172, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.6760616302490234, "logits/rejected": 0.8484926223754883, "logps/chosen": -77.12870788574219, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.90078735351562, "loss": 1.2815, "margin_dpo/margin_mean": 0.25535666942596436, "margin_dpo/margin_std": 0.5372216701507568, "step": 34 }, { "KL/chosen_KL_mean": 0.0320587158203125, "KL/mean": -0.07926470041275024, "KL/rejected_KL_mean": -0.19059371948242188, "KL/std": 0.35021722316741943, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.5210278034210205, "fcm_dpo/delta": 0.28759223222732544, "fcm_dpo/margin": 0.22265750169754028, "fcm_dpo/q_t": 0.47228872776031494, "grad_norm": 207.6819610595703, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.19339677691459656, "logits/rejected": 0.35022592544555664, "logps/chosen": -77.53829193115234, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.379150390625, "loss": 1.2895, "margin_dpo/margin_mean": 0.2226581871509552, "margin_dpo/margin_std": 0.4773765206336975, "step": 35 }, { "KL/chosen_KL_mean": 0.01540374755859375, "KL/mean": -0.10067427903413773, "KL/rejected_KL_mean": -0.21675491333007812, "KL/std": 0.39761775732040405, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.5523529648780823, "fcm_dpo/delta": 0.2747262120246887, "fcm_dpo/margin": 0.23215387761592865, "fcm_dpo/q_t": 0.46974968910217285, "grad_norm": 158.58668518066406, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.5148979425430298, "logits/rejected": 0.7092799544334412, "logps/chosen": -83.27284240722656, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.97416687011719, "loss": 1.2867, "margin_dpo/margin_mean": 0.2321537584066391, "margin_dpo/margin_std": 0.5678343772888184, "step": 36 }, { "KL/chosen_KL_mean": -0.023761749267578125, "KL/mean": -0.16020667552947998, "KL/rejected_KL_mean": -0.29665374755859375, "KL/std": 0.45544636249542236, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.5792882442474365, "fcm_dpo/delta": 0.24607205390930176, "fcm_dpo/margin": 0.272888720035553, "fcm_dpo/q_t": 0.46380048990249634, "grad_norm": 171.94004821777344, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.719542384147644, "logits/rejected": 0.8841136693954468, "logps/chosen": -94.79485321044922, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.013916015625, "loss": 1.2711, "margin_dpo/margin_mean": 0.27288925647735596, "margin_dpo/margin_std": 0.6659706234931946, "step": 37 }, { "KL/chosen_KL_mean": 0.03905487060546875, "KL/mean": -0.09219725430011749, "KL/rejected_KL_mean": -0.22344589233398438, "KL/std": 0.4356527626514435, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.6083833575248718, "fcm_dpo/delta": 0.24445223808288574, "fcm_dpo/margin": 0.2625093162059784, "fcm_dpo/q_t": 0.4641072750091553, "grad_norm": 179.8256072998047, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.45149850845336914, "logits/rejected": 0.5896898508071899, "logps/chosen": -75.939453125, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.32746887207031, "loss": 1.2717, "margin_dpo/margin_mean": 0.2625097930431366, "margin_dpo/margin_std": 0.6502280831336975, "step": 38 }, { "KL/chosen_KL_mean": -0.000461578369140625, "KL/mean": -0.16410335898399353, "KL/rejected_KL_mean": -0.3277473449707031, "KL/std": 0.4187053442001343, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.6325713396072388, "fcm_dpo/delta": 0.197519913315773, "fcm_dpo/margin": 0.32727915048599243, "fcm_dpo/q_t": 0.45179829001426697, "grad_norm": 193.66748046875, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.16119426488876343, "logits/rejected": 0.327913761138916, "logps/chosen": -81.18624114990234, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.52366638183594, "loss": 1.2229, "margin_dpo/margin_mean": 0.3272794485092163, "margin_dpo/margin_std": 0.5553213953971863, "step": 39 }, { "KL/chosen_KL_mean": 0.0396270751953125, "KL/mean": -0.09816452860832214, "KL/rejected_KL_mean": -0.23595428466796875, "KL/std": 0.36204326152801514, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.6648277044296265, "fcm_dpo/delta": 0.22003063559532166, "fcm_dpo/margin": 0.2755813002586365, "fcm_dpo/q_t": 0.456139475107193, "grad_norm": 170.7027587890625, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.5155481696128845, "logits/rejected": 0.7045005559921265, "logps/chosen": -83.2929458618164, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.49186706542969, "loss": 1.2345, "margin_dpo/margin_mean": 0.27558115124702454, "margin_dpo/margin_std": 0.4560346007347107, "step": 40 }, { "KL/chosen_KL_mean": -0.017276763916015625, "KL/mean": -0.15941670536994934, "KL/rejected_KL_mean": -0.3015594482421875, "KL/std": 0.44918596744537354, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.692819356918335, "fcm_dpo/delta": 0.20659089088439941, "fcm_dpo/margin": 0.28428220748901367, "fcm_dpo/q_t": 0.4551916718482971, "grad_norm": 198.27491760253906, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.8287094831466675, "logits/rejected": 1.0262947082519531, "logps/chosen": -93.16593933105469, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.38076782226562, "loss": 1.2461, "margin_dpo/margin_mean": 0.28428229689598083, "margin_dpo/margin_std": 0.634917140007019, "step": 41 }, { "KL/chosen_KL_mean": -0.006748199462890625, "KL/mean": -0.20493735373020172, "KL/rejected_KL_mean": -0.40312957763671875, "KL/std": 0.5089447498321533, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.7113193273544312, "fcm_dpo/delta": 0.12148275971412659, "fcm_dpo/margin": 0.3963768184185028, "fcm_dpo/q_t": 0.4349275827407837, "grad_norm": 219.9290313720703, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.5590070486068726, "logits/rejected": 0.7027156352996826, "logps/chosen": -90.7083740234375, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.26777648925781, "loss": 1.1744, "margin_dpo/margin_mean": 0.396377831697464, "margin_dpo/margin_std": 0.6397839188575745, "step": 42 }, { "KL/chosen_KL_mean": 0.0684661865234375, "KL/mean": -0.1572086364030838, "KL/rejected_KL_mean": -0.38288116455078125, "KL/std": 0.5264471769332886, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.7230080366134644, "fcm_dpo/delta": 0.07622459530830383, "fcm_dpo/margin": 0.45134392380714417, "fcm_dpo/q_t": 0.42625659704208374, "grad_norm": 199.32647705078125, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.6067841053009033, "logits/rejected": 0.7977492809295654, "logps/chosen": -89.5755615234375, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.33473205566406, "loss": 1.1443, "margin_dpo/margin_mean": 0.45134463906288147, "margin_dpo/margin_std": 0.7003409266471863, "step": 43 }, { "KL/chosen_KL_mean": -0.052219390869140625, "KL/mean": -0.2660455107688904, "KL/rejected_KL_mean": -0.4798736572265625, "KL/std": 0.5431427359580994, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.7341824769973755, "fcm_dpo/delta": 0.08888706564903259, "fcm_dpo/margin": 0.42766085267066956, "fcm_dpo/q_t": 0.4287828803062439, "grad_norm": 188.6514129638672, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8414484262466431, "logits/rejected": 0.9624049067497253, "logps/chosen": -81.9305419921875, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.90150451660156, "loss": 1.1605, "margin_dpo/margin_mean": 0.42766109108924866, "margin_dpo/margin_std": 0.7129043340682983, "step": 44 }, { "KL/chosen_KL_mean": -0.06998825073242188, "KL/mean": -0.19522476196289062, "KL/rejected_KL_mean": -0.32045745849609375, "KL/std": 0.4534716010093689, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.7661818265914917, "fcm_dpo/delta": 0.21119216084480286, "fcm_dpo/margin": 0.2504703998565674, "fcm_dpo/q_t": 0.45593202114105225, "grad_norm": 156.6806640625, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.7717897891998291, "logits/rejected": 0.8863900303840637, "logps/chosen": -77.41458129882812, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.56820678710938, "loss": 1.2506, "margin_dpo/margin_mean": 0.25047072768211365, "margin_dpo/margin_std": 0.5778284072875977, "step": 45 }, { "KL/chosen_KL_mean": -0.04897308349609375, "KL/mean": -0.23159198462963104, "KL/rejected_KL_mean": -0.41420745849609375, "KL/std": 0.5217543840408325, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.7852911949157715, "fcm_dpo/delta": 0.11651579290628433, "fcm_dpo/margin": 0.36524051427841187, "fcm_dpo/q_t": 0.4331882894039154, "grad_norm": 206.59815979003906, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.4303792715072632, "logits/rejected": 0.6691064834594727, "logps/chosen": -90.38980102539062, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.09072875976562, "loss": 1.1621, "margin_dpo/margin_mean": 0.36524075269699097, "margin_dpo/margin_std": 0.537470817565918, "step": 46 }, { "KL/chosen_KL_mean": 0.01172637939453125, "KL/mean": -0.18366152048110962, "KL/rejected_KL_mean": -0.37905120849609375, "KL/std": 0.5189104676246643, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.8018202185630798, "fcm_dpo/delta": 0.08926395326852798, "fcm_dpo/margin": 0.3907787799835205, "fcm_dpo/q_t": 0.4319987893104553, "grad_norm": 221.86024475097656, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.3951904773712158, "logits/rejected": 0.57347571849823, "logps/chosen": -104.39957427978516, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.63394927978516, "loss": 1.1868, "margin_dpo/margin_mean": 0.3907780051231384, "margin_dpo/margin_std": 0.7831786870956421, "step": 47 }, { "KL/chosen_KL_mean": 0.010303497314453125, "KL/mean": -0.25155895948410034, "KL/rejected_KL_mean": -0.5134201049804688, "KL/std": 0.6542104482650757, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.800434947013855, "fcm_dpo/delta": -0.020132029429078102, "fcm_dpo/margin": 0.5237245559692383, "fcm_dpo/q_t": 0.4064512848854065, "grad_norm": 202.90330505371094, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.04635071009397507, "logits/rejected": 0.23239943385124207, "logps/chosen": -91.0899658203125, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.593994140625, "loss": 1.1165, "margin_dpo/margin_mean": 0.5237252712249756, "margin_dpo/margin_std": 0.849540114402771, "step": 48 }, { "KL/chosen_KL_mean": 0.050838470458984375, "KL/mean": -0.30068445205688477, "KL/rejected_KL_mean": -0.6522102355957031, "KL/std": 0.7348222732543945, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.7843134999275208, "fcm_dpo/delta": -0.15996122360229492, "fcm_dpo/margin": 0.7030497789382935, "fcm_dpo/q_t": 0.38016554713249207, "grad_norm": 182.75613403320312, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.13615956902503967, "logits/rejected": 0.6091794967651367, "logps/chosen": -92.95284271240234, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.40120697021484, "loss": 1.0168, "margin_dpo/margin_mean": 0.7030496597290039, "margin_dpo/margin_std": 0.8904697895050049, "step": 49 }, { "KL/chosen_KL_mean": -0.09724807739257812, "KL/mean": -0.3749244213104248, "KL/rejected_KL_mean": -0.652587890625, "KL/std": 0.7048405408859253, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.7680332660675049, "fcm_dpo/delta": -0.02851744368672371, "fcm_dpo/margin": 0.5553402304649353, "fcm_dpo/q_t": 0.4067155122756958, "grad_norm": 175.66339111328125, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.4946308135986328, "logits/rejected": 0.6630486249923706, "logps/chosen": -94.72406005859375, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.22694396972656, "loss": 1.1058, "margin_dpo/margin_mean": 0.5553403496742249, "margin_dpo/margin_std": 0.8655717372894287, "step": 50 }, { "KL/chosen_KL_mean": -0.08330535888671875, "KL/mean": -0.3658096194267273, "KL/rejected_KL_mean": -0.6483154296875, "KL/std": 0.7626069188117981, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.7684066295623779, "fcm_dpo/delta": -0.035696156322956085, "fcm_dpo/margin": 0.5650014877319336, "fcm_dpo/q_t": 0.4072089195251465, "grad_norm": 148.68043518066406, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.598242998123169, "logits/rejected": 0.7376997470855713, "logps/chosen": -87.59058380126953, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.12066650390625, "loss": 1.1249, "margin_dpo/margin_mean": 0.5650011301040649, "margin_dpo/margin_std": 0.9893271923065186, "step": 51 }, { "KL/chosen_KL_mean": -0.02032470703125, "KL/mean": -0.4141855239868164, "KL/rejected_KL_mean": -0.8080520629882812, "KL/std": 0.7479879856109619, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.7433894872665405, "fcm_dpo/delta": -0.19721950590610504, "fcm_dpo/margin": 0.787726640701294, "fcm_dpo/q_t": 0.3698500990867615, "grad_norm": 159.13687133789062, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.49312669038772583, "logits/rejected": 0.7081508636474609, "logps/chosen": -90.65058898925781, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.84711456298828, "loss": 0.9758, "margin_dpo/margin_mean": 0.7877264022827148, "margin_dpo/margin_std": 0.8923406004905701, "step": 52 }, { "KL/chosen_KL_mean": -0.06715774536132812, "KL/mean": -0.5487350225448608, "KL/rejected_KL_mean": -1.0303115844726562, "KL/std": 0.9889528751373291, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.7119340300559998, "fcm_dpo/delta": -0.3059566020965576, "fcm_dpo/margin": 0.9631536602973938, "fcm_dpo/q_t": 0.35896795988082886, "grad_norm": 152.50538635253906, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.20951451361179352, "logits/rejected": 0.32455170154571533, "logps/chosen": -81.65023803710938, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.69184112548828, "loss": 0.961, "margin_dpo/margin_mean": 0.9631540775299072, "margin_dpo/margin_std": 1.1812196969985962, "step": 53 }, { "KL/chosen_KL_mean": 0.026172637939453125, "KL/mean": -0.5075909495353699, "KL/rejected_KL_mean": -1.0413589477539062, "KL/std": 1.183929443359375, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.6603946685791016, "fcm_dpo/delta": -0.3278176784515381, "fcm_dpo/margin": 1.0675337314605713, "fcm_dpo/q_t": 0.3495662808418274, "grad_norm": 145.84910583496094, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.542641282081604, "logits/rejected": 0.7238681316375732, "logps/chosen": -88.88399505615234, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.1589126586914, "loss": 0.924, "margin_dpo/margin_mean": 1.067533254623413, "margin_dpo/margin_std": 1.198992133140564, "step": 54 }, { "KL/chosen_KL_mean": 0.12372207641601562, "KL/mean": -0.4373123049736023, "KL/rejected_KL_mean": -0.9983406066894531, "KL/std": 1.1539171934127808, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.6224900484085083, "fcm_dpo/delta": -0.3197006583213806, "fcm_dpo/margin": 1.1220623254776, "fcm_dpo/q_t": 0.3597676455974579, "grad_norm": 120.58650970458984, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.2225342094898224, "logits/rejected": 0.5370264053344727, "logps/chosen": -92.33219909667969, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -93.96927642822266, "loss": 0.9773, "margin_dpo/margin_mean": 1.1220626831054688, "margin_dpo/margin_std": 1.5012993812561035, "step": 55 }, { "KL/chosen_KL_mean": -0.06414413452148438, "KL/mean": -0.5688866376876831, "KL/rejected_KL_mean": -1.0736236572265625, "KL/std": 1.1267895698547363, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.5860604047775269, "fcm_dpo/delta": -0.2059919238090515, "fcm_dpo/margin": 1.009477138519287, "fcm_dpo/q_t": 0.37762561440467834, "grad_norm": 129.25904846191406, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.26163795590400696, "logits/rejected": 0.5993987917900085, "logps/chosen": -87.30079650878906, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.71916198730469, "loss": 1.0267, "margin_dpo/margin_mean": 1.0094773769378662, "margin_dpo/margin_std": 1.490530014038086, "step": 56 }, { "KL/chosen_KL_mean": -0.029575347900390625, "KL/mean": -0.5806881189346313, "KL/rejected_KL_mean": -1.131805419921875, "KL/std": 1.168419361114502, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.5625683069229126, "fcm_dpo/delta": -0.23524609208106995, "fcm_dpo/margin": 1.1022437810897827, "fcm_dpo/q_t": 0.3700264096260071, "grad_norm": 115.05542755126953, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.3815111219882965, "logits/rejected": 0.5993814468383789, "logps/chosen": -98.18031311035156, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.18464660644531, "loss": 0.9967, "margin_dpo/margin_mean": 1.1022440195083618, "margin_dpo/margin_std": 1.4471189975738525, "step": 57 }, { "KL/chosen_KL_mean": 0.2783966064453125, "KL/mean": -0.4530618190765381, "KL/rejected_KL_mean": -1.1845130920410156, "KL/std": 1.4093058109283447, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.5284005999565125, "fcm_dpo/delta": -0.4025019407272339, "fcm_dpo/margin": 1.4629074335098267, "fcm_dpo/q_t": 0.3485648036003113, "grad_norm": 108.73483276367188, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.328408420085907, "logits/rejected": 0.6614660620689392, "logps/chosen": -99.30257415771484, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.40678405761719, "loss": 0.9361, "margin_dpo/margin_mean": 1.4629074335098267, "margin_dpo/margin_std": 1.8542253971099854, "step": 58 }, { "KL/chosen_KL_mean": 0.0319976806640625, "KL/mean": -0.5776711702346802, "KL/rejected_KL_mean": -1.1873359680175781, "KL/std": 1.1382396221160889, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.499040424823761, "fcm_dpo/delta": -0.22148504853248596, "fcm_dpo/margin": 1.219330906867981, "fcm_dpo/q_t": 0.36656421422958374, "grad_norm": 104.87806701660156, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.024204734712839127, "logits/rejected": 0.20756295323371887, "logps/chosen": -89.77032470703125, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.4640121459961, "loss": 0.9696, "margin_dpo/margin_mean": 1.2193305492401123, "margin_dpo/margin_std": 1.3817192316055298, "step": 59 }, { "KL/chosen_KL_mean": 0.07428741455078125, "KL/mean": -0.4971103072166443, "KL/rejected_KL_mean": -1.0685081481933594, "KL/std": 1.1861416101455688, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.47919291257858276, "fcm_dpo/delta": -0.15665188431739807, "fcm_dpo/margin": 1.142797827720642, "fcm_dpo/q_t": 0.37962988018989563, "grad_norm": 95.24152374267578, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.38631966710090637, "logits/rejected": 0.64959716796875, "logps/chosen": -95.0814208984375, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.44094848632812, "loss": 1.0092, "margin_dpo/margin_mean": 1.1427979469299316, "margin_dpo/margin_std": 1.3806625604629517, "step": 60 }, { "KL/chosen_KL_mean": -0.17563247680664062, "KL/mean": -0.6891086101531982, "KL/rejected_KL_mean": -1.2025833129882812, "KL/std": 1.2316420078277588, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.47240257263183594, "fcm_dpo/delta": -0.0895158126950264, "fcm_dpo/margin": 1.0269618034362793, "fcm_dpo/q_t": 0.3972019553184509, "grad_norm": 95.49568176269531, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.3960622251033783, "logits/rejected": 0.5370829105377197, "logps/chosen": -85.90794372558594, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.16834259033203, "loss": 1.0797, "margin_dpo/margin_mean": 1.0269616842269897, "margin_dpo/margin_std": 1.5877673625946045, "step": 61 }, { "KL/chosen_KL_mean": -0.15913772583007812, "KL/mean": -0.7141653299331665, "KL/rejected_KL_mean": -1.2691917419433594, "KL/std": 1.2673290967941284, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.45974329113960266, "fcm_dpo/delta": -0.1163402795791626, "fcm_dpo/margin": 1.1100555658340454, "fcm_dpo/q_t": 0.3886195421218872, "grad_norm": 89.46971130371094, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.25394171476364136, "logits/rejected": 0.4470348358154297, "logps/chosen": -81.79452514648438, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.30750274658203, "loss": 1.0682, "margin_dpo/margin_mean": 1.1100552082061768, "margin_dpo/margin_std": 1.6246373653411865, "step": 62 }, { "KL/chosen_KL_mean": 0.005626678466796875, "KL/mean": -0.6835018396377563, "KL/rejected_KL_mean": -1.3726272583007812, "KL/std": 1.24532949924469, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.43940651416778564, "fcm_dpo/delta": -0.221391499042511, "fcm_dpo/margin": 1.3782570362091064, "fcm_dpo/q_t": 0.36385518312454224, "grad_norm": 97.98200225830078, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.5666729807853699, "logits/rejected": 0.7833958864212036, "logps/chosen": -103.6184310913086, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.7756576538086, "loss": 0.9562, "margin_dpo/margin_mean": 1.3782565593719482, "margin_dpo/margin_std": 1.417145848274231, "step": 63 }, { "KL/chosen_KL_mean": -0.0769195556640625, "KL/mean": -0.8711789846420288, "KL/rejected_KL_mean": -1.6654434204101562, "KL/std": 1.6373652219772339, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.4155277609825134, "fcm_dpo/delta": -0.2829596698284149, "fcm_dpo/margin": 1.588523507118225, "fcm_dpo/q_t": 0.36179184913635254, "grad_norm": 83.01851654052734, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.5544687509536743, "logits/rejected": 0.7661176323890686, "logps/chosen": -87.07850646972656, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.25091552734375, "loss": 0.9577, "margin_dpo/margin_mean": 1.5885233879089355, "margin_dpo/margin_std": 1.8501657247543335, "step": 64 }, { "KL/chosen_KL_mean": -0.0391693115234375, "KL/mean": -0.8181794285774231, "KL/rejected_KL_mean": -1.5971908569335938, "KL/std": 1.4251986742019653, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.399244487285614, "fcm_dpo/delta": -0.2366902381181717, "fcm_dpo/margin": 1.558018445968628, "fcm_dpo/q_t": 0.36404985189437866, "grad_norm": 92.53385925292969, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.005982611328363419, "logits/rejected": 0.13509900867938995, "logps/chosen": -91.2610855102539, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -116.93272399902344, "loss": 0.9615, "margin_dpo/margin_mean": 1.558018684387207, "margin_dpo/margin_std": 1.7730741500854492, "step": 65 }, { "KL/chosen_KL_mean": -0.22301483154296875, "KL/mean": -0.8195175528526306, "KL/rejected_KL_mean": -1.4160194396972656, "KL/std": 1.4110875129699707, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.38683420419692993, "fcm_dpo/delta": -0.06676234304904938, "fcm_dpo/margin": 1.193000316619873, "fcm_dpo/q_t": 0.398768812417984, "grad_norm": 71.78748321533203, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.2919391393661499, "logits/rejected": 0.4938666820526123, "logps/chosen": -84.00724029541016, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.16812133789062, "loss": 1.0702, "margin_dpo/margin_mean": 1.1930006742477417, "margin_dpo/margin_std": 1.6219747066497803, "step": 66 }, { "KL/chosen_KL_mean": -0.6035957336425781, "KL/mean": -1.1078087091445923, "KL/rejected_KL_mean": -1.6120147705078125, "KL/std": 1.6506342887878418, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.38899803161621094, "fcm_dpo/delta": 0.00802034605294466, "fcm_dpo/margin": 1.0084182024002075, "fcm_dpo/q_t": 0.4240454435348511, "grad_norm": 85.2502212524414, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.47742319107055664, "logits/rejected": 0.74605393409729, "logps/chosen": -88.27655792236328, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -80.69876098632812, "loss": 1.1847, "margin_dpo/margin_mean": 1.008418321609497, "margin_dpo/margin_std": 2.208601951599121, "step": 67 }, { "KL/chosen_KL_mean": -0.053302764892578125, "KL/mean": -0.8968609571456909, "KL/rejected_KL_mean": -1.7404136657714844, "KL/std": 1.6660630702972412, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.37521788477897644, "fcm_dpo/delta": -0.24901238083839417, "fcm_dpo/margin": 1.6871089935302734, "fcm_dpo/q_t": 0.3677256107330322, "grad_norm": 71.86233520507812, "learning_rate": 4.855072463768116e-07, "logits/chosen": 0.4419102072715759, "logits/rejected": 0.7661269903182983, "logps/chosen": -97.13214874267578, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -85.85914611816406, "loss": 0.9691, "margin_dpo/margin_mean": 1.6871092319488525, "margin_dpo/margin_std": 2.1078412532806396, "step": 68 }, { "KL/chosen_KL_mean": -0.13350296020507812, "KL/mean": -0.9078932404518127, "KL/rejected_KL_mean": -1.6822853088378906, "KL/std": 1.6212167739868164, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.3620404601097107, "fcm_dpo/delta": -0.17005856335163116, "fcm_dpo/margin": 1.548784613609314, "fcm_dpo/q_t": 0.3792663812637329, "grad_norm": 75.58474731445312, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.3286307454109192, "logits/rejected": 0.4287649095058441, "logps/chosen": -85.85321044921875, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.16249084472656, "loss": 1.0099, "margin_dpo/margin_mean": 1.5487855672836304, "margin_dpo/margin_std": 2.028609275817871, "step": 69 }, { "KL/chosen_KL_mean": -0.2577934265136719, "KL/mean": -1.25699782371521, "KL/rejected_KL_mean": -2.256206512451172, "KL/std": 1.983067512512207, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.3424133360385895, "fcm_dpo/delta": -0.305253267288208, "fcm_dpo/margin": 1.9984153509140015, "fcm_dpo/q_t": 0.35880762338638306, "grad_norm": 66.17256164550781, "learning_rate": 5e-07, "logits/chosen": 0.45389896631240845, "logits/rejected": 0.6233071088790894, "logps/chosen": -95.26774597167969, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -98.46893310546875, "loss": 0.9503, "margin_dpo/margin_mean": 1.9984153509140015, "margin_dpo/margin_std": 2.4461476802825928, "step": 70 }, { "KL/chosen_KL_mean": 0.2728996276855469, "KL/mean": -1.1308135986328125, "KL/rejected_KL_mean": -2.534526824951172, "KL/std": 2.463621139526367, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.31052935123443604, "fcm_dpo/delta": -0.5203126072883606, "fcm_dpo/margin": 2.807429313659668, "fcm_dpo/q_t": 0.31816646456718445, "grad_norm": 67.9230728149414, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.023789262399077415, "logits/rejected": 0.30548790097236633, "logps/chosen": -90.70445251464844, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -105.12555694580078, "loss": 0.8271, "margin_dpo/margin_mean": 2.807429075241089, "margin_dpo/margin_std": 2.8036611080169678, "step": 71 }, { "KL/chosen_KL_mean": -0.12260818481445312, "KL/mean": -1.2512246370315552, "KL/rejected_KL_mean": -2.3798446655273438, "KL/std": 2.084284782409668, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.290529727935791, "fcm_dpo/delta": -0.2751188576221466, "fcm_dpo/margin": 2.257241725921631, "fcm_dpo/q_t": 0.35679197311401367, "grad_norm": 59.97789764404297, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.006599783897399902, "logits/rejected": 0.2854318618774414, "logps/chosen": -98.01412963867188, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -102.57156372070312, "loss": 0.9398, "margin_dpo/margin_mean": 2.2572410106658936, "margin_dpo/margin_std": 2.454249858856201, "step": 72 }, { "KL/chosen_KL_mean": -0.2933235168457031, "KL/mean": -1.4646594524383545, "KL/rejected_KL_mean": -2.6359939575195312, "KL/std": 2.6197996139526367, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.2762339115142822, "fcm_dpo/delta": -0.26447805762290955, "fcm_dpo/margin": 2.342674732208252, "fcm_dpo/q_t": 0.3711169958114624, "grad_norm": 54.54767990112305, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.031873054802417755, "logits/rejected": 0.2578619718551636, "logps/chosen": -96.06234741210938, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -98.56843566894531, "loss": 0.999, "margin_dpo/margin_mean": 2.342674732208252, "margin_dpo/margin_std": 3.3073906898498535, "step": 73 }, { "KL/chosen_KL_mean": -0.5120811462402344, "KL/mean": -1.8381564617156982, "KL/rejected_KL_mean": -3.1642303466796875, "KL/std": 2.7517919540405273, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.2592149078845978, "fcm_dpo/delta": -0.3106447458267212, "fcm_dpo/margin": 2.6521453857421875, "fcm_dpo/q_t": 0.35721421241760254, "grad_norm": 53.50786590576172, "learning_rate": 4.999472998758977e-07, "logits/chosen": 0.02081543393433094, "logits/rejected": 0.15674826502799988, "logps/chosen": -79.32048034667969, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -104.81100463867188, "loss": 0.9548, "margin_dpo/margin_mean": 2.6521453857421875, "margin_dpo/margin_std": 3.2413864135742188, "step": 74 }, { "KL/chosen_KL_mean": -0.542144775390625, "KL/mean": -2.237952709197998, "KL/rejected_KL_mean": -3.9337615966796875, "KL/std": 3.260990858078003, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.24124856293201447, "fcm_dpo/delta": -0.4534810185432434, "fcm_dpo/margin": 3.391617774963379, "fcm_dpo/q_t": 0.3355121612548828, "grad_norm": 52.13274002075195, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.0008343793451786041, "logits/rejected": 0.1901235282421112, "logps/chosen": -78.8239974975586, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -119.33687591552734, "loss": 0.8867, "margin_dpo/margin_mean": 3.391618251800537, "margin_dpo/margin_std": 3.8458786010742188, "step": 75 }, { "KL/chosen_KL_mean": -0.7543144226074219, "KL/mean": -2.0848207473754883, "KL/rejected_KL_mean": -3.415332794189453, "KL/std": 2.962268352508545, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.2264285385608673, "fcm_dpo/delta": -0.21579276025295258, "fcm_dpo/margin": 2.661015033721924, "fcm_dpo/q_t": 0.37436583638191223, "grad_norm": 42.10383605957031, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.264835000038147, "logits/rejected": 0.09487824141979218, "logps/chosen": -88.63145446777344, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -89.135009765625, "loss": 1.0046, "margin_dpo/margin_mean": 2.6610145568847656, "margin_dpo/margin_std": 3.6470561027526855, "step": 76 }, { "KL/chosen_KL_mean": -0.8903884887695312, "KL/mean": -2.2239303588867188, "KL/rejected_KL_mean": -3.557476043701172, "KL/std": 3.1845579147338867, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.21699687838554382, "fcm_dpo/delta": -0.19089357554912567, "fcm_dpo/margin": 2.667083263397217, "fcm_dpo/q_t": 0.3777558207511902, "grad_norm": 49.268943786621094, "learning_rate": 4.998386175651409e-07, "logits/chosen": 0.03124828450381756, "logits/rejected": 0.37664660811424255, "logps/chosen": -100.59073638916016, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -101.76323699951172, "loss": 1.0246, "margin_dpo/margin_mean": 2.667083263397217, "margin_dpo/margin_std": 3.6546473503112793, "step": 77 }, { "KL/chosen_KL_mean": -0.23432540893554688, "KL/mean": -1.726682186126709, "KL/rejected_KL_mean": -3.2190284729003906, "KL/std": 2.82118558883667, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.20831362903118134, "fcm_dpo/delta": -0.2364954948425293, "fcm_dpo/margin": 2.984700918197632, "fcm_dpo/q_t": 0.3623862564563751, "grad_norm": 40.69357681274414, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.5226966142654419, "logits/rejected": 0.7449191808700562, "logps/chosen": -90.53103637695312, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -94.35675048828125, "loss": 0.9508, "margin_dpo/margin_mean": 2.9847006797790527, "margin_dpo/margin_std": 3.220381259918213, "step": 78 }, { "KL/chosen_KL_mean": -0.3572807312011719, "KL/mean": -2.273345947265625, "KL/rejected_KL_mean": -4.189418792724609, "KL/std": 4.044190406799316, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.1950855851173401, "fcm_dpo/delta": -0.37506669759750366, "fcm_dpo/margin": 3.8321354389190674, "fcm_dpo/q_t": 0.34329140186309814, "grad_norm": 41.180912017822266, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.2884211540222168, "logits/rejected": 0.0786387175321579, "logps/chosen": -86.73561096191406, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -98.29718780517578, "loss": 0.913, "margin_dpo/margin_mean": 3.8321352005004883, "margin_dpo/margin_std": 4.393360137939453, "step": 79 }, { "KL/chosen_KL_mean": -0.783355712890625, "KL/mean": -2.3253331184387207, "KL/rejected_KL_mean": -3.8673095703125, "KL/std": 3.307107448577881, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.1855926513671875, "fcm_dpo/delta": -0.1830754280090332, "fcm_dpo/margin": 3.083955764770508, "fcm_dpo/q_t": 0.37388285994529724, "grad_norm": 39.26893997192383, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.4948427975177765, "logits/rejected": -0.17321135103702545, "logps/chosen": -94.75367736816406, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -96.44172668457031, "loss": 0.9814, "margin_dpo/margin_mean": 3.0839552879333496, "margin_dpo/margin_std": 3.4820303916931152, "step": 80 }, { "KL/chosen_KL_mean": -0.38053131103515625, "KL/mean": -2.5421066284179688, "KL/rejected_KL_mean": -4.703678131103516, "KL/std": 4.357947826385498, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.1727968454360962, "fcm_dpo/delta": -0.3788316249847412, "fcm_dpo/margin": 4.323153972625732, "fcm_dpo/q_t": 0.3459482491016388, "grad_norm": 39.12853240966797, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.1797453910112381, "logits/rejected": 0.22782771289348602, "logps/chosen": -100.21065521240234, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -138.3761444091797, "loss": 0.9168, "margin_dpo/margin_mean": 4.323153972625732, "margin_dpo/margin_std": 5.065831184387207, "step": 81 }, { "KL/chosen_KL_mean": -1.3178672790527344, "KL/mean": -2.728947401046753, "KL/rejected_KL_mean": -4.140026092529297, "KL/std": 3.3369452953338623, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.169621080160141, "fcm_dpo/delta": -0.08294699341058731, "fcm_dpo/margin": 2.822159767150879, "fcm_dpo/q_t": 0.39432352781295776, "grad_norm": 39.14413833618164, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.011402931064367294, "logits/rejected": 0.1829192042350769, "logps/chosen": -84.36384582519531, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -98.66597747802734, "loss": 1.0546, "margin_dpo/margin_mean": 2.8221592903137207, "margin_dpo/margin_std": 3.7177491188049316, "step": 82 }, { "KL/chosen_KL_mean": -1.2796897888183594, "KL/mean": -3.0354580879211426, "KL/rejected_KL_mean": -4.791221618652344, "KL/std": 3.7940409183502197, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.16312646865844727, "fcm_dpo/delta": -0.18357455730438232, "fcm_dpo/margin": 3.511536121368408, "fcm_dpo/q_t": 0.37697410583496094, "grad_norm": 37.18784713745117, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.317645788192749, "logits/rejected": 0.07318335771560669, "logps/chosen": -93.45590209960938, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -112.5358657836914, "loss": 1.01, "margin_dpo/margin_mean": 3.511536121368408, "margin_dpo/margin_std": 4.554296970367432, "step": 83 }, { "KL/chosen_KL_mean": -1.7098541259765625, "KL/mean": -2.8803157806396484, "KL/rejected_KL_mean": -4.050777435302734, "KL/std": 3.382716178894043, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.16089318692684174, "fcm_dpo/delta": 0.022845551371574402, "fcm_dpo/margin": 2.3409225940704346, "fcm_dpo/q_t": 0.41750258207321167, "grad_norm": 40.779151916503906, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.11833585798740387, "logits/rejected": 0.2408415973186493, "logps/chosen": -103.23634338378906, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -96.47686767578125, "loss": 1.1283, "margin_dpo/margin_mean": 2.340923309326172, "margin_dpo/margin_std": 3.613403797149658, "step": 84 }, { "KL/chosen_KL_mean": -1.4580307006835938, "KL/mean": -3.009977340698242, "KL/rejected_KL_mean": -4.561923980712891, "KL/std": 3.8440990447998047, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.16076770424842834, "fcm_dpo/delta": -0.10416160523891449, "fcm_dpo/margin": 3.1039013862609863, "fcm_dpo/q_t": 0.3927052915096283, "grad_norm": 36.926841735839844, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.10821367055177689, "logits/rejected": 0.19384482502937317, "logps/chosen": -97.58540344238281, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -89.61711120605469, "loss": 1.0718, "margin_dpo/margin_mean": 3.103902578353882, "margin_dpo/margin_std": 4.639559745788574, "step": 85 }, { "KL/chosen_KL_mean": -1.2114067077636719, "KL/mean": -2.98703932762146, "KL/rejected_KL_mean": -4.7626800537109375, "KL/std": 4.421120643615723, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.15515217185020447, "fcm_dpo/delta": -0.16014575958251953, "fcm_dpo/margin": 3.551274538040161, "fcm_dpo/q_t": 0.38793906569480896, "grad_norm": 31.68059730529785, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.06573524326086044, "logits/rejected": 0.16811884939670563, "logps/chosen": -82.91566467285156, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -98.41818237304688, "loss": 1.0424, "margin_dpo/margin_mean": 3.551274538040161, "margin_dpo/margin_std": 5.3598456382751465, "step": 86 }, { "KL/chosen_KL_mean": -1.3051414489746094, "KL/mean": -3.2421624660491943, "KL/rejected_KL_mean": -5.179176330566406, "KL/std": 4.861509323120117, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.1493651121854782, "fcm_dpo/delta": -0.19046702980995178, "fcm_dpo/margin": 3.874040126800537, "fcm_dpo/q_t": 0.3761615753173828, "grad_norm": 33.96181869506836, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.3948681056499481, "logits/rejected": -0.04236302524805069, "logps/chosen": -93.99110412597656, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -108.09735107421875, "loss": 1.0396, "margin_dpo/margin_mean": 3.8740408420562744, "margin_dpo/margin_std": 5.702277183532715, "step": 87 }, { "KL/chosen_KL_mean": -1.7733688354492188, "KL/mean": -3.346452474594116, "KL/rejected_KL_mean": -4.919532775878906, "KL/std": 4.663633346557617, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.1460544764995575, "fcm_dpo/delta": -0.06362615525722504, "fcm_dpo/margin": 3.1461617946624756, "fcm_dpo/q_t": 0.39994990825653076, "grad_norm": 32.8818244934082, "learning_rate": 4.989335440737586e-07, "logits/chosen": 0.02905421331524849, "logits/rejected": 0.1973274201154709, "logps/chosen": -102.53634643554688, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -118.06990051269531, "loss": 1.0906, "margin_dpo/margin_mean": 3.1461617946624756, "margin_dpo/margin_std": 4.836788177490234, "step": 88 }, { "KL/chosen_KL_mean": -1.4034156799316406, "KL/mean": -3.12723445892334, "KL/rejected_KL_mean": -4.851047515869141, "KL/std": 3.998131036758423, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.14399567246437073, "fcm_dpo/delta": -0.10180637240409851, "fcm_dpo/margin": 3.447629451751709, "fcm_dpo/q_t": 0.3966999351978302, "grad_norm": 30.27601432800293, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.5799098014831543, "logits/rejected": -0.2749943137168884, "logps/chosen": -91.09449768066406, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -93.41938018798828, "loss": 1.0631, "margin_dpo/margin_mean": 3.447628974914551, "margin_dpo/margin_std": 5.332510948181152, "step": 89 }, { "KL/chosen_KL_mean": -1.2806625366210938, "KL/mean": -3.3695731163024902, "KL/rejected_KL_mean": -5.458484649658203, "KL/std": 4.5348100662231445, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.13989216089248657, "fcm_dpo/delta": -0.19548837840557098, "fcm_dpo/margin": 4.177820205688477, "fcm_dpo/q_t": 0.3743078410625458, "grad_norm": 33.04559326171875, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.17052769660949707, "logits/rejected": -0.05637408047914505, "logps/chosen": -82.66322326660156, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -123.23562622070312, "loss": 1.0026, "margin_dpo/margin_mean": 4.177820682525635, "margin_dpo/margin_std": 5.351275444030762, "step": 90 }, { "KL/chosen_KL_mean": -1.3122367858886719, "KL/mean": -3.30027437210083, "KL/rejected_KL_mean": -5.2883148193359375, "KL/std": 4.639225006103516, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.13533362746238708, "fcm_dpo/delta": -0.1457625925540924, "fcm_dpo/margin": 3.976078987121582, "fcm_dpo/q_t": 0.3836142420768738, "grad_norm": 29.33020782470703, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.0014007166028022766, "logits/rejected": 0.3956894874572754, "logps/chosen": -98.53411865234375, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -98.26506042480469, "loss": 1.0175, "margin_dpo/margin_mean": 3.976078748703003, "margin_dpo/margin_std": 5.078838348388672, "step": 91 }, { "KL/chosen_KL_mean": -1.7808036804199219, "KL/mean": -3.4234108924865723, "KL/rejected_KL_mean": -5.066017150878906, "KL/std": 4.5464863777160645, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.1334390938282013, "fcm_dpo/delta": -0.04011496528983116, "fcm_dpo/margin": 3.2852063179016113, "fcm_dpo/q_t": 0.404048353433609, "grad_norm": 29.111675262451172, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.5235368609428406, "logits/rejected": -0.17867718636989594, "logps/chosen": -86.31108093261719, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -89.56733703613281, "loss": 1.0871, "margin_dpo/margin_mean": 3.2852065563201904, "margin_dpo/margin_std": 4.850127220153809, "step": 92 }, { "KL/chosen_KL_mean": -1.7836685180664062, "KL/mean": -3.473155975341797, "KL/rejected_KL_mean": -5.162651062011719, "KL/std": 4.385559558868408, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.13215720653533936, "fcm_dpo/delta": -0.048708368092775345, "fcm_dpo/margin": 3.378983497619629, "fcm_dpo/q_t": 0.4026646018028259, "grad_norm": 30.227449417114258, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7514551877975464, "logits/rejected": -0.22832342982292175, "logps/chosen": -92.03410339355469, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -89.25686645507812, "loss": 1.1015, "margin_dpo/margin_mean": 3.378983497619629, "margin_dpo/margin_std": 5.339705467224121, "step": 93 }, { "KL/chosen_KL_mean": -1.7065467834472656, "KL/mean": -4.178703308105469, "KL/rejected_KL_mean": -6.6508636474609375, "KL/std": 5.104975700378418, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.12592417001724243, "fcm_dpo/delta": -0.2413911074399948, "fcm_dpo/margin": 4.9443182945251465, "fcm_dpo/q_t": 0.3633890450000763, "grad_norm": 29.327829360961914, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.7633987665176392, "logits/rejected": -0.5015383958816528, "logps/chosen": -94.1607666015625, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -111.89815521240234, "loss": 0.9754, "margin_dpo/margin_mean": 4.94431734085083, "margin_dpo/margin_std": 5.780431747436523, "step": 94 }, { "KL/chosen_KL_mean": -1.5247230529785156, "KL/mean": -3.481442451477051, "KL/rejected_KL_mean": -5.4381561279296875, "KL/std": 4.301259517669678, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.12433496862649918, "fcm_dpo/delta": -0.09096968919038773, "fcm_dpo/margin": 3.9134392738342285, "fcm_dpo/q_t": 0.39438313245773315, "grad_norm": 26.467172622680664, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.6517207622528076, "logits/rejected": -0.34143584966659546, "logps/chosen": -80.39842987060547, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -89.02937316894531, "loss": 1.0524, "margin_dpo/margin_mean": 3.9134392738342285, "margin_dpo/margin_std": 5.249029159545898, "step": 95 }, { "KL/chosen_KL_mean": -1.1056632995605469, "KL/mean": -3.5460045337677, "KL/rejected_KL_mean": -5.986351013183594, "KL/std": 5.311269760131836, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.11951266974210739, "fcm_dpo/delta": -0.19510892033576965, "fcm_dpo/margin": 4.8806962966918945, "fcm_dpo/q_t": 0.37236925959587097, "grad_norm": 27.03703498840332, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.5000447630882263, "logits/rejected": -0.09153123199939728, "logps/chosen": -107.69783020019531, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -107.75437927246094, "loss": 0.991, "margin_dpo/margin_mean": 4.880697250366211, "margin_dpo/margin_std": 5.809417724609375, "step": 96 }, { "KL/chosen_KL_mean": -1.9818344116210938, "KL/mean": -4.232769966125488, "KL/rejected_KL_mean": -6.48370361328125, "KL/std": 4.468520164489746, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.11700461059808731, "fcm_dpo/delta": -0.13381992280483246, "fcm_dpo/margin": 4.50186824798584, "fcm_dpo/q_t": 0.38061243295669556, "grad_norm": 27.082752227783203, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.41609373688697815, "logits/rejected": -0.0420740470290184, "logps/chosen": -90.66171264648438, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -91.2959976196289, "loss": 0.9991, "margin_dpo/margin_mean": 4.501868724822998, "margin_dpo/margin_std": 4.681353569030762, "step": 97 }, { "KL/chosen_KL_mean": -1.3799285888671875, "KL/mean": -3.83404541015625, "KL/rejected_KL_mean": -6.2881622314453125, "KL/std": 4.554370403289795, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.11211557686328888, "fcm_dpo/delta": -0.16040676832199097, "fcm_dpo/margin": 4.908233165740967, "fcm_dpo/q_t": 0.3760828673839569, "grad_norm": 29.319568634033203, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.5719818472862244, "logits/rejected": -0.33111733198165894, "logps/chosen": -93.62242126464844, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -107.80764770507812, "loss": 0.9844, "margin_dpo/margin_mean": 4.908232688903809, "margin_dpo/margin_std": 5.150406837463379, "step": 98 }, { "KL/chosen_KL_mean": -1.1330108642578125, "KL/mean": -3.7477898597717285, "KL/rejected_KL_mean": -6.362567901611328, "KL/std": 5.2592854499816895, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.10875675082206726, "fcm_dpo/delta": -0.17906907200813293, "fcm_dpo/margin": 5.229559898376465, "fcm_dpo/q_t": 0.37865716218948364, "grad_norm": 26.637569427490234, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.4274008870124817, "logits/rejected": -0.22159242630004883, "logps/chosen": -73.31765747070312, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -98.24388885498047, "loss": 0.999, "margin_dpo/margin_mean": 5.229560852050781, "margin_dpo/margin_std": 6.6018385887146, "step": 99 }, { "KL/chosen_KL_mean": -1.2738533020019531, "KL/mean": -3.369349241256714, "KL/rejected_KL_mean": -5.464847564697266, "KL/std": 4.277243137359619, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.10662467777729034, "fcm_dpo/delta": -0.049560971558094025, "fcm_dpo/margin": 4.1909894943237305, "fcm_dpo/q_t": 0.39700251817703247, "grad_norm": 23.12835121154785, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.7513393759727478, "logits/rejected": -0.327360063791275, "logps/chosen": -90.78768157958984, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -86.68197631835938, "loss": 1.0423, "margin_dpo/margin_mean": 4.190989017486572, "margin_dpo/margin_std": 4.545284271240234, "step": 100 }, { "KL/chosen_KL_mean": -2.3042755126953125, "KL/mean": -4.12132453918457, "KL/rejected_KL_mean": -5.938377380371094, "KL/std": 5.186249732971191, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.10652109980583191, "fcm_dpo/delta": 0.01308918371796608, "fcm_dpo/margin": 3.6340975761413574, "fcm_dpo/q_t": 0.4125511646270752, "grad_norm": 26.41726303100586, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.5897508859634399, "logits/rejected": -0.3461223244667053, "logps/chosen": -96.70747375488281, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -95.848388671875, "loss": 1.1215, "margin_dpo/margin_mean": 3.6340975761413574, "margin_dpo/margin_std": 5.677130699157715, "step": 101 }, { "KL/chosen_KL_mean": -2.793621063232422, "KL/mean": -4.307377815246582, "KL/rejected_KL_mean": -5.821128845214844, "KL/std": 4.955012321472168, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.10910448431968689, "fcm_dpo/delta": 0.06997931748628616, "fcm_dpo/margin": 3.0275115966796875, "fcm_dpo/q_t": 0.42428526282310486, "grad_norm": 32.145565032958984, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.5195819139480591, "logits/rejected": -0.17109109461307526, "logps/chosen": -94.40304565429688, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -81.59004974365234, "loss": 1.1709, "margin_dpo/margin_mean": 3.0275120735168457, "margin_dpo/margin_std": 5.498409271240234, "step": 102 }, { "KL/chosen_KL_mean": -1.9418563842773438, "KL/mean": -4.5009660720825195, "KL/rejected_KL_mean": -7.0600738525390625, "KL/std": 5.263975620269775, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.10599090158939362, "fcm_dpo/delta": -0.15093779563903809, "fcm_dpo/margin": 5.11821985244751, "fcm_dpo/q_t": 0.3771648406982422, "grad_norm": 26.164382934570312, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6055498123168945, "logits/rejected": -0.2501668632030487, "logps/chosen": -99.34423828125, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -106.04176330566406, "loss": 1.0049, "margin_dpo/margin_mean": 5.118220329284668, "margin_dpo/margin_std": 5.8923797607421875, "step": 103 }, { "KL/chosen_KL_mean": -1.2771339416503906, "KL/mean": -4.144305229187012, "KL/rejected_KL_mean": -7.011478424072266, "KL/std": 5.997805118560791, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.10222253948450089, "fcm_dpo/delta": -0.1979016363620758, "fcm_dpo/margin": 5.73433780670166, "fcm_dpo/q_t": 0.3731772303581238, "grad_norm": 22.169147491455078, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.7495756149291992, "logits/rejected": -0.4333510994911194, "logps/chosen": -96.13349914550781, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -104.03817749023438, "loss": 0.9827, "margin_dpo/margin_mean": 5.734338760375977, "margin_dpo/margin_std": 6.909714698791504, "step": 104 }, { "KL/chosen_KL_mean": -1.8797454833984375, "KL/mean": -4.604125022888184, "KL/rejected_KL_mean": -7.3285064697265625, "KL/std": 5.272136211395264, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.0994778424501419, "fcm_dpo/delta": -0.14991632103919983, "fcm_dpo/margin": 5.448761463165283, "fcm_dpo/q_t": 0.3786480128765106, "grad_norm": 25.41191291809082, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.6824108362197876, "logits/rejected": -0.23769709467887878, "logps/chosen": -98.63764953613281, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -108.90599822998047, "loss": 1.012, "margin_dpo/margin_mean": 5.448760986328125, "margin_dpo/margin_std": 6.593177795410156, "step": 105 }, { "KL/chosen_KL_mean": -2.0093040466308594, "KL/mean": -4.730899810791016, "KL/rejected_KL_mean": -7.452487945556641, "KL/std": 5.623400688171387, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.09672196209430695, "fcm_dpo/delta": -0.13330073654651642, "fcm_dpo/margin": 5.443181037902832, "fcm_dpo/q_t": 0.38387152552604675, "grad_norm": 22.1876277923584, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.49273666739463806, "logits/rejected": -0.07957983016967773, "logps/chosen": -94.44721221923828, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -98.46218872070312, "loss": 1.0153, "margin_dpo/margin_mean": 5.443180561065674, "margin_dpo/margin_std": 6.573175430297852, "step": 106 }, { "KL/chosen_KL_mean": -1.2358551025390625, "KL/mean": -4.335156440734863, "KL/rejected_KL_mean": -7.434467315673828, "KL/std": 7.07717227935791, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.09306496381759644, "fcm_dpo/delta": -0.18765917420387268, "fcm_dpo/margin": 6.198612213134766, "fcm_dpo/q_t": 0.381542444229126, "grad_norm": 22.526670455932617, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.390575110912323, "logits/rejected": 0.10912655293941498, "logps/chosen": -106.3608627319336, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -108.6169662475586, "loss": 1.0146, "margin_dpo/margin_mean": 6.198612213134766, "margin_dpo/margin_std": 8.894733428955078, "step": 107 }, { "KL/chosen_KL_mean": -2.186359405517578, "KL/mean": -5.026329040527344, "KL/rejected_KL_mean": -7.866302490234375, "KL/std": 6.250826835632324, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.0901256576180458, "fcm_dpo/delta": -0.11888940632343292, "fcm_dpo/margin": 5.679939270019531, "fcm_dpo/q_t": 0.38584500551223755, "grad_norm": 22.399633407592773, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.7705433368682861, "logits/rejected": -0.27751266956329346, "logps/chosen": -91.39572143554688, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -109.70414733886719, "loss": 1.0224, "margin_dpo/margin_mean": 5.679939270019531, "margin_dpo/margin_std": 6.741147994995117, "step": 108 }, { "KL/chosen_KL_mean": -1.5366668701171875, "KL/mean": -5.193110466003418, "KL/rejected_KL_mean": -8.849559783935547, "KL/std": 6.799964904785156, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.08651172369718552, "fcm_dpo/delta": -0.24907812476158142, "fcm_dpo/margin": 7.312881946563721, "fcm_dpo/q_t": 0.36125147342681885, "grad_norm": 22.390043258666992, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.3925793170928955, "logits/rejected": -0.1817954182624817, "logps/chosen": -101.7237548828125, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -112.38695526123047, "loss": 0.956, "margin_dpo/margin_mean": 7.3128814697265625, "margin_dpo/margin_std": 8.220879554748535, "step": 109 }, { "KL/chosen_KL_mean": -2.1777801513671875, "KL/mean": -4.449130535125732, "KL/rejected_KL_mean": -6.720485687255859, "KL/std": 5.430877685546875, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.0854112058877945, "fcm_dpo/delta": 0.01219184696674347, "fcm_dpo/margin": 4.542699813842773, "fcm_dpo/q_t": 0.4113576412200928, "grad_norm": 21.119901657104492, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.14766891300678253, "logits/rejected": 0.22106751799583435, "logps/chosen": -109.07350158691406, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -92.53169250488281, "loss": 1.1015, "margin_dpo/margin_mean": 4.542698860168457, "margin_dpo/margin_std": 6.179601669311523, "step": 110 }, { "KL/chosen_KL_mean": -1.9281349182128906, "KL/mean": -6.100839614868164, "KL/rejected_KL_mean": -10.273544311523438, "KL/std": 7.813878059387207, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.08154663443565369, "fcm_dpo/delta": -0.30362704396247864, "fcm_dpo/margin": 8.345413208007812, "fcm_dpo/q_t": 0.3493887782096863, "grad_norm": 21.59543800354004, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.8737378120422363, "logits/rejected": -0.6038230657577515, "logps/chosen": -95.25750732421875, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -114.45103454589844, "loss": 0.9175, "margin_dpo/margin_mean": 8.345413208007812, "margin_dpo/margin_std": 8.296663284301758, "step": 111 }, { "KL/chosen_KL_mean": -2.202289581298828, "KL/mean": -4.474692344665527, "KL/rejected_KL_mean": -6.747093200683594, "KL/std": 4.486100196838379, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.08107158541679382, "fcm_dpo/delta": 0.03276441991329193, "fcm_dpo/margin": 4.544804096221924, "fcm_dpo/q_t": 0.41478365659713745, "grad_norm": 18.852495193481445, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.0286369323730469, "logits/rejected": -0.5779541730880737, "logps/chosen": -84.94933319091797, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -78.3055419921875, "loss": 1.1044, "margin_dpo/margin_mean": 4.544805526733398, "margin_dpo/margin_std": 5.866359710693359, "step": 112 }, { "KL/chosen_KL_mean": -3.0111961364746094, "KL/mean": -5.413173198699951, "KL/rejected_KL_mean": -7.815151214599609, "KL/std": 5.888339996337891, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.08146416395902634, "fcm_dpo/delta": 0.0089817363768816, "fcm_dpo/margin": 4.803958892822266, "fcm_dpo/q_t": 0.4096149206161499, "grad_norm": 22.46959686279297, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.38046419620513916, "logits/rejected": 0.15483920276165009, "logps/chosen": -100.91069030761719, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -92.23474884033203, "loss": 1.0899, "margin_dpo/margin_mean": 4.803958415985107, "margin_dpo/margin_std": 5.769773483276367, "step": 113 }, { "KL/chosen_KL_mean": -2.8191795349121094, "KL/mean": -6.090146064758301, "KL/rejected_KL_mean": -9.36111068725586, "KL/std": 6.9907331466674805, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.07963542640209198, "fcm_dpo/delta": -0.12826590240001678, "fcm_dpo/margin": 6.541933059692383, "fcm_dpo/q_t": 0.3875223696231842, "grad_norm": 21.95738983154297, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.814149022102356, "logits/rejected": -0.2599540948867798, "logps/chosen": -99.31430053710938, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -101.37647247314453, "loss": 1.042, "margin_dpo/margin_mean": 6.541933059692383, "margin_dpo/margin_std": 8.9114990234375, "step": 114 }, { "KL/chosen_KL_mean": -2.7341690063476562, "KL/mean": -6.13348913192749, "KL/rejected_KL_mean": -9.532806396484375, "KL/std": 7.43113374710083, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.07781418412923813, "fcm_dpo/delta": -0.13619017601013184, "fcm_dpo/margin": 6.798635482788086, "fcm_dpo/q_t": 0.38176536560058594, "grad_norm": 19.473194122314453, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.7055402994155884, "logits/rejected": -0.3191353976726532, "logps/chosen": -105.91429138183594, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -115.24876403808594, "loss": 1.0105, "margin_dpo/margin_mean": 6.798635482788086, "margin_dpo/margin_std": 7.930050373077393, "step": 115 }, { "KL/chosen_KL_mean": -2.565032958984375, "KL/mean": -5.362490653991699, "KL/rejected_KL_mean": -8.15994644165039, "KL/std": 5.806205749511719, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.07709239423274994, "fcm_dpo/delta": -0.03281797468662262, "fcm_dpo/margin": 5.594911575317383, "fcm_dpo/q_t": 0.40151000022888184, "grad_norm": 18.70231056213379, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.7291231155395508, "logits/rejected": -0.3037228584289551, "logps/chosen": -83.72340393066406, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -94.46580505371094, "loss": 1.0646, "margin_dpo/margin_mean": 5.594911575317383, "margin_dpo/margin_std": 6.781277656555176, "step": 116 }, { "KL/chosen_KL_mean": -3.2166824340820312, "KL/mean": -5.622782230377197, "KL/rejected_KL_mean": -8.028881072998047, "KL/std": 5.939235687255859, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.07664661109447479, "fcm_dpo/delta": 0.03182479366660118, "fcm_dpo/margin": 4.812201023101807, "fcm_dpo/q_t": 0.4151250123977661, "grad_norm": 23.864423751831055, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.6188455820083618, "logits/rejected": -0.24148431420326233, "logps/chosen": -100.29763793945312, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -83.16879272460938, "loss": 1.1199, "margin_dpo/margin_mean": 4.812200546264648, "margin_dpo/margin_std": 6.941383361816406, "step": 117 }, { "KL/chosen_KL_mean": -2.1093788146972656, "KL/mean": -5.453375816345215, "KL/rejected_KL_mean": -8.797367095947266, "KL/std": 6.573777675628662, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.07581539452075958, "fcm_dpo/delta": -0.11301077157258987, "fcm_dpo/margin": 6.687990188598633, "fcm_dpo/q_t": 0.3824300467967987, "grad_norm": 18.465333938598633, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.9562749862670898, "logits/rejected": -0.6379966735839844, "logps/chosen": -88.64561462402344, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -91.7330093383789, "loss": 1.0033, "margin_dpo/margin_mean": 6.687989234924316, "margin_dpo/margin_std": 6.830928802490234, "step": 118 }, { "KL/chosen_KL_mean": -3.4900360107421875, "KL/mean": -6.8644208908081055, "KL/rejected_KL_mean": -10.23880386352539, "KL/std": 7.6410298347473145, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.07414962351322174, "fcm_dpo/delta": -0.10605673491954803, "fcm_dpo/margin": 6.748763561248779, "fcm_dpo/q_t": 0.3870101273059845, "grad_norm": 20.348169326782227, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.051131248474121, "logits/rejected": -0.7526921033859253, "logps/chosen": -108.09912109375, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -119.39661407470703, "loss": 1.0539, "margin_dpo/margin_mean": 6.7487640380859375, "margin_dpo/margin_std": 9.330079078674316, "step": 119 }, { "KL/chosen_KL_mean": -2.488780975341797, "KL/mean": -7.121270179748535, "KL/rejected_KL_mean": -11.753761291503906, "KL/std": 9.626593589782715, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.07093938440084457, "fcm_dpo/delta": -0.27544552087783813, "fcm_dpo/margin": 9.264982223510742, "fcm_dpo/q_t": 0.3612830936908722, "grad_norm": 22.36029815673828, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.121903896331787, "logits/rejected": -0.5992637276649475, "logps/chosen": -117.17620849609375, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -122.93962860107422, "loss": 0.9823, "margin_dpo/margin_mean": 9.264983177185059, "margin_dpo/margin_std": 11.98548412322998, "step": 120 }, { "KL/chosen_KL_mean": -2.350688934326172, "KL/mean": -6.425556182861328, "KL/rejected_KL_mean": -10.500431060791016, "KL/std": 7.3963775634765625, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.06800815463066101, "fcm_dpo/delta": -0.1642262190580368, "fcm_dpo/margin": 8.149746894836426, "fcm_dpo/q_t": 0.37356841564178467, "grad_norm": 20.85309600830078, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.9912875890731812, "logits/rejected": -0.6719098091125488, "logps/chosen": -94.46734619140625, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -125.47332763671875, "loss": 0.9753, "margin_dpo/margin_mean": 8.149746894836426, "margin_dpo/margin_std": 8.16580867767334, "step": 121 }, { "KL/chosen_KL_mean": -2.729217529296875, "KL/mean": -7.351600646972656, "KL/rejected_KL_mean": -11.973983764648438, "KL/std": 8.095207214355469, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.06574675440788269, "fcm_dpo/delta": -0.22074288129806519, "fcm_dpo/margin": 9.244762420654297, "fcm_dpo/q_t": 0.3615250587463379, "grad_norm": 21.684171676635742, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.873640775680542, "logits/rejected": -0.7553948760032654, "logps/chosen": -87.6067886352539, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -123.05696868896484, "loss": 0.9522, "margin_dpo/margin_mean": 9.24476146697998, "margin_dpo/margin_std": 9.237771987915039, "step": 122 }, { "KL/chosen_KL_mean": -3.208141326904297, "KL/mean": -6.641761302947998, "KL/rejected_KL_mean": -10.075386047363281, "KL/std": 7.525877952575684, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.06396058946847916, "fcm_dpo/delta": -0.04211597889661789, "fcm_dpo/margin": 6.867247581481934, "fcm_dpo/q_t": 0.39957892894744873, "grad_norm": 17.838777542114258, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.8809325695037842, "logits/rejected": -0.5670456886291504, "logps/chosen": -101.6868896484375, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -103.9295654296875, "loss": 1.0642, "margin_dpo/margin_mean": 6.867247581481934, "margin_dpo/margin_std": 8.446205139160156, "step": 123 }, { "KL/chosen_KL_mean": -3.0213241577148438, "KL/mean": -7.201870918273926, "KL/rejected_KL_mean": -11.38241958618164, "KL/std": 9.452424049377441, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.06300877034664154, "fcm_dpo/delta": -0.13378997147083282, "fcm_dpo/margin": 8.361099243164062, "fcm_dpo/q_t": 0.38953661918640137, "grad_norm": 17.68807029724121, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.7758918404579163, "logits/rejected": -0.5132349729537964, "logps/chosen": -105.78422546386719, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -134.38803100585938, "loss": 1.0478, "margin_dpo/margin_mean": 8.361098289489746, "margin_dpo/margin_std": 12.142789840698242, "step": 124 }, { "KL/chosen_KL_mean": -4.513134002685547, "KL/mean": -7.6137518882751465, "KL/rejected_KL_mean": -10.714374542236328, "KL/std": 7.651758670806885, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.06261995434761047, "fcm_dpo/delta": 0.012152507901191711, "fcm_dpo/margin": 6.201233863830566, "fcm_dpo/q_t": 0.41248536109924316, "grad_norm": 18.01238250732422, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.7356467843055725, "logits/rejected": -0.5302629470825195, "logps/chosen": -94.93940734863281, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -99.96339416503906, "loss": 1.1147, "margin_dpo/margin_mean": 6.201234340667725, "margin_dpo/margin_std": 9.3585205078125, "step": 125 }, { "KL/chosen_KL_mean": -3.5181350708007812, "KL/mean": -8.04487419128418, "KL/rejected_KL_mean": -12.571617126464844, "KL/std": 8.39359188079834, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.06122594326734543, "fcm_dpo/delta": -0.16325898468494415, "fcm_dpo/margin": 9.053478240966797, "fcm_dpo/q_t": 0.3715727925300598, "grad_norm": 17.978532791137695, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.7726534008979797, "logits/rejected": -0.500705897808075, "logps/chosen": -97.9410171508789, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -116.45317077636719, "loss": 0.9706, "margin_dpo/margin_mean": 9.053478240966797, "margin_dpo/margin_std": 8.621442794799805, "step": 126 }, { "KL/chosen_KL_mean": -3.5154190063476562, "KL/mean": -7.018926620483398, "KL/rejected_KL_mean": -10.522430419921875, "KL/std": 7.861366271972656, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.060631364583969116, "fcm_dpo/delta": -0.02607840672135353, "fcm_dpo/margin": 7.007016658782959, "fcm_dpo/q_t": 0.404852956533432, "grad_norm": 17.444499969482422, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.6748431921005249, "logits/rejected": -0.33510860800743103, "logps/chosen": -88.26647186279297, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -94.06766510009766, "loss": 1.0866, "margin_dpo/margin_mean": 7.007017612457275, "margin_dpo/margin_std": 9.819450378417969, "step": 127 }, { "KL/chosen_KL_mean": -3.534942626953125, "KL/mean": -7.105337142944336, "KL/rejected_KL_mean": -10.675729751586914, "KL/std": 9.607112884521484, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.06029324606060982, "fcm_dpo/delta": -0.03208436071872711, "fcm_dpo/margin": 7.140791893005371, "fcm_dpo/q_t": 0.40204960107803345, "grad_norm": 19.555091857910156, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5823308229446411, "logits/rejected": -0.1222853884100914, "logps/chosen": -94.15458679199219, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -93.77920532226562, "loss": 1.1039, "margin_dpo/margin_mean": 7.140790939331055, "margin_dpo/margin_std": 11.472566604614258, "step": 128 }, { "KL/chosen_KL_mean": -4.009052276611328, "KL/mean": -7.715317726135254, "KL/rejected_KL_mean": -11.421588897705078, "KL/std": 7.93605899810791, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.059745386242866516, "fcm_dpo/delta": -0.044897668063640594, "fcm_dpo/margin": 7.412540912628174, "fcm_dpo/q_t": 0.4029378890991211, "grad_norm": 15.974428176879883, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.5583959817886353, "logits/rejected": -0.2535550594329834, "logps/chosen": -102.92776489257812, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -114.70578002929688, "loss": 1.0831, "margin_dpo/margin_mean": 7.412540435791016, "margin_dpo/margin_std": 10.480842590332031, "step": 129 }, { "KL/chosen_KL_mean": -3.107044219970703, "KL/mean": -7.962728500366211, "KL/rejected_KL_mean": -12.818416595458984, "KL/std": 9.578262329101562, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.0583450049161911, "fcm_dpo/delta": -0.176468625664711, "fcm_dpo/margin": 9.711376190185547, "fcm_dpo/q_t": 0.3758392632007599, "grad_norm": 15.738917350769043, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.043044090270996, "logits/rejected": -0.7934824824333191, "logps/chosen": -76.29073333740234, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -100.83995819091797, "loss": 0.9959, "margin_dpo/margin_mean": 9.71137523651123, "margin_dpo/margin_std": 11.118294715881348, "step": 130 }, { "KL/chosen_KL_mean": -2.6394577026367188, "KL/mean": -6.865579605102539, "KL/rejected_KL_mean": -11.091705322265625, "KL/std": 9.29486083984375, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.056995637714862823, "fcm_dpo/delta": -0.08613233268260956, "fcm_dpo/margin": 8.452242851257324, "fcm_dpo/q_t": 0.39169618487358093, "grad_norm": 17.735851287841797, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.737106442451477, "logits/rejected": -0.5157066583633423, "logps/chosen": -76.59590148925781, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -105.54753112792969, "loss": 1.0459, "margin_dpo/margin_mean": 8.452242851257324, "margin_dpo/margin_std": 10.481077194213867, "step": 131 }, { "KL/chosen_KL_mean": -4.033527374267578, "KL/mean": -8.433937072753906, "KL/rejected_KL_mean": -12.834342956542969, "KL/std": 9.795948028564453, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.05537623167037964, "fcm_dpo/delta": -0.09262896329164505, "fcm_dpo/margin": 8.800811767578125, "fcm_dpo/q_t": 0.38873109221458435, "grad_norm": 17.487218856811523, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.0891984701156616, "logits/rejected": -0.6797777414321899, "logps/chosen": -96.47300720214844, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -106.96685791015625, "loss": 1.0391, "margin_dpo/margin_mean": 8.800812721252441, "margin_dpo/margin_std": 10.835079193115234, "step": 132 }, { "KL/chosen_KL_mean": -3.1824378967285156, "KL/mean": -7.792255878448486, "KL/rejected_KL_mean": -12.40207290649414, "KL/std": 9.933971405029297, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.05454154685139656, "fcm_dpo/delta": -0.10819597542285919, "fcm_dpo/margin": 9.219639778137207, "fcm_dpo/q_t": 0.3863362669944763, "grad_norm": 16.34535789489746, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.3413530588150024, "logits/rejected": -0.9519087076187134, "logps/chosen": -96.29910278320312, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -114.49698638916016, "loss": 1.0173, "margin_dpo/margin_mean": 9.219639778137207, "margin_dpo/margin_std": 10.403617858886719, "step": 133 }, { "KL/chosen_KL_mean": -3.6070632934570312, "KL/mean": -8.57982063293457, "KL/rejected_KL_mean": -13.55258560180664, "KL/std": 9.596085548400879, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.05341683700680733, "fcm_dpo/delta": -0.13858915865421295, "fcm_dpo/margin": 9.945518493652344, "fcm_dpo/q_t": 0.37848085165023804, "grad_norm": 15.363247871398926, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.5490412712097168, "logits/rejected": -1.2264153957366943, "logps/chosen": -87.15718841552734, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -107.43415832519531, "loss": 0.9962, "margin_dpo/margin_mean": 9.945518493652344, "margin_dpo/margin_std": 10.21273422241211, "step": 134 }, { "KL/chosen_KL_mean": -3.3977317810058594, "KL/mean": -8.151771545410156, "KL/rejected_KL_mean": -12.905815124511719, "KL/std": 10.50531005859375, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.052067242562770844, "fcm_dpo/delta": -0.09989391267299652, "fcm_dpo/margin": 9.508074760437012, "fcm_dpo/q_t": 0.39062702655792236, "grad_norm": 20.39605140686035, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.1915431022644043, "logits/rejected": -1.050886631011963, "logps/chosen": -80.98036193847656, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -133.96229553222656, "loss": 1.0349, "margin_dpo/margin_mean": 9.508075714111328, "margin_dpo/margin_std": 11.69715690612793, "step": 135 }, { "KL/chosen_KL_mean": -3.8504867553710938, "KL/mean": -8.716668128967285, "KL/rejected_KL_mean": -13.582847595214844, "KL/std": 10.540152549743652, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.05084153637290001, "fcm_dpo/delta": -0.09984844923019409, "fcm_dpo/margin": 9.732364654541016, "fcm_dpo/q_t": 0.38632363080978394, "grad_norm": 14.9500093460083, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.4090144634246826, "logits/rejected": -0.9789310693740845, "logps/chosen": -98.02322387695312, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -116.84713745117188, "loss": 1.0127, "margin_dpo/margin_mean": 9.732364654541016, "margin_dpo/margin_std": 10.298114776611328, "step": 136 }, { "KL/chosen_KL_mean": -2.9984588623046875, "KL/mean": -10.609781265258789, "KL/rejected_KL_mean": -18.221107482910156, "KL/std": 13.540325164794922, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.047711655497550964, "fcm_dpo/delta": -0.3551957607269287, "fcm_dpo/margin": 15.222648620605469, "fcm_dpo/q_t": 0.340126633644104, "grad_norm": 18.036914825439453, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.564115285873413, "logits/rejected": -1.338576078414917, "logps/chosen": -81.11720275878906, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -136.1434783935547, "loss": 0.8838, "margin_dpo/margin_mean": 15.222648620605469, "margin_dpo/margin_std": 14.539010047912598, "step": 137 }, { "KL/chosen_KL_mean": -5.4915008544921875, "KL/mean": -10.251014709472656, "KL/rejected_KL_mean": -15.01052474975586, "KL/std": 10.196353912353516, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.046726107597351074, "fcm_dpo/delta": -0.04706428945064545, "fcm_dpo/margin": 9.519028663635254, "fcm_dpo/q_t": 0.3982170820236206, "grad_norm": 13.86099910736084, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.6961441040039062, "logits/rejected": -0.4316865801811218, "logps/chosen": -98.32144165039062, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -113.65892791748047, "loss": 1.0538, "margin_dpo/margin_mean": 9.51902961730957, "margin_dpo/margin_std": 11.278629302978516, "step": 138 }, { "KL/chosen_KL_mean": -3.66119384765625, "KL/mean": -9.192374229431152, "KL/rejected_KL_mean": -14.723556518554688, "KL/std": 11.11378002166748, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.045831307768821716, "fcm_dpo/delta": -0.1130748838186264, "fcm_dpo/margin": 11.06236743927002, "fcm_dpo/q_t": 0.38210391998291016, "grad_norm": 14.037359237670898, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.2379310131072998, "logits/rejected": -0.9135892987251282, "logps/chosen": -82.6642074584961, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -100.55088806152344, "loss": 1.0203, "margin_dpo/margin_mean": 11.06236743927002, "margin_dpo/margin_std": 12.902892112731934, "step": 139 }, { "KL/chosen_KL_mean": -4.105243682861328, "KL/mean": -9.12034797668457, "KL/rejected_KL_mean": -14.135448455810547, "KL/std": 10.56110668182373, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.0453222431242466, "fcm_dpo/delta": -0.05715665966272354, "fcm_dpo/margin": 10.030206680297852, "fcm_dpo/q_t": 0.39480721950531006, "grad_norm": 13.817020416259766, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.4657379388809204, "logits/rejected": -1.0628128051757812, "logps/chosen": -92.45768737792969, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -104.71022033691406, "loss": 1.0413, "margin_dpo/margin_mean": 10.030208587646484, "margin_dpo/margin_std": 11.110319137573242, "step": 140 }, { "KL/chosen_KL_mean": -5.551914215087891, "KL/mean": -11.350290298461914, "KL/rejected_KL_mean": -17.148662567138672, "KL/std": 13.816038131713867, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.044358886778354645, "fcm_dpo/delta": -0.12059544026851654, "fcm_dpo/margin": 11.596753120422363, "fcm_dpo/q_t": 0.38279837369918823, "grad_norm": 16.192533493041992, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.2148659229278564, "logits/rejected": -0.8349431753158569, "logps/chosen": -112.77207946777344, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -121.86089324951172, "loss": 1.0226, "margin_dpo/margin_mean": 11.596752166748047, "margin_dpo/margin_std": 13.970876693725586, "step": 141 }, { "KL/chosen_KL_mean": -6.009120941162109, "KL/mean": -11.613380432128906, "KL/rejected_KL_mean": -17.217639923095703, "KL/std": 12.881781578063965, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.043391939252614975, "fcm_dpo/delta": -0.091046042740345, "fcm_dpo/margin": 11.208529472351074, "fcm_dpo/q_t": 0.39082902669906616, "grad_norm": 13.7327241897583, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.877151608467102, "logits/rejected": -0.6097778677940369, "logps/chosen": -94.73806762695312, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -116.94802856445312, "loss": 1.0292, "margin_dpo/margin_mean": 11.208528518676758, "margin_dpo/margin_std": 13.262828826904297, "step": 142 }, { "KL/chosen_KL_mean": -4.132106781005859, "KL/mean": -8.855437278747559, "KL/rejected_KL_mean": -13.57876205444336, "KL/std": 10.926315307617188, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.04304719716310501, "fcm_dpo/delta": -0.007336262613534927, "fcm_dpo/margin": 9.446649551391602, "fcm_dpo/q_t": 0.4065685272216797, "grad_norm": 13.388407707214355, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.500669240951538, "logits/rejected": -1.0477310419082642, "logps/chosen": -89.63668823242188, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -99.3844223022461, "loss": 1.0842, "margin_dpo/margin_mean": 9.446649551391602, "margin_dpo/margin_std": 12.073646545410156, "step": 143 }, { "KL/chosen_KL_mean": -5.027561187744141, "KL/mean": -9.779726028442383, "KL/rejected_KL_mean": -14.53189468383789, "KL/std": 11.401511192321777, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.0433925986289978, "fcm_dpo/delta": -0.013632982969284058, "fcm_dpo/margin": 9.504331588745117, "fcm_dpo/q_t": 0.40702760219573975, "grad_norm": 17.350440979003906, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.3056565523147583, "logits/rejected": -0.7676533460617065, "logps/chosen": -109.71465301513672, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -102.2509765625, "loss": 1.1042, "margin_dpo/margin_mean": 9.504332542419434, "margin_dpo/margin_std": 14.146889686584473, "step": 144 }, { "KL/chosen_KL_mean": -4.314197540283203, "KL/mean": -9.799956321716309, "KL/rejected_KL_mean": -15.285717010498047, "KL/std": 10.845542907714844, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.042627573013305664, "fcm_dpo/delta": -0.07104075700044632, "fcm_dpo/margin": 10.97152042388916, "fcm_dpo/q_t": 0.3918903172016144, "grad_norm": 13.880906105041504, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.8241848349571228, "logits/rejected": -0.22224263846874237, "logps/chosen": -92.83409118652344, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -99.05085754394531, "loss": 1.0343, "margin_dpo/margin_mean": 10.971521377563477, "margin_dpo/margin_std": 12.20930290222168, "step": 145 }, { "KL/chosen_KL_mean": -4.775188446044922, "KL/mean": -10.942734718322754, "KL/rejected_KL_mean": -17.110279083251953, "KL/std": 12.013012886047363, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.04151751846075058, "fcm_dpo/delta": -0.11940896511077881, "fcm_dpo/margin": 12.335088729858398, "fcm_dpo/q_t": 0.38177433609962463, "grad_norm": 13.543376922607422, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.4403544664382935, "logits/rejected": -0.8069661855697632, "logps/chosen": -107.6810531616211, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -106.05964660644531, "loss": 1.0073, "margin_dpo/margin_mean": 12.335088729858398, "margin_dpo/margin_std": 13.084501266479492, "step": 146 }, { "KL/chosen_KL_mean": -4.617717742919922, "KL/mean": -11.066187858581543, "KL/rejected_KL_mean": -17.514663696289062, "KL/std": 11.671147346496582, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.040687136352062225, "fcm_dpo/delta": -0.13163577020168304, "fcm_dpo/margin": 12.896951675415039, "fcm_dpo/q_t": 0.37846940755844116, "grad_norm": 12.54267692565918, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.2389419078826904, "logits/rejected": -0.6411401033401489, "logps/chosen": -93.8799057006836, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -94.01477813720703, "loss": 0.9875, "margin_dpo/margin_mean": 12.896951675415039, "margin_dpo/margin_std": 12.532159805297852, "step": 147 }, { "KL/chosen_KL_mean": -3.145641326904297, "KL/mean": -8.920623779296875, "KL/rejected_KL_mean": -14.695613861083984, "KL/std": 11.940990447998047, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.04029485583305359, "fcm_dpo/delta": -0.06921286135911942, "fcm_dpo/margin": 11.549966812133789, "fcm_dpo/q_t": 0.39572638273239136, "grad_norm": 13.742793083190918, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.6243813037872314, "logits/rejected": -1.2329108715057373, "logps/chosen": -94.90852355957031, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -125.08114624023438, "loss": 1.0489, "margin_dpo/margin_mean": 11.549967765808105, "margin_dpo/margin_std": 13.511589050292969, "step": 148 }, { "KL/chosen_KL_mean": -6.0254364013671875, "KL/mean": -11.718189239501953, "KL/rejected_KL_mean": -17.410938262939453, "KL/std": 12.825119018554688, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.03940197825431824, "fcm_dpo/delta": -0.05137070640921593, "fcm_dpo/margin": 11.385496139526367, "fcm_dpo/q_t": 0.40207332372665405, "grad_norm": 12.98380184173584, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.1076281070709229, "logits/rejected": -0.7780009508132935, "logps/chosen": -99.50945281982422, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -123.05464172363281, "loss": 1.0736, "margin_dpo/margin_mean": 11.385496139526367, "margin_dpo/margin_std": 15.861934661865234, "step": 149 }, { "KL/chosen_KL_mean": -4.752754211425781, "KL/mean": -10.776357650756836, "KL/rejected_KL_mean": -16.799957275390625, "KL/std": 13.658746719360352, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.03893650323152542, "fcm_dpo/delta": -0.0727292150259018, "fcm_dpo/margin": 12.047204971313477, "fcm_dpo/q_t": 0.3942786455154419, "grad_norm": 12.999213218688965, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.0462074279785156, "logits/rejected": -0.7713516354560852, "logps/chosen": -80.53144073486328, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -106.82672119140625, "loss": 1.0462, "margin_dpo/margin_mean": 12.04720401763916, "margin_dpo/margin_std": 15.017054557800293, "step": 150 }, { "KL/chosen_KL_mean": -6.7592010498046875, "KL/mean": -12.448505401611328, "KL/rejected_KL_mean": -18.137805938720703, "KL/std": 13.44986343383789, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.03863542154431343, "fcm_dpo/delta": -0.04144027829170227, "fcm_dpo/margin": 11.37860107421875, "fcm_dpo/q_t": 0.40086549520492554, "grad_norm": 14.494232177734375, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.8856639862060547, "logits/rejected": -1.2887725830078125, "logps/chosen": -109.49342346191406, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -124.17561340332031, "loss": 1.0714, "margin_dpo/margin_mean": 11.37860107421875, "margin_dpo/margin_std": 15.287328720092773, "step": 151 }, { "KL/chosen_KL_mean": -3.8769454956054688, "KL/mean": -10.776012420654297, "KL/rejected_KL_mean": -17.675086975097656, "KL/std": 13.209709167480469, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.037762466818094254, "fcm_dpo/delta": -0.12799283862113953, "fcm_dpo/margin": 13.798128128051758, "fcm_dpo/q_t": 0.38034120202064514, "grad_norm": 13.032094955444336, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.3569692373275757, "logits/rejected": -0.9675726890563965, "logps/chosen": -101.46768951416016, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -128.17422485351562, "loss": 1.005, "margin_dpo/margin_mean": 13.79813003540039, "margin_dpo/margin_std": 15.226696014404297, "step": 152 }, { "KL/chosen_KL_mean": -6.369403839111328, "KL/mean": -11.416723251342773, "KL/rejected_KL_mean": -16.464046478271484, "KL/std": 13.006369590759277, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.037692636251449585, "fcm_dpo/delta": 0.020278386771678925, "fcm_dpo/margin": 10.094650268554688, "fcm_dpo/q_t": 0.41338086128234863, "grad_norm": 12.772554397583008, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.6836185455322266, "logits/rejected": -1.3619801998138428, "logps/chosen": -93.39500427246094, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -107.53258514404297, "loss": 1.1159, "margin_dpo/margin_mean": 10.094650268554688, "margin_dpo/margin_std": 14.84290885925293, "step": 153 }, { "KL/chosen_KL_mean": -4.76593017578125, "KL/mean": -12.822843551635742, "KL/rejected_KL_mean": -20.8797607421875, "KL/std": 14.076654434204102, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.036855101585388184, "fcm_dpo/delta": -0.20567502081394196, "fcm_dpo/margin": 16.11383628845215, "fcm_dpo/q_t": 0.3641587197780609, "grad_norm": 12.728275299072266, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.2588591575622559, "logits/rejected": -0.9284448623657227, "logps/chosen": -95.87159729003906, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -132.25213623046875, "loss": 0.9489, "margin_dpo/margin_mean": 16.113834381103516, "margin_dpo/margin_std": 14.741827011108398, "step": 154 }, { "KL/chosen_KL_mean": -6.7658843994140625, "KL/mean": -12.946847915649414, "KL/rejected_KL_mean": -19.1278076171875, "KL/std": 14.668111801147461, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.03579120337963104, "fcm_dpo/delta": -0.04514720290899277, "fcm_dpo/margin": 12.361924171447754, "fcm_dpo/q_t": 0.3967772424221039, "grad_norm": 15.131264686584473, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.363485336303711, "logits/rejected": -0.991689920425415, "logps/chosen": -101.32685852050781, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -103.90139770507812, "loss": 1.0876, "margin_dpo/margin_mean": 12.361923217773438, "margin_dpo/margin_std": 18.05032730102539, "step": 155 }, { "KL/chosen_KL_mean": -8.323665618896484, "KL/mean": -13.755014419555664, "KL/rejected_KL_mean": -19.186363220214844, "KL/std": 12.906888961791992, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.035796426236629486, "fcm_dpo/delta": 0.011289432644844055, "fcm_dpo/margin": 10.862693786621094, "fcm_dpo/q_t": 0.4123469591140747, "grad_norm": 13.327481269836426, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.251359224319458, "logits/rejected": -0.7276970148086548, "logps/chosen": -102.97904968261719, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -103.60570526123047, "loss": 1.1089, "margin_dpo/margin_mean": 10.862693786621094, "margin_dpo/margin_std": 15.572434425354004, "step": 156 }, { "KL/chosen_KL_mean": -7.8739166259765625, "KL/mean": -15.43746280670166, "KL/rejected_KL_mean": -23.00101089477539, "KL/std": 13.99744701385498, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.0352681428194046, "fcm_dpo/delta": -0.14088454842567444, "fcm_dpo/margin": 15.127096176147461, "fcm_dpo/q_t": 0.3763526678085327, "grad_norm": 13.725337028503418, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.9000129699707031, "logits/rejected": -1.4369938373565674, "logps/chosen": -104.88630676269531, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -134.70236206054688, "loss": 0.9846, "margin_dpo/margin_mean": 15.127094268798828, "margin_dpo/margin_std": 14.728677749633789, "step": 157 }, { "KL/chosen_KL_mean": -7.259368896484375, "KL/mean": -15.211639404296875, "KL/rejected_KL_mean": -23.163909912109375, "KL/std": 14.975515365600586, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.03423712030053139, "fcm_dpo/delta": -0.1526663601398468, "fcm_dpo/margin": 15.904544830322266, "fcm_dpo/q_t": 0.3750673532485962, "grad_norm": 12.85647201538086, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.3172118663787842, "logits/rejected": -1.0385704040527344, "logps/chosen": -101.97233581542969, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -144.10125732421875, "loss": 0.9801, "margin_dpo/margin_mean": 15.90454387664795, "margin_dpo/margin_std": 15.86979866027832, "step": 158 }, { "KL/chosen_KL_mean": -6.668708801269531, "KL/mean": -13.742225646972656, "KL/rejected_KL_mean": -20.815746307373047, "KL/std": 14.5204496383667, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.03351747244596481, "fcm_dpo/delta": -0.07788847386837006, "fcm_dpo/margin": 14.14704704284668, "fcm_dpo/q_t": 0.3912920355796814, "grad_norm": 11.340263366699219, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.9460320472717285, "logits/rejected": -1.4574239253997803, "logps/chosen": -114.88694763183594, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -115.65476989746094, "loss": 1.0378, "margin_dpo/margin_mean": 14.14704704284668, "margin_dpo/margin_std": 16.608810424804688, "step": 159 }, { "KL/chosen_KL_mean": -5.631401062011719, "KL/mean": -12.632906913757324, "KL/rejected_KL_mean": -19.634410858154297, "KL/std": 13.49333667755127, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.03300575166940689, "fcm_dpo/delta": -0.06539568305015564, "fcm_dpo/margin": 14.002998352050781, "fcm_dpo/q_t": 0.39185240864753723, "grad_norm": 11.865344047546387, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.4524874687194824, "logits/rejected": -0.8901819586753845, "logps/chosen": -103.79481506347656, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -121.91539001464844, "loss": 1.0297, "margin_dpo/margin_mean": 14.003000259399414, "margin_dpo/margin_std": 14.516088485717773, "step": 160 }, { "KL/chosen_KL_mean": -7.712261199951172, "KL/mean": -14.009366035461426, "KL/rejected_KL_mean": -20.306472778320312, "KL/std": 12.555103302001953, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.032761216163635254, "fcm_dpo/delta": -0.01352754607796669, "fcm_dpo/margin": 12.594217300415039, "fcm_dpo/q_t": 0.404840350151062, "grad_norm": 11.787931442260742, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.4999102354049683, "logits/rejected": -0.9853357076644897, "logps/chosen": -94.30413055419922, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -109.32290649414062, "loss": 1.0833, "margin_dpo/margin_mean": 12.594218254089355, "margin_dpo/margin_std": 16.226791381835938, "step": 161 }, { "KL/chosen_KL_mean": -6.864570617675781, "KL/mean": -13.87098503112793, "KL/rejected_KL_mean": -20.877403259277344, "KL/std": 14.554333686828613, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.032504115253686905, "fcm_dpo/delta": -0.058396607637405396, "fcm_dpo/margin": 14.012829780578613, "fcm_dpo/q_t": 0.39356642961502075, "grad_norm": 11.334613800048828, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.1437064409255981, "logits/rejected": -0.7319457530975342, "logps/chosen": -104.2674560546875, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -116.74663543701172, "loss": 1.0395, "margin_dpo/margin_mean": 14.01283073425293, "margin_dpo/margin_std": 15.349771499633789, "step": 162 }, { "KL/chosen_KL_mean": -6.613433837890625, "KL/mean": -15.081428527832031, "KL/rejected_KL_mean": -23.549434661865234, "KL/std": 16.713062286376953, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.031715020537376404, "fcm_dpo/delta": -0.14539991319179535, "fcm_dpo/margin": 16.936002731323242, "fcm_dpo/q_t": 0.3780941069126129, "grad_norm": 10.051616668701172, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.4733749628067017, "logits/rejected": -1.065507411956787, "logps/chosen": -88.78882598876953, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -106.41416931152344, "loss": 0.9895, "margin_dpo/margin_mean": 16.936002731323242, "margin_dpo/margin_std": 17.879697799682617, "step": 163 }, { "KL/chosen_KL_mean": -10.340309143066406, "KL/mean": -15.9946870803833, "KL/rejected_KL_mean": -21.649063110351562, "KL/std": 13.053253173828125, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.03155171871185303, "fcm_dpo/delta": 0.04414636641740799, "fcm_dpo/margin": 11.308747291564941, "fcm_dpo/q_t": 0.41644802689552307, "grad_norm": 11.872608184814453, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.5307713747024536, "logits/rejected": -1.0783181190490723, "logps/chosen": -100.8145751953125, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -108.67715454101562, "loss": 1.1142, "margin_dpo/margin_mean": 11.308748245239258, "margin_dpo/margin_std": 14.296263694763184, "step": 164 }, { "KL/chosen_KL_mean": -7.023193359375, "KL/mean": -15.371489524841309, "KL/rejected_KL_mean": -23.71978759765625, "KL/std": 15.888480186462402, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.031303513795137405, "fcm_dpo/delta": -0.129239022731781, "fcm_dpo/margin": 16.69659423828125, "fcm_dpo/q_t": 0.3821299076080322, "grad_norm": 9.835552215576172, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.7321608066558838, "logits/rejected": -1.3163830041885376, "logps/chosen": -94.33642578125, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -105.62237548828125, "loss": 0.9979, "margin_dpo/margin_mean": 16.69659423828125, "margin_dpo/margin_std": 18.003774642944336, "step": 165 }, { "KL/chosen_KL_mean": -11.375301361083984, "KL/mean": -19.551345825195312, "KL/rejected_KL_mean": -27.72739028930664, "KL/std": 17.02508544921875, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.030543209984898567, "fcm_dpo/delta": -0.10468219220638275, "fcm_dpo/margin": 16.352081298828125, "fcm_dpo/q_t": 0.3900759220123291, "grad_norm": 11.145418167114258, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.6647932529449463, "logits/rejected": -1.4592862129211426, "logps/chosen": -92.96136474609375, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -130.89256286621094, "loss": 1.0289, "margin_dpo/margin_mean": 16.352081298828125, "margin_dpo/margin_std": 20.441665649414062, "step": 166 }, { "KL/chosen_KL_mean": -10.146953582763672, "KL/mean": -18.971742630004883, "KL/rejected_KL_mean": -27.796531677246094, "KL/std": 15.918092727661133, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.02963569015264511, "fcm_dpo/delta": -0.13114698231220245, "fcm_dpo/margin": 17.649572372436523, "fcm_dpo/q_t": 0.3784089684486389, "grad_norm": 10.373214721679688, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.226170301437378, "logits/rejected": -1.0477205514907837, "logps/chosen": -93.70862579345703, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -126.84957885742188, "loss": 0.9843, "margin_dpo/margin_mean": 17.649572372436523, "margin_dpo/margin_std": 16.261445999145508, "step": 167 }, { "KL/chosen_KL_mean": -9.806503295898438, "KL/mean": -17.30911636352539, "KL/rejected_KL_mean": -24.81173324584961, "KL/std": 15.693700790405273, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.029451467096805573, "fcm_dpo/delta": -0.04384768009185791, "fcm_dpo/margin": 15.005227088928223, "fcm_dpo/q_t": 0.3970223367214203, "grad_norm": 11.291738510131836, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.5536309480667114, "logits/rejected": -1.2121269702911377, "logps/chosen": -89.73773956298828, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -104.91389465332031, "loss": 1.0435, "margin_dpo/margin_mean": 15.005227088928223, "margin_dpo/margin_std": 15.853570938110352, "step": 168 }, { "KL/chosen_KL_mean": -8.286487579345703, "KL/mean": -16.949750900268555, "KL/rejected_KL_mean": -25.613014221191406, "KL/std": 16.854068756103516, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.028873208910226822, "fcm_dpo/delta": -0.10558012127876282, "fcm_dpo/margin": 17.326522827148438, "fcm_dpo/q_t": 0.3849586844444275, "grad_norm": 11.121870994567871, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.3366649150848389, "logits/rejected": -1.1020835638046265, "logps/chosen": -91.19019317626953, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -115.73492431640625, "loss": 1.0074, "margin_dpo/margin_mean": 17.326522827148438, "margin_dpo/margin_std": 17.858684539794922, "step": 169 }, { "KL/chosen_KL_mean": -9.984466552734375, "KL/mean": -17.546161651611328, "KL/rejected_KL_mean": -25.10785675048828, "KL/std": 14.148959159851074, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.02845631167292595, "fcm_dpo/delta": -0.03229335695505142, "fcm_dpo/margin": 15.123394012451172, "fcm_dpo/q_t": 0.3979126214981079, "grad_norm": 11.244989395141602, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.3028111457824707, "logits/rejected": -1.077418565750122, "logps/chosen": -84.96643829345703, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -107.2734603881836, "loss": 1.0419, "margin_dpo/margin_mean": 15.123394012451172, "margin_dpo/margin_std": 14.179786682128906, "step": 170 }, { "KL/chosen_KL_mean": -12.63095474243164, "KL/mean": -21.048519134521484, "KL/rejected_KL_mean": -29.46609878540039, "KL/std": 20.310977935791016, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.028235454112291336, "fcm_dpo/delta": -0.07904790341854095, "fcm_dpo/margin": 16.835145950317383, "fcm_dpo/q_t": 0.3972875475883484, "grad_norm": 12.394989967346191, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.692241907119751, "logits/rejected": -1.1498911380767822, "logps/chosen": -131.60948181152344, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -145.23541259765625, "loss": 1.0639, "margin_dpo/margin_mean": 16.835145950317383, "margin_dpo/margin_std": 24.23593521118164, "step": 171 }, { "KL/chosen_KL_mean": -13.519664764404297, "KL/mean": -21.092296600341797, "KL/rejected_KL_mean": -28.664932250976562, "KL/std": 14.584999084472656, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.02791805937886238, "fcm_dpo/delta": -0.024036778137087822, "fcm_dpo/margin": 15.145275115966797, "fcm_dpo/q_t": 0.39855894446372986, "grad_norm": 10.368908882141113, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.268516778945923, "logits/rejected": -1.5174126625061035, "logps/chosen": -114.67345428466797, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -119.67061614990234, "loss": 1.0354, "margin_dpo/margin_mean": 15.145275115966797, "margin_dpo/margin_std": 12.039316177368164, "step": 172 }, { "KL/chosen_KL_mean": -12.286582946777344, "KL/mean": -18.873279571533203, "KL/rejected_KL_mean": -25.459980010986328, "KL/std": 13.893444061279297, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.02811792865395546, "fcm_dpo/delta": 0.030664971098303795, "fcm_dpo/margin": 13.173401832580566, "fcm_dpo/q_t": 0.41260024905204773, "grad_norm": 10.016674041748047, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.5009260177612305, "logits/rejected": -1.020369291305542, "logps/chosen": -111.34326934814453, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -109.31071472167969, "loss": 1.0909, "margin_dpo/margin_mean": 13.17340087890625, "margin_dpo/margin_std": 14.766935348510742, "step": 173 }, { "KL/chosen_KL_mean": -11.913454055786133, "KL/mean": -20.964618682861328, "KL/rejected_KL_mean": -30.015785217285156, "KL/std": 18.987197875976562, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.02772102691233158, "fcm_dpo/delta": -0.10708963125944138, "fcm_dpo/margin": 18.102331161499023, "fcm_dpo/q_t": 0.38403430581092834, "grad_norm": 11.809392929077148, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.4919222593307495, "logits/rejected": -1.4470956325531006, "logps/chosen": -92.32235717773438, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -138.21749877929688, "loss": 1.0086, "margin_dpo/margin_mean": 18.10232925415039, "margin_dpo/margin_std": 18.97220230102539, "step": 174 }, { "KL/chosen_KL_mean": -15.321826934814453, "KL/mean": -24.053287506103516, "KL/rejected_KL_mean": -32.784751892089844, "KL/std": 16.884767532348633, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.02728112041950226, "fcm_dpo/delta": -0.08016402274370193, "fcm_dpo/margin": 17.462921142578125, "fcm_dpo/q_t": 0.38763850927352905, "grad_norm": 11.410508155822754, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.1097466945648193, "logits/rejected": -0.8357747793197632, "logps/chosen": -98.03831481933594, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -129.45187377929688, "loss": 1.0122, "margin_dpo/margin_mean": 17.462921142578125, "margin_dpo/margin_std": 16.00652503967285, "step": 175 }, { "KL/chosen_KL_mean": -12.648033142089844, "KL/mean": -21.527843475341797, "KL/rejected_KL_mean": -30.407657623291016, "KL/std": 18.336122512817383, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.026770778000354767, "fcm_dpo/delta": -0.07922545075416565, "fcm_dpo/margin": 17.759618759155273, "fcm_dpo/q_t": 0.39140811562538147, "grad_norm": 11.144559860229492, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.461488962173462, "logits/rejected": -1.295197606086731, "logps/chosen": -92.27733612060547, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -133.09625244140625, "loss": 1.0384, "margin_dpo/margin_mean": 17.75961685180664, "margin_dpo/margin_std": 20.994644165039062, "step": 176 }, { "KL/chosen_KL_mean": -14.021316528320312, "KL/mean": -24.409223556518555, "KL/rejected_KL_mean": -34.79713439941406, "KL/std": 20.938053131103516, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.026182223111391068, "fcm_dpo/delta": -0.15196071565151215, "fcm_dpo/margin": 20.775821685791016, "fcm_dpo/q_t": 0.37817782163619995, "grad_norm": 9.96103572845459, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.5541965961456299, "logits/rejected": -1.2100471258163452, "logps/chosen": -106.58241271972656, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -138.13552856445312, "loss": 0.9861, "margin_dpo/margin_mean": 20.775821685791016, "margin_dpo/margin_std": 21.845623016357422, "step": 177 }, { "KL/chosen_KL_mean": -12.930191040039062, "KL/mean": -25.335491180419922, "KL/rejected_KL_mean": -37.74079132080078, "KL/std": 20.42729949951172, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.024965349584817886, "fcm_dpo/delta": -0.2341279238462448, "fcm_dpo/margin": 24.81060218811035, "fcm_dpo/q_t": 0.3580138087272644, "grad_norm": 11.220521926879883, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.6707206964492798, "logits/rejected": -1.4424693584442139, "logps/chosen": -97.19598388671875, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -148.53810119628906, "loss": 0.9245, "margin_dpo/margin_mean": 24.81060218811035, "margin_dpo/margin_std": 21.960166931152344, "step": 178 }, { "KL/chosen_KL_mean": -16.608139038085938, "KL/mean": -24.534873962402344, "KL/rejected_KL_mean": -32.46160888671875, "KL/std": 19.812904357910156, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.02464543841779232, "fcm_dpo/delta": 0.009494444355368614, "fcm_dpo/margin": 15.85346794128418, "fcm_dpo/q_t": 0.41128796339035034, "grad_norm": 10.52614688873291, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.4694886207580566, "logits/rejected": -1.0377006530761719, "logps/chosen": -110.05611419677734, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -126.71702575683594, "loss": 1.1087, "margin_dpo/margin_mean": 15.853466033935547, "margin_dpo/margin_std": 22.982295989990234, "step": 179 }, { "KL/chosen_KL_mean": -14.04043197631836, "KL/mean": -24.26563262939453, "KL/rejected_KL_mean": -34.49083709716797, "KL/std": 19.08724594116211, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.024294205009937286, "fcm_dpo/delta": -0.10231912136077881, "fcm_dpo/margin": 20.450401306152344, "fcm_dpo/q_t": 0.38433969020843506, "grad_norm": 12.6638765335083, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.2675533294677734, "logits/rejected": -1.0236175060272217, "logps/chosen": -102.8759765625, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -138.39779663085938, "loss": 0.9992, "margin_dpo/margin_mean": 20.450401306152344, "margin_dpo/margin_std": 19.39406967163086, "step": 180 }, { "KL/chosen_KL_mean": -13.741043090820312, "KL/mean": -21.691543579101562, "KL/rejected_KL_mean": -29.642040252685547, "KL/std": 18.352031707763672, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.024304591119289398, "fcm_dpo/delta": 0.014049563556909561, "fcm_dpo/margin": 15.9010009765625, "fcm_dpo/q_t": 0.4110836088657379, "grad_norm": 9.70184326171875, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.658658504486084, "logits/rejected": -1.2062342166900635, "logps/chosen": -92.97500610351562, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -98.81646728515625, "loss": 1.089, "margin_dpo/margin_mean": 15.9010009765625, "margin_dpo/margin_std": 19.693115234375, "step": 181 }, { "KL/chosen_KL_mean": -17.661022186279297, "KL/mean": -27.84783935546875, "KL/rejected_KL_mean": -38.0346565246582, "KL/std": 21.02090072631836, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.02388744056224823, "fcm_dpo/delta": -0.09168744087219238, "fcm_dpo/margin": 20.373638153076172, "fcm_dpo/q_t": 0.3886273503303528, "grad_norm": 10.246270179748535, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.2804200649261475, "logits/rejected": -1.054466962814331, "logps/chosen": -93.74320983886719, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -133.00326538085938, "loss": 1.0276, "margin_dpo/margin_mean": 20.373638153076172, "margin_dpo/margin_std": 23.208778381347656, "step": 182 }, { "KL/chosen_KL_mean": -18.142189025878906, "KL/mean": -28.192537307739258, "KL/rejected_KL_mean": -38.24288558959961, "KL/std": 19.22570037841797, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.023501737043261528, "fcm_dpo/delta": -0.07655909657478333, "fcm_dpo/margin": 20.100692749023438, "fcm_dpo/q_t": 0.38938093185424805, "grad_norm": 10.65926456451416, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.3344858884811401, "logits/rejected": -0.7089817523956299, "logps/chosen": -112.05009460449219, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -121.80349731445312, "loss": 1.0193, "margin_dpo/margin_mean": 20.100692749023438, "margin_dpo/margin_std": 19.594409942626953, "step": 183 }, { "KL/chosen_KL_mean": -16.359600067138672, "KL/mean": -23.0889892578125, "KL/rejected_KL_mean": -29.818370819091797, "KL/std": 19.86102867126465, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.023806601762771606, "fcm_dpo/delta": 0.08214493095874786, "fcm_dpo/margin": 13.45876693725586, "fcm_dpo/q_t": 0.4248844087123871, "grad_norm": 9.939018249511719, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.6116652488708496, "logits/rejected": -1.3369851112365723, "logps/chosen": -103.22750854492188, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -114.1029281616211, "loss": 1.1547, "margin_dpo/margin_mean": 13.458765983581543, "margin_dpo/margin_std": 21.71732521057129, "step": 184 }, { "KL/chosen_KL_mean": -17.284526824951172, "KL/mean": -25.478012084960938, "KL/rejected_KL_mean": -33.6714973449707, "KL/std": 19.567195892333984, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.023981327190995216, "fcm_dpo/delta": 0.007112087681889534, "fcm_dpo/margin": 16.386974334716797, "fcm_dpo/q_t": 0.41027703881263733, "grad_norm": 11.217276573181152, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.044743776321411, "logits/rejected": -1.4217491149902344, "logps/chosen": -111.30595397949219, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -115.128173828125, "loss": 1.0997, "margin_dpo/margin_mean": 16.386974334716797, "margin_dpo/margin_std": 22.628589630126953, "step": 185 }, { "KL/chosen_KL_mean": -21.392627716064453, "KL/mean": -30.369789123535156, "KL/rejected_KL_mean": -39.34695053100586, "KL/std": 18.875503540039062, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.023750312626361847, "fcm_dpo/delta": -0.02786211669445038, "fcm_dpo/margin": 17.954317092895508, "fcm_dpo/q_t": 0.400560200214386, "grad_norm": 11.679343223571777, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.260082244873047, "logits/rejected": -1.3567298650741577, "logps/chosen": -135.58432006835938, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -132.6860809326172, "loss": 1.0603, "margin_dpo/margin_mean": 17.954315185546875, "margin_dpo/margin_std": 20.43008041381836, "step": 186 }, { "KL/chosen_KL_mean": -18.24752426147461, "KL/mean": -26.850269317626953, "KL/rejected_KL_mean": -35.4530143737793, "KL/std": 21.19013023376465, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.02381810173392296, "fcm_dpo/delta": -0.010355357080698013, "fcm_dpo/margin": 17.205490112304688, "fcm_dpo/q_t": 0.4063573181629181, "grad_norm": 10.371123313903809, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.8111029863357544, "logits/rejected": -1.5738775730133057, "logps/chosen": -95.11707305908203, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -123.61795806884766, "loss": 1.0891, "margin_dpo/margin_mean": 17.205490112304688, "margin_dpo/margin_std": 23.233699798583984, "step": 187 }, { "KL/chosen_KL_mean": -13.921382904052734, "KL/mean": -23.76552963256836, "KL/rejected_KL_mean": -33.60968017578125, "KL/std": 21.078519821166992, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.023423131555318832, "fcm_dpo/delta": -0.06464576721191406, "fcm_dpo/margin": 19.68829345703125, "fcm_dpo/q_t": 0.3912138342857361, "grad_norm": 10.719921112060547, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.0605125427246094, "logits/rejected": -1.8175008296966553, "logps/chosen": -93.11618041992188, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -130.43856811523438, "loss": 1.0315, "margin_dpo/margin_mean": 19.68829345703125, "margin_dpo/margin_std": 19.915063858032227, "step": 188 }, { "KL/chosen_KL_mean": -18.450321197509766, "KL/mean": -28.492952346801758, "KL/rejected_KL_mean": -38.53558349609375, "KL/std": 21.040348052978516, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.023310039192438126, "fcm_dpo/delta": -0.07174272835254669, "fcm_dpo/margin": 20.08526611328125, "fcm_dpo/q_t": 0.39407819509506226, "grad_norm": 9.451172828674316, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.5119017362594604, "logits/rejected": -0.8416706323623657, "logps/chosen": -102.34312438964844, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -118.00148010253906, "loss": 1.0409, "margin_dpo/margin_mean": 20.08526611328125, "margin_dpo/margin_std": 23.536149978637695, "step": 189 }, { "KL/chosen_KL_mean": -14.933372497558594, "KL/mean": -26.990257263183594, "KL/rejected_KL_mean": -39.04713439941406, "KL/std": 23.35231590270996, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.022569432854652405, "fcm_dpo/delta": -0.15262600779533386, "fcm_dpo/margin": 24.113767623901367, "fcm_dpo/q_t": 0.3734833002090454, "grad_norm": 10.411787986755371, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.6398506164550781, "logits/rejected": -1.0240169763565063, "logps/chosen": -118.37483978271484, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -148.2621612548828, "loss": 0.972, "margin_dpo/margin_mean": 24.113767623901367, "margin_dpo/margin_std": 22.282882690429688, "step": 190 }, { "KL/chosen_KL_mean": -14.046867370605469, "KL/mean": -27.55194854736328, "KL/rejected_KL_mean": -41.057037353515625, "KL/std": 25.110118865966797, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.021781057119369507, "fcm_dpo/delta": -0.19984564185142517, "fcm_dpo/margin": 27.010169982910156, "fcm_dpo/q_t": 0.36570775508880615, "grad_norm": 11.090989112854004, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.473405122756958, "logits/rejected": -1.1517852544784546, "logps/chosen": -113.83238220214844, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -157.05731201171875, "loss": 0.9511, "margin_dpo/margin_mean": 27.010169982910156, "margin_dpo/margin_std": 25.67971420288086, "step": 191 }, { "KL/chosen_KL_mean": -23.235088348388672, "KL/mean": -32.735416412353516, "KL/rejected_KL_mean": -42.23574447631836, "KL/std": 20.18899917602539, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.021474026143550873, "fcm_dpo/delta": -0.008402079343795776, "fcm_dpo/margin": 19.000659942626953, "fcm_dpo/q_t": 0.40609294176101685, "grad_norm": 11.45128059387207, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.4406037330627441, "logits/rejected": -1.1839938163757324, "logps/chosen": -124.09541320800781, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -151.68075561523438, "loss": 1.0792, "margin_dpo/margin_mean": 19.000659942626953, "margin_dpo/margin_std": 23.713571548461914, "step": 192 }, { "KL/chosen_KL_mean": -19.17186737060547, "KL/mean": -26.961084365844727, "KL/rejected_KL_mean": -34.75029754638672, "KL/std": 21.411218643188477, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.02175632305443287, "fcm_dpo/delta": 0.06285930424928665, "fcm_dpo/margin": 15.578431129455566, "fcm_dpo/q_t": 0.4219972789287567, "grad_norm": 10.805830955505371, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.212632179260254, "logits/rejected": -1.6488423347473145, "logps/chosen": -112.11018371582031, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -117.85948181152344, "loss": 1.153, "margin_dpo/margin_mean": 15.578432083129883, "margin_dpo/margin_std": 25.95126724243164, "step": 193 }, { "KL/chosen_KL_mean": -19.648685455322266, "KL/mean": -28.676578521728516, "KL/rejected_KL_mean": -37.704471588134766, "KL/std": 22.001331329345703, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.021851275116205215, "fcm_dpo/delta": 0.005449790507555008, "fcm_dpo/margin": 18.0557918548584, "fcm_dpo/q_t": 0.40913453698158264, "grad_norm": 9.149490356445312, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.321040153503418, "logits/rejected": -1.942004680633545, "logps/chosen": -83.44026184082031, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -106.43112182617188, "loss": 1.0912, "margin_dpo/margin_mean": 18.055789947509766, "margin_dpo/margin_std": 23.30768585205078, "step": 194 }, { "KL/chosen_KL_mean": -18.819171905517578, "KL/mean": -30.88967514038086, "KL/rejected_KL_mean": -42.96017837524414, "KL/std": 22.53610610961914, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.021448157727718353, "fcm_dpo/delta": -0.12402576208114624, "fcm_dpo/margin": 24.141006469726562, "fcm_dpo/q_t": 0.38023075461387634, "grad_norm": 9.5504150390625, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.3483912944793701, "logits/rejected": -1.227956771850586, "logps/chosen": -102.70463562011719, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -146.96142578125, "loss": 1.0056, "margin_dpo/margin_mean": 24.141006469726562, "margin_dpo/margin_std": 25.839191436767578, "step": 195 }, { "KL/chosen_KL_mean": -22.072078704833984, "KL/mean": -33.3795280456543, "KL/rejected_KL_mean": -44.68697738647461, "KL/std": 24.02835464477539, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.021000903099775314, "fcm_dpo/delta": -0.07868774235248566, "fcm_dpo/margin": 22.614898681640625, "fcm_dpo/q_t": 0.387977659702301, "grad_norm": 9.134716033935547, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.2337486743927002, "logits/rejected": -0.7770195007324219, "logps/chosen": -111.47344207763672, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -128.03433227539062, "loss": 1.0161, "margin_dpo/margin_mean": 22.614898681640625, "margin_dpo/margin_std": 21.800174713134766, "step": 196 }, { "KL/chosen_KL_mean": -22.311660766601562, "KL/mean": -29.264373779296875, "KL/rejected_KL_mean": -36.21708297729492, "KL/std": 19.539718627929688, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.02125866338610649, "fcm_dpo/delta": 0.10757507383823395, "fcm_dpo/margin": 13.905410766601562, "fcm_dpo/q_t": 0.4302526116371155, "grad_norm": 9.504860877990723, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.4678778648376465, "logits/rejected": -1.141904354095459, "logps/chosen": -109.1523666381836, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -119.00997924804688, "loss": 1.1577, "margin_dpo/margin_mean": 13.905410766601562, "margin_dpo/margin_std": 20.54107666015625, "step": 197 }, { "KL/chosen_KL_mean": -25.893146514892578, "KL/mean": -37.19945526123047, "KL/rejected_KL_mean": -48.505767822265625, "KL/std": 22.798789978027344, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.021130535751581192, "fcm_dpo/delta": -0.08168008923530579, "fcm_dpo/margin": 22.612621307373047, "fcm_dpo/q_t": 0.3892834782600403, "grad_norm": 11.329520225524902, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7221508026123047, "logits/rejected": -1.5456972122192383, "logps/chosen": -110.94139099121094, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -152.6094207763672, "loss": 1.0197, "margin_dpo/margin_mean": 22.612621307373047, "margin_dpo/margin_std": 23.356773376464844, "step": 198 }, { "KL/chosen_KL_mean": -25.148265838623047, "KL/mean": -34.557613372802734, "KL/rejected_KL_mean": -43.96696090698242, "KL/std": 22.277076721191406, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.021068252623081207, "fcm_dpo/delta": 0.003629094921052456, "fcm_dpo/margin": 18.81869888305664, "fcm_dpo/q_t": 0.4078560471534729, "grad_norm": 9.862123489379883, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.7945388555526733, "logits/rejected": -1.5393321514129639, "logps/chosen": -106.93496704101562, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -134.7577362060547, "loss": 1.0856, "margin_dpo/margin_mean": 18.818700790405273, "margin_dpo/margin_std": 23.334117889404297, "step": 199 }, { "KL/chosen_KL_mean": -24.76251983642578, "KL/mean": -35.791648864746094, "KL/rejected_KL_mean": -46.82076644897461, "KL/std": 23.81404685974121, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.020759932696819305, "fcm_dpo/delta": -0.06139633059501648, "fcm_dpo/margin": 22.058250427246094, "fcm_dpo/q_t": 0.39515000581741333, "grad_norm": 9.715496063232422, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.8304107189178467, "logits/rejected": -1.3093968629837036, "logps/chosen": -109.03060913085938, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -133.78863525390625, "loss": 1.0571, "margin_dpo/margin_mean": 22.058250427246094, "margin_dpo/margin_std": 27.491947174072266, "step": 200 }, { "KL/chosen_KL_mean": -26.808879852294922, "KL/mean": -37.616722106933594, "KL/rejected_KL_mean": -48.424564361572266, "KL/std": 26.183542251586914, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.020689569413661957, "fcm_dpo/delta": -0.049439653754234314, "fcm_dpo/margin": 21.615676879882812, "fcm_dpo/q_t": 0.39783158898353577, "grad_norm": 10.074195861816406, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.0864808559417725, "logits/rejected": -0.6829971075057983, "logps/chosen": -134.5813751220703, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -168.217041015625, "loss": 1.0687, "margin_dpo/margin_mean": 21.615676879882812, "margin_dpo/margin_std": 28.592090606689453, "step": 201 }, { "KL/chosen_KL_mean": -24.90392303466797, "KL/mean": -34.891929626464844, "KL/rejected_KL_mean": -44.87993621826172, "KL/std": 21.467811584472656, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.020566390827298164, "fcm_dpo/delta": -0.011303944513201714, "fcm_dpo/margin": 19.976011276245117, "fcm_dpo/q_t": 0.4053710103034973, "grad_norm": 10.26162338256836, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.373791217803955, "logits/rejected": -1.2439234256744385, "logps/chosen": -100.87637329101562, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -139.3399200439453, "loss": 1.0773, "margin_dpo/margin_mean": 19.976011276245117, "margin_dpo/margin_std": 24.29538345336914, "step": 202 }, { "KL/chosen_KL_mean": -22.183849334716797, "KL/mean": -33.478858947753906, "KL/rejected_KL_mean": -44.77387237548828, "KL/std": 22.598102569580078, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.020371519029140472, "fcm_dpo/delta": -0.06304950267076492, "fcm_dpo/margin": 22.590023040771484, "fcm_dpo/q_t": 0.39245957136154175, "grad_norm": 9.22353744506836, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.631709098815918, "logits/rejected": -1.1592448949813843, "logps/chosen": -117.1510009765625, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -137.6615447998047, "loss": 1.0323, "margin_dpo/margin_mean": 22.590023040771484, "margin_dpo/margin_std": 23.636165618896484, "step": 203 }, { "KL/chosen_KL_mean": -28.01312255859375, "KL/mean": -37.74678421020508, "KL/rejected_KL_mean": -47.480438232421875, "KL/std": 22.48832893371582, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0203473512083292, "fcm_dpo/delta": 0.003886604681611061, "fcm_dpo/margin": 19.467315673828125, "fcm_dpo/q_t": 0.4089944362640381, "grad_norm": 10.18489933013916, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.9266104698181152, "logits/rejected": -1.275346279144287, "logps/chosen": -144.3703155517578, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -140.87802124023438, "loss": 1.0976, "margin_dpo/margin_mean": 19.467315673828125, "margin_dpo/margin_std": 26.09893226623535, "step": 204 }, { "KL/chosen_KL_mean": -26.03201675415039, "KL/mean": -38.64739990234375, "KL/rejected_KL_mean": -51.262786865234375, "KL/std": 24.747488021850586, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.01993061602115631, "fcm_dpo/delta": -0.1086086854338646, "fcm_dpo/margin": 25.23077392578125, "fcm_dpo/q_t": 0.38073670864105225, "grad_norm": 9.012118339538574, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.181138515472412, "logits/rejected": -1.745039463043213, "logps/chosen": -114.89136505126953, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -143.11727905273438, "loss": 0.9885, "margin_dpo/margin_mean": 25.23077392578125, "margin_dpo/margin_std": 21.520061492919922, "step": 205 }, { "KL/chosen_KL_mean": -26.65758514404297, "KL/mean": -38.254554748535156, "KL/rejected_KL_mean": -49.85152816772461, "KL/std": 24.151702880859375, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.01961803063750267, "fcm_dpo/delta": -0.05811109393835068, "fcm_dpo/margin": 23.193946838378906, "fcm_dpo/q_t": 0.39182958006858826, "grad_norm": 10.554476737976074, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.7150788307189941, "logits/rejected": -1.5660370588302612, "logps/chosen": -103.20393371582031, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -152.80465698242188, "loss": 1.0218, "margin_dpo/margin_mean": 23.193946838378906, "margin_dpo/margin_std": 20.591449737548828, "step": 206 }, { "KL/chosen_KL_mean": -27.962421417236328, "KL/mean": -40.90303421020508, "KL/rejected_KL_mean": -53.84365463256836, "KL/std": 29.533283233642578, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.019331496208906174, "fcm_dpo/delta": -0.10557013750076294, "fcm_dpo/margin": 25.881229400634766, "fcm_dpo/q_t": 0.38874679803848267, "grad_norm": 10.590154647827148, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.0346693992614746, "logits/rejected": -1.6076472997665405, "logps/chosen": -114.1940689086914, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -144.498779296875, "loss": 1.0416, "margin_dpo/margin_mean": 25.881227493286133, "margin_dpo/margin_std": 33.82598114013672, "step": 207 }, { "KL/chosen_KL_mean": -28.89193344116211, "KL/mean": -42.258174896240234, "KL/rejected_KL_mean": -55.624412536621094, "KL/std": 26.907329559326172, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.01897745206952095, "fcm_dpo/delta": -0.11292941123247147, "fcm_dpo/margin": 26.73247528076172, "fcm_dpo/q_t": 0.380452036857605, "grad_norm": 10.240801811218262, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.7002942562103271, "logits/rejected": -1.3405828475952148, "logps/chosen": -123.05148315429688, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -156.5867462158203, "loss": 0.9955, "margin_dpo/margin_mean": 26.73247528076172, "margin_dpo/margin_std": 24.797590255737305, "step": 208 }, { "KL/chosen_KL_mean": -24.512500762939453, "KL/mean": -36.21591567993164, "KL/rejected_KL_mean": -47.9193229675293, "KL/std": 24.020729064941406, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.018616322427988052, "fcm_dpo/delta": -0.03779434412717819, "fcm_dpo/margin": 23.40682601928711, "fcm_dpo/q_t": 0.3989771008491516, "grad_norm": 9.138090133666992, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -1.8265793323516846, "logits/rejected": -1.33287513256073, "logps/chosen": -108.68306732177734, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -135.53887939453125, "loss": 1.0582, "margin_dpo/margin_mean": 23.40682601928711, "margin_dpo/margin_std": 27.143142700195312, "step": 209 }, { "KL/chosen_KL_mean": -27.933521270751953, "KL/mean": -39.03619384765625, "KL/rejected_KL_mean": -50.13887023925781, "KL/std": 25.4549617767334, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.018580205738544464, "fcm_dpo/delta": -0.01320490799844265, "fcm_dpo/margin": 22.20534896850586, "fcm_dpo/q_t": 0.40422260761260986, "grad_norm": 8.59414291381836, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.7933838367462158, "logits/rejected": -1.3989393711090088, "logps/chosen": -109.10469055175781, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -134.31365966796875, "loss": 1.0694, "margin_dpo/margin_mean": 22.20534896850586, "margin_dpo/margin_std": 25.775720596313477, "step": 210 }, { "KL/chosen_KL_mean": -31.284278869628906, "KL/mean": -42.08905792236328, "KL/rejected_KL_mean": -52.89384078979492, "KL/std": 26.765535354614258, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.018626423552632332, "fcm_dpo/delta": -0.00267762690782547, "fcm_dpo/margin": 21.609561920166016, "fcm_dpo/q_t": 0.40708571672439575, "grad_norm": 10.508427619934082, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.1756157875061035, "logits/rejected": -1.5656003952026367, "logps/chosen": -139.9969940185547, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -146.44949340820312, "loss": 1.0828, "margin_dpo/margin_mean": 21.609561920166016, "margin_dpo/margin_std": 27.137128829956055, "step": 211 }, { "KL/chosen_KL_mean": -21.893096923828125, "KL/mean": -37.793426513671875, "KL/rejected_KL_mean": -53.693748474121094, "KL/std": 30.51824951171875, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.018043115735054016, "fcm_dpo/delta": -0.18492525815963745, "fcm_dpo/margin": 31.800647735595703, "fcm_dpo/q_t": 0.36826813220977783, "grad_norm": 9.051904678344727, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.9232406616210938, "logits/rejected": -1.5143799781799316, "logps/chosen": -120.25505065917969, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -163.583740234375, "loss": 0.9608, "margin_dpo/margin_mean": 31.800647735595703, "margin_dpo/margin_std": 30.649219512939453, "step": 212 }, { "KL/chosen_KL_mean": -25.711559295654297, "KL/mean": -37.599945068359375, "KL/rejected_KL_mean": -49.48832702636719, "KL/std": 27.919334411621094, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.01781335659325123, "fcm_dpo/delta": -0.024716375395655632, "fcm_dpo/margin": 23.776771545410156, "fcm_dpo/q_t": 0.4017530381679535, "grad_norm": 15.527554512023926, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.977824330329895, "logits/rejected": -1.1987173557281494, "logps/chosen": -133.76687622070312, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -149.63247680664062, "loss": 1.0768, "margin_dpo/margin_mean": 23.776771545410156, "margin_dpo/margin_std": 30.699951171875, "step": 213 }, { "KL/chosen_KL_mean": -27.12311553955078, "KL/mean": -38.5416259765625, "KL/rejected_KL_mean": -49.96013641357422, "KL/std": 30.055965423583984, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0178290493786335, "fcm_dpo/delta": -0.007489321753382683, "fcm_dpo/margin": 22.837020874023438, "fcm_dpo/q_t": 0.40794119238853455, "grad_norm": 8.913335800170898, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.6412410736083984, "logits/rejected": -1.5156776905059814, "logps/chosen": -104.9278564453125, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -134.23593139648438, "loss": 1.0994, "margin_dpo/margin_mean": 22.837020874023438, "margin_dpo/margin_std": 32.940284729003906, "step": 214 }, { "KL/chosen_KL_mean": -23.24905776977539, "KL/mean": -38.558685302734375, "KL/rejected_KL_mean": -53.86832046508789, "KL/std": 29.37383270263672, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.017432495951652527, "fcm_dpo/delta": -0.14129571616649628, "fcm_dpo/margin": 30.619260787963867, "fcm_dpo/q_t": 0.37636321783065796, "grad_norm": 8.777982711791992, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.7044813632965088, "logits/rejected": -1.2262623310089111, "logps/chosen": -108.36295318603516, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -150.72982788085938, "loss": 0.979, "margin_dpo/margin_mean": 30.619260787963867, "margin_dpo/margin_std": 28.880966186523438, "step": 215 }, { "KL/chosen_KL_mean": -25.511859893798828, "KL/mean": -38.7379150390625, "KL/rejected_KL_mean": -51.96398162841797, "KL/std": 26.504783630371094, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.01722445711493492, "fcm_dpo/delta": -0.05838002637028694, "fcm_dpo/margin": 26.45212173461914, "fcm_dpo/q_t": 0.3942373991012573, "grad_norm": 8.313742637634277, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.7992920875549316, "logits/rejected": -1.3926150798797607, "logps/chosen": -106.10416412353516, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -133.38381958007812, "loss": 1.0359, "margin_dpo/margin_mean": 26.45212173461914, "margin_dpo/margin_std": 27.684812545776367, "step": 216 }, { "KL/chosen_KL_mean": -30.68423843383789, "KL/mean": -41.72154235839844, "KL/rejected_KL_mean": -52.758846282958984, "KL/std": 27.285358428955078, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.017185745760798454, "fcm_dpo/delta": 0.02135436423122883, "fcm_dpo/margin": 22.074613571166992, "fcm_dpo/q_t": 0.41342997550964355, "grad_norm": 10.161972045898438, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -1.9065253734588623, "logits/rejected": -1.5969821214675903, "logps/chosen": -124.15682220458984, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -156.24737548828125, "loss": 1.1089, "margin_dpo/margin_mean": 22.07461166381836, "margin_dpo/margin_std": 31.20187759399414, "step": 217 }, { "KL/chosen_KL_mean": -30.192646026611328, "KL/mean": -43.36651611328125, "KL/rejected_KL_mean": -56.540382385253906, "KL/std": 27.676029205322266, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.017003249377012253, "fcm_dpo/delta": -0.050517380237579346, "fcm_dpo/margin": 26.34773063659668, "fcm_dpo/q_t": 0.3970297873020172, "grad_norm": 8.849482536315918, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.5811986923217773, "logits/rejected": -1.2073156833648682, "logps/chosen": -119.25148010253906, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -150.84719848632812, "loss": 1.0616, "margin_dpo/margin_mean": 26.347732543945312, "margin_dpo/margin_std": 33.149024963378906, "step": 218 }, { "KL/chosen_KL_mean": -27.384185791015625, "KL/mean": -39.102046966552734, "KL/rejected_KL_mean": -50.819908142089844, "KL/std": 32.13455581665039, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.017092736437916756, "fcm_dpo/delta": -0.0011934302747249603, "fcm_dpo/margin": 23.43572235107422, "fcm_dpo/q_t": 0.40849971771240234, "grad_norm": 10.023874282836914, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.7685015201568604, "logits/rejected": -1.3168234825134277, "logps/chosen": -128.3915252685547, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -143.287841796875, "loss": 1.1072, "margin_dpo/margin_mean": 23.43572235107422, "margin_dpo/margin_std": 34.4747314453125, "step": 219 }, { "KL/chosen_KL_mean": -27.96771240234375, "KL/mean": -39.1359748840332, "KL/rejected_KL_mean": -50.30424499511719, "KL/std": 26.658899307250977, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.01705923303961754, "fcm_dpo/delta": 0.019688602536916733, "fcm_dpo/margin": 22.336532592773438, "fcm_dpo/q_t": 0.41076791286468506, "grad_norm": 10.402189254760742, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5112159252166748, "logits/rejected": -1.1612286567687988, "logps/chosen": -123.50492858886719, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -144.6112823486328, "loss": 1.0902, "margin_dpo/margin_mean": 22.336536407470703, "margin_dpo/margin_std": 26.4658203125, "step": 220 }, { "KL/chosen_KL_mean": -26.349048614501953, "KL/mean": -43.23651885986328, "KL/rejected_KL_mean": -60.12398910522461, "KL/std": 33.15812683105469, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.016738682985305786, "fcm_dpo/delta": -0.17498627305030823, "fcm_dpo/margin": 33.774940490722656, "fcm_dpo/q_t": 0.3693355321884155, "grad_norm": 10.008371353149414, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.2999571561813354, "logits/rejected": -1.1611735820770264, "logps/chosen": -109.19390869140625, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -170.935791015625, "loss": 0.9592, "margin_dpo/margin_mean": 33.774940490722656, "margin_dpo/margin_std": 29.232006072998047, "step": 221 }, { "KL/chosen_KL_mean": -33.69382858276367, "KL/mean": -47.636436462402344, "KL/rejected_KL_mean": -61.57905578613281, "KL/std": 31.92062759399414, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.016399558633565903, "fcm_dpo/delta": -0.06014494597911835, "fcm_dpo/margin": 27.885225296020508, "fcm_dpo/q_t": 0.39561137557029724, "grad_norm": 10.043067932128906, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.7529232501983643, "logits/rejected": -1.3099112510681152, "logps/chosen": -128.8358154296875, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -168.3834686279297, "loss": 1.0711, "margin_dpo/margin_mean": 27.885225296020508, "margin_dpo/margin_std": 37.684295654296875, "step": 222 }, { "KL/chosen_KL_mean": -32.7734489440918, "KL/mean": -42.685646057128906, "KL/rejected_KL_mean": -52.59785079956055, "KL/std": 28.276540756225586, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.016446655616164207, "fcm_dpo/delta": 0.07651456445455551, "fcm_dpo/margin": 19.824403762817383, "fcm_dpo/q_t": 0.4247068762779236, "grad_norm": 9.63951587677002, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.139742374420166, "logits/rejected": -1.7487168312072754, "logps/chosen": -118.34855651855469, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -139.05023193359375, "loss": 1.1483, "margin_dpo/margin_mean": 19.824403762817383, "margin_dpo/margin_std": 30.951217651367188, "step": 223 }, { "KL/chosen_KL_mean": -30.147335052490234, "KL/mean": -44.07826232910156, "KL/rejected_KL_mean": -58.009193420410156, "KL/std": 31.086380004882812, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.01637781411409378, "fcm_dpo/delta": -0.05905860662460327, "fcm_dpo/margin": 27.861860275268555, "fcm_dpo/q_t": 0.39485859870910645, "grad_norm": 9.342782020568848, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.8858660459518433, "logits/rejected": -1.4402472972869873, "logps/chosen": -112.87113952636719, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -140.60458374023438, "loss": 1.0471, "margin_dpo/margin_mean": 27.861862182617188, "margin_dpo/margin_std": 32.6223030090332, "step": 224 }, { "KL/chosen_KL_mean": -33.813716888427734, "KL/mean": -48.38397979736328, "KL/rejected_KL_mean": -62.95423889160156, "KL/std": 32.79917526245117, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.01621045544743538, "fcm_dpo/delta": -0.07594511657953262, "fcm_dpo/margin": 29.140518188476562, "fcm_dpo/q_t": 0.39275485277175903, "grad_norm": 10.12430191040039, "learning_rate": 4.249525076191759e-07, "logits/chosen": -1.8725390434265137, "logits/rejected": -1.3883072137832642, "logps/chosen": -129.49139404296875, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -168.05111694335938, "loss": 1.0491, "margin_dpo/margin_mean": 29.140518188476562, "margin_dpo/margin_std": 36.75519561767578, "step": 225 }, { "KL/chosen_KL_mean": -27.952640533447266, "KL/mean": -41.694252014160156, "KL/rejected_KL_mean": -55.43585205078125, "KL/std": 31.771892547607422, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.015920501202344894, "fcm_dpo/delta": -0.03993773087859154, "fcm_dpo/margin": 27.483211517333984, "fcm_dpo/q_t": 0.397674560546875, "grad_norm": 8.408975601196289, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.798028588294983, "logits/rejected": -1.2596368789672852, "logps/chosen": -121.41356658935547, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -142.13760375976562, "loss": 1.0452, "margin_dpo/margin_mean": 27.48320960998535, "margin_dpo/margin_std": 28.29914665222168, "step": 226 }, { "KL/chosen_KL_mean": -37.9560661315918, "KL/mean": -52.241233825683594, "KL/rejected_KL_mean": -66.52639770507812, "KL/std": 30.800750732421875, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.015840791165828705, "fcm_dpo/delta": -0.05505148321390152, "fcm_dpo/margin": 28.570343017578125, "fcm_dpo/q_t": 0.39661768078804016, "grad_norm": 9.28395938873291, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.3533024787902832, "logits/rejected": -0.99756920337677, "logps/chosen": -130.77032470703125, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -171.26333618164062, "loss": 1.0627, "margin_dpo/margin_mean": 28.570343017578125, "margin_dpo/margin_std": 36.621620178222656, "step": 227 }, { "KL/chosen_KL_mean": -35.35980224609375, "KL/mean": -46.421180725097656, "KL/rejected_KL_mean": -57.48257064819336, "KL/std": 29.063617706298828, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.01589767448604107, "fcm_dpo/delta": 0.05008504539728165, "fcm_dpo/margin": 22.122766494750977, "fcm_dpo/q_t": 0.41822701692581177, "grad_norm": 9.67245101928711, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.9327754974365234, "logits/rejected": -1.5048879384994507, "logps/chosen": -129.3969268798828, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -153.50408935546875, "loss": 1.1309, "margin_dpo/margin_mean": 22.12276840209961, "margin_dpo/margin_std": 32.979713439941406, "step": 228 }, { "KL/chosen_KL_mean": -31.413562774658203, "KL/mean": -49.387718200683594, "KL/rejected_KL_mean": -67.36187744140625, "KL/std": 35.11144256591797, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.01556410826742649, "fcm_dpo/delta": -0.1688627302646637, "fcm_dpo/margin": 35.94831848144531, "fcm_dpo/q_t": 0.37353792786598206, "grad_norm": 9.321662902832031, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.7810207605361938, "logits/rejected": -1.145698070526123, "logps/chosen": -126.99122619628906, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -156.53567504882812, "loss": 0.9825, "margin_dpo/margin_mean": 35.94831848144531, "margin_dpo/margin_std": 38.526058197021484, "step": 229 }, { "KL/chosen_KL_mean": -33.14071273803711, "KL/mean": -51.584842681884766, "KL/rejected_KL_mean": -70.02897644042969, "KL/std": 34.734107971191406, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.01504562795162201, "fcm_dpo/delta": -0.1641485095024109, "fcm_dpo/margin": 36.88825225830078, "fcm_dpo/q_t": 0.37095850706100464, "grad_norm": 10.400053024291992, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.7494392395019531, "logits/rejected": -1.685474157333374, "logps/chosen": -96.85433959960938, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -161.93768310546875, "loss": 0.9686, "margin_dpo/margin_mean": 36.88825225830078, "margin_dpo/margin_std": 34.69062423706055, "step": 230 }, { "KL/chosen_KL_mean": -36.63505554199219, "KL/mean": -52.75994110107422, "KL/rejected_KL_mean": -68.88481903076172, "KL/std": 33.74462890625, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.014682313427329063, "fcm_dpo/delta": -0.07793852686882019, "fcm_dpo/margin": 32.24976348876953, "fcm_dpo/q_t": 0.38834235072135925, "grad_norm": 8.228200912475586, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.6444740295410156, "logits/rejected": -1.0735310316085815, "logps/chosen": -132.09072875976562, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -149.8404998779297, "loss": 1.0118, "margin_dpo/margin_mean": 32.24976348876953, "margin_dpo/margin_std": 29.04229736328125, "step": 231 }, { "KL/chosen_KL_mean": -38.7113037109375, "KL/mean": -52.02202606201172, "KL/rejected_KL_mean": -65.33274841308594, "KL/std": 30.885393142700195, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.014695952646434307, "fcm_dpo/delta": 0.009118199348449707, "fcm_dpo/margin": 26.621437072753906, "fcm_dpo/q_t": 0.40836429595947266, "grad_norm": 9.211897850036621, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.560875415802002, "logits/rejected": -1.0827691555023193, "logps/chosen": -135.6085662841797, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -155.0973663330078, "loss": 1.0834, "margin_dpo/margin_mean": 26.621435165405273, "margin_dpo/margin_std": 31.207420349121094, "step": 232 }, { "KL/chosen_KL_mean": -40.43531799316406, "KL/mean": -59.12599182128906, "KL/rejected_KL_mean": -77.81666564941406, "KL/std": 39.366912841796875, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.01437801867723465, "fcm_dpo/delta": -0.14558997750282288, "fcm_dpo/margin": 37.38134765625, "fcm_dpo/q_t": 0.3759036362171173, "grad_norm": 9.742077827453613, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.6927701234817505, "logits/rejected": -1.4857254028320312, "logps/chosen": -129.4856414794922, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -190.57583618164062, "loss": 0.9875, "margin_dpo/margin_mean": 37.38134765625, "margin_dpo/margin_std": 37.95393371582031, "step": 233 }, { "KL/chosen_KL_mean": -39.749000549316406, "KL/mean": -59.68584442138672, "KL/rejected_KL_mean": -79.62268829345703, "KL/std": 37.470340728759766, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.013850782997906208, "fcm_dpo/delta": -0.16347210109233856, "fcm_dpo/margin": 39.873687744140625, "fcm_dpo/q_t": 0.373555064201355, "grad_norm": 8.648323059082031, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.1206440925598145, "logits/rejected": -1.054878830909729, "logps/chosen": -114.06736755371094, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -186.01028442382812, "loss": 0.9839, "margin_dpo/margin_mean": 39.873687744140625, "margin_dpo/margin_std": 40.945465087890625, "step": 234 }, { "KL/chosen_KL_mean": -48.09256362915039, "KL/mean": -63.11113357543945, "KL/rejected_KL_mean": -78.12970733642578, "KL/std": 33.735877990722656, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.013818752020597458, "fcm_dpo/delta": -0.01573720946907997, "fcm_dpo/margin": 30.037147521972656, "fcm_dpo/q_t": 0.4024432599544525, "grad_norm": 8.748790740966797, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.60084867477417, "logits/rejected": -1.047987461090088, "logps/chosen": -146.30960083007812, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -175.37648010253906, "loss": 1.0594, "margin_dpo/margin_mean": 30.037147521972656, "margin_dpo/margin_std": 31.85127830505371, "step": 235 }, { "KL/chosen_KL_mean": -40.03675079345703, "KL/mean": -59.39972686767578, "KL/rejected_KL_mean": -78.76271057128906, "KL/std": 42.42284393310547, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.013555407524108887, "fcm_dpo/delta": -0.1316768378019333, "fcm_dpo/margin": 38.72595977783203, "fcm_dpo/q_t": 0.38475239276885986, "grad_norm": 9.405375480651855, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.2915515899658203, "logits/rejected": -1.2902464866638184, "logps/chosen": -118.87448120117188, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -187.82614135742188, "loss": 1.0123, "margin_dpo/margin_mean": 38.72595977783203, "margin_dpo/margin_std": 46.44171142578125, "step": 236 }, { "KL/chosen_KL_mean": -48.05778884887695, "KL/mean": -62.22490310668945, "KL/rejected_KL_mean": -76.39202117919922, "KL/std": 37.563907623291016, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.01347154937684536, "fcm_dpo/delta": 0.019020162522792816, "fcm_dpo/margin": 28.334228515625, "fcm_dpo/q_t": 0.41598910093307495, "grad_norm": 8.73369312286377, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.6281380653381348, "logits/rejected": -1.2791506052017212, "logps/chosen": -133.26907348632812, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -176.30201721191406, "loss": 1.1379, "margin_dpo/margin_mean": 28.334230422973633, "margin_dpo/margin_std": 48.64256286621094, "step": 237 }, { "KL/chosen_KL_mean": -52.43962478637695, "KL/mean": -69.14616394042969, "KL/rejected_KL_mean": -85.85270690917969, "KL/std": 38.940345764160156, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.013372497633099556, "fcm_dpo/delta": -0.04908018931746483, "fcm_dpo/margin": 33.41309356689453, "fcm_dpo/q_t": 0.39785057306289673, "grad_norm": 8.163467407226562, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2108381986618042, "logits/rejected": -1.0610544681549072, "logps/chosen": -131.17086791992188, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -181.2711181640625, "loss": 1.0558, "margin_dpo/margin_mean": 33.413089752197266, "margin_dpo/margin_std": 40.59546661376953, "step": 238 }, { "KL/chosen_KL_mean": -47.7218132019043, "KL/mean": -63.04688262939453, "KL/rejected_KL_mean": -78.3719482421875, "KL/std": 34.53764343261719, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.013256929814815521, "fcm_dpo/delta": -0.0074077509343624115, "fcm_dpo/margin": 30.650135040283203, "fcm_dpo/q_t": 0.40614038705825806, "grad_norm": 8.889233589172363, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.8399169445037842, "logits/rejected": -1.3654403686523438, "logps/chosen": -140.32275390625, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -178.8896484375, "loss": 1.0859, "margin_dpo/margin_mean": 30.650135040283203, "margin_dpo/margin_std": 38.49876022338867, "step": 239 }, { "KL/chosen_KL_mean": -49.59743118286133, "KL/mean": -63.33129119873047, "KL/rejected_KL_mean": -77.06515502929688, "KL/std": 34.731285095214844, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.013371091336011887, "fcm_dpo/delta": 0.03394855558872223, "fcm_dpo/margin": 27.467731475830078, "fcm_dpo/q_t": 0.4146925210952759, "grad_norm": 9.891690254211426, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.1117000579833984, "logits/rejected": -1.7577449083328247, "logps/chosen": -135.7572021484375, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -157.52081298828125, "loss": 1.1127, "margin_dpo/margin_mean": 27.467731475830078, "margin_dpo/margin_std": 37.493648529052734, "step": 240 }, { "KL/chosen_KL_mean": -45.3431510925293, "KL/mean": -69.45487976074219, "KL/rejected_KL_mean": -93.56660461425781, "KL/std": 39.753196716308594, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.013015536591410637, "fcm_dpo/delta": -0.24220874905586243, "fcm_dpo/margin": 48.22346115112305, "fcm_dpo/q_t": 0.3570241332054138, "grad_norm": 8.581880569458008, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.712160348892212, "logits/rejected": -1.032771110534668, "logps/chosen": -146.38909912109375, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -187.61595153808594, "loss": 0.9309, "margin_dpo/margin_mean": 48.22346115112305, "margin_dpo/margin_std": 44.58347702026367, "step": 241 }, { "KL/chosen_KL_mean": -53.44758987426758, "KL/mean": -70.82295989990234, "KL/rejected_KL_mean": -88.19832611083984, "KL/std": 35.47185134887695, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.012741124257445335, "fcm_dpo/delta": -0.044785480946302414, "fcm_dpo/margin": 34.75072479248047, "fcm_dpo/q_t": 0.39646115899086, "grad_norm": 9.016071319580078, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.9491791725158691, "logits/rejected": -1.297619342803955, "logps/chosen": -149.12025451660156, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -178.85287475585938, "loss": 1.052, "margin_dpo/margin_mean": 34.75072479248047, "margin_dpo/margin_std": 39.35816955566406, "step": 242 }, { "KL/chosen_KL_mean": -52.94251251220703, "KL/mean": -70.07247924804688, "KL/rejected_KL_mean": -87.20245361328125, "KL/std": 37.95367431640625, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.012701909989118576, "fcm_dpo/delta": -0.037680599838495255, "fcm_dpo/margin": 34.25994110107422, "fcm_dpo/q_t": 0.40059399604797363, "grad_norm": 8.7799072265625, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.9416487216949463, "logits/rejected": -1.1277172565460205, "logps/chosen": -150.97390747070312, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -170.39051818847656, "loss": 1.0793, "margin_dpo/margin_mean": 34.25994110107422, "margin_dpo/margin_std": 44.98371124267578, "step": 243 }, { "KL/chosen_KL_mean": -55.170166015625, "KL/mean": -73.48052215576172, "KL/rejected_KL_mean": -91.79087829589844, "KL/std": 42.053070068359375, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.012448225170373917, "fcm_dpo/delta": -0.05870276689529419, "fcm_dpo/margin": 36.6207160949707, "fcm_dpo/q_t": 0.3947634994983673, "grad_norm": 8.516319274902344, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.0236549377441406, "logits/rejected": -1.6874823570251465, "logps/chosen": -144.0640869140625, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -194.36366271972656, "loss": 1.0446, "margin_dpo/margin_mean": 36.6207160949707, "margin_dpo/margin_std": 42.080657958984375, "step": 244 }, { "KL/chosen_KL_mean": -46.87601852416992, "KL/mean": -66.29254150390625, "KL/rejected_KL_mean": -85.70905303955078, "KL/std": 38.944557189941406, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.012302586808800697, "fcm_dpo/delta": -0.08157012611627579, "fcm_dpo/margin": 38.83303451538086, "fcm_dpo/q_t": 0.3887595534324646, "grad_norm": 8.082144737243652, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.2673592567443848, "logits/rejected": -1.436455249786377, "logps/chosen": -151.07000732421875, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -178.36550903320312, "loss": 1.0211, "margin_dpo/margin_mean": 38.833030700683594, "margin_dpo/margin_std": 39.96971893310547, "step": 245 }, { "KL/chosen_KL_mean": -61.24646759033203, "KL/mean": -76.41091918945312, "KL/rejected_KL_mean": -91.57537841796875, "KL/std": 37.93702697753906, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.01224526483565569, "fcm_dpo/delta": 0.029589543119072914, "fcm_dpo/margin": 30.32891273498535, "fcm_dpo/q_t": 0.41423943638801575, "grad_norm": 8.235289573669434, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.6867619752883911, "logits/rejected": -1.1284260749816895, "logps/chosen": -164.46551513671875, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -182.56759643554688, "loss": 1.1093, "margin_dpo/margin_mean": 30.32891273498535, "margin_dpo/margin_std": 41.101158142089844, "step": 246 }, { "KL/chosen_KL_mean": -54.699066162109375, "KL/mean": -79.83431243896484, "KL/rejected_KL_mean": -104.96955871582031, "KL/std": 42.8199577331543, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.011871191672980785, "fcm_dpo/delta": -0.21066515147686005, "fcm_dpo/margin": 50.27050018310547, "fcm_dpo/q_t": 0.364105224609375, "grad_norm": 8.232104301452637, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.2895286083221436, "logits/rejected": -2.110675811767578, "logps/chosen": -141.6934356689453, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -216.30758666992188, "loss": 0.9505, "margin_dpo/margin_mean": 50.27050018310547, "margin_dpo/margin_std": 48.2497444152832, "step": 247 }, { "KL/chosen_KL_mean": -63.31340408325195, "KL/mean": -80.01724243164062, "KL/rejected_KL_mean": -96.7210922241211, "KL/std": 37.73096466064453, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.011837083846330643, "fcm_dpo/delta": 0.004575330764055252, "fcm_dpo/margin": 33.40768814086914, "fcm_dpo/q_t": 0.407665491104126, "grad_norm": 8.779619216918945, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.8383536338806152, "logits/rejected": -1.8098317384719849, "logps/chosen": -138.0989227294922, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -194.99798583984375, "loss": 1.0799, "margin_dpo/margin_mean": 33.407691955566406, "margin_dpo/margin_std": 38.903045654296875, "step": 248 }, { "KL/chosen_KL_mean": -70.97364807128906, "KL/mean": -85.893798828125, "KL/rejected_KL_mean": -100.81393432617188, "KL/std": 40.044708251953125, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.011848503723740578, "fcm_dpo/delta": 0.04803425818681717, "fcm_dpo/margin": 29.84029769897461, "fcm_dpo/q_t": 0.418906033039093, "grad_norm": 9.96058464050293, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.2508833408355713, "logits/rejected": -1.721149206161499, "logps/chosen": -166.6774444580078, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -187.96041870117188, "loss": 1.1358, "margin_dpo/margin_mean": 29.840301513671875, "margin_dpo/margin_std": 46.1710205078125, "step": 249 }, { "KL/chosen_KL_mean": -62.64890670776367, "KL/mean": -79.58099365234375, "KL/rejected_KL_mean": -96.5130615234375, "KL/std": 40.847076416015625, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.011860568076372147, "fcm_dpo/delta": -0.0022037141025066376, "fcm_dpo/margin": 33.86417007446289, "fcm_dpo/q_t": 0.4063014090061188, "grad_norm": 9.198631286621094, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.7915928363800049, "logits/rejected": -0.8318342566490173, "logps/chosen": -176.70111083984375, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -177.60073852539062, "loss": 1.0933, "margin_dpo/margin_mean": 33.86417007446289, "margin_dpo/margin_std": 43.90589141845703, "step": 250 }, { "KL/chosen_KL_mean": -62.88948059082031, "KL/mean": -78.10136413574219, "KL/rejected_KL_mean": -93.31324768066406, "KL/std": 37.86114501953125, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.011919749900698662, "fcm_dpo/delta": 0.03836161270737648, "fcm_dpo/margin": 30.423765182495117, "fcm_dpo/q_t": 0.41726598143577576, "grad_norm": 9.84812068939209, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.9942355155944824, "logits/rejected": -1.3114838600158691, "logps/chosen": -160.60076904296875, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -176.84066772460938, "loss": 1.1284, "margin_dpo/margin_mean": 30.423765182495117, "margin_dpo/margin_std": 45.350975036621094, "step": 251 }, { "KL/chosen_KL_mean": -63.170867919921875, "KL/mean": -78.29010009765625, "KL/rejected_KL_mean": -93.40933227539062, "KL/std": 40.028690338134766, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.012091992422938347, "fcm_dpo/delta": 0.03554663807153702, "fcm_dpo/margin": 30.23845863342285, "fcm_dpo/q_t": 0.41484642028808594, "grad_norm": 9.685420036315918, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -1.948857069015503, "logits/rejected": -1.4006831645965576, "logps/chosen": -172.156982421875, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -184.97357177734375, "loss": 1.1187, "margin_dpo/margin_mean": 30.23845863342285, "margin_dpo/margin_std": 42.709228515625, "step": 252 }, { "KL/chosen_KL_mean": -69.56097412109375, "KL/mean": -83.72863006591797, "KL/rejected_KL_mean": -97.89629364013672, "KL/std": 43.570762634277344, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.012265942059457302, "fcm_dpo/delta": 0.05322246253490448, "fcm_dpo/margin": 28.33531951904297, "fcm_dpo/q_t": 0.41828811168670654, "grad_norm": 9.7263765335083, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -1.9799484014511108, "logits/rejected": -1.7675567865371704, "logps/chosen": -169.77728271484375, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -203.572998046875, "loss": 1.1238, "margin_dpo/margin_mean": 28.3353214263916, "margin_dpo/margin_std": 38.03700637817383, "step": 253 }, { "KL/chosen_KL_mean": -67.11991882324219, "KL/mean": -81.76282501220703, "KL/rejected_KL_mean": -96.4057388305664, "KL/std": 41.09549331665039, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.012252865359187126, "fcm_dpo/delta": 0.04254695028066635, "fcm_dpo/margin": 29.285818099975586, "fcm_dpo/q_t": 0.4163278639316559, "grad_norm": 9.495330810546875, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.0651493072509766, "logits/rejected": -1.5389602184295654, "logps/chosen": -165.8551025390625, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -190.14398193359375, "loss": 1.1293, "margin_dpo/margin_mean": 29.285818099975586, "margin_dpo/margin_std": 43.65628433227539, "step": 254 }, { "KL/chosen_KL_mean": -68.07154846191406, "KL/mean": -90.30589294433594, "KL/rejected_KL_mean": -112.54022216796875, "KL/std": 46.39250183105469, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.012078619562089443, "fcm_dpo/delta": -0.14477291703224182, "fcm_dpo/margin": 44.46867370605469, "fcm_dpo/q_t": 0.37648898363113403, "grad_norm": 9.482341766357422, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.1224353313446045, "logits/rejected": -1.7933942079544067, "logps/chosen": -158.45411682128906, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -209.61647033691406, "loss": 0.9961, "margin_dpo/margin_mean": 44.46867752075195, "margin_dpo/margin_std": 47.751792907714844, "step": 255 }, { "KL/chosen_KL_mean": -66.11590576171875, "KL/mean": -85.8607177734375, "KL/rejected_KL_mean": -105.60552978515625, "KL/std": 48.1549186706543, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.01187562383711338, "fcm_dpo/delta": -0.07229971885681152, "fcm_dpo/margin": 39.48961639404297, "fcm_dpo/q_t": 0.3944740891456604, "grad_norm": 9.926170349121094, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.8748700618743896, "logits/rejected": -1.5354931354522705, "logps/chosen": -154.86871337890625, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -204.099365234375, "loss": 1.0738, "margin_dpo/margin_mean": 39.48961639404297, "margin_dpo/margin_std": 56.267608642578125, "step": 256 }, { "KL/chosen_KL_mean": -68.25222778320312, "KL/mean": -88.2470703125, "KL/rejected_KL_mean": -108.24192810058594, "KL/std": 49.35713195800781, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.011582026258111, "fcm_dpo/delta": -0.06842642277479172, "fcm_dpo/margin": 39.98970031738281, "fcm_dpo/q_t": 0.3936750888824463, "grad_norm": 8.953132629394531, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.7941615581512451, "logits/rejected": -1.387739896774292, "logps/chosen": -151.05575561523438, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -194.10971069335938, "loss": 1.0537, "margin_dpo/margin_mean": 39.98970031738281, "margin_dpo/margin_std": 47.73744201660156, "step": 257 }, { "KL/chosen_KL_mean": -70.48883819580078, "KL/mean": -88.9354019165039, "KL/rejected_KL_mean": -107.38196563720703, "KL/std": 47.331520080566406, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.01166028156876564, "fcm_dpo/delta": -0.032123927026987076, "fcm_dpo/margin": 36.89311981201172, "fcm_dpo/q_t": 0.40006810426712036, "grad_norm": 9.419631004333496, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.1496171951293945, "logits/rejected": -1.7954916954040527, "logps/chosen": -160.67657470703125, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -210.8526611328125, "loss": 1.0704, "margin_dpo/margin_mean": 36.89311981201172, "margin_dpo/margin_std": 44.96318054199219, "step": 258 }, { "KL/chosen_KL_mean": -76.19316101074219, "KL/mean": -92.1588134765625, "KL/rejected_KL_mean": -108.12445068359375, "KL/std": 47.64023971557617, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.011559716425836086, "fcm_dpo/delta": 0.031615182757377625, "fcm_dpo/margin": 31.931304931640625, "fcm_dpo/q_t": 0.41507571935653687, "grad_norm": 9.327096939086914, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.2722610235214233, "logits/rejected": -1.0177847146987915, "logps/chosen": -166.9656982421875, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -202.71261596679688, "loss": 1.1438, "margin_dpo/margin_mean": 31.931304931640625, "margin_dpo/margin_std": 53.906131744384766, "step": 259 }, { "KL/chosen_KL_mean": -78.35509490966797, "KL/mean": -91.831787109375, "KL/rejected_KL_mean": -105.3084716796875, "KL/std": 48.02167510986328, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.011785298585891724, "fcm_dpo/delta": 0.08508594334125519, "fcm_dpo/margin": 26.953388214111328, "fcm_dpo/q_t": 0.427178293466568, "grad_norm": 11.1622953414917, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.6736125946044922, "logits/rejected": -1.6999796628952026, "logps/chosen": -153.94778442382812, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -206.15402221679688, "loss": 1.1854, "margin_dpo/margin_mean": 26.95339012145996, "margin_dpo/margin_std": 52.84510040283203, "step": 260 }, { "KL/chosen_KL_mean": -59.50114822387695, "KL/mean": -82.83216857910156, "KL/rejected_KL_mean": -106.16319274902344, "KL/std": 48.71497344970703, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.011519616469740868, "fcm_dpo/delta": -0.14688275754451752, "fcm_dpo/margin": 46.66204833984375, "fcm_dpo/q_t": 0.37642771005630493, "grad_norm": 8.849084854125977, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.9554710388183594, "logits/rejected": -1.5876063108444214, "logps/chosen": -153.7560577392578, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -199.8201904296875, "loss": 0.9885, "margin_dpo/margin_mean": 46.66204833984375, "margin_dpo/margin_std": 47.46400451660156, "step": 261 }, { "KL/chosen_KL_mean": -65.17858123779297, "KL/mean": -86.8887939453125, "KL/rejected_KL_mean": -108.59901428222656, "KL/std": 51.2280387878418, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.011294813826680183, "fcm_dpo/delta": -0.0957278236746788, "fcm_dpo/margin": 43.42041778564453, "fcm_dpo/q_t": 0.38740572333335876, "grad_norm": 8.714659690856934, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.6930742263793945, "logits/rejected": -1.1516568660736084, "logps/chosen": -150.44589233398438, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -199.42510986328125, "loss": 1.0314, "margin_dpo/margin_mean": 43.4204216003418, "margin_dpo/margin_std": 50.8759765625, "step": 262 }, { "KL/chosen_KL_mean": -52.22853088378906, "KL/mean": -81.6966323852539, "KL/rejected_KL_mean": -111.16473388671875, "KL/std": 55.87263488769531, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.010878434404730797, "fcm_dpo/delta": -0.2572769820690155, "fcm_dpo/margin": 58.93619918823242, "fcm_dpo/q_t": 0.35422539710998535, "grad_norm": 8.57580280303955, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.9257493019104004, "logits/rejected": -1.3525094985961914, "logps/chosen": -147.98643493652344, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -212.99850463867188, "loss": 0.9386, "margin_dpo/margin_mean": 58.93619918823242, "margin_dpo/margin_std": 59.37754440307617, "step": 263 }, { "KL/chosen_KL_mean": -61.40967559814453, "KL/mean": -86.33154296875, "KL/rejected_KL_mean": -111.25341796875, "KL/std": 52.665924072265625, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.010492851957678795, "fcm_dpo/delta": -0.13006749749183655, "fcm_dpo/margin": 49.84375, "fcm_dpo/q_t": 0.3795081377029419, "grad_norm": 8.445716857910156, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.1388819217681885, "logits/rejected": -1.8574435710906982, "logps/chosen": -152.03233337402344, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -211.57896423339844, "loss": 1.0039, "margin_dpo/margin_mean": 49.84375, "margin_dpo/margin_std": 53.71699905395508, "step": 264 }, { "KL/chosen_KL_mean": -64.32520294189453, "KL/mean": -86.2406234741211, "KL/rejected_KL_mean": -108.15603637695312, "KL/std": 53.91370391845703, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.010308632627129555, "fcm_dpo/delta": -0.054956208914518356, "fcm_dpo/margin": 43.83082580566406, "fcm_dpo/q_t": 0.39772480726242065, "grad_norm": 8.71453857421875, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.1128158569335938, "logits/rejected": -1.7147624492645264, "logps/chosen": -156.69754028320312, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -202.7836151123047, "loss": 1.07, "margin_dpo/margin_mean": 43.83082962036133, "margin_dpo/margin_std": 58.376434326171875, "step": 265 }, { "KL/chosen_KL_mean": -75.28619384765625, "KL/mean": -95.10102844238281, "KL/rejected_KL_mean": -114.91586303710938, "KL/std": 52.021873474121094, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.010293660685420036, "fcm_dpo/delta": -0.008362928405404091, "fcm_dpo/margin": 39.62965393066406, "fcm_dpo/q_t": 0.40690529346466064, "grad_norm": 10.30048942565918, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.245450973510742, "logits/rejected": -1.4254307746887207, "logps/chosen": -177.48622131347656, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -189.28228759765625, "loss": 1.0961, "margin_dpo/margin_mean": 39.62965774536133, "margin_dpo/margin_std": 54.88874816894531, "step": 266 }, { "KL/chosen_KL_mean": -65.93115234375, "KL/mean": -89.29327392578125, "KL/rejected_KL_mean": -112.65538787841797, "KL/std": 52.57661056518555, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.010149003006517887, "fcm_dpo/delta": -0.07853078097105026, "fcm_dpo/margin": 46.724246978759766, "fcm_dpo/q_t": 0.39021795988082886, "grad_norm": 9.02602481842041, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.6002657413482666, "logits/rejected": -0.9585150480270386, "logps/chosen": -170.21714782714844, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -208.642578125, "loss": 1.0285, "margin_dpo/margin_mean": 46.7242431640625, "margin_dpo/margin_std": 49.96333694458008, "step": 267 }, { "KL/chosen_KL_mean": -65.3275375366211, "KL/mean": -92.37986755371094, "KL/rejected_KL_mean": -119.43220520019531, "KL/std": 52.48602294921875, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.009888757020235062, "fcm_dpo/delta": -0.14374177157878876, "fcm_dpo/margin": 54.10466766357422, "fcm_dpo/q_t": 0.37533071637153625, "grad_norm": 9.257969856262207, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6176934242248535, "logits/rejected": -1.4591197967529297, "logps/chosen": -152.0237579345703, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -228.62405395507812, "loss": 0.9834, "margin_dpo/margin_mean": 54.10466384887695, "margin_dpo/margin_std": 51.92957305908203, "step": 268 }, { "KL/chosen_KL_mean": -69.50234985351562, "KL/mean": -96.13490295410156, "KL/rejected_KL_mean": -122.7674560546875, "KL/std": 56.58293151855469, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.009706191718578339, "fcm_dpo/delta": -0.123200923204422, "fcm_dpo/margin": 53.265113830566406, "fcm_dpo/q_t": 0.38010334968566895, "grad_norm": 8.33785629272461, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.9719513654708862, "logits/rejected": -1.525005578994751, "logps/chosen": -174.47415161132812, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -235.24392700195312, "loss": 0.9979, "margin_dpo/margin_mean": 53.265113830566406, "margin_dpo/margin_std": 53.61846923828125, "step": 269 }, { "KL/chosen_KL_mean": -77.90023803710938, "KL/mean": -100.25646209716797, "KL/rejected_KL_mean": -122.61268615722656, "KL/std": 55.810325622558594, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.009606312960386276, "fcm_dpo/delta": -0.03094649501144886, "fcm_dpo/margin": 44.71243667602539, "fcm_dpo/q_t": 0.40143120288848877, "grad_norm": 9.02673625946045, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.8063488006591797, "logits/rejected": -2.417509078979492, "logps/chosen": -178.9740753173828, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -240.36557006835938, "loss": 1.0905, "margin_dpo/margin_mean": 44.71243667602539, "margin_dpo/margin_std": 63.41035461425781, "step": 270 }, { "KL/chosen_KL_mean": -63.445377349853516, "KL/mean": -91.29248809814453, "KL/rejected_KL_mean": -119.13960266113281, "KL/std": 52.185691833496094, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.009368638508021832, "fcm_dpo/delta": -0.12867633998394012, "fcm_dpo/margin": 55.694217681884766, "fcm_dpo/q_t": 0.377975732088089, "grad_norm": 8.589531898498535, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.5635080337524414, "logits/rejected": -1.1165509223937988, "logps/chosen": -160.16998291015625, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -217.6640167236328, "loss": 1.0054, "margin_dpo/margin_mean": 55.69422149658203, "margin_dpo/margin_std": 59.756526947021484, "step": 271 }, { "KL/chosen_KL_mean": -54.87217712402344, "KL/mean": -81.48572540283203, "KL/rejected_KL_mean": -108.09927368164062, "KL/std": 58.01954650878906, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.00918591395020485, "fcm_dpo/delta": -0.09355483949184418, "fcm_dpo/margin": 53.22709274291992, "fcm_dpo/q_t": 0.38814932107925415, "grad_norm": 8.364922523498535, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.6507558822631836, "logits/rejected": -1.5375397205352783, "logps/chosen": -135.84939575195312, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -216.65463256835938, "loss": 1.0336, "margin_dpo/margin_mean": 53.227088928222656, "margin_dpo/margin_std": 62.95185089111328, "step": 272 }, { "KL/chosen_KL_mean": -75.77857208251953, "KL/mean": -103.11614990234375, "KL/rejected_KL_mean": -130.4537353515625, "KL/std": 59.43239974975586, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.008948169648647308, "fcm_dpo/delta": -0.09574626386165619, "fcm_dpo/margin": 54.67515563964844, "fcm_dpo/q_t": 0.38722488284111023, "grad_norm": 8.12210750579834, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.698521614074707, "logits/rejected": -1.2945671081542969, "logps/chosen": -168.003173828125, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -227.81678771972656, "loss": 1.0437, "margin_dpo/margin_mean": 54.67515563964844, "margin_dpo/margin_std": 67.05293273925781, "step": 273 }, { "KL/chosen_KL_mean": -77.55535888671875, "KL/mean": -98.91383361816406, "KL/rejected_KL_mean": -120.27229309082031, "KL/std": 58.502174377441406, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.008975956588983536, "fcm_dpo/delta": 0.017177708446979523, "fcm_dpo/margin": 42.716941833496094, "fcm_dpo/q_t": 0.4110121726989746, "grad_norm": 12.563140869140625, "learning_rate": 3.75e-07, "logits/chosen": -1.9647953510284424, "logits/rejected": -1.569909930229187, "logps/chosen": -159.81625366210938, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -204.1492919921875, "loss": 1.0959, "margin_dpo/margin_mean": 42.716941833496094, "margin_dpo/margin_std": 54.42467498779297, "step": 274 }, { "KL/chosen_KL_mean": -65.2650146484375, "KL/mean": -89.07344055175781, "KL/rejected_KL_mean": -112.8818588256836, "KL/std": 57.23696517944336, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.009011107496917248, "fcm_dpo/delta": -0.030822865664958954, "fcm_dpo/margin": 47.6168212890625, "fcm_dpo/q_t": 0.40134450793266296, "grad_norm": 9.165748596191406, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.2219207286834717, "logits/rejected": -1.90458345413208, "logps/chosen": -144.9519805908203, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -211.63282775878906, "loss": 1.0946, "margin_dpo/margin_mean": 47.616825103759766, "margin_dpo/margin_std": 68.08540344238281, "step": 275 }, { "KL/chosen_KL_mean": -78.42704772949219, "KL/mean": -103.99177551269531, "KL/rejected_KL_mean": -129.5565185546875, "KL/std": 58.043975830078125, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.00881720706820488, "fcm_dpo/delta": -0.05447119101881981, "fcm_dpo/margin": 51.12947463989258, "fcm_dpo/q_t": 0.3968842029571533, "grad_norm": 9.62060832977295, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.0817108154296875, "logits/rejected": -1.5084891319274902, "logps/chosen": -164.9667510986328, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -207.41046142578125, "loss": 1.0656, "margin_dpo/margin_mean": 51.12947463989258, "margin_dpo/margin_std": 65.41534423828125, "step": 276 }, { "KL/chosen_KL_mean": -85.21162414550781, "KL/mean": -103.78577423095703, "KL/rejected_KL_mean": -122.35992431640625, "KL/std": 56.04447937011719, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.008914841338992119, "fcm_dpo/delta": 0.07119029760360718, "fcm_dpo/margin": 37.148292541503906, "fcm_dpo/q_t": 0.4232920706272125, "grad_norm": 9.40386962890625, "learning_rate": 3.71651119641714e-07, "logits/chosen": -1.9751187562942505, "logits/rejected": -1.555647373199463, "logps/chosen": -169.45574951171875, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -221.1941375732422, "loss": 1.1452, "margin_dpo/margin_mean": 37.148292541503906, "margin_dpo/margin_std": 57.01061248779297, "step": 277 }, { "KL/chosen_KL_mean": -81.09117126464844, "KL/mean": -105.03176879882812, "KL/rejected_KL_mean": -128.97235107421875, "KL/std": 60.422584533691406, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.008922533132135868, "fcm_dpo/delta": -0.028566742315888405, "fcm_dpo/margin": 47.881195068359375, "fcm_dpo/q_t": 0.4002891778945923, "grad_norm": 11.13908863067627, "learning_rate": 3.705283756425872e-07, "logits/chosen": -1.8913779258728027, "logits/rejected": -1.6699786186218262, "logps/chosen": -163.52305603027344, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -226.8292694091797, "loss": 1.0751, "margin_dpo/margin_mean": 47.881195068359375, "margin_dpo/margin_std": 60.82353973388672, "step": 278 }, { "KL/chosen_KL_mean": -83.2656478881836, "KL/mean": -112.63936614990234, "KL/rejected_KL_mean": -142.01309204101562, "KL/std": 63.52426528930664, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.008740292862057686, "fcm_dpo/delta": -0.12021353095769882, "fcm_dpo/margin": 58.747432708740234, "fcm_dpo/q_t": 0.38159120082855225, "grad_norm": 10.223775863647461, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.9388704299926758, "logits/rejected": -1.7295448780059814, "logps/chosen": -168.43365478515625, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -236.13973999023438, "loss": 1.0074, "margin_dpo/margin_mean": 58.747432708740234, "margin_dpo/margin_std": 63.17100143432617, "step": 279 }, { "KL/chosen_KL_mean": -86.89634704589844, "KL/mean": -116.44004821777344, "KL/rejected_KL_mean": -145.9837646484375, "KL/std": 58.47175979614258, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.008585210889577866, "fcm_dpo/delta": -0.11287400126457214, "fcm_dpo/margin": 59.08740997314453, "fcm_dpo/q_t": 0.38197898864746094, "grad_norm": 8.83063793182373, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.9569993019104004, "logits/rejected": -1.7618904113769531, "logps/chosen": -172.7527618408203, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -250.10235595703125, "loss": 1.006, "margin_dpo/margin_mean": 59.08740997314453, "margin_dpo/margin_std": 60.63909149169922, "step": 280 }, { "KL/chosen_KL_mean": -92.24554443359375, "KL/mean": -121.67832946777344, "KL/rejected_KL_mean": -151.11111450195312, "KL/std": 68.2264404296875, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.00837787427008152, "fcm_dpo/delta": -0.09831906855106354, "fcm_dpo/margin": 58.865577697753906, "fcm_dpo/q_t": 0.38732707500457764, "grad_norm": 7.093104362487793, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.2819783687591553, "logits/rejected": -1.5650677680969238, "logps/chosen": -178.79635620117188, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -239.73977661132812, "loss": 1.0299, "margin_dpo/margin_mean": 58.865577697753906, "margin_dpo/margin_std": 69.30995178222656, "step": 281 }, { "KL/chosen_KL_mean": -90.77289581298828, "KL/mean": -127.63312530517578, "KL/rejected_KL_mean": -164.49334716796875, "KL/std": 66.58930206298828, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0080888532102108, "fcm_dpo/delta": -0.20906373858451843, "fcm_dpo/margin": 73.72046661376953, "fcm_dpo/q_t": 0.3618050217628479, "grad_norm": 8.634089469909668, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.9191277027130127, "logits/rejected": -1.5473778247833252, "logps/chosen": -195.8780059814453, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -267.34674072265625, "loss": 0.941, "margin_dpo/margin_mean": 73.72046661376953, "margin_dpo/margin_std": 66.39783477783203, "step": 282 }, { "KL/chosen_KL_mean": -106.239013671875, "KL/mean": -131.94285583496094, "KL/rejected_KL_mean": -157.6466827392578, "KL/std": 65.37469482421875, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.00799560733139515, "fcm_dpo/delta": -0.011514941230416298, "fcm_dpo/margin": 51.407676696777344, "fcm_dpo/q_t": 0.4056564271450043, "grad_norm": 10.241144180297852, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.285274028778076, "logits/rejected": -1.9539098739624023, "logps/chosen": -201.29161071777344, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -261.19122314453125, "loss": 1.096, "margin_dpo/margin_mean": 51.40767288208008, "margin_dpo/margin_std": 72.58271789550781, "step": 283 }, { "KL/chosen_KL_mean": -95.861328125, "KL/mean": -129.11343383789062, "KL/rejected_KL_mean": -162.36553955078125, "KL/std": 64.87802124023438, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.0078108301386237144, "fcm_dpo/delta": -0.12691375613212585, "fcm_dpo/margin": 66.50420379638672, "fcm_dpo/q_t": 0.3774507939815521, "grad_norm": 9.422600746154785, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.761216640472412, "logits/rejected": -1.4240851402282715, "logps/chosen": -183.52780151367188, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -261.1165771484375, "loss": 0.9804, "margin_dpo/margin_mean": 66.50420379638672, "margin_dpo/margin_std": 57.62086486816406, "step": 284 }, { "KL/chosen_KL_mean": -108.78124237060547, "KL/mean": -131.8063201904297, "KL/rejected_KL_mean": -154.83139038085938, "KL/std": 59.40949249267578, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.007857291027903557, "fcm_dpo/delta": 0.039404451847076416, "fcm_dpo/margin": 46.050148010253906, "fcm_dpo/q_t": 0.4143316447734833, "grad_norm": 9.832268714904785, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.568901300430298, "logits/rejected": -2.2999329566955566, "logps/chosen": -203.2142791748047, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -258.9033203125, "loss": 1.1043, "margin_dpo/margin_mean": 46.05014419555664, "margin_dpo/margin_std": 56.232540130615234, "step": 285 }, { "KL/chosen_KL_mean": -112.23873901367188, "KL/mean": -143.82821655273438, "KL/rejected_KL_mean": -175.41766357421875, "KL/std": 63.36304473876953, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.0077366577461361885, "fcm_dpo/delta": -0.09350879490375519, "fcm_dpo/margin": 63.178932189941406, "fcm_dpo/q_t": 0.38589632511138916, "grad_norm": 8.29554557800293, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.8425207138061523, "logits/rejected": -1.3709018230438232, "logps/chosen": -215.95913696289062, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -271.6754150390625, "loss": 1.0176, "margin_dpo/margin_mean": 63.178932189941406, "margin_dpo/margin_std": 65.56626892089844, "step": 286 }, { "KL/chosen_KL_mean": -130.4984130859375, "KL/mean": -148.44021606445312, "KL/rejected_KL_mean": -166.38201904296875, "KL/std": 57.74317169189453, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.007886011153459549, "fcm_dpo/delta": 0.11938023567199707, "fcm_dpo/margin": 35.883602142333984, "fcm_dpo/q_t": 0.4355773329734802, "grad_norm": 8.058666229248047, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.444390296936035, "logits/rejected": -1.8761250972747803, "logps/chosen": -224.3883056640625, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -249.71566772460938, "loss": 1.1925, "margin_dpo/margin_mean": 35.883602142333984, "margin_dpo/margin_std": 66.85050964355469, "step": 287 }, { "KL/chosen_KL_mean": -122.82612609863281, "KL/mean": -155.69667053222656, "KL/rejected_KL_mean": -188.5672149658203, "KL/std": 65.55189514160156, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.007747899740934372, "fcm_dpo/delta": -0.11548599600791931, "fcm_dpo/margin": 65.74109649658203, "fcm_dpo/q_t": 0.383929967880249, "grad_norm": 7.766016483306885, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.8564019203186035, "logits/rejected": -1.4454715251922607, "logps/chosen": -210.98214721679688, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -281.84918212890625, "loss": 1.0129, "margin_dpo/margin_mean": 65.74109649658203, "margin_dpo/margin_std": 73.03044128417969, "step": 288 }, { "KL/chosen_KL_mean": -119.78834533691406, "KL/mean": -152.80856323242188, "KL/rejected_KL_mean": -185.82879638671875, "KL/std": 68.64097595214844, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.0076077887788414955, "fcm_dpo/delta": -0.1077081561088562, "fcm_dpo/margin": 66.04045867919922, "fcm_dpo/q_t": 0.3873279392719269, "grad_norm": 7.392016887664795, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.9691574573516846, "logits/rejected": -1.7680737972259521, "logps/chosen": -195.1812744140625, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -278.98309326171875, "loss": 1.0278, "margin_dpo/margin_mean": 66.04045867919922, "margin_dpo/margin_std": 79.19801330566406, "step": 289 }, { "KL/chosen_KL_mean": -129.39089965820312, "KL/mean": -163.91714477539062, "KL/rejected_KL_mean": -198.44338989257812, "KL/std": 72.22532653808594, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.007463677320629358, "fcm_dpo/delta": -0.12171568721532822, "fcm_dpo/margin": 69.052490234375, "fcm_dpo/q_t": 0.383007287979126, "grad_norm": 8.338541030883789, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.2584514617919922, "logits/rejected": -1.3008381128311157, "logps/chosen": -217.43287658691406, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -321.6555480957031, "loss": 1.0108, "margin_dpo/margin_mean": 69.052490234375, "margin_dpo/margin_std": 75.5619888305664, "step": 290 }, { "KL/chosen_KL_mean": -122.27404022216797, "KL/mean": -151.89471435546875, "KL/rejected_KL_mean": -181.51536560058594, "KL/std": 68.71078491210938, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.007340380921959877, "fcm_dpo/delta": -0.03642917051911354, "fcm_dpo/margin": 59.24132537841797, "fcm_dpo/q_t": 0.3981374502182007, "grad_norm": 9.793542861938477, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.146395206451416, "logits/rejected": -1.6699978113174438, "logps/chosen": -216.36929321289062, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -277.5654296875, "loss": 1.0581, "margin_dpo/margin_mean": 59.2413330078125, "margin_dpo/margin_std": 68.63494873046875, "step": 291 }, { "KL/chosen_KL_mean": -125.96492004394531, "KL/mean": -156.39649963378906, "KL/rejected_KL_mean": -186.82806396484375, "KL/std": 66.81610107421875, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0072646355256438255, "fcm_dpo/delta": -0.044123291969299316, "fcm_dpo/margin": 60.86315155029297, "fcm_dpo/q_t": 0.3963738679885864, "grad_norm": 8.494903564453125, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.098069667816162, "logits/rejected": -1.6701104640960693, "logps/chosen": -214.21533203125, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -283.2457275390625, "loss": 1.0403, "margin_dpo/margin_mean": 60.86315155029297, "margin_dpo/margin_std": 62.661094665527344, "step": 292 }, { "KL/chosen_KL_mean": -125.74813842773438, "KL/mean": -150.38894653320312, "KL/rejected_KL_mean": -175.02972412109375, "KL/std": 70.76681518554688, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.0073236445896327496, "fcm_dpo/delta": 0.0400746688246727, "fcm_dpo/margin": 49.28160095214844, "fcm_dpo/q_t": 0.4178940951824188, "grad_norm": 7.975032329559326, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.2657859325408936, "logits/rejected": -1.6891181468963623, "logps/chosen": -213.1246795654297, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -260.7855224609375, "loss": 1.1279, "margin_dpo/margin_mean": 49.28160858154297, "margin_dpo/margin_std": 74.35887145996094, "step": 293 }, { "KL/chosen_KL_mean": -129.1713104248047, "KL/mean": -149.5946044921875, "KL/rejected_KL_mean": -170.01792907714844, "KL/std": 66.95337677001953, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.007439787499606609, "fcm_dpo/delta": 0.09867022186517715, "fcm_dpo/margin": 40.84661865234375, "fcm_dpo/q_t": 0.43083083629608154, "grad_norm": 8.35227108001709, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.6997058391571045, "logits/rejected": -1.5313165187835693, "logps/chosen": -202.67926025390625, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -258.106689453125, "loss": 1.1886, "margin_dpo/margin_mean": 40.84661865234375, "margin_dpo/margin_std": 78.64793395996094, "step": 294 }, { "KL/chosen_KL_mean": -122.537353515625, "KL/mean": -153.89129638671875, "KL/rejected_KL_mean": -185.2452392578125, "KL/std": 68.62663269042969, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.007354273926466703, "fcm_dpo/delta": -0.06472853571176529, "fcm_dpo/margin": 62.7078742980957, "fcm_dpo/q_t": 0.393706738948822, "grad_norm": 8.41625690460205, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.5339140892028809, "logits/rejected": -1.3770533800125122, "logps/chosen": -200.30282592773438, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -279.49249267578125, "loss": 1.0452, "margin_dpo/margin_mean": 62.7078742980957, "margin_dpo/margin_std": 73.267822265625, "step": 295 }, { "KL/chosen_KL_mean": -131.36148071289062, "KL/mean": -164.51416015625, "KL/rejected_KL_mean": -197.66683959960938, "KL/std": 69.16404724121094, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.007242550142109394, "fcm_dpo/delta": -0.0848006159067154, "fcm_dpo/margin": 66.30535888671875, "fcm_dpo/q_t": 0.387763649225235, "grad_norm": 11.64114761352539, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.568387269973755, "logits/rejected": -2.092409610748291, "logps/chosen": -229.2178955078125, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -298.483154296875, "loss": 1.0232, "margin_dpo/margin_mean": 66.30535888671875, "margin_dpo/margin_std": 69.71815490722656, "step": 296 }, { "KL/chosen_KL_mean": -132.50332641601562, "KL/mean": -160.48851013183594, "KL/rejected_KL_mean": -188.4736785888672, "KL/std": 69.14978790283203, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.0072436705231666565, "fcm_dpo/delta": -0.0057238563895225525, "fcm_dpo/margin": 55.9703483581543, "fcm_dpo/q_t": 0.4073626399040222, "grad_norm": 11.232056617736816, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.1676071882247925, "logits/rejected": -0.8563331961631775, "logps/chosen": -221.0691680908203, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -290.0302429199219, "loss": 1.088, "margin_dpo/margin_mean": 55.97035217285156, "margin_dpo/margin_std": 74.02253723144531, "step": 297 }, { "KL/chosen_KL_mean": -111.88188171386719, "KL/mean": -147.65052795410156, "KL/rejected_KL_mean": -183.41917419433594, "KL/std": 75.73405456542969, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.007122003007680178, "fcm_dpo/delta": -0.11520883440971375, "fcm_dpo/margin": 71.53727722167969, "fcm_dpo/q_t": 0.3839992880821228, "grad_norm": 11.892109870910645, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.1314759254455566, "logits/rejected": -2.1025943756103516, "logps/chosen": -206.7623291015625, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -305.7301940917969, "loss": 1.0089, "margin_dpo/margin_mean": 71.53727722167969, "margin_dpo/margin_std": 77.19137573242188, "step": 298 }, { "KL/chosen_KL_mean": -115.81752014160156, "KL/mean": -151.3363494873047, "KL/rejected_KL_mean": -186.85519409179688, "KL/std": 73.16596984863281, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.006954974494874477, "fcm_dpo/delta": -0.09903387725353241, "fcm_dpo/margin": 71.03765106201172, "fcm_dpo/q_t": 0.3857073485851288, "grad_norm": 10.322864532470703, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.0948611497879028, "logits/rejected": -0.8710848689079285, "logps/chosen": -196.2258758544922, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -276.392333984375, "loss": 1.0168, "margin_dpo/margin_mean": 71.03765106201172, "margin_dpo/margin_std": 76.0628662109375, "step": 299 }, { "KL/chosen_KL_mean": -122.85452270507812, "KL/mean": -152.48297119140625, "KL/rejected_KL_mean": -182.11138916015625, "KL/std": 76.38329315185547, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.006886978633701801, "fcm_dpo/delta": -0.008830994367599487, "fcm_dpo/margin": 59.25688171386719, "fcm_dpo/q_t": 0.4059489071369171, "grad_norm": 8.581576347351074, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.4460501670837402, "logits/rejected": -1.2307027578353882, "logps/chosen": -211.013427734375, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -283.05059814453125, "loss": 1.0951, "margin_dpo/margin_mean": 59.25688171386719, "margin_dpo/margin_std": 81.66133117675781, "step": 300 }, { "KL/chosen_KL_mean": -112.50279235839844, "KL/mean": -149.70462036132812, "KL/rejected_KL_mean": -186.90643310546875, "KL/std": 74.78515625, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0067627886310219765, "fcm_dpo/delta": -0.10987886786460876, "fcm_dpo/margin": 74.40364074707031, "fcm_dpo/q_t": 0.38445812463760376, "grad_norm": 9.527485847473145, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -2.031747341156006, "logits/rejected": -1.6179096698760986, "logps/chosen": -198.0076904296875, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -285.8232727050781, "loss": 1.0208, "margin_dpo/margin_mean": 74.40364074707031, "margin_dpo/margin_std": 83.92085266113281, "step": 301 }, { "KL/chosen_KL_mean": -109.37468719482422, "KL/mean": -140.92724609375, "KL/rejected_KL_mean": -172.4798126220703, "KL/std": 77.29283142089844, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.006723775528371334, "fcm_dpo/delta": -0.025502406060695648, "fcm_dpo/margin": 63.10512924194336, "fcm_dpo/q_t": 0.4008913040161133, "grad_norm": 8.481267929077148, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.536202073097229, "logits/rejected": -1.3626537322998047, "logps/chosen": -188.82510375976562, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -275.5389099121094, "loss": 1.067, "margin_dpo/margin_mean": 63.10512924194336, "margin_dpo/margin_std": 75.34263610839844, "step": 302 }, { "KL/chosen_KL_mean": -100.72355651855469, "KL/mean": -135.61778259277344, "KL/rejected_KL_mean": -170.5120086669922, "KL/std": 71.91390991210938, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.00667600566521287, "fcm_dpo/delta": -0.06908593326807022, "fcm_dpo/margin": 69.78845977783203, "fcm_dpo/q_t": 0.39122146368026733, "grad_norm": 8.122322082519531, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -2.0625, "logits/rejected": -1.619120717048645, "logps/chosen": -196.32183837890625, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -262.57659912109375, "loss": 1.0362, "margin_dpo/margin_mean": 69.78846740722656, "margin_dpo/margin_std": 76.34400939941406, "step": 303 }, { "KL/chosen_KL_mean": -100.9998550415039, "KL/mean": -136.94815063476562, "KL/rejected_KL_mean": -172.89642333984375, "KL/std": 74.4554672241211, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.00656186742708087, "fcm_dpo/delta": -0.07532519102096558, "fcm_dpo/margin": 71.89657592773438, "fcm_dpo/q_t": 0.38990721106529236, "grad_norm": 9.038031578063965, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.086533784866333, "logits/rejected": -1.6638517379760742, "logps/chosen": -200.4530029296875, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -270.97869873046875, "loss": 1.0296, "margin_dpo/margin_mean": 71.89657592773438, "margin_dpo/margin_std": 77.61824798583984, "step": 304 }, { "KL/chosen_KL_mean": -110.22999572753906, "KL/mean": -144.32009887695312, "KL/rejected_KL_mean": -178.41021728515625, "KL/std": 77.13314819335938, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.006458980962634087, "fcm_dpo/delta": -0.04308236017823219, "fcm_dpo/margin": 68.18022155761719, "fcm_dpo/q_t": 0.3973541855812073, "grad_norm": 7.6445746421813965, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.0514941215515137, "logits/rejected": -1.5605781078338623, "logps/chosen": -213.49339294433594, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -276.23919677734375, "loss": 1.0649, "margin_dpo/margin_mean": 68.18021392822266, "margin_dpo/margin_std": 82.8209228515625, "step": 305 }, { "KL/chosen_KL_mean": -97.91583251953125, "KL/mean": -137.95533752441406, "KL/rejected_KL_mean": -177.99484252929688, "KL/std": 73.15733337402344, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.006353379227221012, "fcm_dpo/delta": -0.11505059897899628, "fcm_dpo/margin": 80.07898712158203, "fcm_dpo/q_t": 0.3791532516479492, "grad_norm": 10.097332000732422, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.2372548580169678, "logits/rejected": -1.6823768615722656, "logps/chosen": -198.21966552734375, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -270.2257080078125, "loss": 0.9846, "margin_dpo/margin_mean": 80.07899475097656, "margin_dpo/margin_std": 67.21925354003906, "step": 306 }, { "KL/chosen_KL_mean": -102.88835906982422, "KL/mean": -132.9331512451172, "KL/rejected_KL_mean": -162.97796630859375, "KL/std": 74.11896514892578, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0063318549655377865, "fcm_dpo/delta": 0.02015545777976513, "fcm_dpo/margin": 60.08959197998047, "fcm_dpo/q_t": 0.41245776414871216, "grad_norm": 9.011000633239746, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.9037806987762451, "logits/rejected": -1.5543133020401, "logps/chosen": -199.53811645507812, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -256.3265686035156, "loss": 1.1147, "margin_dpo/margin_mean": 60.089595794677734, "margin_dpo/margin_std": 87.40626525878906, "step": 307 }, { "KL/chosen_KL_mean": -107.26225280761719, "KL/mean": -137.48590087890625, "KL/rejected_KL_mean": -167.70953369140625, "KL/std": 75.75179290771484, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.00633701728656888, "fcm_dpo/delta": 0.01700892671942711, "fcm_dpo/margin": 60.44727325439453, "fcm_dpo/q_t": 0.41254347562789917, "grad_norm": 10.339438438415527, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7958993911743164, "logits/rejected": -1.5205187797546387, "logps/chosen": -192.9869384765625, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -254.13272094726562, "loss": 1.1174, "margin_dpo/margin_mean": 60.44727325439453, "margin_dpo/margin_std": 88.1473159790039, "step": 308 }, { "KL/chosen_KL_mean": -121.12330627441406, "KL/mean": -148.36077880859375, "KL/rejected_KL_mean": -175.5982666015625, "KL/std": 80.56492614746094, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.006432904861867428, "fcm_dpo/delta": 0.051328547298908234, "fcm_dpo/margin": 54.4749755859375, "fcm_dpo/q_t": 0.4186372458934784, "grad_norm": 12.407133102416992, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.7447288036346436, "logits/rejected": -1.3156317472457886, "logps/chosen": -204.27476501464844, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -262.18426513671875, "loss": 1.1322, "margin_dpo/margin_mean": 54.4749755859375, "margin_dpo/margin_std": 79.93184661865234, "step": 309 }, { "KL/chosen_KL_mean": -109.10359191894531, "KL/mean": -139.49444580078125, "KL/rejected_KL_mean": -169.88531494140625, "KL/std": 73.42086791992188, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006448490545153618, "fcm_dpo/delta": 0.008371103554964066, "fcm_dpo/margin": 60.781715393066406, "fcm_dpo/q_t": 0.4087228775024414, "grad_norm": 9.113183975219727, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2236886024475098, "logits/rejected": -1.807725191116333, "logps/chosen": -198.84158325195312, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -256.95611572265625, "loss": 1.0913, "margin_dpo/margin_mean": 60.781715393066406, "margin_dpo/margin_std": 77.37232971191406, "step": 310 }, { "KL/chosen_KL_mean": -127.72749328613281, "KL/mean": -166.7416229248047, "KL/rejected_KL_mean": -205.75576782226562, "KL/std": 74.87031555175781, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.006363376043736935, "fcm_dpo/delta": -0.10151693969964981, "fcm_dpo/margin": 78.02826690673828, "fcm_dpo/q_t": 0.38448962569236755, "grad_norm": 8.809056282043457, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.6352646350860596, "logits/rejected": -0.9028536081314087, "logps/chosen": -216.25393676757812, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -288.0218505859375, "loss": 1.0038, "margin_dpo/margin_mean": 78.02826690673828, "margin_dpo/margin_std": 76.16667175292969, "step": 311 }, { "KL/chosen_KL_mean": -123.24789428710938, "KL/mean": -155.43765258789062, "KL/rejected_KL_mean": -187.6273956298828, "KL/std": 82.41438293457031, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.006338041741400957, "fcm_dpo/delta": -0.008545149117708206, "fcm_dpo/margin": 64.37948608398438, "fcm_dpo/q_t": 0.40492916107177734, "grad_norm": 9.301301956176758, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.5656324625015259, "logits/rejected": -1.037259578704834, "logps/chosen": -203.64859008789062, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -270.07452392578125, "loss": 1.0793, "margin_dpo/margin_mean": 64.37948608398438, "margin_dpo/margin_std": 79.09909057617188, "step": 312 }, { "KL/chosen_KL_mean": -117.81941223144531, "KL/mean": -149.75979614257812, "KL/rejected_KL_mean": -181.7001953125, "KL/std": 80.85110473632812, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.006327658426016569, "fcm_dpo/delta": -0.004511205479502678, "fcm_dpo/margin": 63.88078689575195, "fcm_dpo/q_t": 0.4065493941307068, "grad_norm": 9.944908142089844, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.326084613800049, "logits/rejected": -1.915151834487915, "logps/chosen": -201.96131896972656, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -275.12701416015625, "loss": 1.0941, "margin_dpo/margin_mean": 63.88078689575195, "margin_dpo/margin_std": 87.04795837402344, "step": 313 }, { "KL/chosen_KL_mean": -120.00068664550781, "KL/mean": -151.52783203125, "KL/rejected_KL_mean": -183.05496215820312, "KL/std": 81.00984191894531, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.006295610219240189, "fcm_dpo/delta": 0.003010682761669159, "fcm_dpo/margin": 63.054283142089844, "fcm_dpo/q_t": 0.4086865186691284, "grad_norm": 10.149788856506348, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8479301929473877, "logits/rejected": -1.6277765035629272, "logps/chosen": -208.81040954589844, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -287.2138671875, "loss": 1.0913, "margin_dpo/margin_mean": 63.05428695678711, "margin_dpo/margin_std": 81.78738403320312, "step": 314 }, { "KL/chosen_KL_mean": -117.14620971679688, "KL/mean": -149.27627563476562, "KL/rejected_KL_mean": -181.40634155273438, "KL/std": 85.40069580078125, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.006306802853941917, "fcm_dpo/delta": -0.005503036081790924, "fcm_dpo/margin": 64.26014709472656, "fcm_dpo/q_t": 0.4079554080963135, "grad_norm": 8.924033164978027, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.3401310443878174, "logits/rejected": -1.1622647047042847, "logps/chosen": -197.93991088867188, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -273.54345703125, "loss": 1.1103, "margin_dpo/margin_mean": 64.26013946533203, "margin_dpo/margin_std": 97.83670043945312, "step": 315 }, { "KL/chosen_KL_mean": -120.20590209960938, "KL/mean": -154.966796875, "KL/rejected_KL_mean": -189.72769165039062, "KL/std": 83.64166259765625, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.00627762358635664, "fcm_dpo/delta": -0.03807632252573967, "fcm_dpo/margin": 69.52176666259766, "fcm_dpo/q_t": 0.3997570276260376, "grad_norm": 9.011619567871094, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.783485770225525, "logits/rejected": -1.7824616432189941, "logps/chosen": -204.39016723632812, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -295.41851806640625, "loss": 1.0644, "margin_dpo/margin_mean": 69.52175903320312, "margin_dpo/margin_std": 86.36164855957031, "step": 316 }, { "KL/chosen_KL_mean": -125.46996307373047, "KL/mean": -167.7766571044922, "KL/rejected_KL_mean": -210.0833282470703, "KL/std": 92.27151489257812, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.006162088364362717, "fcm_dpo/delta": -0.12787450850009918, "fcm_dpo/margin": 84.61338806152344, "fcm_dpo/q_t": 0.3815971612930298, "grad_norm": 10.0888671875, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.8625136613845825, "logits/rejected": -1.381056785583496, "logps/chosen": -213.34344482421875, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -318.3485107421875, "loss": 1.018, "margin_dpo/margin_mean": 84.61338806152344, "margin_dpo/margin_std": 98.78428649902344, "step": 317 }, { "KL/chosen_KL_mean": -122.20773315429688, "KL/mean": -160.0492401123047, "KL/rejected_KL_mean": -197.8907470703125, "KL/std": 77.17390441894531, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.006041173357516527, "fcm_dpo/delta": -0.060032229870557785, "fcm_dpo/margin": 75.68302917480469, "fcm_dpo/q_t": 0.39369335770606995, "grad_norm": 10.647820472717285, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.1763150691986084, "logits/rejected": -1.4632190465927124, "logps/chosen": -220.41326904296875, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -286.02703857421875, "loss": 1.0419, "margin_dpo/margin_mean": 75.68302917480469, "margin_dpo/margin_std": 84.98448944091797, "step": 318 }, { "KL/chosen_KL_mean": -106.21348571777344, "KL/mean": -147.6033477783203, "KL/rejected_KL_mean": -188.9932098388672, "KL/std": 80.8176040649414, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.005931914784014225, "fcm_dpo/delta": -0.09599410742521286, "fcm_dpo/margin": 82.77972412109375, "fcm_dpo/q_t": 0.386241614818573, "grad_norm": 9.966798782348633, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.7035183906555176, "logits/rejected": -1.6345288753509521, "logps/chosen": -188.71685791015625, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -307.6438903808594, "loss": 1.0222, "margin_dpo/margin_mean": 82.77972412109375, "margin_dpo/margin_std": 91.25111389160156, "step": 319 }, { "KL/chosen_KL_mean": -108.62197875976562, "KL/mean": -142.53286743164062, "KL/rejected_KL_mean": -176.44375610351562, "KL/std": 82.72703552246094, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005904125049710274, "fcm_dpo/delta": -0.0004968177527189255, "fcm_dpo/margin": 67.82178497314453, "fcm_dpo/q_t": 0.40674832463264465, "grad_norm": 8.559760093688965, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.144731044769287, "logits/rejected": -1.9598257541656494, "logps/chosen": -195.7592010498047, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -279.6678466796875, "loss": 1.0776, "margin_dpo/margin_mean": 67.82178497314453, "margin_dpo/margin_std": 79.85848999023438, "step": 320 }, { "KL/chosen_KL_mean": -121.5899658203125, "KL/mean": -160.71722412109375, "KL/rejected_KL_mean": -199.84445190429688, "KL/std": 84.08151245117188, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.00583954993635416, "fcm_dpo/delta": -0.06017662212252617, "fcm_dpo/margin": 78.25450134277344, "fcm_dpo/q_t": 0.3923216462135315, "grad_norm": 8.388371467590332, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.8802385330200195, "logits/rejected": -1.484584093093872, "logps/chosen": -214.69464111328125, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -297.3125915527344, "loss": 1.0385, "margin_dpo/margin_mean": 78.2544937133789, "margin_dpo/margin_std": 84.18301391601562, "step": 321 }, { "KL/chosen_KL_mean": -113.36798095703125, "KL/mean": -155.81613159179688, "KL/rejected_KL_mean": -198.26429748535156, "KL/std": 83.95585632324219, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.005724698305130005, "fcm_dpo/delta": -0.09193172305822372, "fcm_dpo/margin": 84.89630126953125, "fcm_dpo/q_t": 0.3868330717086792, "grad_norm": 13.036797523498535, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.967195987701416, "logits/rejected": -1.6232596635818481, "logps/chosen": -205.56088256835938, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -299.15753173828125, "loss": 1.0244, "margin_dpo/margin_mean": 84.89630126953125, "margin_dpo/margin_std": 90.39353942871094, "step": 322 }, { "KL/chosen_KL_mean": -130.8382568359375, "KL/mean": -166.52455139160156, "KL/rejected_KL_mean": -202.21084594726562, "KL/std": 84.63582611083984, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.005717650055885315, "fcm_dpo/delta": -0.008520756848156452, "fcm_dpo/margin": 71.37257385253906, "fcm_dpo/q_t": 0.40582001209259033, "grad_norm": 10.427461624145508, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.935352087020874, "logits/rejected": -1.551393985748291, "logps/chosen": -211.11036682128906, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -282.5141906738281, "loss": 1.0784, "margin_dpo/margin_mean": 71.37257385253906, "margin_dpo/margin_std": 87.43809509277344, "step": 323 }, { "KL/chosen_KL_mean": -129.5291748046875, "KL/mean": -166.8324432373047, "KL/rejected_KL_mean": -204.13568115234375, "KL/std": 84.86479187011719, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.00571049377322197, "fcm_dpo/delta": -0.0271977037191391, "fcm_dpo/margin": 74.60650634765625, "fcm_dpo/q_t": 0.3996105492115021, "grad_norm": 8.876276016235352, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7400951385498047, "logits/rejected": -1.0933066606521606, "logps/chosen": -225.258544921875, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -298.304931640625, "loss": 1.0546, "margin_dpo/margin_mean": 74.60650634765625, "margin_dpo/margin_std": 80.00389099121094, "step": 324 }, { "KL/chosen_KL_mean": -119.4727783203125, "KL/mean": -159.9676513671875, "KL/rejected_KL_mean": -200.4625244140625, "KL/std": 80.7341537475586, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.005649648606777191, "fcm_dpo/delta": -0.060285307466983795, "fcm_dpo/margin": 80.98974609375, "fcm_dpo/q_t": 0.3924272358417511, "grad_norm": 7.637162208557129, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.516671061515808, "logits/rejected": -0.7524453401565552, "logps/chosen": -218.17965698242188, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -276.87060546875, "loss": 1.033, "margin_dpo/margin_mean": 80.98974609375, "margin_dpo/margin_std": 83.19783020019531, "step": 325 }, { "KL/chosen_KL_mean": -132.51026916503906, "KL/mean": -168.95648193359375, "KL/rejected_KL_mean": -205.4027099609375, "KL/std": 80.45213317871094, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.00561715429648757, "fcm_dpo/delta": -0.009855644777417183, "fcm_dpo/margin": 72.89244079589844, "fcm_dpo/q_t": 0.40386122465133667, "grad_norm": 7.440535068511963, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.5636534690856934, "logits/rejected": -1.9100151062011719, "logps/chosen": -237.17156982421875, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -292.22808837890625, "loss": 1.0744, "margin_dpo/margin_mean": 72.89244079589844, "margin_dpo/margin_std": 86.49795532226562, "step": 326 }, { "KL/chosen_KL_mean": -128.1656036376953, "KL/mean": -162.85736083984375, "KL/rejected_KL_mean": -197.54910278320312, "KL/std": 73.92634582519531, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005610634572803974, "fcm_dpo/delta": 0.011041158810257912, "fcm_dpo/margin": 69.38349151611328, "fcm_dpo/q_t": 0.4095831513404846, "grad_norm": 8.436304092407227, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.7532309293746948, "logits/rejected": -1.1639748811721802, "logps/chosen": -235.6058349609375, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -314.6023254394531, "loss": 1.0856, "margin_dpo/margin_mean": 69.38349151611328, "margin_dpo/margin_std": 83.11146545410156, "step": 327 }, { "KL/chosen_KL_mean": -126.23757934570312, "KL/mean": -171.56521606445312, "KL/rejected_KL_mean": -216.89288330078125, "KL/std": 87.48121643066406, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0055421567521989346, "fcm_dpo/delta": -0.10778771340847015, "fcm_dpo/margin": 90.65531921386719, "fcm_dpo/q_t": 0.3821079134941101, "grad_norm": 9.805124282836914, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.8625245094299316, "logits/rejected": -1.5738158226013184, "logps/chosen": -212.8482666015625, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -314.2947692871094, "loss": 0.9974, "margin_dpo/margin_mean": 90.65531921386719, "margin_dpo/margin_std": 84.93244934082031, "step": 328 }, { "KL/chosen_KL_mean": -130.37579345703125, "KL/mean": -166.36981201171875, "KL/rejected_KL_mean": -202.3638153076172, "KL/std": 83.96554565429688, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.005491404794156551, "fcm_dpo/delta": 0.004606790840625763, "fcm_dpo/margin": 71.988037109375, "fcm_dpo/q_t": 0.4083230495452881, "grad_norm": 9.723923683166504, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.5181224346160889, "logits/rejected": -1.015529751777649, "logps/chosen": -230.98886108398438, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -307.3175048828125, "loss": 1.0883, "margin_dpo/margin_mean": 71.98804473876953, "margin_dpo/margin_std": 90.02657318115234, "step": 329 }, { "KL/chosen_KL_mean": -111.80049133300781, "KL/mean": -150.4501190185547, "KL/rejected_KL_mean": -189.09974670410156, "KL/std": 82.24190521240234, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005486076697707176, "fcm_dpo/delta": -0.025195367634296417, "fcm_dpo/margin": 77.29925537109375, "fcm_dpo/q_t": 0.39854979515075684, "grad_norm": 8.08903694152832, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.918226957321167, "logits/rejected": -1.8406436443328857, "logps/chosen": -188.75892639160156, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -281.9146728515625, "loss": 1.0451, "margin_dpo/margin_mean": 77.29925537109375, "margin_dpo/margin_std": 71.84260559082031, "step": 330 }, { "KL/chosen_KL_mean": -120.31201171875, "KL/mean": -163.87265014648438, "KL/rejected_KL_mean": -207.43328857421875, "KL/std": 80.61764526367188, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.005441469140350819, "fcm_dpo/delta": -0.07773016393184662, "fcm_dpo/margin": 87.12129211425781, "fcm_dpo/q_t": 0.38801753520965576, "grad_norm": 9.582952499389648, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8889548778533936, "logits/rejected": -1.584137201309204, "logps/chosen": -203.37142944335938, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -301.5644226074219, "loss": 1.0112, "margin_dpo/margin_mean": 87.12129211425781, "margin_dpo/margin_std": 78.16541290283203, "step": 331 }, { "KL/chosen_KL_mean": -127.14353942871094, "KL/mean": -157.09716796875, "KL/rejected_KL_mean": -187.05081176757812, "KL/std": 88.99595642089844, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005468487739562988, "fcm_dpo/delta": 0.0747220516204834, "fcm_dpo/margin": 59.907264709472656, "fcm_dpo/q_t": 0.42327484488487244, "grad_norm": 7.6987996101379395, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.256826400756836, "logits/rejected": -1.8374826908111572, "logps/chosen": -209.07443237304688, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -269.896728515625, "loss": 1.1387, "margin_dpo/margin_mean": 59.90726089477539, "margin_dpo/margin_std": 86.51841735839844, "step": 332 }, { "KL/chosen_KL_mean": -148.00527954101562, "KL/mean": -189.53054809570312, "KL/rejected_KL_mean": -231.05580139160156, "KL/std": 92.87278747558594, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.005408711731433868, "fcm_dpo/delta": -0.05247662216424942, "fcm_dpo/margin": 83.05052947998047, "fcm_dpo/q_t": 0.3963466286659241, "grad_norm": 7.914732456207275, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.463102102279663, "logits/rejected": -1.1180977821350098, "logps/chosen": -236.53787231445312, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -333.5218200683594, "loss": 1.0579, "margin_dpo/margin_mean": 83.05052185058594, "margin_dpo/margin_std": 99.88980102539062, "step": 333 }, { "KL/chosen_KL_mean": -134.42222595214844, "KL/mean": -171.78553771972656, "KL/rejected_KL_mean": -209.14886474609375, "KL/std": 87.66520690917969, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.005393003113567829, "fcm_dpo/delta": -0.0037391968071460724, "fcm_dpo/margin": 74.72663879394531, "fcm_dpo/q_t": 0.4054277837276459, "grad_norm": 8.946492195129395, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.2775869369506836, "logits/rejected": -0.973383903503418, "logps/chosen": -232.1800079345703, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -309.1546936035156, "loss": 1.0918, "margin_dpo/margin_mean": 74.72664642333984, "margin_dpo/margin_std": 97.31144714355469, "step": 334 }, { "KL/chosen_KL_mean": -133.57327270507812, "KL/mean": -165.66375732421875, "KL/rejected_KL_mean": -197.75424194335938, "KL/std": 87.28160858154297, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005463001783937216, "fcm_dpo/delta": 0.051169008016586304, "fcm_dpo/margin": 64.18098449707031, "fcm_dpo/q_t": 0.4187089502811432, "grad_norm": 12.676020622253418, "learning_rate": 3.022289525640531e-07, "logits/chosen": -1.998812198638916, "logits/rejected": -1.5522418022155762, "logps/chosen": -229.94931030273438, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -292.6782531738281, "loss": 1.1313, "margin_dpo/margin_mean": 64.18098449707031, "margin_dpo/margin_std": 95.80167388916016, "step": 335 }, { "KL/chosen_KL_mean": -137.13482666015625, "KL/mean": -178.16783142089844, "KL/rejected_KL_mean": -219.20082092285156, "KL/std": 89.95084381103516, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005448919255286455, "fcm_dpo/delta": -0.04937107861042023, "fcm_dpo/margin": 82.06599426269531, "fcm_dpo/q_t": 0.39620181918144226, "grad_norm": 9.993278503417969, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.4624314308166504, "logits/rejected": -1.307457447052002, "logps/chosen": -225.6643524169922, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -328.756591796875, "loss": 1.0501, "margin_dpo/margin_mean": 82.06598663330078, "margin_dpo/margin_std": 95.29290008544922, "step": 336 }, { "KL/chosen_KL_mean": -134.9781494140625, "KL/mean": -174.1922607421875, "KL/rejected_KL_mean": -213.4063720703125, "KL/std": 90.62379455566406, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.005409582518041134, "fcm_dpo/delta": -0.02534061297774315, "fcm_dpo/margin": 78.42822265625, "fcm_dpo/q_t": 0.40265023708343506, "grad_norm": 8.749862670898438, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.535555601119995, "logits/rejected": -2.2590856552124023, "logps/chosen": -232.29052734375, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -323.5377197265625, "loss": 1.0826, "margin_dpo/margin_mean": 78.42822265625, "margin_dpo/margin_std": 105.54985046386719, "step": 337 }, { "KL/chosen_KL_mean": -139.94970703125, "KL/mean": -175.829833984375, "KL/rejected_KL_mean": -211.7099609375, "KL/std": 91.75639343261719, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.00540000107139349, "fcm_dpo/delta": 0.012987496331334114, "fcm_dpo/margin": 71.76025390625, "fcm_dpo/q_t": 0.4089200794696808, "grad_norm": 8.875068664550781, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.809476613998413, "logits/rejected": -1.6544287204742432, "logps/chosen": -223.579833984375, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -305.4023132324219, "loss": 1.0933, "margin_dpo/margin_mean": 71.76025390625, "margin_dpo/margin_std": 90.7015151977539, "step": 338 }, { "KL/chosen_KL_mean": -149.3142852783203, "KL/mean": -187.79037475585938, "KL/rejected_KL_mean": -226.2664794921875, "KL/std": 90.75932312011719, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005429895129054785, "fcm_dpo/delta": -0.019384127110242844, "fcm_dpo/margin": 76.95217895507812, "fcm_dpo/q_t": 0.4030987024307251, "grad_norm": 8.231239318847656, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.8039097785949707, "logits/rejected": -2.3670952320098877, "logps/chosen": -249.16610717773438, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -335.3408203125, "loss": 1.0862, "margin_dpo/margin_mean": 76.95217895507812, "margin_dpo/margin_std": 100.73959350585938, "step": 339 }, { "KL/chosen_KL_mean": -159.7362823486328, "KL/mean": -193.7061309814453, "KL/rejected_KL_mean": -227.67596435546875, "KL/std": 83.59628295898438, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.005416516214609146, "fcm_dpo/delta": 0.03321871906518936, "fcm_dpo/margin": 67.93966674804688, "fcm_dpo/q_t": 0.41373154520988464, "grad_norm": 9.854127883911133, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.0906866788864136, "logits/rejected": -0.5820397734642029, "logps/chosen": -244.7102508544922, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -306.476318359375, "loss": 1.109, "margin_dpo/margin_mean": 67.93966674804688, "margin_dpo/margin_std": 91.23983764648438, "step": 340 }, { "KL/chosen_KL_mean": -166.9191131591797, "KL/mean": -199.38075256347656, "KL/rejected_KL_mean": -231.84237670898438, "KL/std": 87.97468566894531, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.00545103196054697, "fcm_dpo/delta": 0.047796037048101425, "fcm_dpo/margin": 64.92327117919922, "fcm_dpo/q_t": 0.41863012313842773, "grad_norm": 9.257328033447266, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.5616121292114258, "logits/rejected": -1.3117005825042725, "logps/chosen": -252.18310546875, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -337.9283447265625, "loss": 1.1372, "margin_dpo/margin_mean": 64.92327117919922, "margin_dpo/margin_std": 101.15701293945312, "step": 341 }, { "KL/chosen_KL_mean": -133.76583862304688, "KL/mean": -181.11935424804688, "KL/rejected_KL_mean": -228.47288513183594, "KL/std": 87.95306396484375, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.005354674533009529, "fcm_dpo/delta": -0.11411676555871964, "fcm_dpo/margin": 94.70703887939453, "fcm_dpo/q_t": 0.38012343645095825, "grad_norm": 7.661415100097656, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.4052362442016602, "logits/rejected": -0.6431517004966736, "logps/chosen": -234.804443359375, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -327.37139892578125, "loss": 1.0069, "margin_dpo/margin_mean": 94.70703887939453, "margin_dpo/margin_std": 94.16569519042969, "step": 342 }, { "KL/chosen_KL_mean": -145.62693786621094, "KL/mean": -184.94775390625, "KL/rejected_KL_mean": -224.26858520507812, "KL/std": 91.34087371826172, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.005368012003600597, "fcm_dpo/delta": -0.023786328732967377, "fcm_dpo/margin": 78.64164733886719, "fcm_dpo/q_t": 0.4018615484237671, "grad_norm": 10.046955108642578, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -1.9551904201507568, "logits/rejected": -1.842768907546997, "logps/chosen": -229.37429809570312, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -340.84765625, "loss": 1.0734, "margin_dpo/margin_mean": 78.64164733886719, "margin_dpo/margin_std": 94.88910675048828, "step": 343 }, { "KL/chosen_KL_mean": -145.4019012451172, "KL/mean": -186.84324645996094, "KL/rejected_KL_mean": -228.28457641601562, "KL/std": 97.55014038085938, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0053206090815365314, "fcm_dpo/delta": -0.04318472370505333, "fcm_dpo/margin": 82.8826904296875, "fcm_dpo/q_t": 0.39848780632019043, "grad_norm": 9.371628761291504, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.4875856637954712, "logits/rejected": -1.2321722507476807, "logps/chosen": -223.6558837890625, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -323.7132873535156, "loss": 1.0654, "margin_dpo/margin_mean": 82.8826904296875, "margin_dpo/margin_std": 102.56617736816406, "step": 344 }, { "KL/chosen_KL_mean": -149.99685668945312, "KL/mean": -193.83494567871094, "KL/rejected_KL_mean": -237.67303466796875, "KL/std": 97.4256591796875, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.005225942935794592, "fcm_dpo/delta": -0.061128828674554825, "fcm_dpo/margin": 87.67619323730469, "fcm_dpo/q_t": 0.39395445585250854, "grad_norm": 10.034189224243164, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.3537896871566772, "logits/rejected": -1.0489099025726318, "logps/chosen": -248.0763397216797, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -341.70782470703125, "loss": 1.0442, "margin_dpo/margin_mean": 87.67619323730469, "margin_dpo/margin_std": 100.71998596191406, "step": 345 }, { "KL/chosen_KL_mean": -166.44717407226562, "KL/mean": -200.74525451660156, "KL/rejected_KL_mean": -235.04331970214844, "KL/std": 90.31640625, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.005248785950243473, "fcm_dpo/delta": 0.0414545051753521, "fcm_dpo/margin": 68.59615325927734, "fcm_dpo/q_t": 0.4171462059020996, "grad_norm": 7.855130672454834, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.7941935062408447, "logits/rejected": -1.4793498516082764, "logps/chosen": -247.1306610107422, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -320.9359130859375, "loss": 1.1269, "margin_dpo/margin_mean": 68.59615325927734, "margin_dpo/margin_std": 102.90907287597656, "step": 346 }, { "KL/chosen_KL_mean": -156.96719360351562, "KL/mean": -189.70603942871094, "KL/rejected_KL_mean": -222.44491577148438, "KL/std": 89.9362564086914, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.0053193289786577225, "fcm_dpo/delta": 0.053213510662317276, "fcm_dpo/margin": 65.47772216796875, "fcm_dpo/q_t": 0.41974085569381714, "grad_norm": 8.219341278076172, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.7659519910812378, "logits/rejected": -1.4363429546356201, "logps/chosen": -250.64657592773438, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -319.9874267578125, "loss": 1.1325, "margin_dpo/margin_mean": 65.47772216796875, "margin_dpo/margin_std": 97.68600463867188, "step": 347 }, { "KL/chosen_KL_mean": -135.6319580078125, "KL/mean": -173.38446044921875, "KL/rejected_KL_mean": -211.13697814941406, "KL/std": 85.0360107421875, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.0052744075655937195, "fcm_dpo/delta": 0.00045190751552581787, "fcm_dpo/margin": 75.50502014160156, "fcm_dpo/q_t": 0.40630412101745605, "grad_norm": 7.922032356262207, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.8854773044586182, "logits/rejected": -1.6052641868591309, "logps/chosen": -225.88418579101562, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -306.9072570800781, "loss": 1.0843, "margin_dpo/margin_mean": 75.50502014160156, "margin_dpo/margin_std": 87.08931732177734, "step": 348 }, { "KL/chosen_KL_mean": -145.6878204345703, "KL/mean": -181.11033630371094, "KL/rejected_KL_mean": -216.53285217285156, "KL/std": 92.65497589111328, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.005355688277631998, "fcm_dpo/delta": 0.020986618474125862, "fcm_dpo/margin": 70.84503936767578, "fcm_dpo/q_t": 0.4123881161212921, "grad_norm": 9.908736228942871, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.9393155574798584, "logits/rejected": -1.7098970413208008, "logps/chosen": -237.74591064453125, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -320.49810791015625, "loss": 1.106, "margin_dpo/margin_mean": 70.84503936767578, "margin_dpo/margin_std": 95.38746643066406, "step": 349 }, { "KL/chosen_KL_mean": -154.64138793945312, "KL/mean": -184.13565063476562, "KL/rejected_KL_mean": -213.62991333007812, "KL/std": 85.11260986328125, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005402242299169302, "fcm_dpo/delta": 0.08407130092382431, "fcm_dpo/margin": 58.98851776123047, "fcm_dpo/q_t": 0.4260105490684509, "grad_norm": 9.68515396118164, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.780243158340454, "logits/rejected": -1.5518807172775269, "logps/chosen": -241.69558715820312, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -309.35186767578125, "loss": 1.1527, "margin_dpo/margin_mean": 58.9885139465332, "margin_dpo/margin_std": 92.12406921386719, "step": 350 }, { "KL/chosen_KL_mean": -154.7307891845703, "KL/mean": -196.64138793945312, "KL/rejected_KL_mean": -238.55198669433594, "KL/std": 95.42770385742188, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.005397511646151543, "fcm_dpo/delta": -0.05488227307796478, "fcm_dpo/margin": 83.82118225097656, "fcm_dpo/q_t": 0.3950950801372528, "grad_norm": 8.5553617477417, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.6514226198196411, "logits/rejected": -1.3454880714416504, "logps/chosen": -243.8998260498047, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -346.8500061035156, "loss": 1.057, "margin_dpo/margin_mean": 83.82118225097656, "margin_dpo/margin_std": 102.87551879882812, "step": 351 }, { "KL/chosen_KL_mean": -141.7284698486328, "KL/mean": -181.48228454589844, "KL/rejected_KL_mean": -221.23611450195312, "KL/std": 93.90650939941406, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.005351550877094269, "fcm_dpo/delta": -0.026616765186190605, "fcm_dpo/margin": 79.50763702392578, "fcm_dpo/q_t": 0.40291664004325867, "grad_norm": 8.857847213745117, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.7185176610946655, "logits/rejected": -1.151855230331421, "logps/chosen": -238.53994750976562, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -305.64093017578125, "loss": 1.0752, "margin_dpo/margin_mean": 79.50764465332031, "margin_dpo/margin_std": 101.38259887695312, "step": 352 }, { "KL/chosen_KL_mean": -162.337646484375, "KL/mean": -187.32589721679688, "KL/rejected_KL_mean": -212.31411743164062, "KL/std": 82.94154357910156, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.005433902144432068, "fcm_dpo/delta": 0.13223104178905487, "fcm_dpo/margin": 49.976470947265625, "fcm_dpo/q_t": 0.4366387128829956, "grad_norm": 9.842411994934082, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.1574528217315674, "logits/rejected": -1.7746490240097046, "logps/chosen": -247.70013427734375, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -299.68145751953125, "loss": 1.197, "margin_dpo/margin_mean": 49.976470947265625, "margin_dpo/margin_std": 93.39382934570312, "step": 353 }, { "KL/chosen_KL_mean": -141.97149658203125, "KL/mean": -181.6219940185547, "KL/rejected_KL_mean": -221.2725067138672, "KL/std": 85.38575744628906, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.005468826275318861, "fcm_dpo/delta": -0.03522486239671707, "fcm_dpo/margin": 79.30099487304688, "fcm_dpo/q_t": 0.39805829524993896, "grad_norm": 8.893898963928223, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.88273024559021, "logits/rejected": -1.4122051000595093, "logps/chosen": -237.77267456054688, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -316.1386413574219, "loss": 1.0428, "margin_dpo/margin_mean": 79.30099487304688, "margin_dpo/margin_std": 78.92733764648438, "step": 354 }, { "KL/chosen_KL_mean": -140.347900390625, "KL/mean": -177.7858428955078, "KL/rejected_KL_mean": -215.22378540039062, "KL/std": 86.16126251220703, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.005424341186881065, "fcm_dpo/delta": -0.006658285856246948, "fcm_dpo/margin": 74.87589263916016, "fcm_dpo/q_t": 0.40500861406326294, "grad_norm": 9.906431198120117, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.269285202026367, "logits/rejected": -1.7714053392410278, "logps/chosen": -226.533447265625, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -303.68536376953125, "loss": 1.0823, "margin_dpo/margin_mean": 74.87588500976562, "margin_dpo/margin_std": 91.23896026611328, "step": 355 }, { "KL/chosen_KL_mean": -164.81747436523438, "KL/mean": -193.52569580078125, "KL/rejected_KL_mean": -222.2339324951172, "KL/std": 84.24571228027344, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005526228807866573, "fcm_dpo/delta": 0.08498270809650421, "fcm_dpo/margin": 57.41646957397461, "fcm_dpo/q_t": 0.42648112773895264, "grad_norm": 9.189254760742188, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8233134746551514, "logits/rejected": -1.3548599481582642, "logps/chosen": -255.6355743408203, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -304.36126708984375, "loss": 1.1615, "margin_dpo/margin_mean": 57.416473388671875, "margin_dpo/margin_std": 96.01709747314453, "step": 356 }, { "KL/chosen_KL_mean": -137.40328979492188, "KL/mean": -183.76065063476562, "KL/rejected_KL_mean": -230.11801147460938, "KL/std": 93.17790222167969, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0054078539833426476, "fcm_dpo/delta": -0.10845537483692169, "fcm_dpo/margin": 92.7147216796875, "fcm_dpo/q_t": 0.382192999124527, "grad_norm": 8.133981704711914, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.6448888778686523, "logits/rejected": -1.2584431171417236, "logps/chosen": -224.90805053710938, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -322.36737060546875, "loss": 1.0015, "margin_dpo/margin_mean": 92.7147216796875, "margin_dpo/margin_std": 86.50775146484375, "step": 357 }, { "KL/chosen_KL_mean": -150.24693298339844, "KL/mean": -184.45787048339844, "KL/rejected_KL_mean": -218.66880798339844, "KL/std": 86.31697845458984, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0054372139275074005, "fcm_dpo/delta": 0.029062282294034958, "fcm_dpo/margin": 68.421875, "fcm_dpo/q_t": 0.4148765504360199, "grad_norm": 9.304381370544434, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.642162799835205, "logits/rejected": -1.1699724197387695, "logps/chosen": -245.30239868164062, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -315.1449279785156, "loss": 1.1076, "margin_dpo/margin_mean": 68.421875, "margin_dpo/margin_std": 92.87728118896484, "step": 358 }, { "KL/chosen_KL_mean": -167.27499389648438, "KL/mean": -198.10911560058594, "KL/rejected_KL_mean": -228.9432373046875, "KL/std": 80.81256103515625, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.005472412332892418, "fcm_dpo/delta": 0.06453512609004974, "fcm_dpo/margin": 61.66822814941406, "fcm_dpo/q_t": 0.4224627614021301, "grad_norm": 9.250521659851074, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.5949829816818237, "logits/rejected": -1.0296224355697632, "logps/chosen": -254.64833068847656, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -317.715087890625, "loss": 1.1424, "margin_dpo/margin_mean": 61.66822814941406, "margin_dpo/margin_std": 93.9936294555664, "step": 359 }, { "KL/chosen_KL_mean": -152.82562255859375, "KL/mean": -193.58319091796875, "KL/rejected_KL_mean": -234.34075927734375, "KL/std": 81.7760009765625, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.0054725157096982, "fcm_dpo/delta": -0.048309922218322754, "fcm_dpo/margin": 81.51513671875, "fcm_dpo/q_t": 0.3948153257369995, "grad_norm": 12.28376579284668, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.6843130588531494, "logits/rejected": -1.341712236404419, "logps/chosen": -228.049072265625, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -310.43475341796875, "loss": 1.0423, "margin_dpo/margin_mean": 81.51513671875, "margin_dpo/margin_std": 86.58036804199219, "step": 360 }, { "KL/chosen_KL_mean": -156.6403045654297, "KL/mean": -192.22418212890625, "KL/rejected_KL_mean": -227.8080596923828, "KL/std": 92.86901092529297, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005476757884025574, "fcm_dpo/delta": 0.010628725402057171, "fcm_dpo/margin": 71.16775512695312, "fcm_dpo/q_t": 0.4108280837535858, "grad_norm": 9.628510475158691, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.7261899709701538, "logits/rejected": -1.4792550802230835, "logps/chosen": -238.5936279296875, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -324.48724365234375, "loss": 1.1028, "margin_dpo/margin_mean": 71.16775512695312, "margin_dpo/margin_std": 98.97726440429688, "step": 361 }, { "KL/chosen_KL_mean": -162.05145263671875, "KL/mean": -199.1558837890625, "KL/rejected_KL_mean": -236.2602996826172, "KL/std": 87.2686767578125, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0054601687006652355, "fcm_dpo/delta": -0.0055091846734285355, "fcm_dpo/margin": 74.2088394165039, "fcm_dpo/q_t": 0.40531125664711, "grad_norm": 9.084280967712402, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.9063923358917236, "logits/rejected": -1.7023918628692627, "logps/chosen": -251.39117431640625, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -346.76226806640625, "loss": 1.0839, "margin_dpo/margin_mean": 74.2088394165039, "margin_dpo/margin_std": 92.51939392089844, "step": 362 }, { "KL/chosen_KL_mean": -152.24195861816406, "KL/mean": -199.738037109375, "KL/rejected_KL_mean": -247.23410034179688, "KL/std": 91.80551147460938, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.005396299064159393, "fcm_dpo/delta": -0.11854643374681473, "fcm_dpo/margin": 94.99215698242188, "fcm_dpo/q_t": 0.3820420801639557, "grad_norm": 10.372267723083496, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.666197657585144, "logits/rejected": -1.2193057537078857, "logps/chosen": -233.01695251464844, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -326.6667785644531, "loss": 1.0069, "margin_dpo/margin_mean": 94.99217224121094, "margin_dpo/margin_std": 100.57119750976562, "step": 363 }, { "KL/chosen_KL_mean": -151.53665161132812, "KL/mean": -190.2096405029297, "KL/rejected_KL_mean": -228.88262939453125, "KL/std": 84.90563201904297, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.005362963303923607, "fcm_dpo/delta": -0.016168247908353806, "fcm_dpo/margin": 77.34597778320312, "fcm_dpo/q_t": 0.4019864499568939, "grad_norm": 11.372566223144531, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0164690017700195, "logits/rejected": -1.8705050945281982, "logps/chosen": -228.06625366210938, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -326.44439697265625, "loss": 1.063, "margin_dpo/margin_mean": 77.34597778320312, "margin_dpo/margin_std": 80.97306823730469, "step": 364 }, { "KL/chosen_KL_mean": -147.6685791015625, "KL/mean": -190.02432250976562, "KL/rejected_KL_mean": -232.3800811767578, "KL/std": 91.85116577148438, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.005301826633512974, "fcm_dpo/delta": -0.05149694159626961, "fcm_dpo/margin": 84.71150970458984, "fcm_dpo/q_t": 0.3974146842956543, "grad_norm": 8.861852645874023, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.8078622817993164, "logits/rejected": -1.6799579858779907, "logps/chosen": -228.21347045898438, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -331.7032470703125, "loss": 1.0589, "margin_dpo/margin_mean": 84.71150970458984, "margin_dpo/margin_std": 104.69844818115234, "step": 365 }, { "KL/chosen_KL_mean": -148.4163818359375, "KL/mean": -184.220703125, "KL/rejected_KL_mean": -220.0250244140625, "KL/std": 94.8196792602539, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.00528695248067379, "fcm_dpo/delta": 0.022252600640058517, "fcm_dpo/margin": 71.60863494873047, "fcm_dpo/q_t": 0.41365286707878113, "grad_norm": 8.513951301574707, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.6803514957427979, "logits/rejected": -1.7116754055023193, "logps/chosen": -220.6544647216797, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -311.69683837890625, "loss": 1.0976, "margin_dpo/margin_mean": 71.608642578125, "margin_dpo/margin_std": 92.42893981933594, "step": 366 }, { "KL/chosen_KL_mean": -136.1708984375, "KL/mean": -178.52740478515625, "KL/rejected_KL_mean": -220.88388061523438, "KL/std": 84.04374694824219, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0052581848576664925, "fcm_dpo/delta": -0.04753890633583069, "fcm_dpo/margin": 84.71297454833984, "fcm_dpo/q_t": 0.39551252126693726, "grad_norm": 8.14936351776123, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.6967047452926636, "logits/rejected": -1.346043586730957, "logps/chosen": -221.59292602539062, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -304.40167236328125, "loss": 1.0381, "margin_dpo/margin_mean": 84.71296691894531, "margin_dpo/margin_std": 86.83853149414062, "step": 367 }, { "KL/chosen_KL_mean": -126.68898010253906, "KL/mean": -173.33795166015625, "KL/rejected_KL_mean": -219.9869384765625, "KL/std": 92.89498901367188, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.00519681628793478, "fcm_dpo/delta": -0.08917377889156342, "fcm_dpo/margin": 93.29794311523438, "fcm_dpo/q_t": 0.3859795928001404, "grad_norm": 8.813143730163574, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.5978584289550781, "logits/rejected": -1.5630834102630615, "logps/chosen": -200.5301055908203, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -324.6822204589844, "loss": 1.0184, "margin_dpo/margin_mean": 93.29794311523438, "margin_dpo/margin_std": 93.84065246582031, "step": 368 }, { "KL/chosen_KL_mean": -140.17465209960938, "KL/mean": -183.6025390625, "KL/rejected_KL_mean": -227.0304412841797, "KL/std": 85.72183227539062, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0051240986213088036, "fcm_dpo/delta": -0.0471392422914505, "fcm_dpo/margin": 86.85579681396484, "fcm_dpo/q_t": 0.39545130729675293, "grad_norm": 8.485164642333984, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.5366278886795044, "logits/rejected": -1.1615040302276611, "logps/chosen": -238.63661193847656, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -326.1871643066406, "loss": 1.0362, "margin_dpo/margin_mean": 86.85580444335938, "margin_dpo/margin_std": 86.38909912109375, "step": 369 }, { "KL/chosen_KL_mean": -156.02789306640625, "KL/mean": -185.0926513671875, "KL/rejected_KL_mean": -214.15740966796875, "KL/std": 82.3857421875, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.00518408790230751, "fcm_dpo/delta": 0.10163131356239319, "fcm_dpo/margin": 58.129512786865234, "fcm_dpo/q_t": 0.42937690019607544, "grad_norm": 8.62939167022705, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.9700729846954346, "logits/rejected": -1.772528886795044, "logps/chosen": -252.80361938476562, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -316.9015197753906, "loss": 1.1664, "margin_dpo/margin_mean": 58.1295166015625, "margin_dpo/margin_std": 95.37632751464844, "step": 370 }, { "KL/chosen_KL_mean": -147.19085693359375, "KL/mean": -179.70883178710938, "KL/rejected_KL_mean": -212.226806640625, "KL/std": 83.1390609741211, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005267150700092316, "fcm_dpo/delta": 0.059183280915021896, "fcm_dpo/margin": 65.03593444824219, "fcm_dpo/q_t": 0.41952788829803467, "grad_norm": 11.146612167358398, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.83547043800354, "logits/rejected": -1.4654693603515625, "logps/chosen": -250.18106079101562, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -317.5774230957031, "loss": 1.1272, "margin_dpo/margin_mean": 65.03593444824219, "margin_dpo/margin_std": 89.75439453125, "step": 371 }, { "KL/chosen_KL_mean": -125.7117691040039, "KL/mean": -174.0099639892578, "KL/rejected_KL_mean": -222.30816650390625, "KL/std": 96.7072982788086, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.005190849304199219, "fcm_dpo/delta": -0.10669172555208206, "fcm_dpo/margin": 96.59637451171875, "fcm_dpo/q_t": 0.38376638293266296, "grad_norm": 7.496921539306641, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.833418846130371, "logits/rejected": -1.1931160688400269, "logps/chosen": -216.73800659179688, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -307.91839599609375, "loss": 1.008, "margin_dpo/margin_mean": 96.59638214111328, "margin_dpo/margin_std": 98.76528930664062, "step": 372 }, { "KL/chosen_KL_mean": -145.17608642578125, "KL/mean": -180.71939086914062, "KL/rejected_KL_mean": -216.26266479492188, "KL/std": 91.71344757080078, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.005177260376513004, "fcm_dpo/delta": 0.03319840878248215, "fcm_dpo/margin": 71.08657836914062, "fcm_dpo/q_t": 0.41447025537490845, "grad_norm": 9.826135635375977, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.2021379470825195, "logits/rejected": -1.6891120672225952, "logps/chosen": -245.24916076660156, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -307.3790283203125, "loss": 1.1141, "margin_dpo/margin_mean": 71.08657836914062, "margin_dpo/margin_std": 98.74791717529297, "step": 373 }, { "KL/chosen_KL_mean": -125.90779113769531, "KL/mean": -165.10006713867188, "KL/rejected_KL_mean": -204.29232788085938, "KL/std": 87.17729187011719, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.005177012644708157, "fcm_dpo/delta": -0.006137605756521225, "fcm_dpo/margin": 78.38453674316406, "fcm_dpo/q_t": 0.4048568606376648, "grad_norm": 8.15467643737793, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.176445484161377, "logits/rejected": -1.7800202369689941, "logps/chosen": -223.92013549804688, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -305.0445556640625, "loss": 1.0658, "margin_dpo/margin_mean": 78.38453674316406, "margin_dpo/margin_std": 83.94216918945312, "step": 374 }, { "KL/chosen_KL_mean": -127.96849060058594, "KL/mean": -173.17376708984375, "KL/rejected_KL_mean": -218.37908935546875, "KL/std": 84.58993530273438, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.005104624666273594, "fcm_dpo/delta": -0.06562300771474838, "fcm_dpo/margin": 90.41058349609375, "fcm_dpo/q_t": 0.39113014936447144, "grad_norm": 9.366206169128418, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.9287617206573486, "logits/rejected": -1.7064990997314453, "logps/chosen": -212.36477661132812, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -314.0979309082031, "loss": 1.0299, "margin_dpo/margin_mean": 90.41058349609375, "margin_dpo/margin_std": 89.9800796508789, "step": 375 }, { "KL/chosen_KL_mean": -123.11735534667969, "KL/mean": -173.1202392578125, "KL/rejected_KL_mean": -223.12313842773438, "KL/std": 96.39289855957031, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0050306483171880245, "fcm_dpo/delta": -0.10869846493005753, "fcm_dpo/margin": 100.00576782226562, "fcm_dpo/q_t": 0.38280025124549866, "grad_norm": 9.342599868774414, "learning_rate": 2.5e-07, "logits/chosen": -1.462032675743103, "logits/rejected": -1.1308530569076538, "logps/chosen": -218.8550567626953, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -335.096435546875, "loss": 1.0065, "margin_dpo/margin_mean": 100.00576782226562, "margin_dpo/margin_std": 102.14617919921875, "step": 376 }, { "KL/chosen_KL_mean": -131.0883331298828, "KL/mean": -182.3998565673828, "KL/rejected_KL_mean": -233.7113800048828, "KL/std": 97.50601959228516, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004909324925392866, "fcm_dpo/delta": -0.10997127741575241, "fcm_dpo/margin": 102.623046875, "fcm_dpo/q_t": 0.3815486431121826, "grad_norm": 8.022355079650879, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5088937282562256, "logits/rejected": -1.3906641006469727, "logps/chosen": -210.01528930664062, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -339.61029052734375, "loss": 0.9976, "margin_dpo/margin_mean": 102.623046875, "margin_dpo/margin_std": 96.38182067871094, "step": 377 }, { "KL/chosen_KL_mean": -126.9972152709961, "KL/mean": -167.9139862060547, "KL/rejected_KL_mean": -208.83074951171875, "KL/std": 92.5882797241211, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004917294718325138, "fcm_dpo/delta": -0.002833399921655655, "fcm_dpo/margin": 81.83354187011719, "fcm_dpo/q_t": 0.404682457447052, "grad_norm": 9.506821632385254, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.026639223098755, "logits/rejected": -1.7472724914550781, "logps/chosen": -202.36962890625, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -302.53485107421875, "loss": 1.0786, "margin_dpo/margin_mean": 81.83354187011719, "margin_dpo/margin_std": 95.84074401855469, "step": 378 }, { "KL/chosen_KL_mean": -148.89407348632812, "KL/mean": -185.17724609375, "KL/rejected_KL_mean": -221.46041870117188, "KL/std": 95.03437805175781, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.00492641469463706, "fcm_dpo/delta": 0.04410012811422348, "fcm_dpo/margin": 72.56633758544922, "fcm_dpo/q_t": 0.41811734437942505, "grad_norm": 8.999001502990723, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.7931835651397705, "logits/rejected": -1.7129929065704346, "logps/chosen": -237.749755859375, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -342.841552734375, "loss": 1.1287, "margin_dpo/margin_mean": 72.56633758544922, "margin_dpo/margin_std": 108.98214721679688, "step": 379 }, { "KL/chosen_KL_mean": -164.6917724609375, "KL/mean": -207.85540771484375, "KL/rejected_KL_mean": -251.01904296875, "KL/std": 97.53778076171875, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004919442348182201, "fcm_dpo/delta": -0.025785475969314575, "fcm_dpo/margin": 86.3272476196289, "fcm_dpo/q_t": 0.402924507856369, "grad_norm": 12.320979118347168, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.2749626636505127, "logits/rejected": -1.2483978271484375, "logps/chosen": -244.5249786376953, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -361.7637939453125, "loss": 1.0743, "margin_dpo/margin_mean": 86.32725524902344, "margin_dpo/margin_std": 111.27102661132812, "step": 380 }, { "KL/chosen_KL_mean": -142.97860717773438, "KL/mean": -181.2846221923828, "KL/rejected_KL_mean": -219.59066772460938, "KL/std": 93.70805358886719, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004931316711008549, "fcm_dpo/delta": 0.023073265329003334, "fcm_dpo/margin": 76.61205291748047, "fcm_dpo/q_t": 0.4112129211425781, "grad_norm": 10.13085651397705, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.6752389669418335, "logits/rejected": -1.1148834228515625, "logps/chosen": -229.54542541503906, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -304.0074157714844, "loss": 1.0981, "margin_dpo/margin_mean": 76.612060546875, "margin_dpo/margin_std": 96.80157470703125, "step": 381 }, { "KL/chosen_KL_mean": -140.80160522460938, "KL/mean": -184.47848510742188, "KL/rejected_KL_mean": -228.1553497314453, "KL/std": 92.23736572265625, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004887628369033337, "fcm_dpo/delta": -0.028999362140893936, "fcm_dpo/margin": 87.353759765625, "fcm_dpo/q_t": 0.3999081254005432, "grad_norm": 8.960593223571777, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -2.0231058597564697, "logits/rejected": -1.7611430883407593, "logps/chosen": -245.26339721679688, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -342.757568359375, "loss": 1.0573, "margin_dpo/margin_mean": 87.353759765625, "margin_dpo/margin_std": 94.21865844726562, "step": 382 }, { "KL/chosen_KL_mean": -153.98794555664062, "KL/mean": -191.03469848632812, "KL/rejected_KL_mean": -228.08145141601562, "KL/std": 95.11100769042969, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.004925740882754326, "fcm_dpo/delta": 0.03635592758655548, "fcm_dpo/margin": 74.09354400634766, "fcm_dpo/q_t": 0.414497435092926, "grad_norm": 9.340737342834473, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.8000679016113281, "logits/rejected": -1.6811119318008423, "logps/chosen": -236.10400390625, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -326.8586120605469, "loss": 1.1056, "margin_dpo/margin_mean": 74.09353637695312, "margin_dpo/margin_std": 92.6292724609375, "step": 383 }, { "KL/chosen_KL_mean": -165.64297485351562, "KL/mean": -202.04885864257812, "KL/rejected_KL_mean": -238.45472717285156, "KL/std": 98.910400390625, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.005002918653190136, "fcm_dpo/delta": 0.03601492941379547, "fcm_dpo/margin": 72.811767578125, "fcm_dpo/q_t": 0.4151723086833954, "grad_norm": 10.860148429870605, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7357726097106934, "logits/rejected": -1.5083496570587158, "logps/chosen": -246.2282257080078, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -329.9026184082031, "loss": 1.1335, "margin_dpo/margin_mean": 72.811767578125, "margin_dpo/margin_std": 110.22061157226562, "step": 384 }, { "KL/chosen_KL_mean": -135.8887939453125, "KL/mean": -184.25875854492188, "KL/rejected_KL_mean": -232.62869262695312, "KL/std": 89.09498596191406, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004898593761026859, "fcm_dpo/delta": -0.07842236757278442, "fcm_dpo/margin": 96.7398681640625, "fcm_dpo/q_t": 0.389860063791275, "grad_norm": 9.847590446472168, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.254437208175659, "logits/rejected": -1.6457946300506592, "logps/chosen": -228.6489715576172, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -317.08868408203125, "loss": 1.0283, "margin_dpo/margin_mean": 96.7398681640625, "margin_dpo/margin_std": 102.67733001708984, "step": 385 }, { "KL/chosen_KL_mean": -144.69778442382812, "KL/mean": -190.39231872558594, "KL/rejected_KL_mean": -236.0868682861328, "KL/std": 91.43205261230469, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.00486674252897501, "fcm_dpo/delta": -0.0468706451356411, "fcm_dpo/margin": 91.38908386230469, "fcm_dpo/q_t": 0.3946182131767273, "grad_norm": 7.611448764801025, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.5964020490646362, "logits/rejected": -1.3086998462677002, "logps/chosen": -230.05674743652344, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -333.0775146484375, "loss": 1.0472, "margin_dpo/margin_mean": 91.38907623291016, "margin_dpo/margin_std": 100.32862854003906, "step": 386 }, { "KL/chosen_KL_mean": -168.80209350585938, "KL/mean": -210.01596069335938, "KL/rejected_KL_mean": -251.2298583984375, "KL/std": 89.16094970703125, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0048636822029948235, "fcm_dpo/delta": -0.000987769104540348, "fcm_dpo/margin": 82.42776489257812, "fcm_dpo/q_t": 0.405638188123703, "grad_norm": 10.385807991027832, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.4562149047851562, "logits/rejected": -1.7730436325073242, "logps/chosen": -271.490234375, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -350.658447265625, "loss": 1.0729, "margin_dpo/margin_mean": 82.42776489257812, "margin_dpo/margin_std": 92.15877532958984, "step": 387 }, { "KL/chosen_KL_mean": -170.18836975097656, "KL/mean": -215.16651916503906, "KL/rejected_KL_mean": -260.1446838378906, "KL/std": 97.95915222167969, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.004825222305953503, "fcm_dpo/delta": -0.03566283732652664, "fcm_dpo/margin": 89.95632934570312, "fcm_dpo/q_t": 0.398121178150177, "grad_norm": 9.08930778503418, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.7228989601135254, "logits/rejected": -1.6006195545196533, "logps/chosen": -253.77359008789062, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -370.24505615234375, "loss": 1.0542, "margin_dpo/margin_mean": 89.95632934570312, "margin_dpo/margin_std": 100.20621490478516, "step": 388 }, { "KL/chosen_KL_mean": -174.91867065429688, "KL/mean": -214.07839965820312, "KL/rejected_KL_mean": -253.23812866210938, "KL/std": 100.02484130859375, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004834345541894436, "fcm_dpo/delta": 0.022223729640245438, "fcm_dpo/margin": 78.31947326660156, "fcm_dpo/q_t": 0.4121132493019104, "grad_norm": 11.054903984069824, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.4482569694519043, "logits/rejected": -2.004437208175659, "logps/chosen": -264.7409362792969, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -338.0918884277344, "loss": 1.1073, "margin_dpo/margin_mean": 78.31946563720703, "margin_dpo/margin_std": 108.02644348144531, "step": 389 }, { "KL/chosen_KL_mean": -176.46243286132812, "KL/mean": -213.52572631835938, "KL/rejected_KL_mean": -250.58901977539062, "KL/std": 97.96418762207031, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.0048496313393116, "fcm_dpo/delta": 0.04174743592739105, "fcm_dpo/margin": 74.1265869140625, "fcm_dpo/q_t": 0.41865724325180054, "grad_norm": 8.849761009216309, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.6630160808563232, "logits/rejected": -1.4556221961975098, "logps/chosen": -253.994873046875, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -343.1729431152344, "loss": 1.1299, "margin_dpo/margin_mean": 74.1265869140625, "margin_dpo/margin_std": 112.54557800292969, "step": 390 }, { "KL/chosen_KL_mean": -159.74839782714844, "KL/mean": -200.30535888671875, "KL/rejected_KL_mean": -240.8623046875, "KL/std": 94.0231704711914, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004872615449130535, "fcm_dpo/delta": 0.004796572029590607, "fcm_dpo/margin": 81.11389923095703, "fcm_dpo/q_t": 0.40769338607788086, "grad_norm": 8.958074569702148, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.0480916500091553, "logits/rejected": -1.6030235290527344, "logps/chosen": -247.9813232421875, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -333.869873046875, "loss": 1.087, "margin_dpo/margin_mean": 81.11389923095703, "margin_dpo/margin_std": 99.68278503417969, "step": 391 }, { "KL/chosen_KL_mean": -161.91433715820312, "KL/mean": -211.00848388671875, "KL/rejected_KL_mean": -260.1026611328125, "KL/std": 105.43682861328125, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0048458147794008255, "fcm_dpo/delta": -0.07954007387161255, "fcm_dpo/margin": 98.18830871582031, "fcm_dpo/q_t": 0.38951417803764343, "grad_norm": 9.177162170410156, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.7627452611923218, "logits/rejected": -1.6444153785705566, "logps/chosen": -249.65875244140625, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -368.67333984375, "loss": 1.037, "margin_dpo/margin_mean": 98.18830871582031, "margin_dpo/margin_std": 112.88900756835938, "step": 392 }, { "KL/chosen_KL_mean": -144.3315887451172, "KL/mean": -188.53359985351562, "KL/rejected_KL_mean": -232.73562622070312, "KL/std": 99.40591430664062, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004789086990058422, "fcm_dpo/delta": -0.024448102340102196, "fcm_dpo/margin": 88.404052734375, "fcm_dpo/q_t": 0.40084224939346313, "grad_norm": 7.92706298828125, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.7752118110656738, "logits/rejected": -1.4555728435516357, "logps/chosen": -220.8953857421875, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -312.4618225097656, "loss": 1.0594, "margin_dpo/margin_mean": 88.404052734375, "margin_dpo/margin_std": 98.24198913574219, "step": 393 }, { "KL/chosen_KL_mean": -151.8074493408203, "KL/mean": -201.11688232421875, "KL/rejected_KL_mean": -250.4263153076172, "KL/std": 95.06200408935547, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004738909192383289, "fcm_dpo/delta": -0.07062850892543793, "fcm_dpo/margin": 98.61886596679688, "fcm_dpo/q_t": 0.3893735408782959, "grad_norm": 8.252213478088379, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.4194071292877197, "logits/rejected": -1.1813889741897583, "logps/chosen": -236.50611877441406, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -347.88409423828125, "loss": 1.0183, "margin_dpo/margin_mean": 98.61886596679688, "margin_dpo/margin_std": 92.53298950195312, "step": 394 }, { "KL/chosen_KL_mean": -159.1632080078125, "KL/mean": -195.93316650390625, "KL/rejected_KL_mean": -232.70315551757812, "KL/std": 99.40000915527344, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004736945964396, "fcm_dpo/delta": 0.05333995074033737, "fcm_dpo/margin": 73.53996276855469, "fcm_dpo/q_t": 0.4218969941139221, "grad_norm": 8.962035179138184, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.439274311065674, "logits/rejected": -2.0289759635925293, "logps/chosen": -248.05084228515625, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -323.7486877441406, "loss": 1.1651, "margin_dpo/margin_mean": 73.53997039794922, "margin_dpo/margin_std": 134.47781372070312, "step": 395 }, { "KL/chosen_KL_mean": -168.6678924560547, "KL/mean": -201.9315643310547, "KL/rejected_KL_mean": -235.1952362060547, "KL/std": 88.22920227050781, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0048286197707057, "fcm_dpo/delta": 0.08139054477214813, "fcm_dpo/margin": 66.52733612060547, "fcm_dpo/q_t": 0.42538249492645264, "grad_norm": 11.346397399902344, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8249086141586304, "logits/rejected": -1.325066328048706, "logps/chosen": -271.13922119140625, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -328.4300231933594, "loss": 1.1508, "margin_dpo/margin_mean": 66.52733612060547, "margin_dpo/margin_std": 103.82200622558594, "step": 396 }, { "KL/chosen_KL_mean": -150.9079132080078, "KL/mean": -210.27688598632812, "KL/rejected_KL_mean": -269.6458740234375, "KL/std": 102.24638366699219, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.00472081545740366, "fcm_dpo/delta": -0.17039062082767487, "fcm_dpo/margin": 118.73796844482422, "fcm_dpo/q_t": 0.3702618479728699, "grad_norm": 8.876315116882324, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.171036720275879, "logits/rejected": -1.5591509342193604, "logps/chosen": -251.95233154296875, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -378.4970703125, "loss": 0.9786, "margin_dpo/margin_mean": 118.73797607421875, "margin_dpo/margin_std": 121.48551940917969, "step": 397 }, { "KL/chosen_KL_mean": -159.30555725097656, "KL/mean": -202.65855407714844, "KL/rejected_KL_mean": -246.01153564453125, "KL/std": 100.65144348144531, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.0046756695955991745, "fcm_dpo/delta": -0.005694168619811535, "fcm_dpo/margin": 86.70599365234375, "fcm_dpo/q_t": 0.4065457284450531, "grad_norm": 8.015686988830566, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.0055484771728516, "logits/rejected": -1.7164894342422485, "logps/chosen": -254.3547821044922, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -360.01715087890625, "loss": 1.0929, "margin_dpo/margin_mean": 86.70599365234375, "margin_dpo/margin_std": 118.29430389404297, "step": 398 }, { "KL/chosen_KL_mean": -140.49114990234375, "KL/mean": -195.11947631835938, "KL/rejected_KL_mean": -249.74778747558594, "KL/std": 90.586669921875, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.004613353870809078, "fcm_dpo/delta": -0.10943984985351562, "fcm_dpo/margin": 109.25662231445312, "fcm_dpo/q_t": 0.38111087679862976, "grad_norm": 8.334001541137695, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.05356502532959, "logits/rejected": -1.2444149255752563, "logps/chosen": -237.12969970703125, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -333.28424072265625, "loss": 0.9913, "margin_dpo/margin_mean": 109.25662231445312, "margin_dpo/margin_std": 96.77314758300781, "step": 399 }, { "KL/chosen_KL_mean": -145.95570373535156, "KL/mean": -201.29440307617188, "KL/rejected_KL_mean": -256.6330871582031, "KL/std": 100.0023422241211, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.004509554244577885, "fcm_dpo/delta": -0.1043153703212738, "fcm_dpo/margin": 110.67739868164062, "fcm_dpo/q_t": 0.38377487659454346, "grad_norm": 12.381619453430176, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.1808836460113525, "logits/rejected": -1.6959240436553955, "logps/chosen": -251.25885009765625, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -365.28863525390625, "loss": 1.0125, "margin_dpo/margin_mean": 110.67739868164062, "margin_dpo/margin_std": 114.68898010253906, "step": 400 }, { "KL/chosen_KL_mean": -161.7530059814453, "KL/mean": -196.468505859375, "KL/rejected_KL_mean": -231.1840057373047, "KL/std": 101.47819519042969, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.004532516933977604, "fcm_dpo/delta": 0.08816267549991608, "fcm_dpo/margin": 69.43101501464844, "fcm_dpo/q_t": 0.4295598268508911, "grad_norm": 10.592202186584473, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.510354995727539, "logits/rejected": -1.263162612915039, "logps/chosen": -246.04217529296875, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -322.12713623046875, "loss": 1.1702, "margin_dpo/margin_mean": 69.4310073852539, "margin_dpo/margin_std": 124.412841796875, "step": 401 }, { "KL/chosen_KL_mean": -144.58724975585938, "KL/mean": -209.13165283203125, "KL/rejected_KL_mean": -273.6760559082031, "KL/std": 107.18785095214844, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.004448581486940384, "fcm_dpo/delta": -0.18465420603752136, "fcm_dpo/margin": 129.08880615234375, "fcm_dpo/q_t": 0.36431384086608887, "grad_norm": 9.081015586853027, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.7447412014007568, "logits/rejected": -1.261171579360962, "logps/chosen": -228.18038940429688, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -361.486328125, "loss": 0.943, "margin_dpo/margin_mean": 129.08880615234375, "margin_dpo/margin_std": 103.77088928222656, "step": 402 }, { "KL/chosen_KL_mean": -146.17965698242188, "KL/mean": -194.3623046875, "KL/rejected_KL_mean": -242.54498291015625, "KL/std": 103.90478515625, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004375974647700787, "fcm_dpo/delta": -0.022779569029808044, "fcm_dpo/margin": 96.36531066894531, "fcm_dpo/q_t": 0.4012417197227478, "grad_norm": 8.615886688232422, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.293342113494873, "logits/rejected": -1.8972268104553223, "logps/chosen": -219.9327392578125, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -326.465087890625, "loss": 1.0646, "margin_dpo/margin_mean": 96.36531066894531, "margin_dpo/margin_std": 110.81733703613281, "step": 403 }, { "KL/chosen_KL_mean": -174.89939880371094, "KL/mean": -215.69873046875, "KL/rejected_KL_mean": -256.4980773925781, "KL/std": 100.1719970703125, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0043987310491502285, "fcm_dpo/delta": 0.04259982705116272, "fcm_dpo/margin": 81.59866333007812, "fcm_dpo/q_t": 0.41711509227752686, "grad_norm": 12.259238243103027, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.016555070877075, "logits/rejected": -1.678879737854004, "logps/chosen": -254.57557678222656, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -340.7785949707031, "loss": 1.1284, "margin_dpo/margin_mean": 81.59866333007812, "margin_dpo/margin_std": 122.86048889160156, "step": 404 }, { "KL/chosen_KL_mean": -177.46945190429688, "KL/mean": -218.869384765625, "KL/rejected_KL_mean": -260.269287109375, "KL/std": 94.34796142578125, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004458615556359291, "fcm_dpo/delta": 0.031429924070835114, "fcm_dpo/margin": 82.79981994628906, "fcm_dpo/q_t": 0.4136407673358917, "grad_norm": 8.184670448303223, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.5115833282470703, "logits/rejected": -1.0338497161865234, "logps/chosen": -273.931396484375, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -353.1400146484375, "loss": 1.0986, "margin_dpo/margin_mean": 82.79981994628906, "margin_dpo/margin_std": 98.9716796875, "step": 405 }, { "KL/chosen_KL_mean": -168.008056640625, "KL/mean": -205.4975128173828, "KL/rejected_KL_mean": -242.9869842529297, "KL/std": 100.87611389160156, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004489736631512642, "fcm_dpo/delta": 0.06555839627981186, "fcm_dpo/margin": 74.97892761230469, "fcm_dpo/q_t": 0.42203694581985474, "grad_norm": 14.583476066589355, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.953465223312378, "logits/rejected": -1.7835934162139893, "logps/chosen": -246.14202880859375, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -341.27056884765625, "loss": 1.1455, "margin_dpo/margin_mean": 74.97891998291016, "margin_dpo/margin_std": 118.68801879882812, "step": 406 }, { "KL/chosen_KL_mean": -174.8277587890625, "KL/mean": -207.92494201660156, "KL/rejected_KL_mean": -241.02212524414062, "KL/std": 99.51219177246094, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0045715514570474625, "fcm_dpo/delta": 0.10043603926897049, "fcm_dpo/margin": 66.19436645507812, "fcm_dpo/q_t": 0.42941027879714966, "grad_norm": 9.290252685546875, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.202202320098877, "logits/rejected": -1.6825331449508667, "logps/chosen": -258.8406066894531, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -323.80316162109375, "loss": 1.1621, "margin_dpo/margin_mean": 66.19436645507812, "margin_dpo/margin_std": 106.06536865234375, "step": 407 }, { "KL/chosen_KL_mean": -166.00537109375, "KL/mean": -199.56875610351562, "KL/rejected_KL_mean": -233.13214111328125, "KL/std": 94.72920227050781, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0046527329832315445, "fcm_dpo/delta": 0.09057843685150146, "fcm_dpo/margin": 67.12677764892578, "fcm_dpo/q_t": 0.4284248948097229, "grad_norm": 9.585349082946777, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.1878819465637207, "logits/rejected": -1.5595223903656006, "logps/chosen": -270.46710205078125, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -329.50433349609375, "loss": 1.17, "margin_dpo/margin_mean": 67.12677764892578, "margin_dpo/margin_std": 118.27676391601562, "step": 408 }, { "KL/chosen_KL_mean": -177.67068481445312, "KL/mean": -212.9954833984375, "KL/rejected_KL_mean": -248.3203125, "KL/std": 97.73112487792969, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.004727277904748917, "fcm_dpo/delta": 0.06829928606748581, "fcm_dpo/margin": 70.64959716796875, "fcm_dpo/q_t": 0.42098987102508545, "grad_norm": 9.010188102722168, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.494247555732727, "logits/rejected": -1.0498709678649902, "logps/chosen": -275.3389892578125, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -338.36614990234375, "loss": 1.1442, "margin_dpo/margin_mean": 70.64959716796875, "margin_dpo/margin_std": 107.58717346191406, "step": 409 }, { "KL/chosen_KL_mean": -172.11546325683594, "KL/mean": -214.48101806640625, "KL/rejected_KL_mean": -256.8465576171875, "KL/std": 103.37193298339844, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.004740777425467968, "fcm_dpo/delta": -0.0017648276407271624, "fcm_dpo/margin": 84.73109436035156, "fcm_dpo/q_t": 0.40531495213508606, "grad_norm": 8.575937271118164, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.676349401473999, "logits/rejected": -1.5421899557113647, "logps/chosen": -248.58470153808594, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -345.4871826171875, "loss": 1.082, "margin_dpo/margin_mean": 84.73109436035156, "margin_dpo/margin_std": 104.03684997558594, "step": 410 }, { "KL/chosen_KL_mean": -183.56411743164062, "KL/mean": -222.9620361328125, "KL/rejected_KL_mean": -262.3599853515625, "KL/std": 96.22058868408203, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004746856167912483, "fcm_dpo/delta": 0.026875976473093033, "fcm_dpo/margin": 78.79586029052734, "fcm_dpo/q_t": 0.41382747888565063, "grad_norm": 8.8742094039917, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.5183920860290527, "logits/rejected": -2.1247384548187256, "logps/chosen": -270.73040771484375, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -349.45599365234375, "loss": 1.105, "margin_dpo/margin_mean": 78.79586029052734, "margin_dpo/margin_std": 103.61910247802734, "step": 411 }, { "KL/chosen_KL_mean": -174.05380249023438, "KL/mean": -212.7039794921875, "KL/rejected_KL_mean": -251.35414123535156, "KL/std": 96.21929931640625, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004778586328029633, "fcm_dpo/delta": 0.03174576163291931, "fcm_dpo/margin": 77.30035400390625, "fcm_dpo/q_t": 0.4163166880607605, "grad_norm": 10.245725631713867, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.4107422828674316, "logits/rejected": -2.1927762031555176, "logps/chosen": -253.00112915039062, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -357.4596862792969, "loss": 1.1346, "margin_dpo/margin_mean": 77.30035400390625, "margin_dpo/margin_std": 124.76705932617188, "step": 412 }, { "KL/chosen_KL_mean": -187.42189025878906, "KL/mean": -230.55059814453125, "KL/rejected_KL_mean": -273.6793212890625, "KL/std": 102.32810974121094, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004774519708007574, "fcm_dpo/delta": -0.012568188831210136, "fcm_dpo/margin": 86.2574234008789, "fcm_dpo/q_t": 0.40402060747146606, "grad_norm": 9.546736717224121, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.0735152959823608, "logits/rejected": -0.8094683289527893, "logps/chosen": -283.1166076660156, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -381.6401672363281, "loss": 1.0963, "margin_dpo/margin_mean": 86.2574234008789, "margin_dpo/margin_std": 122.58245849609375, "step": 413 }, { "KL/chosen_KL_mean": -172.40631103515625, "KL/mean": -213.39834594726562, "KL/rejected_KL_mean": -254.39035034179688, "KL/std": 99.13764953613281, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.004797955974936485, "fcm_dpo/delta": 0.006859011948108673, "fcm_dpo/margin": 81.98403930664062, "fcm_dpo/q_t": 0.41008490324020386, "grad_norm": 12.533997535705566, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.4911670684814453, "logits/rejected": -1.1272027492523193, "logps/chosen": -260.6829833984375, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -347.2604064941406, "loss": 1.1161, "margin_dpo/margin_mean": 81.98403930664062, "margin_dpo/margin_std": 125.67837524414062, "step": 414 }, { "KL/chosen_KL_mean": -187.02206420898438, "KL/mean": -221.4931182861328, "KL/rejected_KL_mean": -255.9641571044922, "KL/std": 90.66765594482422, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.004844988696277142, "fcm_dpo/delta": 0.06819400191307068, "fcm_dpo/margin": 68.94209289550781, "fcm_dpo/q_t": 0.42303377389907837, "grad_norm": 10.716426849365234, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.4162585735321045, "logits/rejected": -1.0269473791122437, "logps/chosen": -295.93798828125, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -363.4355163574219, "loss": 1.1495, "margin_dpo/margin_mean": 68.94209289550781, "margin_dpo/margin_std": 111.09007263183594, "step": 415 }, { "KL/chosen_KL_mean": -187.31051635742188, "KL/mean": -234.16177368164062, "KL/rejected_KL_mean": -281.0130310058594, "KL/std": 104.05574035644531, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0048531400971114635, "fcm_dpo/delta": -0.05808945372700691, "fcm_dpo/margin": 93.70252990722656, "fcm_dpo/q_t": 0.3950579762458801, "grad_norm": 9.00806999206543, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.7572053670883179, "logits/rejected": -1.2453339099884033, "logps/chosen": -280.70941162109375, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -378.68603515625, "loss": 1.0547, "margin_dpo/margin_mean": 93.7025375366211, "margin_dpo/margin_std": 109.36782836914062, "step": 416 }, { "KL/chosen_KL_mean": -173.19534301757812, "KL/mean": -216.7273406982422, "KL/rejected_KL_mean": -260.25933837890625, "KL/std": 97.21144104003906, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.004793957807123661, "fcm_dpo/delta": -0.01814088225364685, "fcm_dpo/margin": 87.06402587890625, "fcm_dpo/q_t": 0.4023556709289551, "grad_norm": 9.227706909179688, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -1.9824455976486206, "logits/rejected": -1.027436375617981, "logps/chosen": -256.73065185546875, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -334.7012023925781, "loss": 1.0664, "margin_dpo/margin_mean": 87.06402587890625, "margin_dpo/margin_std": 99.82061767578125, "step": 417 }, { "KL/chosen_KL_mean": -166.27293395996094, "KL/mean": -221.5467529296875, "KL/rejected_KL_mean": -276.820556640625, "KL/std": 104.506103515625, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0047359587624669075, "fcm_dpo/delta": -0.1308847963809967, "fcm_dpo/margin": 110.54762268066406, "fcm_dpo/q_t": 0.37852245569229126, "grad_norm": 10.712328910827637, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.213711738586426, "logits/rejected": -1.5592008829116821, "logps/chosen": -274.49444580078125, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -388.6851806640625, "loss": 0.9923, "margin_dpo/margin_mean": 110.54762268066406, "margin_dpo/margin_std": 102.14996337890625, "step": 418 }, { "KL/chosen_KL_mean": -173.810302734375, "KL/mean": -221.94039916992188, "KL/rejected_KL_mean": -270.07049560546875, "KL/std": 93.09687805175781, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.004647375550121069, "fcm_dpo/delta": -0.04975828528404236, "fcm_dpo/margin": 96.26017761230469, "fcm_dpo/q_t": 0.3954481780529022, "grad_norm": 11.0460205078125, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.6209633350372314, "logits/rejected": -1.1824194192886353, "logps/chosen": -270.99359130859375, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -368.25579833984375, "loss": 1.0435, "margin_dpo/margin_mean": 96.26018524169922, "margin_dpo/margin_std": 102.55694580078125, "step": 419 }, { "KL/chosen_KL_mean": -177.99838256835938, "KL/mean": -220.69728088378906, "KL/rejected_KL_mean": -263.39617919921875, "KL/std": 97.59660339355469, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.00463168416172266, "fcm_dpo/delta": 0.0044556912034749985, "fcm_dpo/margin": 85.39776611328125, "fcm_dpo/q_t": 0.4084246754646301, "grad_norm": 14.477088928222656, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.2283992767333984, "logits/rejected": -1.1268627643585205, "logps/chosen": -292.306884765625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -339.0797119140625, "loss": 1.0948, "margin_dpo/margin_mean": 85.39776611328125, "margin_dpo/margin_std": 113.13612365722656, "step": 420 }, { "KL/chosen_KL_mean": -169.5975799560547, "KL/mean": -217.27993774414062, "KL/rejected_KL_mean": -264.9622802734375, "KL/std": 100.90658569335938, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.004585624672472477, "fcm_dpo/delta": -0.03911059722304344, "fcm_dpo/margin": 95.36471557617188, "fcm_dpo/q_t": 0.3978048861026764, "grad_norm": 8.032257080078125, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8425661325454712, "logits/rejected": -1.4680731296539307, "logps/chosen": -255.47743225097656, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -361.29876708984375, "loss": 1.0581, "margin_dpo/margin_mean": 95.36471557617188, "margin_dpo/margin_std": 111.18939208984375, "step": 421 }, { "KL/chosen_KL_mean": -167.20547485351562, "KL/mean": -212.86624145507812, "KL/rejected_KL_mean": -258.5269775390625, "KL/std": 93.10214233398438, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004575836006551981, "fcm_dpo/delta": -0.018658917397260666, "fcm_dpo/margin": 91.32149505615234, "fcm_dpo/q_t": 0.40252822637557983, "grad_norm": 12.471305847167969, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.3677308559417725, "logits/rejected": -1.998854398727417, "logps/chosen": -267.68609619140625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -352.9351806640625, "loss": 1.0655, "margin_dpo/margin_mean": 91.32150268554688, "margin_dpo/margin_std": 104.91423034667969, "step": 422 }, { "KL/chosen_KL_mean": -156.3128662109375, "KL/mean": -202.14404296875, "KL/rejected_KL_mean": -247.97525024414062, "KL/std": 111.38519287109375, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.004570307210087776, "fcm_dpo/delta": -0.01998838037252426, "fcm_dpo/margin": 91.66236877441406, "fcm_dpo/q_t": 0.4046717882156372, "grad_norm": 10.115958213806152, "learning_rate": 1.902669377503756e-07, "logits/chosen": -2.008469343185425, "logits/rejected": -1.8909437656402588, "logps/chosen": -234.7628173828125, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -340.02178955078125, "loss": 1.0896, "margin_dpo/margin_mean": 91.6623764038086, "margin_dpo/margin_std": 126.29459381103516, "step": 423 }, { "KL/chosen_KL_mean": -175.62274169921875, "KL/mean": -220.45480346679688, "KL/rejected_KL_mean": -265.286865234375, "KL/std": 98.69284057617188, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0045412215404212475, "fcm_dpo/delta": -0.007493988145142794, "fcm_dpo/margin": 89.66412353515625, "fcm_dpo/q_t": 0.4041185677051544, "grad_norm": 8.760427474975586, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7677505016326904, "logits/rejected": -1.1717780828475952, "logps/chosen": -263.26507568359375, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -337.65252685546875, "loss": 1.077, "margin_dpo/margin_mean": 89.66412353515625, "margin_dpo/margin_std": 107.58222961425781, "step": 424 }, { "KL/chosen_KL_mean": -183.1480255126953, "KL/mean": -213.7548370361328, "KL/rejected_KL_mean": -244.36163330078125, "KL/std": 102.36970520019531, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004616462625563145, "fcm_dpo/delta": 0.1209649071097374, "fcm_dpo/margin": 61.213600158691406, "fcm_dpo/q_t": 0.43734681606292725, "grad_norm": 10.410018920898438, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.5168066024780273, "logits/rejected": -1.4469282627105713, "logps/chosen": -261.6678161621094, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -347.11029052734375, "loss": 1.2065, "margin_dpo/margin_mean": 61.213600158691406, "margin_dpo/margin_std": 128.76568603515625, "step": 425 }, { "KL/chosen_KL_mean": -155.92095947265625, "KL/mean": -200.76290893554688, "KL/rejected_KL_mean": -245.60488891601562, "KL/std": 94.21288299560547, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0046510761603713036, "fcm_dpo/delta": -0.017950644716620445, "fcm_dpo/margin": 89.68394470214844, "fcm_dpo/q_t": 0.4026781916618347, "grad_norm": 11.241652488708496, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.6900911331176758, "logits/rejected": -0.9477226734161377, "logps/chosen": -264.4267578125, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -333.9049377441406, "loss": 1.0746, "margin_dpo/margin_mean": 89.6839370727539, "margin_dpo/margin_std": 109.84078216552734, "step": 426 }, { "KL/chosen_KL_mean": -128.3109893798828, "KL/mean": -169.61354064941406, "KL/rejected_KL_mean": -210.9160919189453, "KL/std": 90.0490951538086, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.004651614464819431, "fcm_dpo/delta": 0.0163428857922554, "fcm_dpo/margin": 82.60509490966797, "fcm_dpo/q_t": 0.4097248911857605, "grad_norm": 9.085285186767578, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.468514919281006, "logits/rejected": -1.8400628566741943, "logps/chosen": -227.43145751953125, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -296.64068603515625, "loss": 1.0878, "margin_dpo/margin_mean": 82.60509490966797, "margin_dpo/margin_std": 97.83792114257812, "step": 427 }, { "KL/chosen_KL_mean": -181.90237426757812, "KL/mean": -217.5048828125, "KL/rejected_KL_mean": -253.107421875, "KL/std": 101.43812561035156, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.00467848964035511, "fcm_dpo/delta": 0.06908674538135529, "fcm_dpo/margin": 71.20503997802734, "fcm_dpo/q_t": 0.42313146591186523, "grad_norm": 9.920881271362305, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.7187827825546265, "logits/rejected": -1.457289457321167, "logps/chosen": -287.87164306640625, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -362.2095947265625, "loss": 1.1478, "margin_dpo/margin_mean": 71.20503234863281, "margin_dpo/margin_std": 112.039794921875, "step": 428 }, { "KL/chosen_KL_mean": -166.12811279296875, "KL/mean": -211.89231872558594, "KL/rejected_KL_mean": -257.656494140625, "KL/std": 105.9439468383789, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0047252061776816845, "fcm_dpo/delta": -0.034777235239744186, "fcm_dpo/margin": 91.52838897705078, "fcm_dpo/q_t": 0.40114396810531616, "grad_norm": 7.9994964599609375, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.8210684061050415, "logits/rejected": -1.0894170999526978, "logps/chosen": -279.6729736328125, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -355.8985290527344, "loss": 1.0801, "margin_dpo/margin_mean": 91.52839660644531, "margin_dpo/margin_std": 120.09689331054688, "step": 429 }, { "KL/chosen_KL_mean": -159.72084045410156, "KL/mean": -194.3505859375, "KL/rejected_KL_mean": -228.9803466796875, "KL/std": 97.73844909667969, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.004749178886413574, "fcm_dpo/delta": 0.0730467140674591, "fcm_dpo/margin": 69.25948333740234, "fcm_dpo/q_t": 0.4245675206184387, "grad_norm": 9.102964401245117, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4612743854522705, "logits/rejected": -1.4245764017105103, "logps/chosen": -251.0402069091797, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -339.0899658203125, "loss": 1.1546, "margin_dpo/margin_mean": 69.25948333740234, "margin_dpo/margin_std": 114.2397232055664, "step": 430 }, { "KL/chosen_KL_mean": -160.57656860351562, "KL/mean": -202.2850341796875, "KL/rejected_KL_mean": -243.99349975585938, "KL/std": 99.40290832519531, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004747520200908184, "fcm_dpo/delta": 0.0041028838604688644, "fcm_dpo/margin": 83.41691589355469, "fcm_dpo/q_t": 0.40803444385528564, "grad_norm": 10.148834228515625, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.9923008680343628, "logits/rejected": -1.6088130474090576, "logps/chosen": -253.75778198242188, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -342.1257629394531, "loss": 1.0983, "margin_dpo/margin_mean": 83.41691589355469, "margin_dpo/margin_std": 113.73867797851562, "step": 431 }, { "KL/chosen_KL_mean": -140.80722045898438, "KL/mean": -191.23757934570312, "KL/rejected_KL_mean": -241.66793823242188, "KL/std": 104.65936279296875, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.00471523217856884, "fcm_dpo/delta": -0.07931604236364365, "fcm_dpo/margin": 100.8607177734375, "fcm_dpo/q_t": 0.38866737484931946, "grad_norm": 8.945375442504883, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.3543646335601807, "logits/rejected": -0.7846279144287109, "logps/chosen": -245.24200439453125, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -346.75750732421875, "loss": 1.0243, "margin_dpo/margin_mean": 100.86072540283203, "margin_dpo/margin_std": 102.28563690185547, "step": 432 }, { "KL/chosen_KL_mean": -167.19729614257812, "KL/mean": -207.17645263671875, "KL/rejected_KL_mean": -247.15560913085938, "KL/std": 98.71083068847656, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0047178310342133045, "fcm_dpo/delta": 0.02315293624997139, "fcm_dpo/margin": 79.95830535888672, "fcm_dpo/q_t": 0.4129111170768738, "grad_norm": 9.064740180969238, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.5438282489776611, "logits/rejected": -1.2140048742294312, "logps/chosen": -257.04052734375, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -348.8890686035156, "loss": 1.1135, "margin_dpo/margin_mean": 79.95829772949219, "margin_dpo/margin_std": 113.09304809570312, "step": 433 }, { "KL/chosen_KL_mean": -144.45108032226562, "KL/mean": -190.02691650390625, "KL/rejected_KL_mean": -235.60275268554688, "KL/std": 109.56161499023438, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004687263164669275, "fcm_dpo/delta": -0.028460538014769554, "fcm_dpo/margin": 91.15167999267578, "fcm_dpo/q_t": 0.40426331758499146, "grad_norm": 12.152153015136719, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.4360790252685547, "logits/rejected": -1.7645423412322998, "logps/chosen": -242.14361572265625, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -319.69403076171875, "loss": 1.0834, "margin_dpo/margin_mean": 91.15167999267578, "margin_dpo/margin_std": 127.11630249023438, "step": 434 }, { "KL/chosen_KL_mean": -143.2596893310547, "KL/mean": -189.9921417236328, "KL/rejected_KL_mean": -236.724609375, "KL/std": 101.49556732177734, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.004656041041016579, "fcm_dpo/delta": -0.03676321357488632, "fcm_dpo/margin": 93.46492004394531, "fcm_dpo/q_t": 0.39958512783050537, "grad_norm": 10.107874870300293, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.2688841819763184, "logits/rejected": -1.807559847831726, "logps/chosen": -229.43161010742188, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -330.476318359375, "loss": 1.061, "margin_dpo/margin_mean": 93.46491241455078, "margin_dpo/margin_std": 112.02484893798828, "step": 435 }, { "KL/chosen_KL_mean": -118.83677673339844, "KL/mean": -167.28875732421875, "KL/rejected_KL_mean": -215.74073791503906, "KL/std": 86.99470520019531, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004603767301887274, "fcm_dpo/delta": -0.04834093153476715, "fcm_dpo/margin": 96.90396118164062, "fcm_dpo/q_t": 0.39422690868377686, "grad_norm": 10.199516296386719, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.6534345149993896, "logits/rejected": -0.7731667757034302, "logps/chosen": -224.62387084960938, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -304.3654479980469, "loss": 1.0299, "margin_dpo/margin_mean": 96.90396118164062, "margin_dpo/margin_std": 90.08195495605469, "step": 436 }, { "KL/chosen_KL_mean": -137.89990234375, "KL/mean": -177.02557373046875, "KL/rejected_KL_mean": -216.15122985839844, "KL/std": 92.63902282714844, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004630147013813257, "fcm_dpo/delta": 0.039004355669021606, "fcm_dpo/margin": 78.2513198852539, "fcm_dpo/q_t": 0.4136236608028412, "grad_norm": 9.94968032836914, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.822760820388794, "logits/rejected": -1.1495966911315918, "logps/chosen": -233.9619598388672, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -305.1634521484375, "loss": 1.0994, "margin_dpo/margin_mean": 78.2513198852539, "margin_dpo/margin_std": 91.09196472167969, "step": 437 }, { "KL/chosen_KL_mean": -142.96383666992188, "KL/mean": -182.01824951171875, "KL/rejected_KL_mean": -221.07264709472656, "KL/std": 92.85961151123047, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.00464106909930706, "fcm_dpo/delta": 0.03873150050640106, "fcm_dpo/margin": 78.10881042480469, "fcm_dpo/q_t": 0.4149070978164673, "grad_norm": 10.079015731811523, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.0734620094299316, "logits/rejected": -1.2479709386825562, "logps/chosen": -244.8192138671875, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -310.52032470703125, "loss": 1.1097, "margin_dpo/margin_mean": 78.10880279541016, "margin_dpo/margin_std": 99.98898315429688, "step": 438 }, { "KL/chosen_KL_mean": -152.73745727539062, "KL/mean": -193.06832885742188, "KL/rejected_KL_mean": -233.399169921875, "KL/std": 101.15681457519531, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004656674340367317, "fcm_dpo/delta": 0.024682722985744476, "fcm_dpo/margin": 80.66172790527344, "fcm_dpo/q_t": 0.41236525774002075, "grad_norm": 9.800536155700684, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.3348183631896973, "logits/rejected": -1.0861692428588867, "logps/chosen": -234.4930877685547, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -323.98553466796875, "loss": 1.0995, "margin_dpo/margin_mean": 80.66172790527344, "margin_dpo/margin_std": 98.39764404296875, "step": 439 }, { "KL/chosen_KL_mean": -137.61875915527344, "KL/mean": -187.0274658203125, "KL/rejected_KL_mean": -236.4361572265625, "KL/std": 102.08338928222656, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.004625425208359957, "fcm_dpo/delta": -0.06068401783704758, "fcm_dpo/margin": 98.81739807128906, "fcm_dpo/q_t": 0.39424169063568115, "grad_norm": 7.836474418640137, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.468109130859375, "logits/rejected": -1.9737708568572998, "logps/chosen": -243.2598419189453, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -339.837158203125, "loss": 1.045, "margin_dpo/margin_mean": 98.81739807128906, "margin_dpo/margin_std": 111.9505615234375, "step": 440 }, { "KL/chosen_KL_mean": -176.85140991210938, "KL/mean": -219.15643310546875, "KL/rejected_KL_mean": -261.46148681640625, "KL/std": 96.879638671875, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0046339211985468864, "fcm_dpo/delta": 0.00821245089173317, "fcm_dpo/margin": 84.61009216308594, "fcm_dpo/q_t": 0.4096594452857971, "grad_norm": 8.728558540344238, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.693657398223877, "logits/rejected": -1.3290272951126099, "logps/chosen": -268.38092041015625, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -365.08087158203125, "loss": 1.0856, "margin_dpo/margin_mean": 84.61009216308594, "margin_dpo/margin_std": 104.14657592773438, "step": 441 }, { "KL/chosen_KL_mean": -173.42959594726562, "KL/mean": -226.28509521484375, "KL/rejected_KL_mean": -279.140625, "KL/std": 102.36576843261719, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.004582737572491169, "fcm_dpo/delta": -0.08873237669467926, "fcm_dpo/margin": 105.7110595703125, "fcm_dpo/q_t": 0.3862221837043762, "grad_norm": 10.921019554138184, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.3892748355865479, "logits/rejected": -0.8945022821426392, "logps/chosen": -258.2071533203125, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -362.96478271484375, "loss": 1.0115, "margin_dpo/margin_mean": 105.7110595703125, "margin_dpo/margin_std": 103.02040100097656, "step": 442 }, { "KL/chosen_KL_mean": -168.42718505859375, "KL/mean": -201.20840454101562, "KL/rejected_KL_mean": -233.9896240234375, "KL/std": 100.1496353149414, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004613466560840607, "fcm_dpo/delta": 0.10066770017147064, "fcm_dpo/margin": 65.56242370605469, "fcm_dpo/q_t": 0.4293175935745239, "grad_norm": 10.033644676208496, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.8556716442108154, "logits/rejected": -1.2979168891906738, "logps/chosen": -271.07647705078125, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -327.0277099609375, "loss": 1.1568, "margin_dpo/margin_mean": 65.56242370605469, "margin_dpo/margin_std": 96.83353424072266, "step": 443 }, { "KL/chosen_KL_mean": -174.11630249023438, "KL/mean": -212.68429565429688, "KL/rejected_KL_mean": -251.25228881835938, "KL/std": 96.96403503417969, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.004685916006565094, "fcm_dpo/delta": 0.03994153439998627, "fcm_dpo/margin": 77.135986328125, "fcm_dpo/q_t": 0.4166090488433838, "grad_norm": 10.845218658447266, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6384479999542236, "logits/rejected": -1.4690241813659668, "logps/chosen": -262.0360107421875, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -354.57574462890625, "loss": 1.1237, "margin_dpo/margin_mean": 77.135986328125, "margin_dpo/margin_std": 113.18402862548828, "step": 444 }, { "KL/chosen_KL_mean": -153.62908935546875, "KL/mean": -198.50738525390625, "KL/rejected_KL_mean": -243.3856658935547, "KL/std": 95.09761047363281, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.004662311635911465, "fcm_dpo/delta": -0.019565189257264137, "fcm_dpo/margin": 89.75657653808594, "fcm_dpo/q_t": 0.40169984102249146, "grad_norm": 8.93307113647461, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.796631097793579, "logits/rejected": -1.370347499847412, "logps/chosen": -255.02996826171875, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -342.423583984375, "loss": 1.0635, "margin_dpo/margin_mean": 89.75657653808594, "margin_dpo/margin_std": 100.39181518554688, "step": 445 }, { "KL/chosen_KL_mean": -147.80299377441406, "KL/mean": -187.58853149414062, "KL/rejected_KL_mean": -227.37405395507812, "KL/std": 100.59852600097656, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.004699447192251682, "fcm_dpo/delta": 0.026974519714713097, "fcm_dpo/margin": 79.57105255126953, "fcm_dpo/q_t": 0.4123729467391968, "grad_norm": 12.827736854553223, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.964587926864624, "logits/rejected": -1.8238766193389893, "logps/chosen": -235.225341796875, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -334.0748291015625, "loss": 1.0959, "margin_dpo/margin_mean": 79.57106018066406, "margin_dpo/margin_std": 96.24667358398438, "step": 446 }, { "KL/chosen_KL_mean": -144.75604248046875, "KL/mean": -197.7760009765625, "KL/rejected_KL_mean": -250.79592895507812, "KL/std": 103.86825561523438, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004630739800632, "fcm_dpo/delta": -0.09576896578073502, "fcm_dpo/margin": 106.03988647460938, "fcm_dpo/q_t": 0.38633495569229126, "grad_norm": 8.873396873474121, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.0275819301605225, "logits/rejected": -1.9809596538543701, "logps/chosen": -228.37130737304688, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -355.70831298828125, "loss": 1.025, "margin_dpo/margin_mean": 106.03988647460938, "margin_dpo/margin_std": 118.22628021240234, "step": 447 }, { "KL/chosen_KL_mean": -156.91534423828125, "KL/mean": -207.868896484375, "KL/rejected_KL_mean": -258.82244873046875, "KL/std": 108.90815734863281, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0045827897265553474, "fcm_dpo/delta": -0.0704253539443016, "fcm_dpo/margin": 101.9071044921875, "fcm_dpo/q_t": 0.3929058909416199, "grad_norm": 9.922281265258789, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.6751248836517334, "logits/rejected": -1.4024133682250977, "logps/chosen": -249.4911346435547, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -366.5122375488281, "loss": 1.0564, "margin_dpo/margin_mean": 101.9071044921875, "margin_dpo/margin_std": 128.04771423339844, "step": 448 }, { "KL/chosen_KL_mean": -154.03697204589844, "KL/mean": -206.83367919921875, "KL/rejected_KL_mean": -259.63037109375, "KL/std": 101.68843078613281, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.004490693565458059, "fcm_dpo/delta": -0.07791125774383545, "fcm_dpo/margin": 105.59336853027344, "fcm_dpo/q_t": 0.3894463777542114, "grad_norm": 8.836973190307617, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4389851093292236, "logits/rejected": -0.991568922996521, "logps/chosen": -256.4359130859375, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -354.77923583984375, "loss": 1.0314, "margin_dpo/margin_mean": 105.59336853027344, "margin_dpo/margin_std": 116.36212158203125, "step": 449 }, { "KL/chosen_KL_mean": -137.90805053710938, "KL/mean": -187.26747131347656, "KL/rejected_KL_mean": -236.62692260742188, "KL/std": 103.17265319824219, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.00444291764870286, "fcm_dpo/delta": -0.040434639900922775, "fcm_dpo/margin": 98.71885681152344, "fcm_dpo/q_t": 0.3971881866455078, "grad_norm": 9.737565040588379, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3982181549072266, "logits/rejected": -2.204925060272217, "logps/chosen": -224.90090942382812, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -345.158935546875, "loss": 1.0554, "margin_dpo/margin_mean": 98.71885681152344, "margin_dpo/margin_std": 113.610595703125, "step": 450 }, { "KL/chosen_KL_mean": -156.49842834472656, "KL/mean": -196.3057861328125, "KL/rejected_KL_mean": -236.11312866210938, "KL/std": 102.35255432128906, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.00447550555691123, "fcm_dpo/delta": 0.0451735183596611, "fcm_dpo/margin": 79.61471557617188, "fcm_dpo/q_t": 0.4183828830718994, "grad_norm": 13.66801643371582, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.3130745887756348, "logits/rejected": -1.7973562479019165, "logps/chosen": -243.30970764160156, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -315.9687194824219, "loss": 1.1385, "margin_dpo/margin_mean": 79.61471557617188, "margin_dpo/margin_std": 127.30015563964844, "step": 451 }, { "KL/chosen_KL_mean": -136.47021484375, "KL/mean": -181.12271118164062, "KL/rejected_KL_mean": -225.77523803710938, "KL/std": 98.40079498291016, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004448451101779938, "fcm_dpo/delta": 0.0018386095762252808, "fcm_dpo/margin": 89.30503845214844, "fcm_dpo/q_t": 0.4062986671924591, "grad_norm": 8.492249488830566, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.877152442932129, "logits/rejected": -1.8168025016784668, "logps/chosen": -215.849853515625, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -329.4906311035156, "loss": 1.0733, "margin_dpo/margin_mean": 89.30503845214844, "margin_dpo/margin_std": 91.54463958740234, "step": 452 }, { "KL/chosen_KL_mean": -164.87271118164062, "KL/mean": -218.70626831054688, "KL/rejected_KL_mean": -272.539794921875, "KL/std": 100.2962646484375, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.004430997185409069, "fcm_dpo/delta": -0.08087509870529175, "fcm_dpo/margin": 107.66709899902344, "fcm_dpo/q_t": 0.38885021209716797, "grad_norm": 11.910933494567871, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.9328951835632324, "logits/rejected": -1.4588098526000977, "logps/chosen": -252.56790161132812, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -362.5980224609375, "loss": 1.0183, "margin_dpo/margin_mean": 107.66709899902344, "margin_dpo/margin_std": 107.01710510253906, "step": 453 }, { "KL/chosen_KL_mean": -171.87547302246094, "KL/mean": -215.76513671875, "KL/rejected_KL_mean": -259.6548156738281, "KL/std": 102.61280822753906, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0044363029301166534, "fcm_dpo/delta": 0.010505832731723785, "fcm_dpo/margin": 87.77934265136719, "fcm_dpo/q_t": 0.4108082354068756, "grad_norm": 9.4187593460083, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.8055365085601807, "logits/rejected": -1.3379814624786377, "logps/chosen": -261.44171142578125, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -352.57586669921875, "loss": 1.1183, "margin_dpo/margin_mean": 87.77934265136719, "margin_dpo/margin_std": 132.85498046875, "step": 454 }, { "KL/chosen_KL_mean": -162.83786010742188, "KL/mean": -203.79742431640625, "KL/rejected_KL_mean": -244.7570037841797, "KL/std": 100.5754165649414, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.004455722868442535, "fcm_dpo/delta": 0.035991039127111435, "fcm_dpo/margin": 81.91914367675781, "fcm_dpo/q_t": 0.41520917415618896, "grad_norm": 9.709168434143066, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8785185813903809, "logits/rejected": -1.837470293045044, "logps/chosen": -240.46783447265625, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -363.7349548339844, "loss": 1.1182, "margin_dpo/margin_mean": 81.91914367675781, "margin_dpo/margin_std": 115.2054672241211, "step": 455 }, { "KL/chosen_KL_mean": -165.49813842773438, "KL/mean": -206.7355194091797, "KL/rejected_KL_mean": -247.972900390625, "KL/std": 102.82202911376953, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.004477779380977154, "fcm_dpo/delta": 0.03177221119403839, "fcm_dpo/margin": 82.47476196289062, "fcm_dpo/q_t": 0.41370248794555664, "grad_norm": 11.29566478729248, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.0610709190368652, "logits/rejected": -1.9293882846832275, "logps/chosen": -255.11500549316406, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -357.5325927734375, "loss": 1.103, "margin_dpo/margin_mean": 82.47476196289062, "margin_dpo/margin_std": 104.74467468261719, "step": 456 }, { "KL/chosen_KL_mean": -133.69277954101562, "KL/mean": -188.53192138671875, "KL/rejected_KL_mean": -243.37106323242188, "KL/std": 102.7159194946289, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.004388490691781044, "fcm_dpo/delta": -0.08713261783123016, "fcm_dpo/margin": 109.67828369140625, "fcm_dpo/q_t": 0.38561081886291504, "grad_norm": 10.466789245605469, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.076032876968384, "logits/rejected": -1.756617546081543, "logps/chosen": -221.01446533203125, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -334.1376953125, "loss": 1.0048, "margin_dpo/margin_mean": 109.67828369140625, "margin_dpo/margin_std": 92.21528625488281, "step": 457 }, { "KL/chosen_KL_mean": -155.75685119628906, "KL/mean": -195.49868774414062, "KL/rejected_KL_mean": -235.24050903320312, "KL/std": 102.09293365478516, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004442187957465649, "fcm_dpo/delta": 0.04843265935778618, "fcm_dpo/margin": 79.48365020751953, "fcm_dpo/q_t": 0.419257789850235, "grad_norm": 9.556123733520508, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.7406861782073975, "logits/rejected": -1.5061578750610352, "logps/chosen": -249.27731323242188, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -338.6094970703125, "loss": 1.1288, "margin_dpo/margin_mean": 79.48365020751953, "margin_dpo/margin_std": 118.32780456542969, "step": 458 }, { "KL/chosen_KL_mean": -149.04302978515625, "KL/mean": -198.4646759033203, "KL/rejected_KL_mean": -247.88632202148438, "KL/std": 106.40361022949219, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004399778321385384, "fcm_dpo/delta": -0.037025414407253265, "fcm_dpo/margin": 98.8432846069336, "fcm_dpo/q_t": 0.3993379473686218, "grad_norm": 7.558887004852295, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.873838186264038, "logits/rejected": -1.6275604963302612, "logps/chosen": -224.73123168945312, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -340.05682373046875, "loss": 1.0589, "margin_dpo/margin_mean": 98.8432846069336, "margin_dpo/margin_std": 114.51268005371094, "step": 459 }, { "KL/chosen_KL_mean": -152.25970458984375, "KL/mean": -198.50230407714844, "KL/rejected_KL_mean": -244.74490356445312, "KL/std": 103.75729370117188, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.004390698857605457, "fcm_dpo/delta": -0.006613072007894516, "fcm_dpo/margin": 92.48521423339844, "fcm_dpo/q_t": 0.405759334564209, "grad_norm": 10.447441101074219, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0694446563720703, "logits/rejected": -2.0931429862976074, "logps/chosen": -233.37759399414062, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -355.05731201171875, "loss": 1.0868, "margin_dpo/margin_mean": 92.48521423339844, "margin_dpo/margin_std": 118.99238586425781, "step": 460 }, { "KL/chosen_KL_mean": -188.20130920410156, "KL/mean": -225.4494171142578, "KL/rejected_KL_mean": -262.6975402832031, "KL/std": 90.56214904785156, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004440647549927235, "fcm_dpo/delta": 0.0716044008731842, "fcm_dpo/margin": 74.4962158203125, "fcm_dpo/q_t": 0.42321351170539856, "grad_norm": 8.55777359008789, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.4381260871887207, "logits/rejected": -1.2400624752044678, "logps/chosen": -269.78485107421875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -356.57464599609375, "loss": 1.1383, "margin_dpo/margin_mean": 74.4962158203125, "margin_dpo/margin_std": 107.42152404785156, "step": 461 }, { "KL/chosen_KL_mean": -142.82247924804688, "KL/mean": -190.51254272460938, "KL/rejected_KL_mean": -238.20257568359375, "KL/std": 97.91545104980469, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004457816481590271, "fcm_dpo/delta": -0.026313908398151398, "fcm_dpo/margin": 95.38008880615234, "fcm_dpo/q_t": 0.40053996443748474, "grad_norm": 8.933396339416504, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.097012758255005, "logits/rejected": -1.5782849788665771, "logps/chosen": -237.02102661132812, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -323.834228515625, "loss": 1.0573, "margin_dpo/margin_mean": 95.38009643554688, "margin_dpo/margin_std": 104.89161682128906, "step": 462 }, { "KL/chosen_KL_mean": -138.82077026367188, "KL/mean": -196.11627197265625, "KL/rejected_KL_mean": -253.41177368164062, "KL/std": 105.18852233886719, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.004400912672281265, "fcm_dpo/delta": -0.10997207462787628, "fcm_dpo/margin": 114.59103393554688, "fcm_dpo/q_t": 0.38168489933013916, "grad_norm": 13.457008361816406, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.7130553722381592, "logits/rejected": -1.2750639915466309, "logps/chosen": -224.74551391601562, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -350.3136291503906, "loss": 1.0096, "margin_dpo/margin_mean": 114.59102630615234, "margin_dpo/margin_std": 113.39146423339844, "step": 463 }, { "KL/chosen_KL_mean": -171.52142333984375, "KL/mean": -218.31216430664062, "KL/rejected_KL_mean": -265.1029052734375, "KL/std": 104.57394409179688, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004334311466664076, "fcm_dpo/delta": -0.005971504375338554, "fcm_dpo/margin": 93.58149719238281, "fcm_dpo/q_t": 0.40574368834495544, "grad_norm": 12.651374816894531, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.681307077407837, "logits/rejected": -1.761479139328003, "logps/chosen": -251.2106170654297, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -372.3952331542969, "loss": 1.0837, "margin_dpo/margin_mean": 93.58149719238281, "margin_dpo/margin_std": 118.85334777832031, "step": 464 }, { "KL/chosen_KL_mean": -162.05316162109375, "KL/mean": -216.677734375, "KL/rejected_KL_mean": -271.30230712890625, "KL/std": 112.308837890625, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0042906817980110645, "fcm_dpo/delta": -0.07227225601673126, "fcm_dpo/margin": 109.24917602539062, "fcm_dpo/q_t": 0.38954222202301025, "grad_norm": 8.757061004638672, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.491240382194519, "logits/rejected": -1.1825611591339111, "logps/chosen": -253.9133758544922, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -390.0123291015625, "loss": 1.0207, "margin_dpo/margin_mean": 109.24917602539062, "margin_dpo/margin_std": 106.33027648925781, "step": 465 }, { "KL/chosen_KL_mean": -144.02633666992188, "KL/mean": -199.85086059570312, "KL/rejected_KL_mean": -255.67539978027344, "KL/std": 108.0897216796875, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0042280163615942, "fcm_dpo/delta": -0.07573378086090088, "fcm_dpo/margin": 111.6490478515625, "fcm_dpo/q_t": 0.38930946588516235, "grad_norm": 6.903109073638916, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.754224181175232, "logits/rejected": -1.4421706199645996, "logps/chosen": -228.72775268554688, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -351.72625732421875, "loss": 1.0229, "margin_dpo/margin_mean": 111.6490478515625, "margin_dpo/margin_std": 113.49149322509766, "step": 466 }, { "KL/chosen_KL_mean": -184.72225952148438, "KL/mean": -231.1077880859375, "KL/rejected_KL_mean": -277.49334716796875, "KL/std": 112.62197875976562, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.004203619435429573, "fcm_dpo/delta": 0.010059058666229248, "fcm_dpo/margin": 92.77107238769531, "fcm_dpo/q_t": 0.4093745946884155, "grad_norm": 11.449848175048828, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.2802906036376953, "logits/rejected": -1.695613145828247, "logps/chosen": -294.02056884765625, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -386.33697509765625, "loss": 1.1046, "margin_dpo/margin_mean": 92.77107238769531, "margin_dpo/margin_std": 128.1747283935547, "step": 467 }, { "KL/chosen_KL_mean": -144.11880493164062, "KL/mean": -187.19973754882812, "KL/rejected_KL_mean": -230.28067016601562, "KL/std": 110.61858367919922, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.004242611117660999, "fcm_dpo/delta": 0.03577013313770294, "fcm_dpo/margin": 86.161865234375, "fcm_dpo/q_t": 0.41642695665359497, "grad_norm": 9.1831693649292, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.8781862258911133, "logits/rejected": -1.6742807626724243, "logps/chosen": -242.44044494628906, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -336.96112060546875, "loss": 1.1216, "margin_dpo/margin_mean": 86.16184997558594, "margin_dpo/margin_std": 126.76406860351562, "step": 468 }, { "KL/chosen_KL_mean": -176.19862365722656, "KL/mean": -229.56784057617188, "KL/rejected_KL_mean": -282.93707275390625, "KL/std": 110.72152709960938, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.004201997071504593, "fcm_dpo/delta": -0.05153747648000717, "fcm_dpo/margin": 106.73843383789062, "fcm_dpo/q_t": 0.39544564485549927, "grad_norm": 9.682819366455078, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.2673251628875732, "logits/rejected": -1.846580982208252, "logps/chosen": -272.96282958984375, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -392.5320739746094, "loss": 1.0485, "margin_dpo/margin_mean": 106.73843383789062, "margin_dpo/margin_std": 119.74537658691406, "step": 469 }, { "KL/chosen_KL_mean": -162.69357299804688, "KL/mean": -211.05499267578125, "KL/rejected_KL_mean": -259.4164123535156, "KL/std": 102.70069885253906, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.004204724915325642, "fcm_dpo/delta": -0.006987440399825573, "fcm_dpo/margin": 96.72284698486328, "fcm_dpo/q_t": 0.4041072130203247, "grad_norm": 9.136495590209961, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.4848805665969849, "logits/rejected": -1.050535798072815, "logps/chosen": -246.19021606445312, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -347.9022216796875, "loss": 1.0646, "margin_dpo/margin_mean": 96.72285461425781, "margin_dpo/margin_std": 102.51174926757812, "step": 470 }, { "KL/chosen_KL_mean": -187.5118408203125, "KL/mean": -233.1875762939453, "KL/rejected_KL_mean": -278.86334228515625, "KL/std": 101.22312927246094, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004214408807456493, "fcm_dpo/delta": 0.015611214563250542, "fcm_dpo/margin": 91.3514404296875, "fcm_dpo/q_t": 0.40924835205078125, "grad_norm": 8.949760437011719, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.5139249563217163, "logits/rejected": -1.0920445919036865, "logps/chosen": -275.98614501953125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -369.34503173828125, "loss": 1.0821, "margin_dpo/margin_mean": 91.3514404296875, "margin_dpo/margin_std": 102.51415252685547, "step": 471 }, { "KL/chosen_KL_mean": -175.53160095214844, "KL/mean": -229.99415588378906, "KL/rejected_KL_mean": -284.4566955566406, "KL/std": 102.31436920166016, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.004172901157289743, "fcm_dpo/delta": -0.057385604828596115, "fcm_dpo/margin": 108.92509460449219, "fcm_dpo/q_t": 0.39371198415756226, "grad_norm": 7.639924049377441, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.0315184593200684, "logits/rejected": -1.3163987398147583, "logps/chosen": -300.76531982421875, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -405.51019287109375, "loss": 1.0309, "margin_dpo/margin_mean": 108.92509460449219, "margin_dpo/margin_std": 107.9698257446289, "step": 472 }, { "KL/chosen_KL_mean": -198.89633178710938, "KL/mean": -232.30702209472656, "KL/rejected_KL_mean": -265.7177429199219, "KL/std": 103.43218994140625, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.004250361118465662, "fcm_dpo/delta": 0.11921582370996475, "fcm_dpo/margin": 66.82140350341797, "fcm_dpo/q_t": 0.43505141139030457, "grad_norm": 9.713085174560547, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.6531858444213867, "logits/rejected": -1.1448001861572266, "logps/chosen": -294.5076904296875, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -353.868896484375, "loss": 1.1834, "margin_dpo/margin_mean": 66.82140350341797, "margin_dpo/margin_std": 119.812255859375, "step": 473 }, { "KL/chosen_KL_mean": -185.41751098632812, "KL/mean": -220.27420043945312, "KL/rejected_KL_mean": -255.13088989257812, "KL/std": 98.22425842285156, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.004362854175269604, "fcm_dpo/delta": 0.09729725122451782, "fcm_dpo/margin": 69.71336364746094, "fcm_dpo/q_t": 0.42978012561798096, "grad_norm": 14.79588508605957, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.4752180576324463, "logits/rejected": -1.2828636169433594, "logps/chosen": -266.89727783203125, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -351.5965270996094, "loss": 1.1765, "margin_dpo/margin_mean": 69.71337127685547, "margin_dpo/margin_std": 119.95606994628906, "step": 474 }, { "KL/chosen_KL_mean": -140.1126251220703, "KL/mean": -194.25946044921875, "KL/rejected_KL_mean": -248.40628051757812, "KL/std": 105.05824279785156, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.004311840049922466, "fcm_dpo/delta": -0.0701977089047432, "fcm_dpo/margin": 108.29365539550781, "fcm_dpo/q_t": 0.391788125038147, "grad_norm": 8.319985389709473, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.1498570442199707, "logits/rejected": -2.0944597721099854, "logps/chosen": -214.30860900878906, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -343.09869384765625, "loss": 1.0345, "margin_dpo/margin_mean": 108.29365539550781, "margin_dpo/margin_std": 119.30381774902344, "step": 475 }, { "KL/chosen_KL_mean": -161.61013793945312, "KL/mean": -210.32421875, "KL/rejected_KL_mean": -259.0382995605469, "KL/std": 107.36123657226562, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.004271647892892361, "fcm_dpo/delta": -0.017088403925299644, "fcm_dpo/margin": 97.42816162109375, "fcm_dpo/q_t": 0.4038185477256775, "grad_norm": 10.314826965332031, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.9973870515823364, "logits/rejected": -1.6167161464691162, "logps/chosen": -233.58123779296875, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -339.300537109375, "loss": 1.0714, "margin_dpo/margin_mean": 97.42816162109375, "margin_dpo/margin_std": 116.62994384765625, "step": 476 }, { "KL/chosen_KL_mean": -163.53321838378906, "KL/mean": -211.38165283203125, "KL/rejected_KL_mean": -259.2300720214844, "KL/std": 94.99380493164062, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004277168307453394, "fcm_dpo/delta": -0.009714744053781033, "fcm_dpo/margin": 95.69686126708984, "fcm_dpo/q_t": 0.4039818048477173, "grad_norm": 8.73729419708252, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -1.954722285270691, "logits/rejected": -0.9965189695358276, "logps/chosen": -268.53875732421875, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -341.1085205078125, "loss": 1.0784, "margin_dpo/margin_mean": 95.69686126708984, "margin_dpo/margin_std": 117.61109924316406, "step": 477 }, { "KL/chosen_KL_mean": -159.64190673828125, "KL/mean": -208.98333740234375, "KL/rejected_KL_mean": -258.32476806640625, "KL/std": 102.79658508300781, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.004257073160260916, "fcm_dpo/delta": -0.02098863199353218, "fcm_dpo/margin": 98.68284606933594, "fcm_dpo/q_t": 0.40163323283195496, "grad_norm": 8.659882545471191, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.5665512084960938, "logits/rejected": -1.3891569375991821, "logps/chosen": -236.43011474609375, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -348.76470947265625, "loss": 1.065, "margin_dpo/margin_mean": 98.68284606933594, "margin_dpo/margin_std": 113.59212493896484, "step": 478 }, { "KL/chosen_KL_mean": -161.76971435546875, "KL/mean": -211.23358154296875, "KL/rejected_KL_mean": -260.6974182128906, "KL/std": 103.62519836425781, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004255016800016165, "fcm_dpo/delta": -0.02204390987753868, "fcm_dpo/margin": 98.92770385742188, "fcm_dpo/q_t": 0.4006671905517578, "grad_norm": 12.021432876586914, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.5033169984817505, "logits/rejected": -1.3602721691131592, "logps/chosen": -243.1320343017578, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -361.78857421875, "loss": 1.0594, "margin_dpo/margin_mean": 98.92770385742188, "margin_dpo/margin_std": 106.42539978027344, "step": 479 }, { "KL/chosen_KL_mean": -188.68167114257812, "KL/mean": -227.40780639648438, "KL/rejected_KL_mean": -266.1339416503906, "KL/std": 98.58780670166016, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004281127825379372, "fcm_dpo/delta": 0.07074100524187088, "fcm_dpo/margin": 77.45226287841797, "fcm_dpo/q_t": 0.42254307866096497, "grad_norm": 10.398234367370605, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.5236461162567139, "logits/rejected": -0.8150526285171509, "logps/chosen": -285.45556640625, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -352.5386962890625, "loss": 1.134, "margin_dpo/margin_mean": 77.45226287841797, "margin_dpo/margin_std": 108.92544555664062, "step": 480 }, { "KL/chosen_KL_mean": -182.20396423339844, "KL/mean": -228.6483917236328, "KL/rejected_KL_mean": -275.09283447265625, "KL/std": 102.94124603271484, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004294467158615589, "fcm_dpo/delta": 0.0011350437998771667, "fcm_dpo/margin": 92.88883972167969, "fcm_dpo/q_t": 0.40663182735443115, "grad_norm": 12.176377296447754, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.8859056234359741, "logits/rejected": -1.4411334991455078, "logps/chosen": -273.8741455078125, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -373.7877197265625, "loss": 1.0838, "margin_dpo/margin_mean": 92.88883972167969, "margin_dpo/margin_std": 114.39848327636719, "step": 481 }, { "KL/chosen_KL_mean": -178.15420532226562, "KL/mean": -222.48150634765625, "KL/rejected_KL_mean": -266.8088073730469, "KL/std": 101.30116271972656, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0042976755648851395, "fcm_dpo/delta": 0.01965637505054474, "fcm_dpo/margin": 88.65458679199219, "fcm_dpo/q_t": 0.41027307510375977, "grad_norm": 12.173494338989258, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.6402223110198975, "logits/rejected": -1.2847137451171875, "logps/chosen": -276.67431640625, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -361.63824462890625, "loss": 1.0976, "margin_dpo/margin_mean": 88.65459442138672, "margin_dpo/margin_std": 112.55567932128906, "step": 482 }, { "KL/chosen_KL_mean": -182.713134765625, "KL/mean": -230.06314086914062, "KL/rejected_KL_mean": -277.41314697265625, "KL/std": 107.43142700195312, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0043305903673172, "fcm_dpo/delta": -0.011132415384054184, "fcm_dpo/margin": 94.70001983642578, "fcm_dpo/q_t": 0.40351366996765137, "grad_norm": 7.658377170562744, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.0464303493499756, "logits/rejected": -1.4039239883422852, "logps/chosen": -289.83172607421875, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -378.52813720703125, "loss": 1.0711, "margin_dpo/margin_mean": 94.70002746582031, "margin_dpo/margin_std": 106.70809936523438, "step": 483 }, { "KL/chosen_KL_mean": -168.58837890625, "KL/mean": -220.83389282226562, "KL/rejected_KL_mean": -273.07940673828125, "KL/std": 108.21234130859375, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004285591654479504, "fcm_dpo/delta": -0.0501004159450531, "fcm_dpo/margin": 104.49101257324219, "fcm_dpo/q_t": 0.39516744017601013, "grad_norm": 8.942955017089844, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.4415156841278076, "logits/rejected": -1.9646785259246826, "logps/chosen": -255.56829833984375, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -363.8030700683594, "loss": 1.0501, "margin_dpo/margin_mean": 104.49101257324219, "margin_dpo/margin_std": 118.79875946044922, "step": 484 }, { "KL/chosen_KL_mean": -158.73268127441406, "KL/mean": -216.30560302734375, "KL/rejected_KL_mean": -273.8785400390625, "KL/std": 113.1194076538086, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.004205911420285702, "fcm_dpo/delta": -0.08856441080570221, "fcm_dpo/margin": 115.1458969116211, "fcm_dpo/q_t": 0.38669541478157043, "grad_norm": 8.671127319335938, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.8215713500976562, "logits/rejected": -1.4023644924163818, "logps/chosen": -252.7914276123047, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -382.4415283203125, "loss": 1.0129, "margin_dpo/margin_mean": 115.1458969116211, "margin_dpo/margin_std": 113.7135009765625, "step": 485 }, { "KL/chosen_KL_mean": -182.59799194335938, "KL/mean": -232.90505981445312, "KL/rejected_KL_mean": -283.2121276855469, "KL/std": 109.02668762207031, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.004182810429483652, "fcm_dpo/delta": -0.021840248256921768, "fcm_dpo/margin": 100.61415100097656, "fcm_dpo/q_t": 0.4013470709323883, "grad_norm": 10.29386043548584, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.1073474884033203, "logits/rejected": -1.40671706199646, "logps/chosen": -276.3438720703125, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -381.28277587890625, "loss": 1.0664, "margin_dpo/margin_mean": 100.61414337158203, "margin_dpo/margin_std": 116.4154052734375, "step": 486 }, { "KL/chosen_KL_mean": -174.9798583984375, "KL/mean": -221.1657257080078, "KL/rejected_KL_mean": -267.3515930175781, "KL/std": 104.32818603515625, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0041753677651286125, "fcm_dpo/delta": 0.014891051687300205, "fcm_dpo/margin": 92.37171936035156, "fcm_dpo/q_t": 0.4094817638397217, "grad_norm": 9.363518714904785, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.0738561153411865, "logits/rejected": -1.75709867477417, "logps/chosen": -263.013427734375, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -365.82366943359375, "loss": 1.0876, "margin_dpo/margin_mean": 92.37171936035156, "margin_dpo/margin_std": 110.0530014038086, "step": 487 }, { "KL/chosen_KL_mean": -159.5416717529297, "KL/mean": -210.58035278320312, "KL/rejected_KL_mean": -261.6190185546875, "KL/std": 110.64993286132812, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.004169368650764227, "fcm_dpo/delta": -0.026739154011011124, "fcm_dpo/margin": 102.07734680175781, "fcm_dpo/q_t": 0.39899271726608276, "grad_norm": 9.236041069030762, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.407935619354248, "logits/rejected": -1.1366381645202637, "logps/chosen": -244.33132934570312, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -351.8924865722656, "loss": 1.0471, "margin_dpo/margin_mean": 102.07734680175781, "margin_dpo/margin_std": 98.0372314453125, "step": 488 }, { "KL/chosen_KL_mean": -180.39723205566406, "KL/mean": -226.07943725585938, "KL/rejected_KL_mean": -271.7616271972656, "KL/std": 101.01782989501953, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.004172194749116898, "fcm_dpo/delta": 0.01953832432627678, "fcm_dpo/margin": 91.36439514160156, "fcm_dpo/q_t": 0.4109130799770355, "grad_norm": 8.77079963684082, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6397719383239746, "logits/rejected": -1.1476274728775024, "logps/chosen": -270.86651611328125, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -358.15924072265625, "loss": 1.0929, "margin_dpo/margin_mean": 91.36439514160156, "margin_dpo/margin_std": 112.29541015625, "step": 489 }, { "KL/chosen_KL_mean": -180.53622436523438, "KL/mean": -223.69851684570312, "KL/rejected_KL_mean": -266.86083984375, "KL/std": 108.9730224609375, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004224471747875214, "fcm_dpo/delta": 0.03550705313682556, "fcm_dpo/margin": 86.32461547851562, "fcm_dpo/q_t": 0.41705867648124695, "grad_norm": 8.552720069885254, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.0475924015045166, "logits/rejected": -2.127312183380127, "logps/chosen": -265.8563537597656, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -382.85467529296875, "loss": 1.1448, "margin_dpo/margin_mean": 86.32462310791016, "margin_dpo/margin_std": 143.2669677734375, "step": 490 }, { "KL/chosen_KL_mean": -182.50453186035156, "KL/mean": -223.37008666992188, "KL/rejected_KL_mean": -264.235595703125, "KL/std": 111.55133056640625, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004237424116581678, "fcm_dpo/delta": 0.05561234429478645, "fcm_dpo/margin": 81.73106384277344, "fcm_dpo/q_t": 0.41889604926109314, "grad_norm": 9.530388832092285, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8479841947555542, "logits/rejected": -1.35367751121521, "logps/chosen": -268.6480712890625, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -344.9150695800781, "loss": 1.1243, "margin_dpo/margin_mean": 81.73106384277344, "margin_dpo/margin_std": 112.85596466064453, "step": 491 }, { "KL/chosen_KL_mean": -172.70272827148438, "KL/mean": -204.25213623046875, "KL/rejected_KL_mean": -235.801513671875, "KL/std": 109.11627197265625, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0043297139927744865, "fcm_dpo/delta": 0.13048246502876282, "fcm_dpo/margin": 63.09876251220703, "fcm_dpo/q_t": 0.4376440644264221, "grad_norm": 14.199357986450195, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.9896970987319946, "logits/rejected": -1.596346378326416, "logps/chosen": -253.81031799316406, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -316.5534973144531, "loss": 1.2137, "margin_dpo/margin_mean": 63.098758697509766, "margin_dpo/margin_std": 135.66433715820312, "step": 492 }, { "KL/chosen_KL_mean": -169.57354736328125, "KL/mean": -212.96392822265625, "KL/rejected_KL_mean": -256.35430908203125, "KL/std": 99.1092529296875, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004358625039458275, "fcm_dpo/delta": 0.022315405309200287, "fcm_dpo/margin": 86.78076171875, "fcm_dpo/q_t": 0.41095107793807983, "grad_norm": 14.43386459350586, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.4859514236450195, "logits/rejected": -1.759881854057312, "logps/chosen": -281.7808837890625, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -349.96148681640625, "loss": 1.0954, "margin_dpo/margin_mean": 86.78076171875, "margin_dpo/margin_std": 104.81170654296875, "step": 493 }, { "KL/chosen_KL_mean": -165.4573516845703, "KL/mean": -222.26031494140625, "KL/rejected_KL_mean": -279.0632629394531, "KL/std": 109.7476806640625, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004298219457268715, "fcm_dpo/delta": -0.09397023171186447, "fcm_dpo/margin": 113.60591125488281, "fcm_dpo/q_t": 0.38596662878990173, "grad_norm": 9.855114936828613, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.443009376525879, "logits/rejected": -1.3638818264007568, "logps/chosen": -238.57225036621094, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -371.22625732421875, "loss": 1.0196, "margin_dpo/margin_mean": 113.60591125488281, "margin_dpo/margin_std": 118.70089721679688, "step": 494 }, { "KL/chosen_KL_mean": -169.92913818359375, "KL/mean": -218.70953369140625, "KL/rejected_KL_mean": -267.4899597167969, "KL/std": 103.83397674560547, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.004286076873540878, "fcm_dpo/delta": -0.018967675045132637, "fcm_dpo/margin": 97.56082916259766, "fcm_dpo/q_t": 0.4030217230319977, "grad_norm": 10.89179801940918, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.322678565979004, "logits/rejected": -1.8634378910064697, "logps/chosen": -269.449462890625, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -365.42083740234375, "loss": 1.0774, "margin_dpo/margin_mean": 97.56083679199219, "margin_dpo/margin_std": 124.4035415649414, "step": 495 }, { "KL/chosen_KL_mean": -189.00218200683594, "KL/mean": -230.71426391601562, "KL/rejected_KL_mean": -272.4263610839844, "KL/std": 104.65789794921875, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004322127439081669, "fcm_dpo/delta": 0.040660686790943146, "fcm_dpo/margin": 83.42417907714844, "fcm_dpo/q_t": 0.41494500637054443, "grad_norm": 8.902202606201172, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.2882652282714844, "logits/rejected": -1.6567872762680054, "logps/chosen": -296.85894775390625, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -365.1968994140625, "loss": 1.1092, "margin_dpo/margin_mean": 83.42418670654297, "margin_dpo/margin_std": 105.33882141113281, "step": 496 }, { "KL/chosen_KL_mean": -202.7794189453125, "KL/mean": -229.59872436523438, "KL/rejected_KL_mean": -256.41796875, "KL/std": 105.1229248046875, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004420091398060322, "fcm_dpo/delta": 0.16712862253189087, "fcm_dpo/margin": 53.638553619384766, "fcm_dpo/q_t": 0.4455042779445648, "grad_norm": 8.972331047058105, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.8063628673553467, "logits/rejected": -1.3605537414550781, "logps/chosen": -294.8626708984375, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -338.2130126953125, "loss": 1.2238, "margin_dpo/margin_mean": 53.6385498046875, "margin_dpo/margin_std": 111.85783386230469, "step": 497 }, { "KL/chosen_KL_mean": -171.8790283203125, "KL/mean": -216.97442626953125, "KL/rejected_KL_mean": -262.06982421875, "KL/std": 103.26524353027344, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.00446145236492157, "fcm_dpo/delta": -0.0024965116754174232, "fcm_dpo/margin": 90.1907730102539, "fcm_dpo/q_t": 0.40573376417160034, "grad_norm": 11.530007362365723, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.6761226654052734, "logits/rejected": -1.1653426885604858, "logps/chosen": -270.0733947753906, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -352.75726318359375, "loss": 1.0835, "margin_dpo/margin_mean": 90.19078063964844, "margin_dpo/margin_std": 112.57320404052734, "step": 498 }, { "KL/chosen_KL_mean": -178.70278930664062, "KL/mean": -226.40213012695312, "KL/rejected_KL_mean": -274.10150146484375, "KL/std": 112.76808166503906, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0044516269117593765, "fcm_dpo/delta": -0.025768263265490532, "fcm_dpo/margin": 95.39872741699219, "fcm_dpo/q_t": 0.4016485810279846, "grad_norm": 10.140097618103027, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.9434070587158203, "logits/rejected": -1.8906807899475098, "logps/chosen": -259.74267578125, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -381.023193359375, "loss": 1.0724, "margin_dpo/margin_mean": 95.39872741699219, "margin_dpo/margin_std": 119.58555603027344, "step": 499 }, { "KL/chosen_KL_mean": -200.19589233398438, "KL/mean": -240.61325073242188, "KL/rejected_KL_mean": -281.0306396484375, "KL/std": 107.68091583251953, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004462596029043198, "fcm_dpo/delta": 0.04075472801923752, "fcm_dpo/margin": 80.83474731445312, "fcm_dpo/q_t": 0.4177528917789459, "grad_norm": 11.536323547363281, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.712573766708374, "logits/rejected": -1.541154384613037, "logps/chosen": -289.4439392089844, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -381.44085693359375, "loss": 1.126, "margin_dpo/margin_mean": 80.83474731445312, "margin_dpo/margin_std": 120.57972717285156, "step": 500 }, { "KL/chosen_KL_mean": -173.2012939453125, "KL/mean": -218.013671875, "KL/rejected_KL_mean": -262.8260498046875, "KL/std": 117.09432220458984, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004507631529122591, "fcm_dpo/delta": -0.0052390992641448975, "fcm_dpo/margin": 89.624755859375, "fcm_dpo/q_t": 0.40776869654655457, "grad_norm": 9.372475624084473, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.5126514434814453, "logits/rejected": -1.067846417427063, "logps/chosen": -263.7982177734375, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -348.123291015625, "loss": 1.1099, "margin_dpo/margin_mean": 89.624755859375, "margin_dpo/margin_std": 131.2595977783203, "step": 501 }, { "KL/chosen_KL_mean": -175.78323364257812, "KL/mean": -215.92579650878906, "KL/rejected_KL_mean": -256.068359375, "KL/std": 97.46456146240234, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.004516517743468285, "fcm_dpo/delta": 0.038330912590026855, "fcm_dpo/margin": 80.28514099121094, "fcm_dpo/q_t": 0.41557759046554565, "grad_norm": 10.09169864654541, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9631787538528442, "logits/rejected": -1.2309739589691162, "logps/chosen": -279.25396728515625, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -345.112548828125, "loss": 1.1284, "margin_dpo/margin_mean": 80.28514862060547, "margin_dpo/margin_std": 120.4079818725586, "step": 502 }, { "KL/chosen_KL_mean": -173.8798828125, "KL/mean": -219.30441284179688, "KL/rejected_KL_mean": -264.7289733886719, "KL/std": 104.55940246582031, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004519544541835785, "fcm_dpo/delta": -0.011385608464479446, "fcm_dpo/margin": 90.84909057617188, "fcm_dpo/q_t": 0.40464019775390625, "grad_norm": 9.46459674835205, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.6617162227630615, "logits/rejected": -1.1523778438568115, "logps/chosen": -267.60986328125, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -352.33795166015625, "loss": 1.0837, "margin_dpo/margin_mean": 90.84909057617188, "margin_dpo/margin_std": 115.51527404785156, "step": 503 }, { "KL/chosen_KL_mean": -176.68438720703125, "KL/mean": -235.08334350585938, "KL/rejected_KL_mean": -293.4822998046875, "KL/std": 108.36630249023438, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.004421335645020008, "fcm_dpo/delta": -0.12262585759162903, "fcm_dpo/margin": 116.79791259765625, "fcm_dpo/q_t": 0.3785170912742615, "grad_norm": 9.867627143859863, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.136824131011963, "logits/rejected": -1.802694320678711, "logps/chosen": -268.97625732421875, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -402.51995849609375, "loss": 0.9899, "margin_dpo/margin_mean": 116.79791259765625, "margin_dpo/margin_std": 109.12785339355469, "step": 504 }, { "KL/chosen_KL_mean": -193.335205078125, "KL/mean": -235.89027404785156, "KL/rejected_KL_mean": -278.4453125, "KL/std": 107.72501373291016, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.004403563216328621, "fcm_dpo/delta": 0.02620157226920128, "fcm_dpo/margin": 85.11011505126953, "fcm_dpo/q_t": 0.4122451841831207, "grad_norm": 9.247615814208984, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.2966201305389404, "logits/rejected": -1.7281837463378906, "logps/chosen": -299.6121520996094, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -372.65142822265625, "loss": 1.0978, "margin_dpo/margin_mean": 85.11011505126953, "margin_dpo/margin_std": 105.02202606201172, "step": 505 }, { "KL/chosen_KL_mean": -187.6168212890625, "KL/mean": -235.25750732421875, "KL/rejected_KL_mean": -282.8982238769531, "KL/std": 101.24746704101562, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.00438356027007103, "fcm_dpo/delta": -0.018764697015285492, "fcm_dpo/margin": 95.28142547607422, "fcm_dpo/q_t": 0.40083739161491394, "grad_norm": 9.065443992614746, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9555237293243408, "logits/rejected": -1.5921567678451538, "logps/chosen": -269.05767822265625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -368.5626220703125, "loss": 1.0637, "margin_dpo/margin_mean": 95.28142547607422, "margin_dpo/margin_std": 105.31832885742188, "step": 506 }, { "KL/chosen_KL_mean": -196.50265502929688, "KL/mean": -243.9690704345703, "KL/rejected_KL_mean": -291.4355163574219, "KL/std": 105.69239807128906, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004384808242321014, "fcm_dpo/delta": -0.016965841874480247, "fcm_dpo/margin": 94.93284606933594, "fcm_dpo/q_t": 0.4039403796195984, "grad_norm": 9.301901817321777, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.2580103874206543, "logits/rejected": -1.821134090423584, "logps/chosen": -295.92962646484375, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -393.6884765625, "loss": 1.0845, "margin_dpo/margin_mean": 94.93284606933594, "margin_dpo/margin_std": 124.48847961425781, "step": 507 }, { "KL/chosen_KL_mean": -190.44009399414062, "KL/mean": -230.9735870361328, "KL/rejected_KL_mean": -271.507080078125, "KL/std": 114.4671630859375, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0044097150675952435, "fcm_dpo/delta": 0.044084805995225906, "fcm_dpo/margin": 81.06700897216797, "fcm_dpo/q_t": 0.41875186562538147, "grad_norm": 15.364039421081543, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.0249698162078857, "logits/rejected": -1.5393774509429932, "logps/chosen": -288.3557434082031, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -375.3049011230469, "loss": 1.1442, "margin_dpo/margin_mean": 81.06700134277344, "margin_dpo/margin_std": 132.73289489746094, "step": 508 }, { "KL/chosen_KL_mean": -176.5628662109375, "KL/mean": -214.0605926513672, "KL/rejected_KL_mean": -251.55831909179688, "KL/std": 90.35609436035156, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0044694021344184875, "fcm_dpo/delta": 0.06692862510681152, "fcm_dpo/margin": 74.99545288085938, "fcm_dpo/q_t": 0.4209263324737549, "grad_norm": 16.093881607055664, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9652047157287598, "logits/rejected": -1.2944450378417969, "logps/chosen": -277.292724609375, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -338.5016784667969, "loss": 1.149, "margin_dpo/margin_mean": 74.99545288085938, "margin_dpo/margin_std": 120.26268768310547, "step": 509 }, { "KL/chosen_KL_mean": -208.69131469726562, "KL/mean": -253.7987518310547, "KL/rejected_KL_mean": -298.90618896484375, "KL/std": 107.82176208496094, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004482119344174862, "fcm_dpo/delta": -0.0045778825879096985, "fcm_dpo/margin": 90.21487426757812, "fcm_dpo/q_t": 0.4054366946220398, "grad_norm": 12.207052230834961, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.6524683237075806, "logits/rejected": -1.4665682315826416, "logps/chosen": -297.8607177734375, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -406.8255615234375, "loss": 1.0802, "margin_dpo/margin_mean": 90.21487426757812, "margin_dpo/margin_std": 108.30242156982422, "step": 510 }, { "KL/chosen_KL_mean": -191.255126953125, "KL/mean": -240.3212127685547, "KL/rejected_KL_mean": -289.3872985839844, "KL/std": 105.80136108398438, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004457438830286264, "fcm_dpo/delta": -0.03913607820868492, "fcm_dpo/margin": 98.13220977783203, "fcm_dpo/q_t": 0.3989560008049011, "grad_norm": 11.157567024230957, "learning_rate": 9.028829858700973e-08, "logits/chosen": -1.8863470554351807, "logits/rejected": -1.2959043979644775, "logps/chosen": -281.0049133300781, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -387.11859130859375, "loss": 1.0605, "margin_dpo/margin_mean": 98.13221740722656, "margin_dpo/margin_std": 117.58805847167969, "step": 511 }, { "KL/chosen_KL_mean": -181.25848388671875, "KL/mean": -228.00717163085938, "KL/rejected_KL_mean": -274.755859375, "KL/std": 103.86884307861328, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004451374057680368, "fcm_dpo/delta": -0.017371851950883865, "fcm_dpo/margin": 93.49739074707031, "fcm_dpo/q_t": 0.40322166681289673, "grad_norm": 14.337342262268066, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.7796235084533691, "logits/rejected": -1.3310644626617432, "logps/chosen": -266.9936218261719, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -362.3807373046875, "loss": 1.0865, "margin_dpo/margin_mean": 93.49739074707031, "margin_dpo/margin_std": 122.97364807128906, "step": 512 }, { "KL/chosen_KL_mean": -192.3274688720703, "KL/mean": -232.06869506835938, "KL/rejected_KL_mean": -271.8099365234375, "KL/std": 99.91929626464844, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004437715746462345, "fcm_dpo/delta": 0.04885413870215416, "fcm_dpo/margin": 79.4824447631836, "fcm_dpo/q_t": 0.416756272315979, "grad_norm": 11.109336853027344, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.397455930709839, "logits/rejected": -1.7112340927124023, "logps/chosen": -288.5179748535156, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -354.7276611328125, "loss": 1.1271, "margin_dpo/margin_mean": 79.48245239257812, "margin_dpo/margin_std": 112.80994415283203, "step": 513 }, { "KL/chosen_KL_mean": -187.5226287841797, "KL/mean": -229.59466552734375, "KL/rejected_KL_mean": -271.66668701171875, "KL/std": 99.78309631347656, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004484020639210939, "fcm_dpo/delta": 0.02358059585094452, "fcm_dpo/margin": 84.1440658569336, "fcm_dpo/q_t": 0.41340702772140503, "grad_norm": 11.585857391357422, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.7057015895843506, "logits/rejected": -1.129355549812317, "logps/chosen": -285.34625244140625, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -357.0365295410156, "loss": 1.1105, "margin_dpo/margin_mean": 84.14407348632812, "margin_dpo/margin_std": 118.48342895507812, "step": 514 }, { "KL/chosen_KL_mean": -179.34185791015625, "KL/mean": -221.0348663330078, "KL/rejected_KL_mean": -262.7278747558594, "KL/std": 103.38040161132812, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.004505247808992863, "fcm_dpo/delta": 0.02527589723467827, "fcm_dpo/margin": 83.38601684570312, "fcm_dpo/q_t": 0.4125024974346161, "grad_norm": 12.045883178710938, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.8969168663024902, "logits/rejected": -1.669182300567627, "logps/chosen": -261.2754211425781, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -356.304443359375, "loss": 1.1069, "margin_dpo/margin_mean": 83.38601684570312, "margin_dpo/margin_std": 111.5902328491211, "step": 515 }, { "KL/chosen_KL_mean": -183.20175170898438, "KL/mean": -223.60276794433594, "KL/rejected_KL_mean": -264.0037841796875, "KL/std": 118.85980987548828, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.004540680907666683, "fcm_dpo/delta": 0.03427726775407791, "fcm_dpo/margin": 80.8020248413086, "fcm_dpo/q_t": 0.41744059324264526, "grad_norm": 12.238958358764648, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.2081520557403564, "logits/rejected": -2.0285532474517822, "logps/chosen": -280.2934265136719, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -382.5604553222656, "loss": 1.148, "margin_dpo/margin_mean": 80.8020248413086, "margin_dpo/margin_std": 140.58233642578125, "step": 516 }, { "KL/chosen_KL_mean": -195.7447509765625, "KL/mean": -245.92376708984375, "KL/rejected_KL_mean": -296.1028137207031, "KL/std": 107.224365234375, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004477737471461296, "fcm_dpo/delta": -0.052891723811626434, "fcm_dpo/margin": 100.35807800292969, "fcm_dpo/q_t": 0.3962159752845764, "grad_norm": 9.364889144897461, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.822260856628418, "logits/rejected": -1.3895546197891235, "logps/chosen": -288.9313049316406, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -391.69561767578125, "loss": 1.0601, "margin_dpo/margin_mean": 100.35807800292969, "margin_dpo/margin_std": 121.15934753417969, "step": 517 }, { "KL/chosen_KL_mean": -203.60220336914062, "KL/mean": -240.69248962402344, "KL/rejected_KL_mean": -277.78277587890625, "KL/std": 101.64203643798828, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004536496475338936, "fcm_dpo/delta": 0.06571877002716064, "fcm_dpo/margin": 74.1805648803711, "fcm_dpo/q_t": 0.42324918508529663, "grad_norm": 10.165836334228516, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.8491461277008057, "logits/rejected": -1.514328122138977, "logps/chosen": -290.6235656738281, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -369.02734375, "loss": 1.1536, "margin_dpo/margin_mean": 74.1805648803711, "margin_dpo/margin_std": 125.096435546875, "step": 518 }, { "KL/chosen_KL_mean": -189.370849609375, "KL/mean": -230.2154541015625, "KL/rejected_KL_mean": -271.06005859375, "KL/std": 94.64813232421875, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.004563508555293083, "fcm_dpo/delta": 0.02823273092508316, "fcm_dpo/margin": 81.68917846679688, "fcm_dpo/q_t": 0.4131203293800354, "grad_norm": 10.305850982666016, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.8172502517700195, "logits/rejected": -1.433316707611084, "logps/chosen": -290.05841064453125, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -380.0904541015625, "loss": 1.0973, "margin_dpo/margin_mean": 81.68917083740234, "margin_dpo/margin_std": 99.68930053710938, "step": 519 }, { "KL/chosen_KL_mean": -175.08575439453125, "KL/mean": -223.44949340820312, "KL/rejected_KL_mean": -271.8132019042969, "KL/std": 106.18504333496094, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004545171745121479, "fcm_dpo/delta": -0.041563037782907486, "fcm_dpo/margin": 96.7274398803711, "fcm_dpo/q_t": 0.39662373065948486, "grad_norm": 9.463604927062988, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.5657912492752075, "logits/rejected": -1.1228694915771484, "logps/chosen": -273.8634033203125, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -378.7589111328125, "loss": 1.0474, "margin_dpo/margin_mean": 96.7274398803711, "margin_dpo/margin_std": 104.15180969238281, "step": 520 }, { "KL/chosen_KL_mean": -177.5872802734375, "KL/mean": -226.48211669921875, "KL/rejected_KL_mean": -275.3769836425781, "KL/std": 99.78184509277344, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004531391896307468, "fcm_dpo/delta": -0.045322783291339874, "fcm_dpo/margin": 97.78966522216797, "fcm_dpo/q_t": 0.3961653709411621, "grad_norm": 11.437718391418457, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.913017749786377, "logits/rejected": -1.57076096534729, "logps/chosen": -263.9383544921875, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -365.7701416015625, "loss": 1.0431, "margin_dpo/margin_mean": 97.78966522216797, "margin_dpo/margin_std": 101.42819213867188, "step": 521 }, { "KL/chosen_KL_mean": -184.2644500732422, "KL/mean": -228.94483947753906, "KL/rejected_KL_mean": -273.625244140625, "KL/std": 109.41970825195312, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0045060026459395885, "fcm_dpo/delta": -0.002817177213728428, "fcm_dpo/margin": 89.36080169677734, "fcm_dpo/q_t": 0.4064469337463379, "grad_norm": 10.242158889770508, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.628753900527954, "logits/rejected": -1.2537212371826172, "logps/chosen": -260.2724914550781, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -348.0664978027344, "loss": 1.0875, "margin_dpo/margin_mean": 89.36080169677734, "margin_dpo/margin_std": 115.18787384033203, "step": 522 }, { "KL/chosen_KL_mean": -205.2162628173828, "KL/mean": -236.92050170898438, "KL/rejected_KL_mean": -268.624755859375, "KL/std": 106.57084655761719, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.004568861797451973, "fcm_dpo/delta": 0.11370746791362762, "fcm_dpo/margin": 63.408470153808594, "fcm_dpo/q_t": 0.4339453876018524, "grad_norm": 12.369627952575684, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.0099997520446777, "logits/rejected": -1.3055813312530518, "logps/chosen": -301.7125244140625, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -359.6400146484375, "loss": 1.1953, "margin_dpo/margin_mean": 63.408477783203125, "margin_dpo/margin_std": 125.25432586669922, "step": 523 }, { "KL/chosen_KL_mean": -187.12193298339844, "KL/mean": -233.27914428710938, "KL/rejected_KL_mean": -279.43634033203125, "KL/std": 106.93254089355469, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.004582815803587437, "fcm_dpo/delta": -0.024097716435790062, "fcm_dpo/margin": 92.3144302368164, "fcm_dpo/q_t": 0.40254366397857666, "grad_norm": 12.780837059020996, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.181407928466797, "logits/rejected": -1.486419916152954, "logps/chosen": -290.5274353027344, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -367.34649658203125, "loss": 1.0712, "margin_dpo/margin_mean": 92.31442260742188, "margin_dpo/margin_std": 114.20622253417969, "step": 524 }, { "KL/chosen_KL_mean": -175.25985717773438, "KL/mean": -227.3384246826172, "KL/rejected_KL_mean": -279.4169921875, "KL/std": 107.57228088378906, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0044871047139167786, "fcm_dpo/delta": -0.07322761416435242, "fcm_dpo/margin": 104.15715026855469, "fcm_dpo/q_t": 0.39399170875549316, "grad_norm": 10.430622100830078, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.4496898651123047, "logits/rejected": -1.3611853122711182, "logps/chosen": -259.4663391113281, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -385.6785888671875, "loss": 1.0523, "margin_dpo/margin_mean": 104.15715026855469, "margin_dpo/margin_std": 124.57701873779297, "step": 525 }, { "KL/chosen_KL_mean": -181.1079864501953, "KL/mean": -228.55181884765625, "KL/rejected_KL_mean": -275.9956359863281, "KL/std": 101.96849822998047, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0044958521611988544, "fcm_dpo/delta": -0.02777956798672676, "fcm_dpo/margin": 94.88765716552734, "fcm_dpo/q_t": 0.40113669633865356, "grad_norm": 8.551830291748047, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7754673957824707, "logits/rejected": -1.121049404144287, "logps/chosen": -283.96209716796875, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -369.6470947265625, "loss": 1.0726, "margin_dpo/margin_mean": 94.88765716552734, "margin_dpo/margin_std": 119.47691345214844, "step": 526 }, { "KL/chosen_KL_mean": -183.05426025390625, "KL/mean": -231.21240234375, "KL/rejected_KL_mean": -279.37054443359375, "KL/std": 106.92764282226562, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004467691294848919, "fcm_dpo/delta": -0.03166987746953964, "fcm_dpo/margin": 96.3163070678711, "fcm_dpo/q_t": 0.3980027735233307, "grad_norm": 10.16421890258789, "learning_rate": 7.504749238082414e-08, "logits/chosen": -2.956820249557495, "logits/rejected": -2.110201835632324, "logps/chosen": -279.8197021484375, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -364.7657470703125, "loss": 1.0426, "margin_dpo/margin_mean": 96.3163070678711, "margin_dpo/margin_std": 93.04641723632812, "step": 527 }, { "KL/chosen_KL_mean": -171.95046997070312, "KL/mean": -219.2411651611328, "KL/rejected_KL_mean": -266.5318603515625, "KL/std": 108.68387603759766, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004452135413885117, "fcm_dpo/delta": -0.02203201875090599, "fcm_dpo/margin": 94.58135986328125, "fcm_dpo/q_t": 0.4025321304798126, "grad_norm": 9.160587310791016, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.223809242248535, "logits/rejected": -1.7165277004241943, "logps/chosen": -266.1153869628906, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -366.9041748046875, "loss": 1.079, "margin_dpo/margin_mean": 94.58135986328125, "margin_dpo/margin_std": 122.56964111328125, "step": 528 }, { "KL/chosen_KL_mean": -198.1875, "KL/mean": -241.72164916992188, "KL/rejected_KL_mean": -285.25579833984375, "KL/std": 106.8153305053711, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.00445692939683795, "fcm_dpo/delta": 0.01232403889298439, "fcm_dpo/margin": 87.06829833984375, "fcm_dpo/q_t": 0.41113021969795227, "grad_norm": 11.087881088256836, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.6715788841247559, "logits/rejected": -1.4915525913238525, "logps/chosen": -278.96429443359375, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -382.44696044921875, "loss": 1.1088, "margin_dpo/margin_mean": 87.06829833984375, "margin_dpo/margin_std": 125.41436767578125, "step": 529 }, { "KL/chosen_KL_mean": -182.67910766601562, "KL/mean": -226.37266540527344, "KL/rejected_KL_mean": -270.06622314453125, "KL/std": 104.20526123046875, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.004445292986929417, "fcm_dpo/delta": 0.011848794296383858, "fcm_dpo/margin": 87.38712310791016, "fcm_dpo/q_t": 0.4094581604003906, "grad_norm": 9.684823036193848, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.4816317558288574, "logits/rejected": -1.1531944274902344, "logps/chosen": -262.454833984375, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -362.1201171875, "loss": 1.0828, "margin_dpo/margin_mean": 87.38712310791016, "margin_dpo/margin_std": 100.83465576171875, "step": 530 }, { "KL/chosen_KL_mean": -176.0400390625, "KL/mean": -229.50112915039062, "KL/rejected_KL_mean": -282.9621887207031, "KL/std": 108.418701171875, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004407747648656368, "fcm_dpo/delta": -0.07490825653076172, "fcm_dpo/margin": 106.92216491699219, "fcm_dpo/q_t": 0.3887737989425659, "grad_norm": 10.738329887390137, "learning_rate": 7.141774982445147e-08, "logits/chosen": -2.0028958320617676, "logits/rejected": -1.4151712656021118, "logps/chosen": -255.35923767089844, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -361.0705261230469, "loss": 1.0233, "margin_dpo/margin_mean": 106.92218017578125, "margin_dpo/margin_std": 108.35570526123047, "step": 531 }, { "KL/chosen_KL_mean": -189.6221923828125, "KL/mean": -247.69671630859375, "KL/rejected_KL_mean": -305.771240234375, "KL/std": 104.10749053955078, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.00433534849435091, "fcm_dpo/delta": -0.10889368504285812, "fcm_dpo/margin": 116.14905548095703, "fcm_dpo/q_t": 0.38238006830215454, "grad_norm": 10.81988525390625, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.8841519355773926, "logits/rejected": -1.2753493785858154, "logps/chosen": -291.25909423828125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -399.11663818359375, "loss": 1.0051, "margin_dpo/margin_mean": 116.14905548095703, "margin_dpo/margin_std": 115.18002319335938, "step": 532 }, { "KL/chosen_KL_mean": -183.7340850830078, "KL/mean": -217.82095336914062, "KL/rejected_KL_mean": -251.90780639648438, "KL/std": 110.21793365478516, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0043735564686357975, "fcm_dpo/delta": 0.10481753945350647, "fcm_dpo/margin": 68.17376708984375, "fcm_dpo/q_t": 0.4322153329849243, "grad_norm": 10.750551223754883, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.420991897583008, "logits/rejected": -1.820906639099121, "logps/chosen": -274.480712890625, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -334.79632568359375, "loss": 1.1732, "margin_dpo/margin_mean": 68.17376708984375, "margin_dpo/margin_std": 118.08873748779297, "step": 533 }, { "KL/chosen_KL_mean": -165.21194458007812, "KL/mean": -214.8966827392578, "KL/rejected_KL_mean": -264.5814208984375, "KL/std": 103.86346435546875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004373771138489246, "fcm_dpo/delta": -0.03617957979440689, "fcm_dpo/margin": 99.36946105957031, "fcm_dpo/q_t": 0.397424578666687, "grad_norm": 9.37254524230957, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0398013591766357, "logits/rejected": -1.6476473808288574, "logps/chosen": -251.84278869628906, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -349.453369140625, "loss": 1.0457, "margin_dpo/margin_mean": 99.36946105957031, "margin_dpo/margin_std": 101.8693618774414, "step": 534 }, { "KL/chosen_KL_mean": -166.20179748535156, "KL/mean": -219.27928161621094, "KL/rejected_KL_mean": -272.35675048828125, "KL/std": 104.82968139648438, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.004336371086537838, "fcm_dpo/delta": -0.06330778449773788, "fcm_dpo/margin": 106.15494537353516, "fcm_dpo/q_t": 0.39332616329193115, "grad_norm": 8.537967681884766, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.7115788459777832, "logits/rejected": -1.522179126739502, "logps/chosen": -256.54718017578125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -373.13470458984375, "loss": 1.0422, "margin_dpo/margin_mean": 106.15495300292969, "margin_dpo/margin_std": 119.60867309570312, "step": 535 }, { "KL/chosen_KL_mean": -165.18682861328125, "KL/mean": -217.56430053710938, "KL/rejected_KL_mean": -269.9417724609375, "KL/std": 105.67327880859375, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.004264752380549908, "fcm_dpo/delta": -0.049136798828840256, "fcm_dpo/margin": 104.75494384765625, "fcm_dpo/q_t": 0.39479589462280273, "grad_norm": 8.633084297180176, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.999756097793579, "logits/rejected": -1.7516398429870605, "logps/chosen": -244.1464080810547, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -363.0295715332031, "loss": 1.0416, "margin_dpo/margin_mean": 104.75494384765625, "margin_dpo/margin_std": 110.50715637207031, "step": 536 }, { "KL/chosen_KL_mean": -144.1810302734375, "KL/mean": -199.71536254882812, "KL/rejected_KL_mean": -255.24969482421875, "KL/std": 108.87037658691406, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.004240803420543671, "fcm_dpo/delta": -0.07479295879602432, "fcm_dpo/margin": 111.06864166259766, "fcm_dpo/q_t": 0.38911178708076477, "grad_norm": 9.882691383361816, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.486198902130127, "logits/rejected": -0.7812190055847168, "logps/chosen": -230.8904266357422, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -334.1573486328125, "loss": 1.0218, "margin_dpo/margin_mean": 111.06864929199219, "margin_dpo/margin_std": 106.49443817138672, "step": 537 }, { "KL/chosen_KL_mean": -196.70611572265625, "KL/mean": -241.97186279296875, "KL/rejected_KL_mean": -287.23760986328125, "KL/std": 97.18284606933594, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.004212346859276295, "fcm_dpo/delta": 0.019387295469641685, "fcm_dpo/margin": 90.53150939941406, "fcm_dpo/q_t": 0.4101918637752533, "grad_norm": 9.117573738098145, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.8425400257110596, "logits/rejected": -1.3030366897583008, "logps/chosen": -295.14825439453125, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -378.332275390625, "loss": 1.0864, "margin_dpo/margin_mean": 90.53150939941406, "margin_dpo/margin_std": 103.84486389160156, "step": 538 }, { "KL/chosen_KL_mean": -170.6702117919922, "KL/mean": -218.27163696289062, "KL/rejected_KL_mean": -265.87310791015625, "KL/std": 106.44011688232422, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.004216569475829601, "fcm_dpo/delta": -0.0014912369661033154, "fcm_dpo/margin": 95.20291137695312, "fcm_dpo/q_t": 0.40625715255737305, "grad_norm": 8.18448543548584, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.408295154571533, "logits/rejected": -1.924803614616394, "logps/chosen": -261.6023254394531, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -363.3185729980469, "loss": 1.0867, "margin_dpo/margin_mean": 95.20291137695312, "margin_dpo/margin_std": 121.64412689208984, "step": 539 }, { "KL/chosen_KL_mean": -182.34368896484375, "KL/mean": -240.3431396484375, "KL/rejected_KL_mean": -298.34259033203125, "KL/std": 116.13557434082031, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.00417669489979744, "fcm_dpo/delta": -0.08877268433570862, "fcm_dpo/margin": 115.9989013671875, "fcm_dpo/q_t": 0.3892328143119812, "grad_norm": 10.299283981323242, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.827339768409729, "logits/rejected": -1.5797513723373413, "logps/chosen": -290.2135925292969, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -419.736083984375, "loss": 1.0343, "margin_dpo/margin_mean": 115.9989013671875, "margin_dpo/margin_std": 135.59751892089844, "step": 540 }, { "KL/chosen_KL_mean": -202.21739196777344, "KL/mean": -238.46121215820312, "KL/rejected_KL_mean": -274.70501708984375, "KL/std": 96.1559066772461, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.00417899526655674, "fcm_dpo/delta": 0.10000735521316528, "fcm_dpo/margin": 72.48764038085938, "fcm_dpo/q_t": 0.42953577637672424, "grad_norm": 9.691296577453613, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.340578556060791, "logits/rejected": -1.873331069946289, "logps/chosen": -308.7848815917969, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -383.7713623046875, "loss": 1.1566, "margin_dpo/margin_mean": 72.48764038085938, "margin_dpo/margin_std": 103.72265625, "step": 541 }, { "KL/chosen_KL_mean": -171.25457763671875, "KL/mean": -226.94854736328125, "KL/rejected_KL_mean": -282.64251708984375, "KL/std": 108.79530334472656, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.004180562682449818, "fcm_dpo/delta": -0.06888352334499359, "fcm_dpo/margin": 111.38790893554688, "fcm_dpo/q_t": 0.390401154756546, "grad_norm": 9.971014022827148, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.1458849906921387, "logits/rejected": -2.018155813217163, "logps/chosen": -258.6238708496094, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -385.46282958984375, "loss": 1.0306, "margin_dpo/margin_mean": 111.38790893554688, "margin_dpo/margin_std": 116.9708480834961, "step": 542 }, { "KL/chosen_KL_mean": -197.70362854003906, "KL/mean": -226.99652099609375, "KL/rejected_KL_mean": -256.2894287109375, "KL/std": 114.54349517822266, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.004249558784067631, "fcm_dpo/delta": 0.15518760681152344, "fcm_dpo/margin": 58.585784912109375, "fcm_dpo/q_t": 0.4437216520309448, "grad_norm": 12.362500190734863, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.829298496246338, "logits/rejected": -1.8703577518463135, "logps/chosen": -284.856201171875, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -367.8211669921875, "loss": 1.2363, "margin_dpo/margin_mean": 58.58578872680664, "margin_dpo/margin_std": 138.1220703125, "step": 543 }, { "KL/chosen_KL_mean": -206.58737182617188, "KL/mean": -243.78817749023438, "KL/rejected_KL_mean": -280.98895263671875, "KL/std": 107.40501403808594, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.004346227273344994, "fcm_dpo/delta": 0.07923734933137894, "fcm_dpo/margin": 74.40159606933594, "fcm_dpo/q_t": 0.4252041280269623, "grad_norm": 9.33056354522705, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.7389962673187256, "logits/rejected": -1.642477035522461, "logps/chosen": -292.7342529296875, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -387.38287353515625, "loss": 1.1435, "margin_dpo/margin_mean": 74.40158081054688, "margin_dpo/margin_std": 111.08627319335938, "step": 544 }, { "KL/chosen_KL_mean": -184.91485595703125, "KL/mean": -230.21438598632812, "KL/rejected_KL_mean": -275.513916015625, "KL/std": 107.12690734863281, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004374153912067413, "fcm_dpo/delta": 0.0038262177258729935, "fcm_dpo/margin": 90.59904479980469, "fcm_dpo/q_t": 0.40843692421913147, "grad_norm": 9.320859909057617, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.5578718185424805, "logits/rejected": -1.1321461200714111, "logps/chosen": -281.339111328125, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -376.2479553222656, "loss": 1.1088, "margin_dpo/margin_mean": 90.59903717041016, "margin_dpo/margin_std": 132.9342498779297, "step": 545 }, { "KL/chosen_KL_mean": -189.75888061523438, "KL/mean": -230.74534606933594, "KL/rejected_KL_mean": -271.7318115234375, "KL/std": 108.58887481689453, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004388246685266495, "fcm_dpo/delta": 0.04178601875901222, "fcm_dpo/margin": 81.97291564941406, "fcm_dpo/q_t": 0.4155253767967224, "grad_norm": 9.124309539794922, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.566489815711975, "logits/rejected": -1.2093701362609863, "logps/chosen": -275.780029296875, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -359.10443115234375, "loss": 1.1237, "margin_dpo/margin_mean": 81.97291564941406, "margin_dpo/margin_std": 118.75297546386719, "step": 546 }, { "KL/chosen_KL_mean": -181.7330780029297, "KL/mean": -231.38246154785156, "KL/rejected_KL_mean": -281.0318603515625, "KL/std": 105.85003662109375, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004393605515360832, "fcm_dpo/delta": -0.03798150271177292, "fcm_dpo/margin": 99.2988052368164, "fcm_dpo/q_t": 0.39672988653182983, "grad_norm": 9.289485931396484, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.6159342527389526, "logits/rejected": -1.134981632232666, "logps/chosen": -282.38409423828125, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -378.943603515625, "loss": 1.0401, "margin_dpo/margin_mean": 99.2988052368164, "margin_dpo/margin_std": 96.06673431396484, "step": 547 }, { "KL/chosen_KL_mean": -178.38525390625, "KL/mean": -218.84912109375, "KL/rejected_KL_mean": -259.31298828125, "KL/std": 107.95115661621094, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.004404093138873577, "fcm_dpo/delta": 0.04518217593431473, "fcm_dpo/margin": 80.927734375, "fcm_dpo/q_t": 0.4178776741027832, "grad_norm": 11.02549934387207, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.3908488750457764, "logits/rejected": -2.0413856506347656, "logps/chosen": -274.41534423828125, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -357.7261962890625, "loss": 1.1464, "margin_dpo/margin_mean": 80.92774200439453, "margin_dpo/margin_std": 135.3878936767578, "step": 548 }, { "KL/chosen_KL_mean": -168.83697509765625, "KL/mean": -211.09909057617188, "KL/rejected_KL_mean": -253.36119079589844, "KL/std": 102.84603881835938, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004449094645678997, "fcm_dpo/delta": 0.02434992790222168, "fcm_dpo/margin": 84.52420043945312, "fcm_dpo/q_t": 0.41241341829299927, "grad_norm": 10.834419250488281, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.870591163635254, "logits/rejected": -1.4499731063842773, "logps/chosen": -252.28794860839844, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -334.7414245605469, "loss": 1.1244, "margin_dpo/margin_mean": 84.52420043945312, "margin_dpo/margin_std": 126.6233901977539, "step": 549 }, { "KL/chosen_KL_mean": -189.51902770996094, "KL/mean": -229.87698364257812, "KL/rejected_KL_mean": -270.23492431640625, "KL/std": 97.13681030273438, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004470706917345524, "fcm_dpo/delta": 0.0404314249753952, "fcm_dpo/margin": 80.71589660644531, "fcm_dpo/q_t": 0.4159596264362335, "grad_norm": 9.497106552124023, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.5237830877304077, "logits/rejected": -1.3048580884933472, "logps/chosen": -279.1133728027344, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -371.5504150390625, "loss": 1.1128, "margin_dpo/margin_mean": 80.71589660644531, "margin_dpo/margin_std": 105.56082916259766, "step": 550 }, { "KL/chosen_KL_mean": -166.9997100830078, "KL/mean": -216.350830078125, "KL/rejected_KL_mean": -265.70196533203125, "KL/std": 110.61848449707031, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004423696082085371, "fcm_dpo/delta": -0.03894048184156418, "fcm_dpo/margin": 98.7022705078125, "fcm_dpo/q_t": 0.39955443143844604, "grad_norm": 10.917253494262695, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.5451164245605469, "logits/rejected": -1.3988494873046875, "logps/chosen": -262.08184814453125, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -383.42120361328125, "loss": 1.0561, "margin_dpo/margin_mean": 98.70226287841797, "margin_dpo/margin_std": 113.38111877441406, "step": 551 }, { "KL/chosen_KL_mean": -160.71292114257812, "KL/mean": -230.98123168945312, "KL/rejected_KL_mean": -301.2495422363281, "KL/std": 120.07585906982422, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.004264539107680321, "fcm_dpo/delta": -0.21479392051696777, "fcm_dpo/margin": 140.5365753173828, "fcm_dpo/q_t": 0.3597748279571533, "grad_norm": 12.746831893920898, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.1285765171051025, "logits/rejected": -1.767514944076538, "logps/chosen": -262.12213134765625, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -412.0879821777344, "loss": 0.9422, "margin_dpo/margin_mean": 140.53659057617188, "margin_dpo/margin_std": 123.90788269042969, "step": 552 }, { "KL/chosen_KL_mean": -173.68243408203125, "KL/mean": -216.2352752685547, "KL/rejected_KL_mean": -258.7881164550781, "KL/std": 110.9793701171875, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.004277697764337063, "fcm_dpo/delta": 0.03723409026861191, "fcm_dpo/margin": 85.1056900024414, "fcm_dpo/q_t": 0.41469109058380127, "grad_norm": 10.279136657714844, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.4291285276412964, "logits/rejected": -1.0934220552444458, "logps/chosen": -271.64630126953125, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -362.36529541015625, "loss": 1.1167, "margin_dpo/margin_mean": 85.10569763183594, "margin_dpo/margin_std": 118.37693786621094, "step": 553 }, { "KL/chosen_KL_mean": -180.52255249023438, "KL/mean": -227.71743774414062, "KL/rejected_KL_mean": -274.91229248046875, "KL/std": 111.34757995605469, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0042935581877827644, "fcm_dpo/delta": -0.005759958177804947, "fcm_dpo/margin": 94.38973999023438, "fcm_dpo/q_t": 0.4068376421928406, "grad_norm": 9.488258361816406, "learning_rate": 5.205293880283551e-08, "logits/chosen": -2.0539755821228027, "logits/rejected": -1.2982670068740845, "logps/chosen": -284.2093505859375, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -367.9044189453125, "loss": 1.1104, "margin_dpo/margin_mean": 94.38973999023438, "margin_dpo/margin_std": 142.1490478515625, "step": 554 }, { "KL/chosen_KL_mean": -189.2399139404297, "KL/mean": -232.7355194091797, "KL/rejected_KL_mean": -276.23114013671875, "KL/std": 115.61207580566406, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.004292121157050133, "fcm_dpo/delta": 0.0276632122695446, "fcm_dpo/margin": 86.99124145507812, "fcm_dpo/q_t": 0.4146351218223572, "grad_norm": 9.494473457336426, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.6900362968444824, "logits/rejected": -1.5958009958267212, "logps/chosen": -275.44354248046875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -390.7783203125, "loss": 1.124, "margin_dpo/margin_mean": 86.9912338256836, "margin_dpo/margin_std": 133.06875610351562, "step": 555 }, { "KL/chosen_KL_mean": -170.6045684814453, "KL/mean": -217.66329956054688, "KL/rejected_KL_mean": -264.7220458984375, "KL/std": 102.9560546875, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004285210277885199, "fcm_dpo/delta": -0.003630075603723526, "fcm_dpo/margin": 94.11747741699219, "fcm_dpo/q_t": 0.40587538480758667, "grad_norm": 10.4158296585083, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.4844380617141724, "logits/rejected": -1.1750332117080688, "logps/chosen": -259.008056640625, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -371.36114501953125, "loss": 1.0832, "margin_dpo/margin_mean": 94.11747741699219, "margin_dpo/margin_std": 117.14349365234375, "step": 556 }, { "KL/chosen_KL_mean": -183.38870239257812, "KL/mean": -236.58395385742188, "KL/rejected_KL_mean": -289.77923583984375, "KL/std": 119.20575714111328, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.004243585281074047, "fcm_dpo/delta": -0.054528672248125076, "fcm_dpo/margin": 106.39053344726562, "fcm_dpo/q_t": 0.39626383781433105, "grad_norm": 8.483499526977539, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9370801448822021, "logits/rejected": -1.7629197835922241, "logps/chosen": -282.22393798828125, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -405.4281311035156, "loss": 1.0573, "margin_dpo/margin_mean": 106.39053344726562, "margin_dpo/margin_std": 129.9913330078125, "step": 557 }, { "KL/chosen_KL_mean": -179.78900146484375, "KL/mean": -237.1547088623047, "KL/rejected_KL_mean": -294.5204162597656, "KL/std": 102.80654907226562, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.004188914783298969, "fcm_dpo/delta": -0.08489947021007538, "fcm_dpo/margin": 114.73141479492188, "fcm_dpo/q_t": 0.3867698311805725, "grad_norm": 10.982680320739746, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.386615753173828, "logits/rejected": -1.7741432189941406, "logps/chosen": -285.80084228515625, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -402.9117431640625, "loss": 1.0093, "margin_dpo/margin_mean": 114.73141479492188, "margin_dpo/margin_std": 105.31607055664062, "step": 558 }, { "KL/chosen_KL_mean": -195.19715881347656, "KL/mean": -240.44778442382812, "KL/rejected_KL_mean": -285.69842529296875, "KL/std": 104.59542846679688, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0041953567415475845, "fcm_dpo/delta": 0.021082421764731407, "fcm_dpo/margin": 90.50125885009766, "fcm_dpo/q_t": 0.4106181859970093, "grad_norm": 10.299842834472656, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.265432596206665, "logits/rejected": -1.7740048170089722, "logps/chosen": -295.62188720703125, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -386.1111145019531, "loss": 1.102, "margin_dpo/margin_mean": 90.50125122070312, "margin_dpo/margin_std": 118.8939437866211, "step": 559 }, { "KL/chosen_KL_mean": -185.36474609375, "KL/mean": -230.42721557617188, "KL/rejected_KL_mean": -275.48968505859375, "KL/std": 103.67749786376953, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.00420765858143568, "fcm_dpo/delta": 0.02161007560789585, "fcm_dpo/margin": 90.12492370605469, "fcm_dpo/q_t": 0.41156259179115295, "grad_norm": 12.098847389221191, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.3371694087982178, "logits/rejected": -0.8856902122497559, "logps/chosen": -273.0002746582031, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -362.8695373535156, "loss": 1.102, "margin_dpo/margin_mean": 90.12492370605469, "margin_dpo/margin_std": 118.99862670898438, "step": 560 }, { "KL/chosen_KL_mean": -194.5346221923828, "KL/mean": -236.059814453125, "KL/rejected_KL_mean": -277.5849914550781, "KL/std": 101.62269592285156, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004252566955983639, "fcm_dpo/delta": 0.04838232696056366, "fcm_dpo/margin": 83.05037689208984, "fcm_dpo/q_t": 0.41681933403015137, "grad_norm": 8.182480812072754, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.8306050300598145, "logits/rejected": -1.8003395795822144, "logps/chosen": -269.416015625, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -383.6329650878906, "loss": 1.1052, "margin_dpo/margin_mean": 83.05038452148438, "margin_dpo/margin_std": 97.7025146484375, "step": 561 }, { "KL/chosen_KL_mean": -177.01300048828125, "KL/mean": -218.54989624023438, "KL/rejected_KL_mean": -260.0867919921875, "KL/std": 102.91455078125, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.004271417856216431, "fcm_dpo/delta": 0.04675827920436859, "fcm_dpo/margin": 83.07379913330078, "fcm_dpo/q_t": 0.41813695430755615, "grad_norm": 13.39061450958252, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.22745943069458, "logits/rejected": -1.779007911682129, "logps/chosen": -277.67620849609375, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -362.3276672363281, "loss": 1.1333, "margin_dpo/margin_mean": 83.07379913330078, "margin_dpo/margin_std": 127.95989990234375, "step": 562 }, { "KL/chosen_KL_mean": -188.07696533203125, "KL/mean": -223.07386779785156, "KL/rejected_KL_mean": -258.0707702636719, "KL/std": 103.37786865234375, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004369831643998623, "fcm_dpo/delta": 0.09668529778718948, "fcm_dpo/margin": 69.99378967285156, "fcm_dpo/q_t": 0.4286757707595825, "grad_norm": 10.401561737060547, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.5786824226379395, "logits/rejected": -1.2332818508148193, "logps/chosen": -284.336181640625, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -359.27581787109375, "loss": 1.1605, "margin_dpo/margin_mean": 69.99378967285156, "margin_dpo/margin_std": 110.95360565185547, "step": 563 }, { "KL/chosen_KL_mean": -182.75799560546875, "KL/mean": -222.05392456054688, "KL/rejected_KL_mean": -261.349853515625, "KL/std": 103.97802734375, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.004423552192747593, "fcm_dpo/delta": 0.054037436842918396, "fcm_dpo/margin": 78.59185028076172, "fcm_dpo/q_t": 0.41792136430740356, "grad_norm": 11.201470375061035, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.3370800018310547, "logits/rejected": -1.7632567882537842, "logps/chosen": -295.7183837890625, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -360.8778991699219, "loss": 1.1128, "margin_dpo/margin_mean": 78.59185028076172, "margin_dpo/margin_std": 96.12423706054688, "step": 564 }, { "KL/chosen_KL_mean": -179.71966552734375, "KL/mean": -228.51596069335938, "KL/rejected_KL_mean": -277.312255859375, "KL/std": 112.7188949584961, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.004401536658406258, "fcm_dpo/delta": -0.03094322606921196, "fcm_dpo/margin": 97.59255981445312, "fcm_dpo/q_t": 0.4008091688156128, "grad_norm": 8.58334732055664, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.6090036630630493, "logits/rejected": -1.0922868251800537, "logps/chosen": -275.08526611328125, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -369.52960205078125, "loss": 1.0723, "margin_dpo/margin_mean": 97.59256744384766, "margin_dpo/margin_std": 124.1192398071289, "step": 565 }, { "KL/chosen_KL_mean": -187.79727172851562, "KL/mean": -234.00418090820312, "KL/rejected_KL_mean": -280.2110900878906, "KL/std": 115.17008209228516, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004405232146382332, "fcm_dpo/delta": -0.007507972419261932, "fcm_dpo/margin": 92.41383361816406, "fcm_dpo/q_t": 0.40445494651794434, "grad_norm": 8.392843246459961, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -2.017458915710449, "logits/rejected": -1.5882503986358643, "logps/chosen": -286.7704772949219, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -391.031982421875, "loss": 1.0818, "margin_dpo/margin_mean": 92.413818359375, "margin_dpo/margin_std": 114.69636535644531, "step": 566 }, { "KL/chosen_KL_mean": -182.63616943359375, "KL/mean": -223.82237243652344, "KL/rejected_KL_mean": -265.0085754394531, "KL/std": 103.75431823730469, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.00439921161159873, "fcm_dpo/delta": 0.0388229638338089, "fcm_dpo/margin": 82.37238311767578, "fcm_dpo/q_t": 0.4144902229309082, "grad_norm": 9.997818946838379, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.4186668395996094, "logits/rejected": -1.7616794109344482, "logps/chosen": -275.33978271484375, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -356.23291015625, "loss": 1.1139, "margin_dpo/margin_mean": 82.37238311767578, "margin_dpo/margin_std": 108.8014144897461, "step": 567 }, { "KL/chosen_KL_mean": -172.8264617919922, "KL/mean": -226.33932495117188, "KL/rejected_KL_mean": -279.8521728515625, "KL/std": 109.44441223144531, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.004365639295428991, "fcm_dpo/delta": -0.07101192325353622, "fcm_dpo/margin": 107.0257339477539, "fcm_dpo/q_t": 0.3908562660217285, "grad_norm": 8.690141677856445, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.5026994943618774, "logits/rejected": -1.3707489967346191, "logps/chosen": -245.95872497558594, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -374.8048400878906, "loss": 1.0279, "margin_dpo/margin_mean": 107.0257339477539, "margin_dpo/margin_std": 110.63449096679688, "step": 568 }, { "KL/chosen_KL_mean": -182.23785400390625, "KL/mean": -226.85336303710938, "KL/rejected_KL_mean": -271.4689025878906, "KL/std": 110.89201354980469, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004377290606498718, "fcm_dpo/delta": 0.009735957719385624, "fcm_dpo/margin": 89.23106384277344, "fcm_dpo/q_t": 0.4092269837856293, "grad_norm": 13.152180671691895, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.9286937713623047, "logits/rejected": -1.5488288402557373, "logps/chosen": -271.4549560546875, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -357.87994384765625, "loss": 1.1163, "margin_dpo/margin_mean": 89.2310562133789, "margin_dpo/margin_std": 134.2045135498047, "step": 569 }, { "KL/chosen_KL_mean": -163.598388671875, "KL/mean": -221.22177124023438, "KL/rejected_KL_mean": -278.8451232910156, "KL/std": 105.20042419433594, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.004304665606468916, "fcm_dpo/delta": -0.10120187699794769, "fcm_dpo/margin": 115.2467269897461, "fcm_dpo/q_t": 0.3827279806137085, "grad_norm": 8.949538230895996, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.3504157066345215, "logits/rejected": -1.6680021286010742, "logps/chosen": -258.3611755371094, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -367.6920471191406, "loss": 1.0027, "margin_dpo/margin_mean": 115.2467269897461, "margin_dpo/margin_std": 109.26953125, "step": 570 }, { "KL/chosen_KL_mean": -171.41952514648438, "KL/mean": -221.5180206298828, "KL/rejected_KL_mean": -271.61651611328125, "KL/std": 103.1860580444336, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.004275895655155182, "fcm_dpo/delta": -0.029736077412962914, "fcm_dpo/margin": 100.19698333740234, "fcm_dpo/q_t": 0.400624543428421, "grad_norm": 11.767889976501465, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.5820577144622803, "logits/rejected": -1.4186947345733643, "logps/chosen": -252.24081420898438, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -368.6318054199219, "loss": 1.0702, "margin_dpo/margin_mean": 100.19699096679688, "margin_dpo/margin_std": 124.23790740966797, "step": 571 }, { "KL/chosen_KL_mean": -174.42630004882812, "KL/mean": -225.53070068359375, "KL/rejected_KL_mean": -276.6351318359375, "KL/std": 110.47218322753906, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0042340317741036415, "fcm_dpo/delta": -0.03429486230015755, "fcm_dpo/margin": 102.2088394165039, "fcm_dpo/q_t": 0.3999590575695038, "grad_norm": 8.759450912475586, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.686682939529419, "logits/rejected": -1.5133495330810547, "logps/chosen": -251.80621337890625, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -367.20111083984375, "loss": 1.0576, "margin_dpo/margin_mean": 102.2088394165039, "margin_dpo/margin_std": 117.53616333007812, "step": 572 }, { "KL/chosen_KL_mean": -169.524169921875, "KL/mean": -217.20022583007812, "KL/rejected_KL_mean": -264.87628173828125, "KL/std": 109.38301849365234, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.004253046587109566, "fcm_dpo/delta": -0.006501439958810806, "fcm_dpo/margin": 95.35211181640625, "fcm_dpo/q_t": 0.4046524167060852, "grad_norm": 8.648233413696289, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.070194959640503, "logits/rejected": -1.8358365297317505, "logps/chosen": -253.23025512695312, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -355.57373046875, "loss": 1.0825, "margin_dpo/margin_mean": 95.35211181640625, "margin_dpo/margin_std": 115.69882202148438, "step": 573 }, { "KL/chosen_KL_mean": -179.2760467529297, "KL/mean": -230.6185760498047, "KL/rejected_KL_mean": -281.96112060546875, "KL/std": 108.52104949951172, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.004215835593640804, "fcm_dpo/delta": -0.03445819020271301, "fcm_dpo/margin": 102.68505096435547, "fcm_dpo/q_t": 0.39896538853645325, "grad_norm": 10.317168235778809, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.359853744506836, "logits/rejected": -2.0430755615234375, "logps/chosen": -264.405517578125, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -381.01513671875, "loss": 1.0633, "margin_dpo/margin_mean": 102.68504333496094, "margin_dpo/margin_std": 122.41567993164062, "step": 574 }, { "KL/chosen_KL_mean": -185.7866973876953, "KL/mean": -233.1434326171875, "KL/rejected_KL_mean": -280.50018310546875, "KL/std": 111.42221069335938, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.004198268987238407, "fcm_dpo/delta": 0.002465156838297844, "fcm_dpo/margin": 94.71348571777344, "fcm_dpo/q_t": 0.40797144174575806, "grad_norm": 11.8463134765625, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.4198956489562988, "logits/rejected": -0.9541828632354736, "logps/chosen": -291.55755615234375, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -381.4496154785156, "loss": 1.0975, "margin_dpo/margin_mean": 94.71348571777344, "margin_dpo/margin_std": 129.92039489746094, "step": 575 }, { "KL/chosen_KL_mean": -185.154541015625, "KL/mean": -230.7563934326172, "KL/rejected_KL_mean": -276.35821533203125, "KL/std": 103.05194091796875, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.00421521533280611, "fcm_dpo/delta": 0.016104161739349365, "fcm_dpo/margin": 91.20368957519531, "fcm_dpo/q_t": 0.4102553129196167, "grad_norm": 11.08985710144043, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.97794508934021, "logits/rejected": -1.9129674434661865, "logps/chosen": -266.35906982421875, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -381.76763916015625, "loss": 1.1045, "margin_dpo/margin_mean": 91.20368957519531, "margin_dpo/margin_std": 124.54684448242188, "step": 576 }, { "KL/chosen_KL_mean": -164.9627227783203, "KL/mean": -222.4231414794922, "KL/rejected_KL_mean": -279.883544921875, "KL/std": 111.75218200683594, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.004147332161664963, "fcm_dpo/delta": -0.08097266405820847, "fcm_dpo/margin": 114.92082214355469, "fcm_dpo/q_t": 0.3887852430343628, "grad_norm": 8.267738342285156, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.1905946731567383, "logits/rejected": -1.5091333389282227, "logps/chosen": -266.42694091796875, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -369.20989990234375, "loss": 1.0318, "margin_dpo/margin_mean": 114.92082214355469, "margin_dpo/margin_std": 127.32967376708984, "step": 577 }, { "KL/chosen_KL_mean": -198.081787109375, "KL/mean": -239.19117736816406, "KL/rejected_KL_mean": -280.3005676269531, "KL/std": 106.03045654296875, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.00417992752045393, "fcm_dpo/delta": 0.05834614485502243, "fcm_dpo/margin": 82.21878814697266, "fcm_dpo/q_t": 0.41980624198913574, "grad_norm": 13.139318466186523, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.3220551013946533, "logits/rejected": -1.0795323848724365, "logps/chosen": -275.37420654296875, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -368.5475769042969, "loss": 1.1327, "margin_dpo/margin_mean": 82.21878051757812, "margin_dpo/margin_std": 120.80241394042969, "step": 578 }, { "KL/chosen_KL_mean": -176.2196502685547, "KL/mean": -218.53793334960938, "KL/rejected_KL_mean": -260.8562316894531, "KL/std": 106.35812377929688, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004204971715807915, "fcm_dpo/delta": 0.045591168105602264, "fcm_dpo/margin": 84.63656616210938, "fcm_dpo/q_t": 0.4171333909034729, "grad_norm": 8.100046157836914, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7175426483154297, "logits/rejected": -1.4137418270111084, "logps/chosen": -263.56268310546875, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -362.97723388671875, "loss": 1.1157, "margin_dpo/margin_mean": 84.63656616210938, "margin_dpo/margin_std": 108.45112609863281, "step": 579 }, { "KL/chosen_KL_mean": -169.61868286132812, "KL/mean": -221.5994415283203, "KL/rejected_KL_mean": -273.5802001953125, "KL/std": 106.5303955078125, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0041906265541911125, "fcm_dpo/delta": -0.03778352588415146, "fcm_dpo/margin": 103.96150207519531, "fcm_dpo/q_t": 0.3998372554779053, "grad_norm": 10.134312629699707, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.5881640911102295, "logits/rejected": -1.2752701044082642, "logps/chosen": -258.5474853515625, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -362.27679443359375, "loss": 1.0662, "margin_dpo/margin_mean": 103.96150207519531, "margin_dpo/margin_std": 128.63319396972656, "step": 580 }, { "KL/chosen_KL_mean": -178.313232421875, "KL/mean": -225.20724487304688, "KL/rejected_KL_mean": -272.10125732421875, "KL/std": 106.96881103515625, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004206322133541107, "fcm_dpo/delta": 0.005707694683223963, "fcm_dpo/margin": 93.78804016113281, "fcm_dpo/q_t": 0.4083898663520813, "grad_norm": 11.736775398254395, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.945720911026001, "logits/rejected": -1.5940684080123901, "logps/chosen": -270.53643798828125, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -371.4906921386719, "loss": 1.1067, "margin_dpo/margin_mean": 93.78804016113281, "margin_dpo/margin_std": 134.733154296875, "step": 581 }, { "KL/chosen_KL_mean": -158.80577087402344, "KL/mean": -201.68701171875, "KL/rejected_KL_mean": -244.5682373046875, "KL/std": 111.82752990722656, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0042627835646271706, "fcm_dpo/delta": 0.03387906402349472, "fcm_dpo/margin": 85.76246643066406, "fcm_dpo/q_t": 0.4166504442691803, "grad_norm": 10.634055137634277, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.9575109481811523, "logits/rejected": -1.9459798336029053, "logps/chosen": -246.14138793945312, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -359.72265625, "loss": 1.1253, "margin_dpo/margin_mean": 85.76245880126953, "margin_dpo/margin_std": 123.87232971191406, "step": 582 }, { "KL/chosen_KL_mean": -191.86199951171875, "KL/mean": -238.78067016601562, "KL/rejected_KL_mean": -285.6993408203125, "KL/std": 104.87452697753906, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0042190393432974815, "fcm_dpo/delta": 0.003969091922044754, "fcm_dpo/margin": 93.83732604980469, "fcm_dpo/q_t": 0.40856677293777466, "grad_norm": 11.256776809692383, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4651384353637695, "logits/rejected": -1.228807806968689, "logps/chosen": -275.5790710449219, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -380.28399658203125, "loss": 1.0938, "margin_dpo/margin_mean": 93.83732604980469, "margin_dpo/margin_std": 123.25637817382812, "step": 583 }, { "KL/chosen_KL_mean": -168.99411010742188, "KL/mean": -219.1605224609375, "KL/rejected_KL_mean": -269.32696533203125, "KL/std": 108.93672180175781, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0042480602860450745, "fcm_dpo/delta": -0.02816908434033394, "fcm_dpo/margin": 100.33282470703125, "fcm_dpo/q_t": 0.40033382177352905, "grad_norm": 8.464076042175293, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9195505380630493, "logits/rejected": -1.4584009647369385, "logps/chosen": -263.253662109375, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -373.80963134765625, "loss": 1.0743, "margin_dpo/margin_mean": 100.33283233642578, "margin_dpo/margin_std": 122.82672119140625, "step": 584 }, { "KL/chosen_KL_mean": -180.874267578125, "KL/mean": -236.4208984375, "KL/rejected_KL_mean": -291.967529296875, "KL/std": 124.6435546875, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.004183872137218714, "fcm_dpo/delta": -0.06795414537191391, "fcm_dpo/margin": 111.09324645996094, "fcm_dpo/q_t": 0.39398035407066345, "grad_norm": 8.973194122314453, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.891883373260498, "logits/rejected": -1.6121163368225098, "logps/chosen": -266.51104736328125, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -382.25299072265625, "loss": 1.0609, "margin_dpo/margin_mean": 111.09324645996094, "margin_dpo/margin_std": 145.177978515625, "step": 585 }, { "KL/chosen_KL_mean": -199.71792602539062, "KL/mean": -237.19326782226562, "KL/rejected_KL_mean": -274.66864013671875, "KL/std": 109.09827423095703, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0042098951525986195, "fcm_dpo/delta": 0.0872235894203186, "fcm_dpo/margin": 74.95069885253906, "fcm_dpo/q_t": 0.42590606212615967, "grad_norm": 10.290909767150879, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.7771236896514893, "logits/rejected": -1.5089643001556396, "logps/chosen": -283.3990478515625, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -373.02777099609375, "loss": 1.1412, "margin_dpo/margin_mean": 74.95069885253906, "margin_dpo/margin_std": 102.84270477294922, "step": 586 }, { "KL/chosen_KL_mean": -162.37432861328125, "KL/mean": -203.58932495117188, "KL/rejected_KL_mean": -244.80429077148438, "KL/std": 105.31354522705078, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0042643677443265915, "fcm_dpo/delta": 0.05019497871398926, "fcm_dpo/margin": 82.42999267578125, "fcm_dpo/q_t": 0.4178653359413147, "grad_norm": 10.766550064086914, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.314816474914551, "logits/rejected": -2.0250282287597656, "logps/chosen": -239.95184326171875, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -334.1246337890625, "loss": 1.118, "margin_dpo/margin_mean": 82.42999267578125, "margin_dpo/margin_std": 110.36050415039062, "step": 587 }, { "KL/chosen_KL_mean": -175.93988037109375, "KL/mean": -213.95477294921875, "KL/rejected_KL_mean": -251.96969604492188, "KL/std": 107.20919036865234, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004311534576117992, "fcm_dpo/delta": 0.0747101679444313, "fcm_dpo/margin": 76.02982330322266, "fcm_dpo/q_t": 0.42524850368499756, "grad_norm": 13.94801139831543, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.8377118110656738, "logits/rejected": -1.7964608669281006, "logps/chosen": -259.744140625, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -365.089111328125, "loss": 1.1587, "margin_dpo/margin_mean": 76.02982330322266, "margin_dpo/margin_std": 128.85101318359375, "step": 588 }, { "KL/chosen_KL_mean": -189.06582641601562, "KL/mean": -239.5032196044922, "KL/rejected_KL_mean": -289.94061279296875, "KL/std": 105.45673370361328, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004313983954489231, "fcm_dpo/delta": -0.03676054626703262, "fcm_dpo/margin": 100.87478637695312, "fcm_dpo/q_t": 0.3972448408603668, "grad_norm": 9.115158081054688, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.5690743923187256, "logits/rejected": -1.16725492477417, "logps/chosen": -285.5308532714844, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -405.977783203125, "loss": 1.0439, "margin_dpo/margin_mean": 100.87478637695312, "margin_dpo/margin_std": 101.99276733398438, "step": 589 }, { "KL/chosen_KL_mean": -174.5926055908203, "KL/mean": -223.87564086914062, "KL/rejected_KL_mean": -273.15869140625, "KL/std": 98.19406127929688, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0043053654953837395, "fcm_dpo/delta": -0.025682669132947922, "fcm_dpo/margin": 98.56605529785156, "fcm_dpo/q_t": 0.40057677030563354, "grad_norm": 12.320378303527832, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.928055763244629, "logits/rejected": -1.7779767513275146, "logps/chosen": -263.02044677734375, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -383.357421875, "loss": 1.0531, "margin_dpo/margin_mean": 98.56605529785156, "margin_dpo/margin_std": 102.39292907714844, "step": 590 }, { "KL/chosen_KL_mean": -164.58212280273438, "KL/mean": -218.1695556640625, "KL/rejected_KL_mean": -271.7569885253906, "KL/std": 106.45458984375, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.004236600361764431, "fcm_dpo/delta": -0.05685260891914368, "fcm_dpo/margin": 107.17488098144531, "fcm_dpo/q_t": 0.3929981589317322, "grad_norm": 7.285908222198486, "learning_rate": 2.678415274939408e-08, "logits/chosen": -2.0067553520202637, "logits/rejected": -1.392752766609192, "logps/chosen": -252.4318084716797, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -361.336181640625, "loss": 1.0284, "margin_dpo/margin_mean": 107.17488098144531, "margin_dpo/margin_std": 102.96257781982422, "step": 591 }, { "KL/chosen_KL_mean": -195.96109008789062, "KL/mean": -236.76358032226562, "KL/rejected_KL_mean": -277.56610107421875, "KL/std": 103.02247619628906, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004269970115274191, "fcm_dpo/delta": 0.053371451795101166, "fcm_dpo/margin": 81.60496520996094, "fcm_dpo/q_t": 0.4189314544200897, "grad_norm": 12.482398986816406, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.9382035732269287, "logits/rejected": -1.6139354705810547, "logps/chosen": -270.93682861328125, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -359.11871337890625, "loss": 1.1396, "margin_dpo/margin_mean": 81.60496520996094, "margin_dpo/margin_std": 128.4751434326172, "step": 592 }, { "KL/chosen_KL_mean": -179.55296325683594, "KL/mean": -214.88059997558594, "KL/rejected_KL_mean": -250.20822143554688, "KL/std": 107.84858703613281, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.00432781595736742, "fcm_dpo/delta": 0.0972917228937149, "fcm_dpo/margin": 70.6552505493164, "fcm_dpo/q_t": 0.4296596646308899, "grad_norm": 14.465426445007324, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8841742277145386, "logits/rejected": -0.5799588561058044, "logps/chosen": -251.7919921875, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -320.30474853515625, "loss": 1.1683, "margin_dpo/margin_mean": 70.65524291992188, "margin_dpo/margin_std": 119.11581420898438, "step": 593 }, { "KL/chosen_KL_mean": -161.83184814453125, "KL/mean": -216.22406005859375, "KL/rejected_KL_mean": -270.61627197265625, "KL/std": 105.09785461425781, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.004329598508775234, "fcm_dpo/delta": -0.07448194921016693, "fcm_dpo/margin": 108.78439331054688, "fcm_dpo/q_t": 0.38918572664260864, "grad_norm": 9.850162506103516, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.921159267425537, "logits/rejected": -1.3922364711761475, "logps/chosen": -252.26956176757812, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -362.88385009765625, "loss": 1.0167, "margin_dpo/margin_mean": 108.78439331054688, "margin_dpo/margin_std": 101.5719223022461, "step": 594 }, { "KL/chosen_KL_mean": -172.48667907714844, "KL/mean": -217.11819458007812, "KL/rejected_KL_mean": -261.7497253417969, "KL/std": 104.20787048339844, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0043189432471990585, "fcm_dpo/delta": 0.014893664047122002, "fcm_dpo/margin": 89.26303100585938, "fcm_dpo/q_t": 0.41106534004211426, "grad_norm": 9.714381217956543, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.860266923904419, "logits/rejected": -1.4491722583770752, "logps/chosen": -254.00149536132812, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -348.27020263671875, "loss": 1.1019, "margin_dpo/margin_mean": 89.26303100585938, "margin_dpo/margin_std": 120.37528228759766, "step": 595 }, { "KL/chosen_KL_mean": -180.03810119628906, "KL/mean": -219.53648376464844, "KL/rejected_KL_mean": -259.03485107421875, "KL/std": 103.36459350585938, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.004352291580289602, "fcm_dpo/delta": 0.05812724679708481, "fcm_dpo/margin": 78.99674224853516, "fcm_dpo/q_t": 0.4211881756782532, "grad_norm": 12.717914581298828, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.6751766204833984, "logits/rejected": -1.0960326194763184, "logps/chosen": -268.24615478515625, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -345.12860107421875, "loss": 1.149, "margin_dpo/margin_mean": 78.99673461914062, "margin_dpo/margin_std": 130.70559692382812, "step": 596 }, { "KL/chosen_KL_mean": -174.64950561523438, "KL/mean": -227.98043823242188, "KL/rejected_KL_mean": -281.31134033203125, "KL/std": 110.90653991699219, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.004312833305448294, "fcm_dpo/delta": -0.06310343742370605, "fcm_dpo/margin": 106.66182708740234, "fcm_dpo/q_t": 0.3933037221431732, "grad_norm": 9.114588737487793, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.4767816066741943, "logits/rejected": -1.3591156005859375, "logps/chosen": -258.239501953125, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -388.2557373046875, "loss": 1.0415, "margin_dpo/margin_mean": 106.66181945800781, "margin_dpo/margin_std": 120.53350067138672, "step": 597 }, { "KL/chosen_KL_mean": -165.45730590820312, "KL/mean": -210.39547729492188, "KL/rejected_KL_mean": -255.33363342285156, "KL/std": 104.1739273071289, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.004299808293581009, "fcm_dpo/delta": 0.01385500654578209, "fcm_dpo/margin": 89.87632751464844, "fcm_dpo/q_t": 0.4092050790786743, "grad_norm": 9.22042465209961, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.8813738822937012, "logits/rejected": -1.5640136003494263, "logps/chosen": -255.63992309570312, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -350.1322021484375, "loss": 1.101, "margin_dpo/margin_mean": 89.87632751464844, "margin_dpo/margin_std": 118.50434875488281, "step": 598 }, { "KL/chosen_KL_mean": -168.58370971679688, "KL/mean": -208.65478515625, "KL/rejected_KL_mean": -248.72586059570312, "KL/std": 109.25886535644531, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.004333799239248037, "fcm_dpo/delta": 0.05437636375427246, "fcm_dpo/margin": 80.14217376708984, "fcm_dpo/q_t": 0.4191525876522064, "grad_norm": 9.098594665527344, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.8562637567520142, "logits/rejected": -1.734307050704956, "logps/chosen": -246.4259796142578, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -337.9234924316406, "loss": 1.1371, "margin_dpo/margin_mean": 80.14218139648438, "margin_dpo/margin_std": 120.73688507080078, "step": 599 }, { "KL/chosen_KL_mean": -188.48699951171875, "KL/mean": -233.4007568359375, "KL/rejected_KL_mean": -278.3144836425781, "KL/std": 102.82810974121094, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004373063333332539, "fcm_dpo/delta": 0.007465180940926075, "fcm_dpo/margin": 89.82748413085938, "fcm_dpo/q_t": 0.4086702764034271, "grad_norm": 9.798473358154297, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.31777286529541, "logits/rejected": -1.9659358263015747, "logps/chosen": -280.6748046875, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -371.268798828125, "loss": 1.0883, "margin_dpo/margin_mean": 89.82748413085938, "margin_dpo/margin_std": 111.00724792480469, "step": 600 }, { "KL/chosen_KL_mean": -189.9231414794922, "KL/mean": -237.50106811523438, "KL/rejected_KL_mean": -285.0790100097656, "KL/std": 104.11033630371094, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004350216127932072, "fcm_dpo/delta": -0.01477745920419693, "fcm_dpo/margin": 95.1558837890625, "fcm_dpo/q_t": 0.4026733636856079, "grad_norm": 9.49589729309082, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.8752753734588623, "logits/rejected": -1.5543054342269897, "logps/chosen": -279.1392822265625, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -385.24957275390625, "loss": 1.0599, "margin_dpo/margin_mean": 95.1558837890625, "margin_dpo/margin_std": 99.71797180175781, "step": 601 }, { "KL/chosen_KL_mean": -150.4300994873047, "KL/mean": -212.35382080078125, "KL/rejected_KL_mean": -274.27752685546875, "KL/std": 114.67559814453125, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.004271931946277618, "fcm_dpo/delta": -0.13629910349845886, "fcm_dpo/margin": 123.84742736816406, "fcm_dpo/q_t": 0.37789055705070496, "grad_norm": 9.402480125427246, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.241452693939209, "logits/rejected": -1.9841909408569336, "logps/chosen": -230.48770141601562, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -367.47503662109375, "loss": 0.9978, "margin_dpo/margin_mean": 123.84742736816406, "margin_dpo/margin_std": 131.26661682128906, "step": 602 }, { "KL/chosen_KL_mean": -160.18783569335938, "KL/mean": -211.21900939941406, "KL/rejected_KL_mean": -262.25018310546875, "KL/std": 102.81280517578125, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.004225675016641617, "fcm_dpo/delta": -0.0326792448759079, "fcm_dpo/margin": 102.06233215332031, "fcm_dpo/q_t": 0.39945757389068604, "grad_norm": 8.41208267211914, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1938681602478027, "logits/rejected": -1.614495038986206, "logps/chosen": -262.4974365234375, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -370.31903076171875, "loss": 1.0591, "margin_dpo/margin_mean": 102.06233215332031, "margin_dpo/margin_std": 118.58842468261719, "step": 603 }, { "KL/chosen_KL_mean": -168.473388671875, "KL/mean": -221.04898071289062, "KL/rejected_KL_mean": -273.62457275390625, "KL/std": 108.11251068115234, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.00417605135589838, "fcm_dpo/delta": -0.04124688729643822, "fcm_dpo/margin": 105.15118408203125, "fcm_dpo/q_t": 0.39571934938430786, "grad_norm": 9.022107124328613, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.6538655757904053, "logits/rejected": -1.4965479373931885, "logps/chosen": -246.6474609375, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -373.12066650390625, "loss": 1.046, "margin_dpo/margin_mean": 105.15118408203125, "margin_dpo/margin_std": 110.25424194335938, "step": 604 }, { "KL/chosen_KL_mean": -183.71475219726562, "KL/mean": -227.09571838378906, "KL/rejected_KL_mean": -270.4766540527344, "KL/std": 96.80925750732422, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004201376810669899, "fcm_dpo/delta": 0.036836035549640656, "fcm_dpo/margin": 86.76190185546875, "fcm_dpo/q_t": 0.4148518443107605, "grad_norm": 9.611822128295898, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.7811270952224731, "logits/rejected": -1.2775591611862183, "logps/chosen": -278.48809814453125, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -377.78155517578125, "loss": 1.1054, "margin_dpo/margin_mean": 86.76190185546875, "margin_dpo/margin_std": 109.87071990966797, "step": 605 }, { "KL/chosen_KL_mean": -176.44921875, "KL/mean": -221.9599609375, "KL/rejected_KL_mean": -267.47076416015625, "KL/std": 106.51266479492188, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0042410679161548615, "fcm_dpo/delta": 0.01396404579281807, "fcm_dpo/margin": 91.02154541015625, "fcm_dpo/q_t": 0.40915483236312866, "grad_norm": 8.051018714904785, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.8539748191833496, "logits/rejected": -1.4603512287139893, "logps/chosen": -264.2026062011719, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -371.71295166015625, "loss": 1.0948, "margin_dpo/margin_mean": 91.02154541015625, "margin_dpo/margin_std": 113.89401245117188, "step": 606 }, { "KL/chosen_KL_mean": -171.43820190429688, "KL/mean": -222.60394287109375, "KL/rejected_KL_mean": -273.7696838378906, "KL/std": 113.2490463256836, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.004205677658319473, "fcm_dpo/delta": -0.03172750398516655, "fcm_dpo/margin": 102.33146667480469, "fcm_dpo/q_t": 0.40078580379486084, "grad_norm": 10.999119758605957, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7105859518051147, "logits/rejected": -1.172386646270752, "logps/chosen": -259.7672424316406, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -360.53778076171875, "loss": 1.083, "margin_dpo/margin_mean": 102.33146667480469, "margin_dpo/margin_std": 139.82839965820312, "step": 607 }, { "KL/chosen_KL_mean": -157.019775390625, "KL/mean": -210.65646362304688, "KL/rejected_KL_mean": -264.29315185546875, "KL/std": 111.853271484375, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.004140463657677174, "fcm_dpo/delta": -0.04734822362661362, "fcm_dpo/margin": 107.27337646484375, "fcm_dpo/q_t": 0.3952040374279022, "grad_norm": 9.225273132324219, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -2.000462055206299, "logits/rejected": -1.7081737518310547, "logps/chosen": -242.7891387939453, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -364.52593994140625, "loss": 1.0545, "margin_dpo/margin_mean": 107.27337646484375, "margin_dpo/margin_std": 120.58857727050781, "step": 608 }, { "KL/chosen_KL_mean": -174.8145751953125, "KL/mean": -224.6392822265625, "KL/rejected_KL_mean": -274.4639892578125, "KL/std": 112.5068130493164, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0041318414732813835, "fcm_dpo/delta": -0.012644831091165543, "fcm_dpo/margin": 99.64942169189453, "fcm_dpo/q_t": 0.4043305516242981, "grad_norm": 10.773375511169434, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.29479718208313, "logits/rejected": -1.8740206956863403, "logps/chosen": -267.7811584472656, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -370.3821716308594, "loss": 1.0796, "margin_dpo/margin_mean": 99.64942932128906, "margin_dpo/margin_std": 119.782470703125, "step": 609 }, { "KL/chosen_KL_mean": -159.88327026367188, "KL/mean": -213.76116943359375, "KL/rejected_KL_mean": -267.63909912109375, "KL/std": 107.151123046875, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.004109634552150965, "fcm_dpo/delta": -0.04501873627305031, "fcm_dpo/margin": 107.75581359863281, "fcm_dpo/q_t": 0.3950701951980591, "grad_norm": 8.607884407043457, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.9883440732955933, "logits/rejected": -1.46561861038208, "logps/chosen": -254.58355712890625, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -357.32647705078125, "loss": 1.0435, "margin_dpo/margin_mean": 107.75581359863281, "margin_dpo/margin_std": 113.17021179199219, "step": 610 }, { "KL/chosen_KL_mean": -168.1598358154297, "KL/mean": -221.14157104492188, "KL/rejected_KL_mean": -274.123291015625, "KL/std": 111.67532348632812, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.004104941617697477, "fcm_dpo/delta": -0.03680243343114853, "fcm_dpo/margin": 105.96348571777344, "fcm_dpo/q_t": 0.3984890580177307, "grad_norm": 9.99014949798584, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.116386890411377, "logits/rejected": -1.7690290212631226, "logps/chosen": -254.94317626953125, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -363.96636962890625, "loss": 1.0507, "margin_dpo/margin_mean": 105.96348571777344, "margin_dpo/margin_std": 113.29161834716797, "step": 611 }, { "KL/chosen_KL_mean": -185.91986083984375, "KL/mean": -239.24208068847656, "KL/rejected_KL_mean": -292.5643310546875, "KL/std": 104.82804870605469, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.004053760785609484, "fcm_dpo/delta": -0.03383946791291237, "fcm_dpo/margin": 106.64447021484375, "fcm_dpo/q_t": 0.398134708404541, "grad_norm": 8.903263092041016, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.031095027923584, "logits/rejected": -1.1773366928100586, "logps/chosen": -286.94000244140625, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -386.34735107421875, "loss": 1.046, "margin_dpo/margin_mean": 106.64447021484375, "margin_dpo/margin_std": 109.18930053710938, "step": 612 }, { "KL/chosen_KL_mean": -148.76959228515625, "KL/mean": -209.52529907226562, "KL/rejected_KL_mean": -270.281005859375, "KL/std": 107.76298522949219, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.00399150513112545, "fcm_dpo/delta": -0.08954423666000366, "fcm_dpo/margin": 121.51141357421875, "fcm_dpo/q_t": 0.38492441177368164, "grad_norm": 7.955501556396484, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.783284306526184, "logits/rejected": -1.5796582698822021, "logps/chosen": -237.7582244873047, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -380.27655029296875, "loss": 1.0002, "margin_dpo/margin_mean": 121.51141357421875, "margin_dpo/margin_std": 104.87442016601562, "step": 613 }, { "KL/chosen_KL_mean": -188.04705810546875, "KL/mean": -237.00653076171875, "KL/rejected_KL_mean": -285.9659729003906, "KL/std": 102.0472412109375, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.003981379792094231, "fcm_dpo/delta": 0.010545169934630394, "fcm_dpo/margin": 97.91890716552734, "fcm_dpo/q_t": 0.40836700797080994, "grad_norm": 10.090545654296875, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.5359504222869873, "logits/rejected": -1.5232903957366943, "logps/chosen": -268.24713134765625, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -395.828369140625, "loss": 1.0942, "margin_dpo/margin_mean": 97.91890716552734, "margin_dpo/margin_std": 125.21676635742188, "step": 614 }, { "KL/chosen_KL_mean": -174.45416259765625, "KL/mean": -215.08612060546875, "KL/rejected_KL_mean": -255.718017578125, "KL/std": 103.18727111816406, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.004056040663272142, "fcm_dpo/delta": 0.07155247777700424, "fcm_dpo/margin": 81.26383972167969, "fcm_dpo/q_t": 0.4217025637626648, "grad_norm": 8.936718940734863, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.8792959451675415, "logits/rejected": -1.4824169874191284, "logps/chosen": -274.8894348144531, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -356.8980712890625, "loss": 1.1265, "margin_dpo/margin_mean": 81.26383972167969, "margin_dpo/margin_std": 99.34938049316406, "step": 615 }, { "KL/chosen_KL_mean": -176.4322967529297, "KL/mean": -232.07098388671875, "KL/rejected_KL_mean": -287.7096862792969, "KL/std": 103.13184356689453, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.004029741510748863, "fcm_dpo/delta": -0.050778210163116455, "fcm_dpo/margin": 111.27738952636719, "fcm_dpo/q_t": 0.394789457321167, "grad_norm": 8.606926918029785, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.9342925548553467, "logits/rejected": -1.444275975227356, "logps/chosen": -268.92523193359375, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -369.77032470703125, "loss": 1.0436, "margin_dpo/margin_mean": 111.27738952636719, "margin_dpo/margin_std": 120.34280395507812, "step": 616 }, { "KL/chosen_KL_mean": -183.44967651367188, "KL/mean": -227.59857177734375, "KL/rejected_KL_mean": -271.7474365234375, "KL/std": 100.19087982177734, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.004030309617519379, "fcm_dpo/delta": 0.04577726870775223, "fcm_dpo/margin": 88.29776000976562, "fcm_dpo/q_t": 0.4160616397857666, "grad_norm": 8.682917594909668, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6037209033966064, "logits/rejected": -0.9608767628669739, "logps/chosen": -274.54669189453125, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -342.1575012207031, "loss": 1.1127, "margin_dpo/margin_mean": 88.29776000976562, "margin_dpo/margin_std": 113.307373046875, "step": 617 }, { "KL/chosen_KL_mean": -185.59774780273438, "KL/mean": -234.3432159423828, "KL/rejected_KL_mean": -283.08868408203125, "KL/std": 110.95719909667969, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.00404013879597187, "fcm_dpo/delta": 0.0063490960747003555, "fcm_dpo/margin": 97.49095153808594, "fcm_dpo/q_t": 0.4068368077278137, "grad_norm": 8.946409225463867, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.079545259475708, "logits/rejected": -1.7203644514083862, "logps/chosen": -287.61834716796875, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -382.889892578125, "loss": 1.0901, "margin_dpo/margin_mean": 97.49095153808594, "margin_dpo/margin_std": 122.50668334960938, "step": 618 }, { "KL/chosen_KL_mean": -180.13711547851562, "KL/mean": -223.49220275878906, "KL/rejected_KL_mean": -266.8472900390625, "KL/std": 109.09359741210938, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004075068049132824, "fcm_dpo/delta": 0.04836799204349518, "fcm_dpo/margin": 86.71013641357422, "fcm_dpo/q_t": 0.419109046459198, "grad_norm": 8.616724014282227, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.5446152687072754, "logits/rejected": -1.1776957511901855, "logps/chosen": -269.87847900390625, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -366.7486572265625, "loss": 1.1299, "margin_dpo/margin_mean": 86.71014404296875, "margin_dpo/margin_std": 130.36773681640625, "step": 619 }, { "KL/chosen_KL_mean": -171.87953186035156, "KL/mean": -219.24049377441406, "KL/rejected_KL_mean": -266.6014404296875, "KL/std": 115.50912475585938, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.004058824852108955, "fcm_dpo/delta": 0.014680266380310059, "fcm_dpo/margin": 94.72191619873047, "fcm_dpo/q_t": 0.41041290760040283, "grad_norm": 9.722071647644043, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.09255313873291, "logits/rejected": -1.5560133457183838, "logps/chosen": -257.00384521484375, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -352.01397705078125, "loss": 1.1074, "margin_dpo/margin_mean": 94.72191619873047, "margin_dpo/margin_std": 124.33357238769531, "step": 620 }, { "KL/chosen_KL_mean": -174.81802368164062, "KL/mean": -221.49380493164062, "KL/rejected_KL_mean": -268.16961669921875, "KL/std": 111.01587677001953, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.004140703938901424, "fcm_dpo/delta": 0.012420468032360077, "fcm_dpo/margin": 93.35159301757812, "fcm_dpo/q_t": 0.41013553738594055, "grad_norm": 9.438796043395996, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -2.0897107124328613, "logits/rejected": -1.8312872648239136, "logps/chosen": -264.0664367675781, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -363.6342468261719, "loss": 1.1117, "margin_dpo/margin_mean": 93.35159301757812, "margin_dpo/margin_std": 128.92581176757812, "step": 621 }, { "KL/chosen_KL_mean": -174.06613159179688, "KL/mean": -224.1903076171875, "KL/rejected_KL_mean": -274.3144836425781, "KL/std": 99.35678100585938, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.004092497285455465, "fcm_dpo/delta": -0.010828034952282906, "fcm_dpo/margin": 100.24832153320312, "fcm_dpo/q_t": 0.403218537569046, "grad_norm": 10.59797191619873, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.3817815780639648, "logits/rejected": -0.706696093082428, "logps/chosen": -273.8602600097656, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -365.1427001953125, "loss": 1.0652, "margin_dpo/margin_mean": 100.24832153320312, "margin_dpo/margin_std": 108.61425018310547, "step": 622 }, { "KL/chosen_KL_mean": -180.638916015625, "KL/mean": -215.4715118408203, "KL/rejected_KL_mean": -250.30410766601562, "KL/std": 106.71815490722656, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.004157062619924545, "fcm_dpo/delta": 0.11384863406419754, "fcm_dpo/margin": 69.6651611328125, "fcm_dpo/q_t": 0.43373483419418335, "grad_norm": 9.711257934570312, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.3194761276245117, "logits/rejected": -1.8842746019363403, "logps/chosen": -271.094482421875, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -341.6268615722656, "loss": 1.1901, "margin_dpo/margin_mean": 69.6651611328125, "margin_dpo/margin_std": 131.64199829101562, "step": 623 }, { "KL/chosen_KL_mean": -187.072509765625, "KL/mean": -243.29673767089844, "KL/rejected_KL_mean": -299.5209655761719, "KL/std": 106.83299255371094, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.004150357097387314, "fcm_dpo/delta": -0.06996473670005798, "fcm_dpo/margin": 112.44845581054688, "fcm_dpo/q_t": 0.39122235774993896, "grad_norm": 9.773256301879883, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.601799488067627, "logits/rejected": -1.6906918287277222, "logps/chosen": -296.94775390625, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -404.294189453125, "loss": 1.0331, "margin_dpo/margin_mean": 112.44845581054688, "margin_dpo/margin_std": 121.98001861572266, "step": 624 }, { "KL/chosen_KL_mean": -193.63681030273438, "KL/mean": -234.26168823242188, "KL/rejected_KL_mean": -274.8865661621094, "KL/std": 105.54098510742188, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.004183284007012844, "fcm_dpo/delta": 0.061978187412023544, "fcm_dpo/margin": 81.2497329711914, "fcm_dpo/q_t": 0.4206922650337219, "grad_norm": 9.173751831054688, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.5411264896392822, "logits/rejected": -1.1539995670318604, "logps/chosen": -280.80499267578125, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -366.748046875, "loss": 1.1279, "margin_dpo/margin_mean": 81.24972534179688, "margin_dpo/margin_std": 112.50981140136719, "step": 625 }, { "KL/chosen_KL_mean": -182.89813232421875, "KL/mean": -224.0612335205078, "KL/rejected_KL_mean": -265.22430419921875, "KL/std": 116.43057250976562, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.00423016119748354, "fcm_dpo/delta": 0.053382907062768936, "fcm_dpo/margin": 82.32618713378906, "fcm_dpo/q_t": 0.4199950695037842, "grad_norm": 11.278299331665039, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.4163728952407837, "logits/rejected": -0.9786391258239746, "logps/chosen": -287.1223449707031, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -369.40179443359375, "loss": 1.1366, "margin_dpo/margin_mean": 82.32618713378906, "margin_dpo/margin_std": 127.23822021484375, "step": 626 }, { "KL/chosen_KL_mean": -161.75430297851562, "KL/mean": -213.190673828125, "KL/rejected_KL_mean": -264.6270751953125, "KL/std": 119.61892700195312, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004210334271192551, "fcm_dpo/delta": -0.034626998007297516, "fcm_dpo/margin": 102.87274169921875, "fcm_dpo/q_t": 0.40086621046066284, "grad_norm": 7.527867794036865, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.4474297761917114, "logits/rejected": -1.5963165760040283, "logps/chosen": -236.68890380859375, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -377.199951171875, "loss": 1.0703, "margin_dpo/margin_mean": 102.87274169921875, "margin_dpo/margin_std": 131.97962951660156, "step": 627 }, { "KL/chosen_KL_mean": -171.47882080078125, "KL/mean": -224.18923950195312, "KL/rejected_KL_mean": -276.899658203125, "KL/std": 107.798095703125, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.004177641589194536, "fcm_dpo/delta": -0.04226092994213104, "fcm_dpo/margin": 105.42080688476562, "fcm_dpo/q_t": 0.3973705768585205, "grad_norm": 10.758256912231445, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.0678889751434326, "logits/rejected": -1.806017518043518, "logps/chosen": -265.169921875, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -384.24359130859375, "loss": 1.0535, "margin_dpo/margin_mean": 105.4207992553711, "margin_dpo/margin_std": 120.00865173339844, "step": 628 }, { "KL/chosen_KL_mean": -183.7495574951172, "KL/mean": -231.84295654296875, "KL/rejected_KL_mean": -279.93634033203125, "KL/std": 108.46688842773438, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.004155931994318962, "fcm_dpo/delta": 0.00010988861322402954, "fcm_dpo/margin": 96.18677520751953, "fcm_dpo/q_t": 0.4057689309120178, "grad_norm": 9.045262336730957, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.0339741706848145, "logits/rejected": -1.5949912071228027, "logps/chosen": -286.97991943359375, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -377.104736328125, "loss": 1.0821, "margin_dpo/margin_mean": 96.18677520751953, "margin_dpo/margin_std": 115.80900573730469, "step": 629 }, { "KL/chosen_KL_mean": -167.43609619140625, "KL/mean": -198.08203125, "KL/rejected_KL_mean": -228.7279510498047, "KL/std": 101.45230102539062, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.004269762896001339, "fcm_dpo/delta": 0.1413707435131073, "fcm_dpo/margin": 61.29182434082031, "fcm_dpo/q_t": 0.4389322102069855, "grad_norm": 12.557343482971191, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.9854458570480347, "logits/rejected": -1.4519288539886475, "logps/chosen": -261.3336486816406, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -318.10235595703125, "loss": 1.2009, "margin_dpo/margin_mean": 61.29182434082031, "margin_dpo/margin_std": 115.09196472167969, "step": 630 }, { "KL/chosen_KL_mean": -164.99685668945312, "KL/mean": -211.46713256835938, "KL/rejected_KL_mean": -257.9373779296875, "KL/std": 100.16387939453125, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.00428355485200882, "fcm_dpo/delta": 0.001940676011145115, "fcm_dpo/margin": 92.94052124023438, "fcm_dpo/q_t": 0.406375527381897, "grad_norm": 9.504329681396484, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.560692310333252, "logits/rejected": -2.135763645172119, "logps/chosen": -252.76766967773438, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -346.61981201171875, "loss": 1.0843, "margin_dpo/margin_mean": 92.9405288696289, "margin_dpo/margin_std": 114.0899658203125, "step": 631 }, { "KL/chosen_KL_mean": -162.4783935546875, "KL/mean": -213.02767944335938, "KL/rejected_KL_mean": -263.57696533203125, "KL/std": 105.57902526855469, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.004248114302754402, "fcm_dpo/delta": -0.031425122171640396, "fcm_dpo/margin": 101.09854125976562, "fcm_dpo/q_t": 0.4008200764656067, "grad_norm": 11.339091300964355, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.0973618030548096, "logits/rejected": -1.6718213558197021, "logps/chosen": -251.10491943359375, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -355.0278625488281, "loss": 1.0762, "margin_dpo/margin_mean": 101.09854125976562, "margin_dpo/margin_std": 130.494140625, "step": 632 }, { "KL/chosen_KL_mean": -202.85232543945312, "KL/mean": -232.87542724609375, "KL/rejected_KL_mean": -262.8985290527344, "KL/std": 107.8766860961914, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.004374593961983919, "fcm_dpo/delta": 0.13972653448581696, "fcm_dpo/margin": 60.04619598388672, "fcm_dpo/q_t": 0.4414445161819458, "grad_norm": 10.52823257446289, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.3626136779785156, "logits/rejected": -2.176546812057495, "logps/chosen": -284.22674560546875, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -361.52423095703125, "loss": 1.2371, "margin_dpo/margin_mean": 60.04619598388672, "margin_dpo/margin_std": 145.0777130126953, "step": 633 }, { "KL/chosen_KL_mean": -186.38748168945312, "KL/mean": -225.2864227294922, "KL/rejected_KL_mean": -264.18536376953125, "KL/std": 97.32357788085938, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.004452942870557308, "fcm_dpo/delta": 0.0537932924926281, "fcm_dpo/margin": 77.7978515625, "fcm_dpo/q_t": 0.4188125729560852, "grad_norm": 10.029311180114746, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.299551010131836, "logits/rejected": -1.1210371255874634, "logps/chosen": -282.5103454589844, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -377.03314208984375, "loss": 1.1231, "margin_dpo/margin_mean": 77.7978515625, "margin_dpo/margin_std": 100.34674072265625, "step": 634 }, { "KL/chosen_KL_mean": -195.6399688720703, "KL/mean": -225.45809936523438, "KL/rejected_KL_mean": -255.27621459960938, "KL/std": 106.08648681640625, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.004540085792541504, "fcm_dpo/delta": 0.13171057403087616, "fcm_dpo/margin": 59.63624954223633, "fcm_dpo/q_t": 0.4384138584136963, "grad_norm": 12.148545265197754, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.7242467403411865, "logits/rejected": -1.1807423830032349, "logps/chosen": -294.3240966796875, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -345.1754150390625, "loss": 1.2146, "margin_dpo/margin_mean": 59.636253356933594, "margin_dpo/margin_std": 127.42637634277344, "step": 635 }, { "KL/chosen_KL_mean": -173.47174072265625, "KL/mean": -225.20436096191406, "KL/rejected_KL_mean": -276.93695068359375, "KL/std": 108.93263244628906, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004507560282945633, "fcm_dpo/delta": -0.06956586241722107, "fcm_dpo/margin": 103.46524047851562, "fcm_dpo/q_t": 0.3912629187107086, "grad_norm": 9.340991020202637, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.3699270486831665, "logits/rejected": -0.8769779205322266, "logps/chosen": -263.88970947265625, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -364.64385986328125, "loss": 1.029, "margin_dpo/margin_mean": 103.46524047851562, "margin_dpo/margin_std": 107.89678955078125, "step": 636 }, { "KL/chosen_KL_mean": -174.69369506835938, "KL/mean": -229.0177001953125, "KL/rejected_KL_mean": -283.3416748046875, "KL/std": 120.6225357055664, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0044349804520606995, "fcm_dpo/delta": -0.08591315150260925, "fcm_dpo/margin": 108.64799499511719, "fcm_dpo/q_t": 0.3873834013938904, "grad_norm": 8.479291915893555, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.9402339458465576, "logits/rejected": -1.8354425430297852, "logps/chosen": -257.1434020996094, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -387.3702697753906, "loss": 1.0222, "margin_dpo/margin_mean": 108.64799499511719, "margin_dpo/margin_std": 113.47599792480469, "step": 637 }, { "KL/chosen_KL_mean": -164.08584594726562, "KL/mean": -216.81350708007812, "KL/rejected_KL_mean": -269.5411682128906, "KL/std": 107.09508514404297, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.004388597793877125, "fcm_dpo/delta": -0.06605572253465652, "fcm_dpo/margin": 105.45530700683594, "fcm_dpo/q_t": 0.3909299969673157, "grad_norm": 8.886940956115723, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.604865550994873, "logits/rejected": -0.9216387271881104, "logps/chosen": -256.0108337402344, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -355.82818603515625, "loss": 1.0245, "margin_dpo/margin_mean": 105.45530700683594, "margin_dpo/margin_std": 99.3663101196289, "step": 638 }, { "KL/chosen_KL_mean": -216.526123046875, "KL/mean": -241.65321350097656, "KL/rejected_KL_mean": -266.78033447265625, "KL/std": 99.76943969726562, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.004483419004827738, "fcm_dpo/delta": 0.17788483202457428, "fcm_dpo/margin": 50.25420379638672, "fcm_dpo/q_t": 0.4476528465747833, "grad_norm": 9.79828929901123, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3666374683380127, "logits/rejected": -0.5976688861846924, "logps/chosen": -321.05364990234375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -348.26068115234375, "loss": 1.248, "margin_dpo/margin_mean": 50.25420379638672, "margin_dpo/margin_std": 123.69154357910156, "step": 639 }, { "KL/chosen_KL_mean": -183.24485778808594, "KL/mean": -231.66278076171875, "KL/rejected_KL_mean": -280.0807189941406, "KL/std": 107.36080932617188, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.004521852359175682, "fcm_dpo/delta": -0.04088723659515381, "fcm_dpo/margin": 96.83586120605469, "fcm_dpo/q_t": 0.39756637811660767, "grad_norm": 11.99178409576416, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8822128772735596, "logits/rejected": -1.3627707958221436, "logps/chosen": -281.130126953125, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -392.7857666015625, "loss": 1.0609, "margin_dpo/margin_mean": 96.83586120605469, "margin_dpo/margin_std": 107.74942779541016, "step": 640 }, { "KL/chosen_KL_mean": -168.01910400390625, "KL/mean": -218.22300720214844, "KL/rejected_KL_mean": -268.42694091796875, "KL/std": 124.62991333007812, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.004443425685167313, "fcm_dpo/delta": -0.048284679651260376, "fcm_dpo/margin": 100.40784454345703, "fcm_dpo/q_t": 0.4002114236354828, "grad_norm": 9.527145385742188, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.2532856464385986, "logits/rejected": -1.704717755317688, "logps/chosen": -264.46478271484375, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -363.5626220703125, "loss": 1.0728, "margin_dpo/margin_mean": 100.4078369140625, "margin_dpo/margin_std": 138.1864471435547, "step": 641 }, { "KL/chosen_KL_mean": -188.26605224609375, "KL/mean": -235.17617797851562, "KL/rejected_KL_mean": -282.0863037109375, "KL/std": 106.93321990966797, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0044301776215434074, "fcm_dpo/delta": -0.016425499692559242, "fcm_dpo/margin": 93.82025146484375, "fcm_dpo/q_t": 0.40602976083755493, "grad_norm": 11.96419620513916, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.7101149559020996, "logits/rejected": -1.3848456144332886, "logps/chosen": -289.02587890625, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -396.7939453125, "loss": 1.0932, "margin_dpo/margin_mean": 93.82025146484375, "margin_dpo/margin_std": 133.53408813476562, "step": 642 }, { "KL/chosen_KL_mean": -175.73898315429688, "KL/mean": -220.31219482421875, "KL/rejected_KL_mean": -264.88543701171875, "KL/std": 99.95890808105469, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.004388836212456226, "fcm_dpo/delta": 0.008150741457939148, "fcm_dpo/margin": 89.14643859863281, "fcm_dpo/q_t": 0.40838927030563354, "grad_norm": 11.917402267456055, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.0531067848205566, "logits/rejected": -1.665812611579895, "logps/chosen": -260.4826354980469, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -359.203857421875, "loss": 1.0993, "margin_dpo/margin_mean": 89.14643859863281, "margin_dpo/margin_std": 115.64948272705078, "step": 643 }, { "KL/chosen_KL_mean": -182.49514770507812, "KL/mean": -219.21829223632812, "KL/rejected_KL_mean": -255.94143676757812, "KL/std": 101.03888702392578, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004469976760447025, "fcm_dpo/delta": 0.07414352893829346, "fcm_dpo/margin": 73.44628143310547, "fcm_dpo/q_t": 0.42345404624938965, "grad_norm": 10.9982271194458, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.9498875141143799, "logits/rejected": -1.6831095218658447, "logps/chosen": -267.8179016113281, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -350.550048828125, "loss": 1.1413, "margin_dpo/margin_mean": 73.44628143310547, "margin_dpo/margin_std": 108.98300170898438, "step": 644 }, { "KL/chosen_KL_mean": -195.01116943359375, "KL/mean": -218.31643676757812, "KL/rejected_KL_mean": -241.6217041015625, "KL/std": 109.52326202392578, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.004626215901225805, "fcm_dpo/delta": 0.1878928244113922, "fcm_dpo/margin": 46.61053466796875, "fcm_dpo/q_t": 0.45087456703186035, "grad_norm": 11.338079452514648, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.5269402265548706, "logits/rejected": -1.3572487831115723, "logps/chosen": -277.6014099121094, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -344.6254577636719, "loss": 1.262, "margin_dpo/margin_mean": 46.61053466796875, "margin_dpo/margin_std": 124.05409240722656, "step": 645 }, { "KL/chosen_KL_mean": -170.1150360107422, "KL/mean": -217.32394409179688, "KL/rejected_KL_mean": -264.5328369140625, "KL/std": 95.34092712402344, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0046125296503305435, "fcm_dpo/delta": -0.037710029631853104, "fcm_dpo/margin": 94.41781616210938, "fcm_dpo/q_t": 0.39786261320114136, "grad_norm": 9.53659725189209, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.9524312019348145, "logits/rejected": -1.3558188676834106, "logps/chosen": -261.30706787109375, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -348.3111572265625, "loss": 1.0529, "margin_dpo/margin_mean": 94.41781616210938, "margin_dpo/margin_std": 103.21369171142578, "step": 646 }, { "KL/chosen_KL_mean": -184.60826110839844, "KL/mean": -226.87014770507812, "KL/rejected_KL_mean": -269.13201904296875, "KL/std": 115.20654296875, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004629869945347309, "fcm_dpo/delta": 0.00901979859918356, "fcm_dpo/margin": 84.52377319335938, "fcm_dpo/q_t": 0.4111616015434265, "grad_norm": 10.474075317382812, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.4215582609176636, "logits/rejected": -1.2805519104003906, "logps/chosen": -273.45269775390625, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -368.63037109375, "loss": 1.1153, "margin_dpo/margin_mean": 84.52377319335938, "margin_dpo/margin_std": 128.322021484375, "step": 647 }, { "KL/chosen_KL_mean": -170.83575439453125, "KL/mean": -228.85169982910156, "KL/rejected_KL_mean": -286.8676452636719, "KL/std": 118.4449234008789, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.004551419522613287, "fcm_dpo/delta": -0.1350504457950592, "fcm_dpo/margin": 116.03186798095703, "fcm_dpo/q_t": 0.3778263330459595, "grad_norm": 10.007070541381836, "learning_rate": 3.798061746947995e-09, "logits/chosen": -2.0374584197998047, "logits/rejected": -1.9625945091247559, "logps/chosen": -258.6838684082031, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -391.5377197265625, "loss": 1.0021, "margin_dpo/margin_mean": 116.0318603515625, "margin_dpo/margin_std": 124.9610595703125, "step": 648 }, { "KL/chosen_KL_mean": -157.08895874023438, "KL/mean": -205.72122192382812, "KL/rejected_KL_mean": -254.35348510742188, "KL/std": 103.1243896484375, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.004462903365492821, "fcm_dpo/delta": -0.03631171956658363, "fcm_dpo/margin": 97.26454162597656, "fcm_dpo/q_t": 0.39803099632263184, "grad_norm": 12.086915969848633, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.454956531524658, "logits/rejected": -2.052997589111328, "logps/chosen": -246.78146362304688, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -343.06005859375, "loss": 1.0486, "margin_dpo/margin_mean": 97.26454162597656, "margin_dpo/margin_std": 100.09803771972656, "step": 649 }, { "KL/chosen_KL_mean": -185.568115234375, "KL/mean": -231.56504821777344, "KL/rejected_KL_mean": -277.5619812011719, "KL/std": 111.64608764648438, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004472729749977589, "fcm_dpo/delta": -0.011960911564528942, "fcm_dpo/margin": 91.99385070800781, "fcm_dpo/q_t": 0.4052048921585083, "grad_norm": 10.047531127929688, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.4529550075531006, "logits/rejected": -1.257375717163086, "logps/chosen": -275.49871826171875, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -380.1748046875, "loss": 1.098, "margin_dpo/margin_mean": 91.99385833740234, "margin_dpo/margin_std": 132.83770751953125, "step": 650 }, { "KL/chosen_KL_mean": -154.87271118164062, "KL/mean": -210.5814208984375, "KL/rejected_KL_mean": -266.2901611328125, "KL/std": 107.39662170410156, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.004400956444442272, "fcm_dpo/delta": -0.09506052732467651, "fcm_dpo/margin": 111.4174575805664, "fcm_dpo/q_t": 0.38420408964157104, "grad_norm": 9.239805221557617, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.10825777053833, "logits/rejected": -1.9325135946273804, "logps/chosen": -234.06002807617188, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -372.2234802246094, "loss": 1.0065, "margin_dpo/margin_mean": 111.4174575805664, "margin_dpo/margin_std": 105.76499938964844, "step": 651 }, { "KL/chosen_KL_mean": -162.40609741210938, "KL/mean": -208.78880310058594, "KL/rejected_KL_mean": -255.17152404785156, "KL/std": 100.2474365234375, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.004387515597045422, "fcm_dpo/delta": -0.007404278963804245, "fcm_dpo/margin": 92.76543426513672, "fcm_dpo/q_t": 0.4040106236934662, "grad_norm": 9.714406967163086, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.4744515419006348, "logits/rejected": -2.175802230834961, "logps/chosen": -264.1963195800781, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -371.49609375, "loss": 1.0618, "margin_dpo/margin_mean": 92.76543426513672, "margin_dpo/margin_std": 95.42292785644531, "step": 652 }, { "KL/chosen_KL_mean": -186.83547973632812, "KL/mean": -231.97906494140625, "KL/rejected_KL_mean": -277.1226501464844, "KL/std": 109.63093566894531, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004385844804346561, "fcm_dpo/delta": 0.004109857603907585, "fcm_dpo/margin": 90.28716278076172, "fcm_dpo/q_t": 0.40589216351509094, "grad_norm": 10.306207656860352, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.045562744140625, "logits/rejected": -1.787302017211914, "logps/chosen": -280.1890869140625, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -386.24591064453125, "loss": 1.0822, "margin_dpo/margin_mean": 90.28716278076172, "margin_dpo/margin_std": 106.67852783203125, "step": 653 }, { "KL/chosen_KL_mean": -187.41168212890625, "KL/mean": -226.28250122070312, "KL/rejected_KL_mean": -265.1533508300781, "KL/std": 105.0422592163086, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.004416775889694691, "fcm_dpo/delta": 0.05865585058927536, "fcm_dpo/margin": 77.74168395996094, "fcm_dpo/q_t": 0.42258337140083313, "grad_norm": 12.478866577148438, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.168185234069824, "logits/rejected": -1.9800138473510742, "logps/chosen": -276.5272216796875, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -370.07330322265625, "loss": 1.1524, "margin_dpo/margin_mean": 77.74168395996094, "margin_dpo/margin_std": 133.13494873046875, "step": 654 }, { "KL/chosen_KL_mean": -169.9119873046875, "KL/mean": -217.1782989501953, "KL/rejected_KL_mean": -264.4446105957031, "KL/std": 114.43123626708984, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.00442303903400898, "fcm_dpo/delta": -0.018915394321084023, "fcm_dpo/margin": 94.53258514404297, "fcm_dpo/q_t": 0.40254008769989014, "grad_norm": 9.140869140625, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.9657845497131348, "logits/rejected": -1.966432809829712, "logps/chosen": -250.94808959960938, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -367.2469482421875, "loss": 1.082, "margin_dpo/margin_mean": 94.5325927734375, "margin_dpo/margin_std": 122.88005065917969, "step": 655 }, { "KL/chosen_KL_mean": -171.25003051757812, "KL/mean": -231.27334594726562, "KL/rejected_KL_mean": -291.2966613769531, "KL/std": 114.2488021850586, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.004326796159148216, "fcm_dpo/delta": -0.12614339590072632, "fcm_dpo/margin": 120.04662322998047, "fcm_dpo/q_t": 0.37941867113113403, "grad_norm": 10.014375686645508, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.2565488815307617, "logits/rejected": -1.3280973434448242, "logps/chosen": -256.57537841796875, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -409.63531494140625, "loss": 0.9923, "margin_dpo/margin_mean": 120.04662322998047, "margin_dpo/margin_std": 117.7344970703125, "step": 656 }, { "KL/chosen_KL_mean": -186.79159545898438, "KL/mean": -231.02471923828125, "KL/rejected_KL_mean": -275.25787353515625, "KL/std": 105.8082275390625, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.004297615960240364, "fcm_dpo/delta": 0.02026219666004181, "fcm_dpo/margin": 88.46627807617188, "fcm_dpo/q_t": 0.41280561685562134, "grad_norm": 11.36693000793457, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.8130652904510498, "logits/rejected": -1.6404776573181152, "logps/chosen": -266.98931884765625, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -377.83917236328125, "loss": 1.1147, "margin_dpo/margin_mean": 88.46627807617188, "margin_dpo/margin_std": 127.48248291015625, "step": 657 }, { "KL/chosen_KL_mean": -184.22882080078125, "KL/mean": -225.01657104492188, "KL/rejected_KL_mean": -265.8043212890625, "KL/std": 99.30429077148438, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.004363303072750568, "fcm_dpo/delta": 0.04539173096418381, "fcm_dpo/margin": 81.57552337646484, "fcm_dpo/q_t": 0.4165422320365906, "grad_norm": 9.098910331726074, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.948915719985962, "logits/rejected": -1.472639560699463, "logps/chosen": -278.86334228515625, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -367.43597412109375, "loss": 1.1154, "margin_dpo/margin_mean": 81.57552337646484, "margin_dpo/margin_std": 107.57491302490234, "step": 658 }, { "KL/chosen_KL_mean": -166.60247802734375, "KL/mean": -216.20681762695312, "KL/rejected_KL_mean": -265.8111572265625, "KL/std": 107.3541259765625, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.004323400557041168, "fcm_dpo/delta": -0.030679544433951378, "fcm_dpo/margin": 99.20862579345703, "fcm_dpo/q_t": 0.4017835259437561, "grad_norm": 8.725358963012695, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.144078254699707, "logits/rejected": -1.1767905950546265, "logps/chosen": -279.1583557128906, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -359.73272705078125, "loss": 1.0918, "margin_dpo/margin_mean": 99.20863342285156, "margin_dpo/margin_std": 139.6153106689453, "step": 659 }, { "KL/chosen_KL_mean": -174.6622314453125, "KL/mean": -225.94349670410156, "KL/rejected_KL_mean": -277.22479248046875, "KL/std": 104.17098999023438, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.004306299611926079, "fcm_dpo/delta": -0.04358954727649689, "fcm_dpo/margin": 102.56255340576172, "fcm_dpo/q_t": 0.39684873819351196, "grad_norm": 10.937690734863281, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.6687394380569458, "logits/rejected": -1.0184828042984009, "logps/chosen": -268.0396728515625, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -358.6195983886719, "loss": 1.0487, "margin_dpo/margin_mean": 102.56254577636719, "margin_dpo/margin_std": 113.94944763183594, "step": 660 }, { "KL/chosen_KL_mean": -173.62655639648438, "KL/mean": -231.12680053710938, "KL/rejected_KL_mean": -288.6270446777344, "KL/std": 116.85284423828125, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.004246284253895283, "fcm_dpo/delta": -0.0927559956908226, "fcm_dpo/margin": 115.00049591064453, "fcm_dpo/q_t": 0.3874090909957886, "grad_norm": 9.980668067932129, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.8860403299331665, "logits/rejected": -1.3731896877288818, "logps/chosen": -261.4817199707031, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -381.03033447265625, "loss": 1.028, "margin_dpo/margin_mean": 115.00050354003906, "margin_dpo/margin_std": 129.46385192871094, "step": 661 }, { "KL/chosen_KL_mean": -169.61715698242188, "KL/mean": -221.73794555664062, "KL/rejected_KL_mean": -273.85870361328125, "KL/std": 101.70012664794922, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.004199195187538862, "fcm_dpo/delta": -0.039444342255592346, "fcm_dpo/margin": 104.2415771484375, "fcm_dpo/q_t": 0.3971138596534729, "grad_norm": 10.189535140991211, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.8295040130615234, "logits/rejected": -1.2653377056121826, "logps/chosen": -272.3969421386719, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -369.08404541015625, "loss": 1.0451, "margin_dpo/margin_mean": 104.2415771484375, "margin_dpo/margin_std": 107.52664184570312, "step": 662 }, { "KL/chosen_KL_mean": -170.70413208007812, "KL/mean": -218.66213989257812, "KL/rejected_KL_mean": -266.62017822265625, "KL/std": 105.8056640625, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.0041909716092050076, "fcm_dpo/delta": -0.002130841836333275, "fcm_dpo/margin": 95.91603088378906, "fcm_dpo/q_t": 0.4065453112125397, "grad_norm": 11.681427001953125, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.1285619735717773, "logits/rejected": -1.9050403833389282, "logps/chosen": -267.05072021484375, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -387.1466064453125, "loss": 1.0884, "margin_dpo/margin_mean": 95.91603088378906, "margin_dpo/margin_std": 123.58525085449219, "step": 663 }, { "KL/chosen_KL_mean": -189.58758544921875, "KL/mean": -232.40335083007812, "KL/rejected_KL_mean": -275.2191162109375, "KL/std": 108.37821960449219, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.004217204637825489, "fcm_dpo/delta": 0.04014057666063309, "fcm_dpo/margin": 85.63155364990234, "fcm_dpo/q_t": 0.415271520614624, "grad_norm": 9.036176681518555, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.5783047676086426, "logits/rejected": -1.4012095928192139, "logps/chosen": -281.42999267578125, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -387.0558166503906, "loss": 1.1083, "margin_dpo/margin_mean": 85.63154602050781, "margin_dpo/margin_std": 107.4930191040039, "step": 664 }, { "KL/chosen_KL_mean": -163.0349578857422, "KL/mean": -209.53807067871094, "KL/rejected_KL_mean": -256.04119873046875, "KL/std": 104.78343963623047, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004230777733027935, "fcm_dpo/delta": 0.006644213572144508, "fcm_dpo/margin": 93.00625610351562, "fcm_dpo/q_t": 0.406880259513855, "grad_norm": 8.035690307617188, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.4916818141937256, "logits/rejected": -2.06754732131958, "logps/chosen": -251.22113037109375, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -347.9532165527344, "loss": 1.0809, "margin_dpo/margin_mean": 93.00625610351562, "margin_dpo/margin_std": 107.66532897949219, "step": 665 }, { "KL/chosen_KL_mean": -178.6190948486328, "KL/mean": -227.08990478515625, "KL/rejected_KL_mean": -275.5607604980469, "KL/std": 102.16664123535156, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0042055207304656506, "fcm_dpo/delta": -0.008188748732209206, "fcm_dpo/margin": 96.94166564941406, "fcm_dpo/q_t": 0.4053737223148346, "grad_norm": 11.269975662231445, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.6178851127624512, "logits/rejected": -1.2336347103118896, "logps/chosen": -279.59368896484375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -376.8106689453125, "loss": 1.0825, "margin_dpo/margin_mean": 96.94166564941406, "margin_dpo/margin_std": 120.70773315429688, "step": 666 }, { "KL/chosen_KL_mean": -195.40504455566406, "KL/mean": -244.5108184814453, "KL/rejected_KL_mean": -293.6166076660156, "KL/std": 106.94903564453125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.004201361909508705, "fcm_dpo/delta": -0.01321760006248951, "fcm_dpo/margin": 98.21154022216797, "fcm_dpo/q_t": 0.4023793935775757, "grad_norm": 9.015604019165039, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.4838597774505615, "logits/rejected": -0.8236222267150879, "logps/chosen": -292.9761962890625, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -387.20135498046875, "loss": 1.0603, "margin_dpo/margin_mean": 98.21154022216797, "margin_dpo/margin_std": 102.80162048339844, "step": 667 }, { "KL/chosen_KL_mean": -172.4447021484375, "KL/mean": -229.94741821289062, "KL/rejected_KL_mean": -287.45013427734375, "KL/std": 114.24057006835938, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.004157195799052715, "fcm_dpo/delta": -0.08196092396974564, "fcm_dpo/margin": 115.00544738769531, "fcm_dpo/q_t": 0.38974809646606445, "grad_norm": 10.060585021972656, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.194495439529419, "logits/rejected": -1.6606438159942627, "logps/chosen": -275.019775390625, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -396.267822265625, "loss": 1.0264, "margin_dpo/margin_mean": 115.00544738769531, "margin_dpo/margin_std": 124.88902282714844, "step": 668 }, { "KL/chosen_KL_mean": -158.1702880859375, "KL/mean": -213.0489501953125, "KL/rejected_KL_mean": -267.9276123046875, "KL/std": 111.3969497680664, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0041051278822124004, "fcm_dpo/delta": -0.05293355509638786, "fcm_dpo/margin": 109.75730895996094, "fcm_dpo/q_t": 0.39458465576171875, "grad_norm": 9.844964981079102, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.543347716331482, "logits/rejected": -1.1332920789718628, "logps/chosen": -247.66506958007812, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -350.44708251953125, "loss": 1.0414, "margin_dpo/margin_mean": 109.75730895996094, "margin_dpo/margin_std": 118.66370391845703, "step": 669 }, { "KL/chosen_KL_mean": -158.92910766601562, "KL/mean": -219.20706176757812, "KL/rejected_KL_mean": -279.4850158691406, "KL/std": 113.46476745605469, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.004042153246700764, "fcm_dpo/delta": -0.0916917622089386, "fcm_dpo/margin": 120.55589294433594, "fcm_dpo/q_t": 0.38462913036346436, "grad_norm": 7.63702392578125, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.6413823366165161, "logits/rejected": -1.0921692848205566, "logps/chosen": -254.38369750976562, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -381.0179443359375, "loss": 1.0118, "margin_dpo/margin_mean": 120.55589294433594, "margin_dpo/margin_std": 118.21768188476562, "step": 670 }, { "KL/chosen_KL_mean": -193.25917053222656, "KL/mean": -234.752685546875, "KL/rejected_KL_mean": -276.2462158203125, "KL/std": 105.33341979980469, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.004076777957379818, "fcm_dpo/delta": 0.06296457350254059, "fcm_dpo/margin": 82.98706817626953, "fcm_dpo/q_t": 0.4212629795074463, "grad_norm": 9.265609741210938, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.652955174446106, "logits/rejected": -1.4274795055389404, "logps/chosen": -275.3822937011719, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -366.4659118652344, "loss": 1.1322, "margin_dpo/margin_mean": 82.98706817626953, "margin_dpo/margin_std": 116.15106201171875, "step": 671 }, { "KL/chosen_KL_mean": -192.88467407226562, "KL/mean": -228.9210968017578, "KL/rejected_KL_mean": -264.95751953125, "KL/std": 101.35043334960938, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0041283369064331055, "fcm_dpo/delta": 0.10570075362920761, "fcm_dpo/margin": 72.07286071777344, "fcm_dpo/q_t": 0.4304293394088745, "grad_norm": 12.976770401000977, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.122283458709717, "logits/rejected": -1.7947906255722046, "logps/chosen": -282.94659423828125, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -365.4107666015625, "loss": 1.1608, "margin_dpo/margin_mean": 72.0728530883789, "margin_dpo/margin_std": 107.15641784667969, "step": 672 }, { "KL/chosen_KL_mean": -171.73419189453125, "KL/mean": -212.39743041992188, "KL/rejected_KL_mean": -253.0606689453125, "KL/std": 104.537353515625, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004202650394290686, "fcm_dpo/delta": 0.06002630665898323, "fcm_dpo/margin": 81.32649993896484, "fcm_dpo/q_t": 0.4198816418647766, "grad_norm": 11.422228813171387, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.858733892440796, "logits/rejected": -1.4860217571258545, "logps/chosen": -250.99732971191406, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -344.409912109375, "loss": 1.1214, "margin_dpo/margin_mean": 81.32649993896484, "margin_dpo/margin_std": 106.33506774902344, "step": 673 }, { "KL/chosen_KL_mean": -160.55062866210938, "KL/mean": -217.969482421875, "KL/rejected_KL_mean": -275.38836669921875, "KL/std": 103.95561218261719, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.004151826724410057, "fcm_dpo/delta": -0.08072085678577423, "fcm_dpo/margin": 114.83770751953125, "fcm_dpo/q_t": 0.388614296913147, "grad_norm": 8.431386947631836, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.5086874961853027, "logits/rejected": -1.0984456539154053, "logps/chosen": -236.00894165039062, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -351.59197998046875, "loss": 1.0156, "margin_dpo/margin_mean": 114.83769989013672, "margin_dpo/margin_std": 112.45167541503906, "step": 674 }, { "KL/chosen_KL_mean": -177.26548767089844, "KL/mean": -227.87405395507812, "KL/rejected_KL_mean": -278.482666015625, "KL/std": 112.3695068359375, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.004103890620172024, "fcm_dpo/delta": -0.016857296228408813, "fcm_dpo/margin": 101.2171630859375, "fcm_dpo/q_t": 0.4032216966152191, "grad_norm": 10.911612510681152, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.408381462097168, "logits/rejected": -1.0604931116104126, "logps/chosen": -257.1750183105469, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -359.7008972167969, "loss": 1.0914, "margin_dpo/margin_mean": 101.21717071533203, "margin_dpo/margin_std": 137.26824951171875, "step": 675 }, { "KL/chosen_KL_mean": -190.56793212890625, "KL/mean": -240.6352996826172, "KL/rejected_KL_mean": -290.70263671875, "KL/std": 107.76188659667969, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.004108218476176262, "fcm_dpo/delta": -0.011985639110207558, "fcm_dpo/margin": 100.13470458984375, "fcm_dpo/q_t": 0.40339577198028564, "grad_norm": 8.443227767944336, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.9168659448623657, "logits/rejected": -1.3254802227020264, "logps/chosen": -288.73907470703125, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -385.727294921875, "loss": 1.0781, "margin_dpo/margin_mean": 100.13468933105469, "margin_dpo/margin_std": 122.0142822265625, "step": 676 }, { "KL/chosen_KL_mean": -170.41064453125, "KL/mean": -220.74853515625, "KL/rejected_KL_mean": -271.08642578125, "KL/std": 106.60063934326172, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.004097428172826767, "fcm_dpo/delta": -0.013180763460695744, "fcm_dpo/margin": 100.67578125, "fcm_dpo/q_t": 0.40272462368011475, "grad_norm": 8.313207626342773, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.7041046619415283, "logits/rejected": -1.0235017538070679, "logps/chosen": -261.7899169921875, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -353.9642028808594, "loss": 1.061, "margin_dpo/margin_mean": 100.67578125, "margin_dpo/margin_std": 104.80182647705078, "step": 677 }, { "KL/chosen_KL_mean": -160.97210693359375, "KL/mean": -220.61158752441406, "KL/rejected_KL_mean": -280.2510681152344, "KL/std": 114.27841186523438, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.004067711532115936, "fcm_dpo/delta": -0.08954396843910217, "fcm_dpo/margin": 119.27896118164062, "fcm_dpo/q_t": 0.3859301805496216, "grad_norm": 9.758649826049805, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.6789861917495728, "logits/rejected": -1.3497157096862793, "logps/chosen": -236.61798095703125, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -367.2171630859375, "loss": 1.0123, "margin_dpo/margin_mean": 119.27896118164062, "margin_dpo/margin_std": 114.88993835449219, "step": 678 }, { "KL/chosen_KL_mean": -185.6637420654297, "KL/mean": -235.24606323242188, "KL/rejected_KL_mean": -284.828369140625, "KL/std": 95.44699096679688, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0040285103023052216, "fcm_dpo/delta": 0.0005263802595436573, "fcm_dpo/margin": 99.16463470458984, "fcm_dpo/q_t": 0.40586063265800476, "grad_norm": 8.407492637634277, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.2022433280944824, "logits/rejected": -1.6799986362457275, "logps/chosen": -266.43719482421875, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -367.7068786621094, "loss": 1.0647, "margin_dpo/margin_mean": 99.16462707519531, "margin_dpo/margin_std": 100.74436950683594, "step": 679 }, { "KL/chosen_KL_mean": -188.06173706054688, "KL/mean": -240.95181274414062, "KL/rejected_KL_mean": -293.8419189453125, "KL/std": 112.56369018554688, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.004040123894810677, "fcm_dpo/delta": -0.029258504509925842, "fcm_dpo/margin": 105.78022003173828, "fcm_dpo/q_t": 0.4006451368331909, "grad_norm": 7.158690929412842, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9014968872070312, "logits/rejected": -1.5674934387207031, "logps/chosen": -295.7446594238281, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -409.9367980957031, "loss": 1.0648, "margin_dpo/margin_mean": 105.78022003173828, "margin_dpo/margin_std": 121.61329650878906, "step": 680 }, { "KL/chosen_KL_mean": -171.42578125, "KL/mean": -217.99835205078125, "KL/rejected_KL_mean": -264.5709228515625, "KL/std": 106.85641479492188, "epoch": 1.0, "fcm_dpo/beta": 0.004029108677059412, "fcm_dpo/delta": 0.025589467957615852, "fcm_dpo/margin": 93.1451416015625, "fcm_dpo/q_t": 0.412333607673645, "grad_norm": 9.155862808227539, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.285475730895996, "logits/rejected": -1.9735426902770996, "logps/chosen": -264.4368591308594, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -359.3930969238281, "loss": 1.0987, "margin_dpo/margin_mean": 93.1451416015625, "margin_dpo/margin_std": 116.73313903808594, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0842852252870578, "train_runtime": 2064.7767, "train_samples_per_second": 21.115, "train_steps_per_second": 0.33 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }