{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.8192024230957, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.758508682250977, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.02330780029296875, "KL/mean": 0.009138144552707672, "KL/rejected_KL_mean": -0.00502777099609375, "KL/std": 0.31077951192855835, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028335824608802795, "fcm_dpo/q_t": 0.4992920756340027, "grad_norm": 26.035297393798828, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42693132162094116, "logits/rejected": 0.6501660346984863, "logps/chosen": -93.78767395019531, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.23452758789062, "loss": 1.384, "margin_dpo/margin_mean": 0.028335824608802795, "margin_dpo/margin_std": 0.47666221857070923, "step": 3 }, { "KL/chosen_KL_mean": -0.056209564208984375, "KL/mean": -0.03549990803003311, "KL/rejected_KL_mean": -0.0147857666015625, "KL/std": 0.3047260046005249, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.041424140334129333, "fcm_dpo/q_t": 0.5010337829589844, "grad_norm": 32.8336181640625, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6466473937034607, "logits/rejected": 0.7785443067550659, "logps/chosen": -87.34867858886719, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.83904266357422, "loss": 1.3908, "margin_dpo/margin_mean": -0.0414242297410965, "margin_dpo/margin_std": 0.40186232328414917, "step": 4 }, { "KL/chosen_KL_mean": -0.0432891845703125, "KL/mean": -0.04466918110847473, "KL/rejected_KL_mean": -0.046051025390625, "KL/std": 0.30040794610977173, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.002764463424682617, "fcm_dpo/q_t": 0.4999305009841919, "grad_norm": 35.040618896484375, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.44582608342170715, "logits/rejected": 0.6573205590248108, "logps/chosen": -89.38005065917969, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.79388427734375, "loss": 1.3865, "margin_dpo/margin_mean": 0.002764970064163208, "margin_dpo/margin_std": 0.44222182035446167, "step": 5 }, { "KL/chosen_KL_mean": 0.018367767333984375, "KL/mean": -0.004490651190280914, "KL/rejected_KL_mean": -0.02734375, "KL/std": 0.32685428857803345, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04571045935153961, "fcm_dpo/q_t": 0.49885666370391846, "grad_norm": 35.97358322143555, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.48600590229034424, "logits/rejected": 0.8011856079101562, "logps/chosen": -97.30310821533203, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.91079711914062, "loss": 1.3823, "margin_dpo/margin_mean": 0.04571050405502319, "margin_dpo/margin_std": 0.4451420307159424, "step": 6 }, { "KL/chosen_KL_mean": -0.08198165893554688, "KL/mean": -0.06608044356107712, "KL/rejected_KL_mean": -0.050182342529296875, "KL/std": 0.3357203006744385, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.031802937388420105, "fcm_dpo/q_t": 0.500794529914856, "grad_norm": 38.11608123779297, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.48728084564208984, "logits/rejected": 0.5350579023361206, "logps/chosen": -86.73051452636719, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.66636657714844, "loss": 1.39, "margin_dpo/margin_mean": -0.03180357813835144, "margin_dpo/margin_std": 0.45320549607276917, "step": 7 }, { "KL/chosen_KL_mean": -0.07541656494140625, "KL/mean": -0.043665312230587006, "KL/rejected_KL_mean": -0.01190948486328125, "KL/std": 0.30540022253990173, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06350617110729218, "fcm_dpo/q_t": 0.5015870332717896, "grad_norm": 32.80784225463867, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.40473976731300354, "logits/rejected": 0.7172644734382629, "logps/chosen": -89.99909973144531, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.23994445800781, "loss": 1.3932, "margin_dpo/margin_mean": -0.06350578367710114, "margin_dpo/margin_std": 0.47104012966156006, "step": 8 }, { "KL/chosen_KL_mean": 0.05530548095703125, "KL/mean": 0.041829898953437805, "KL/rejected_KL_mean": 0.028354644775390625, "KL/std": 0.3528507649898529, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.026947706937789917, "fcm_dpo/q_t": 0.4993263781070709, "grad_norm": 36.18634796142578, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7192440629005432, "logits/rejected": 0.9026556015014648, "logps/chosen": -103.80182647705078, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.29096984863281, "loss": 1.3841, "margin_dpo/margin_mean": 0.02694791555404663, "margin_dpo/margin_std": 0.43218350410461426, "step": 9 }, { "KL/chosen_KL_mean": 0.018463134765625, "KL/mean": 0.04360641539096832, "KL/rejected_KL_mean": 0.06874465942382812, "KL/std": 0.2318560928106308, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05028638243675232, "fcm_dpo/q_t": 0.5012569427490234, "grad_norm": 33.6259765625, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.7003529071807861, "logits/rejected": 0.9214102029800415, "logps/chosen": -76.18589782714844, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.08335876464844, "loss": 1.3916, "margin_dpo/margin_mean": -0.050286561250686646, "margin_dpo/margin_std": 0.3299436867237091, "step": 10 }, { "KL/chosen_KL_mean": 0.06085205078125, "KL/mean": 0.010579630732536316, "KL/rejected_KL_mean": -0.03969573974609375, "KL/std": 0.3111981749534607, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10054662823677063, "fcm_dpo/q_t": 0.49748775362968445, "grad_norm": 36.601646423339844, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.39947575330734253, "logits/rejected": 0.6218962073326111, "logps/chosen": -82.30564880371094, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.3043212890625, "loss": 1.3767, "margin_dpo/margin_mean": 0.10054698586463928, "margin_dpo/margin_std": 0.4267161190509796, "step": 11 }, { "KL/chosen_KL_mean": 0.004955291748046875, "KL/mean": 0.0026900023221969604, "KL/rejected_KL_mean": 0.00041961669921875, "KL/std": 0.3910931348800659, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.004530102014541626, "fcm_dpo/q_t": 0.49988049268722534, "grad_norm": 40.19815444946289, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.3247011601924896, "logits/rejected": 0.45036810636520386, "logps/chosen": -99.10054016113281, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.27099609375, "loss": 1.3869, "margin_dpo/margin_mean": 0.0045309364795684814, "margin_dpo/margin_std": 0.6360738277435303, "step": 12 }, { "KL/chosen_KL_mean": 0.013027191162109375, "KL/mean": 0.0053391531109809875, "KL/rejected_KL_mean": -0.002349853515625, "KL/std": 0.3627878427505493, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.015384957194328308, "fcm_dpo/q_t": 0.49961382150650024, "grad_norm": 41.3858642578125, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.5008239150047302, "logits/rejected": 0.7580915093421936, "logps/chosen": -90.54669952392578, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.69345092773438, "loss": 1.3855, "margin_dpo/margin_mean": 0.015384703874588013, "margin_dpo/margin_std": 0.5249772667884827, "step": 13 }, { "KL/chosen_KL_mean": -0.08678817749023438, "KL/mean": 0.0002853795886039734, "KL/rejected_KL_mean": 0.08736038208007812, "KL/std": 0.33923521637916565, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.17415352165699005, "fcm_dpo/q_t": 0.5043511986732483, "grad_norm": 36.161678314208984, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.7832696437835693, "logits/rejected": 0.8684687614440918, "logps/chosen": -99.9139633178711, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.85464477539062, "loss": 1.4043, "margin_dpo/margin_mean": -0.17415401339530945, "margin_dpo/margin_std": 0.45376187562942505, "step": 14 }, { "KL/chosen_KL_mean": -0.0480499267578125, "KL/mean": -0.058234065771102905, "KL/rejected_KL_mean": -0.06841278076171875, "KL/std": 0.2973819673061371, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020366668701171875, "fcm_dpo/q_t": 0.49949193000793457, "grad_norm": 31.69318962097168, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.4585314393043518, "logits/rejected": 0.6295017600059509, "logps/chosen": -78.95802307128906, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.13075256347656, "loss": 1.3847, "margin_dpo/margin_mean": 0.02036646008491516, "margin_dpo/margin_std": 0.39507579803466797, "step": 15 }, { "KL/chosen_KL_mean": 0.015193939208984375, "KL/mean": -0.008750393986701965, "KL/rejected_KL_mean": -0.032703399658203125, "KL/std": 0.3349078893661499, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.047898560762405396, "fcm_dpo/q_t": 0.49880218505859375, "grad_norm": 34.50259780883789, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.5198068022727966, "logits/rejected": 0.7727633118629456, "logps/chosen": -97.4080810546875, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.63216400146484, "loss": 1.382, "margin_dpo/margin_mean": 0.047898441553115845, "margin_dpo/margin_std": 0.436043918132782, "step": 16 }, { "KL/chosen_KL_mean": -0.030658721923828125, "KL/mean": -0.015300802886486053, "KL/rejected_KL_mean": 4.9591064453125e-05, "KL/std": 0.3361964821815491, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.030706480145454407, "fcm_dpo/q_t": 0.5007667541503906, "grad_norm": 35.97813415527344, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.40684938430786133, "logits/rejected": 0.6264126300811768, "logps/chosen": -104.39496612548828, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.4676742553711, "loss": 1.3899, "margin_dpo/margin_mean": -0.030707508325576782, "margin_dpo/margin_std": 0.46864497661590576, "step": 17 }, { "KL/chosen_KL_mean": -0.01566314697265625, "KL/mean": -0.008798554539680481, "KL/rejected_KL_mean": -0.00193023681640625, "KL/std": 0.31606343388557434, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01373407244682312, "fcm_dpo/q_t": 0.5003408789634705, "grad_norm": 41.84980773925781, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.2614307701587677, "logits/rejected": 0.590286135673523, "logps/chosen": -87.10762023925781, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.85264587402344, "loss": 1.3883, "margin_dpo/margin_mean": -0.013733908534049988, "margin_dpo/margin_std": 0.4857482314109802, "step": 18 }, { "KL/chosen_KL_mean": 0.001953125, "KL/mean": -1.1622905731201172e-05, "KL/rejected_KL_mean": -0.00197601318359375, "KL/std": 0.299930214881897, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003920197486877441, "fcm_dpo/q_t": 0.49990367889404297, "grad_norm": 31.42831802368164, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.741286039352417, "logits/rejected": 1.0292823314666748, "logps/chosen": -105.87158966064453, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.93220520019531, "loss": 1.3864, "margin_dpo/margin_mean": 0.003920108079910278, "margin_dpo/margin_std": 0.42319223284721375, "step": 19 }, { "KL/chosen_KL_mean": -0.0027923583984375, "KL/mean": -0.011641770601272583, "KL/rejected_KL_mean": -0.02050018310546875, "KL/std": 0.2890578508377075, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.017707884311676025, "fcm_dpo/q_t": 0.4995581805706024, "grad_norm": 32.586029052734375, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.20901216566562653, "logits/rejected": 0.4820350408554077, "logps/chosen": -90.76091003417969, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.93281555175781, "loss": 1.385, "margin_dpo/margin_mean": 0.017707467079162598, "margin_dpo/margin_std": 0.41423308849334717, "step": 20 }, { "KL/chosen_KL_mean": 0.07179641723632812, "KL/mean": 0.04993293434381485, "KL/rejected_KL_mean": 0.028072357177734375, "KL/std": 0.24836955964565277, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.043725207448005676, "fcm_dpo/q_t": 0.49890732765197754, "grad_norm": 31.51675796508789, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.6238024830818176, "logits/rejected": 0.8094606399536133, "logps/chosen": -80.26165771484375, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.9056625366211, "loss": 1.3822, "margin_dpo/margin_mean": 0.04372560977935791, "margin_dpo/margin_std": 0.34140628576278687, "step": 21 }, { "KL/chosen_KL_mean": -0.0052642822265625, "KL/mean": -0.027231022715568542, "KL/rejected_KL_mean": -0.0491943359375, "KL/std": 0.31837520003318787, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.043921083211898804, "fcm_dpo/q_t": 0.4989031255245209, "grad_norm": 42.433528900146484, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.573004961013794, "logits/rejected": 0.690398633480072, "logps/chosen": -95.40056610107422, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.52271270751953, "loss": 1.3823, "margin_dpo/margin_mean": 0.0439208447933197, "margin_dpo/margin_std": 0.3951013386249542, "step": 22 }, { "KL/chosen_KL_mean": 0.04568290710449219, "KL/mean": 0.029423080384731293, "KL/rejected_KL_mean": 0.0131683349609375, "KL/std": 0.278354674577713, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03251884877681732, "fcm_dpo/q_t": 0.4991888403892517, "grad_norm": 31.292600631713867, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.6836845874786377, "logits/rejected": 0.8588317036628723, "logps/chosen": -90.59182739257812, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.58108520507812, "loss": 1.3835, "margin_dpo/margin_mean": 0.0325181782245636, "margin_dpo/margin_std": 0.39437204599380493, "step": 23 }, { "KL/chosen_KL_mean": -0.011766433715820312, "KL/mean": -0.010381445288658142, "KL/rejected_KL_mean": -0.009002685546875, "KL/std": 0.3026145100593567, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.002769380807876587, "fcm_dpo/q_t": 0.5000700354576111, "grad_norm": 44.274871826171875, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.7027078866958618, "logits/rejected": 0.7322394847869873, "logps/chosen": -69.92906188964844, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.61890411376953, "loss": 1.387, "margin_dpo/margin_mean": -0.0027685612440109253, "margin_dpo/margin_std": 0.4081639051437378, "step": 24 }, { "KL/chosen_KL_mean": 0.0265655517578125, "KL/mean": 0.0038614422082901, "KL/rejected_KL_mean": -0.0188446044921875, "KL/std": 0.3287331461906433, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04540279507637024, "fcm_dpo/q_t": 0.4988662004470825, "grad_norm": 37.205039978027344, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.6150128841400146, "logits/rejected": 0.7869346141815186, "logps/chosen": -80.79891967773438, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -95.9759521484375, "loss": 1.3824, "margin_dpo/margin_mean": 0.0454026460647583, "margin_dpo/margin_std": 0.5020095109939575, "step": 25 }, { "KL/chosen_KL_mean": -0.01483917236328125, "KL/mean": 0.009972691535949707, "KL/rejected_KL_mean": 0.034786224365234375, "KL/std": 0.32532280683517456, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.049622952938079834, "fcm_dpo/q_t": 0.5012384057044983, "grad_norm": 41.307411193847656, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.4694328010082245, "logits/rejected": 0.6820614337921143, "logps/chosen": -88.916015625, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.79341125488281, "loss": 1.3918, "margin_dpo/margin_mean": -0.049622029066085815, "margin_dpo/margin_std": 0.47193443775177, "step": 26 }, { "KL/chosen_KL_mean": 0.021839141845703125, "KL/mean": -0.03469548374414444, "KL/rejected_KL_mean": -0.0912322998046875, "KL/std": 0.3446305990219116, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11307112872600555, "fcm_dpo/q_t": 0.4971747100353241, "grad_norm": 44.24214553833008, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.2833951711654663, "logits/rejected": 0.47397956252098083, "logps/chosen": -77.57417297363281, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.02973937988281, "loss": 1.3755, "margin_dpo/margin_mean": 0.11307081580162048, "margin_dpo/margin_std": 0.43208199739456177, "step": 27 }, { "KL/chosen_KL_mean": 0.05237579345703125, "KL/mean": 0.0013619661331176758, "KL/rejected_KL_mean": -0.049652099609375, "KL/std": 0.3748849928379059, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10202693939208984, "fcm_dpo/q_t": 0.49744531512260437, "grad_norm": 36.43778610229492, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.6846746206283569, "logits/rejected": 0.9133214950561523, "logps/chosen": -102.17619323730469, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.00914764404297, "loss": 1.3769, "margin_dpo/margin_mean": 0.10202699899673462, "margin_dpo/margin_std": 0.552767276763916, "step": 28 }, { "KL/chosen_KL_mean": 0.01308441162109375, "KL/mean": -0.03650215268135071, "KL/rejected_KL_mean": -0.08608627319335938, "KL/std": 0.3483700752258301, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09916925430297852, "fcm_dpo/q_t": 0.49752184748649597, "grad_norm": 42.388092041015625, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.3394951820373535, "logits/rejected": 0.5994234085083008, "logps/chosen": -88.63397216796875, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.04620361328125, "loss": 1.377, "margin_dpo/margin_mean": 0.09917005896568298, "margin_dpo/margin_std": 0.4736003279685974, "step": 29 }, { "KL/chosen_KL_mean": 0.043468475341796875, "KL/mean": -0.004196256399154663, "KL/rejected_KL_mean": -0.05187225341796875, "KL/std": 0.29179471731185913, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09533864259719849, "fcm_dpo/q_t": 0.49761784076690674, "grad_norm": 38.92412185668945, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.395405650138855, "logits/rejected": 0.5024744272232056, "logps/chosen": -88.34491729736328, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.37076568603516, "loss": 1.3772, "margin_dpo/margin_mean": 0.09533926844596863, "margin_dpo/margin_std": 0.3916522264480591, "step": 30 }, { "KL/chosen_KL_mean": -0.043487548828125, "KL/mean": -0.07502669841051102, "KL/rejected_KL_mean": -0.10656356811523438, "KL/std": 0.3913864493370056, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06307275593280792, "fcm_dpo/q_t": 0.49842557311058044, "grad_norm": 31.32901954650879, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.9302603006362915, "logits/rejected": 1.2011293172836304, "logps/chosen": -101.16914367675781, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.96498107910156, "loss": 1.3806, "margin_dpo/margin_mean": 0.06307333707809448, "margin_dpo/margin_std": 0.48693209886550903, "step": 31 }, { "KL/chosen_KL_mean": -0.0228118896484375, "KL/mean": -0.08408394455909729, "KL/rejected_KL_mean": -0.14535903930664062, "KL/std": 0.3374728560447693, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12255328893661499, "fcm_dpo/q_t": 0.4969385266304016, "grad_norm": 35.152156829833984, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.30440640449523926, "logits/rejected": 0.4380635619163513, "logps/chosen": -96.63983917236328, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.36972045898438, "loss": 1.3747, "margin_dpo/margin_mean": 0.12255391478538513, "margin_dpo/margin_std": 0.48258599638938904, "step": 32 }, { "KL/chosen_KL_mean": 0.026966094970703125, "KL/mean": -0.016704894602298737, "KL/rejected_KL_mean": -0.060375213623046875, "KL/std": 0.33754000067710876, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08734075725078583, "fcm_dpo/q_t": 0.49781739711761475, "grad_norm": 33.65288543701172, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5697014331817627, "logits/rejected": 0.7506155967712402, "logps/chosen": -81.49404907226562, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.8663330078125, "loss": 1.3782, "margin_dpo/margin_mean": 0.08734112977981567, "margin_dpo/margin_std": 0.4746260643005371, "step": 33 }, { "KL/chosen_KL_mean": 0.0410614013671875, "KL/mean": -0.04497279226779938, "KL/rejected_KL_mean": -0.13100814819335938, "KL/std": 0.37820515036582947, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17207476496696472, "fcm_dpo/q_t": 0.4957064390182495, "grad_norm": 40.896297454833984, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5837531089782715, "logits/rejected": 0.7564134001731873, "logps/chosen": -77.16098022460938, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.84977722167969, "loss": 1.3699, "margin_dpo/margin_mean": 0.17207470536231995, "margin_dpo/margin_std": 0.5390475988388062, "step": 34 }, { "KL/chosen_KL_mean": -0.026233673095703125, "KL/mean": -0.10484717786312103, "KL/rejected_KL_mean": -0.18346023559570312, "KL/std": 0.39960581064224243, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15722736716270447, "fcm_dpo/q_t": 0.49607330560684204, "grad_norm": 42.801918029785156, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.0944751501083374, "logits/rejected": 0.2516512870788574, "logps/chosen": -77.59658813476562, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.37200927734375, "loss": 1.3714, "margin_dpo/margin_mean": 0.15722721815109253, "margin_dpo/margin_std": 0.5298490524291992, "step": 35 }, { "KL/chosen_KL_mean": -0.004619598388671875, "KL/mean": -0.08657992631196976, "KL/rejected_KL_mean": -0.16854095458984375, "KL/std": 0.3824528455734253, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16391386091709137, "fcm_dpo/q_t": 0.4959065318107605, "grad_norm": 32.43760299682617, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.5151327252388, "logits/rejected": 0.710216760635376, "logps/chosen": -83.29286193847656, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.92595672607422, "loss": 1.3707, "margin_dpo/margin_mean": 0.16391432285308838, "margin_dpo/margin_std": 0.5054072141647339, "step": 36 }, { "KL/chosen_KL_mean": 0.00460052490234375, "KL/mean": -0.14736080169677734, "KL/rejected_KL_mean": -0.2993278503417969, "KL/std": 0.38851290941238403, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3039228916168213, "fcm_dpo/q_t": 0.49241024255752563, "grad_norm": 36.01972961425781, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.8982394337654114, "logits/rejected": 1.0781348943710327, "logps/chosen": -94.76649475097656, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01659393310547, "loss": 1.3569, "margin_dpo/margin_mean": 0.30392324924468994, "margin_dpo/margin_std": 0.5396698713302612, "step": 37 }, { "KL/chosen_KL_mean": 0.00562286376953125, "KL/mean": -0.08792141824960709, "KL/rejected_KL_mean": -0.18146514892578125, "KL/std": 0.40408000349998474, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18708886206150055, "fcm_dpo/q_t": 0.49533015489578247, "grad_norm": 37.13588333129883, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.5153862237930298, "logits/rejected": 0.6536558866500854, "logps/chosen": -75.97288513183594, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.28548431396484, "loss": 1.3684, "margin_dpo/margin_mean": 0.1870894432067871, "margin_dpo/margin_std": 0.5322318077087402, "step": 38 }, { "KL/chosen_KL_mean": -0.020404815673828125, "KL/mean": -0.16962403059005737, "KL/rejected_KL_mean": -0.3188438415527344, "KL/std": 0.43626800179481506, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2984335422515869, "fcm_dpo/q_t": 0.4925464987754822, "grad_norm": 36.16465759277344, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.4460335373878479, "logits/rejected": 0.6312836408615112, "logps/chosen": -81.20618438720703, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.51476287841797, "loss": 1.3576, "margin_dpo/margin_mean": 0.29843345284461975, "margin_dpo/margin_std": 0.5757360458374023, "step": 39 }, { "KL/chosen_KL_mean": 0.043304443359375, "KL/mean": -0.06933881342411041, "KL/rejected_KL_mean": -0.181976318359375, "KL/std": 0.3281630873680115, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22529026865959167, "fcm_dpo/q_t": 0.49437105655670166, "grad_norm": 29.10408592224121, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.8706713914871216, "logits/rejected": 1.0857394933700562, "logps/chosen": -83.28926849365234, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.4378890991211, "loss": 1.3644, "margin_dpo/margin_mean": 0.22528982162475586, "margin_dpo/margin_std": 0.4404880404472351, "step": 40 }, { "KL/chosen_KL_mean": -0.09333038330078125, "KL/mean": -0.2238127440214157, "KL/rejected_KL_mean": -0.35430145263671875, "KL/std": 0.49110132455825806, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26097288727760315, "fcm_dpo/q_t": 0.49348461627960205, "grad_norm": 33.301025390625, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.5516754388809204, "logits/rejected": 0.7344405651092529, "logps/chosen": -93.24198913574219, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.43350982666016, "loss": 1.3615, "margin_dpo/margin_mean": 0.2609729766845703, "margin_dpo/margin_std": 0.6405251622200012, "step": 41 }, { "KL/chosen_KL_mean": 0.046142578125, "KL/mean": -0.180605947971344, "KL/rejected_KL_mean": -0.40735626220703125, "KL/std": 0.5021203756332397, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.45349544286727905, "fcm_dpo/q_t": 0.4886767864227295, "grad_norm": 40.685630798339844, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.30138301849365234, "logits/rejected": 0.4416188597679138, "logps/chosen": -90.65547943115234, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.27200317382812, "loss": 1.3424, "margin_dpo/margin_mean": 0.4534958004951477, "margin_dpo/margin_std": 0.6032987833023071, "step": 42 }, { "KL/chosen_KL_mean": -0.044330596923828125, "KL/mean": -0.22814278304576874, "KL/rejected_KL_mean": -0.4119606018066406, "KL/std": 0.4922180771827698, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.36762091517448425, "fcm_dpo/q_t": 0.4908219575881958, "grad_norm": 38.23686981201172, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.39107900857925415, "logits/rejected": 0.5723750591278076, "logps/chosen": -89.68836212158203, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.36381530761719, "loss": 1.3509, "margin_dpo/margin_mean": 0.3676205277442932, "margin_dpo/margin_std": 0.6384631395339966, "step": 43 }, { "KL/chosen_KL_mean": -0.05371856689453125, "KL/mean": -0.25897446274757385, "KL/rejected_KL_mean": -0.4642295837402344, "KL/std": 0.46895867586135864, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.41051214933395386, "fcm_dpo/q_t": 0.48974817991256714, "grad_norm": 34.523948669433594, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.6663105487823486, "logits/rejected": 0.7913767099380493, "logps/chosen": -81.93203735351562, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.8858642578125, "loss": 1.3464, "margin_dpo/margin_mean": 0.4105132520198822, "margin_dpo/margin_std": 0.5515778064727783, "step": 44 }, { "KL/chosen_KL_mean": -0.11534500122070312, "KL/mean": -0.22582048177719116, "KL/rejected_KL_mean": -0.3362922668457031, "KL/std": 0.39818382263183594, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22094972431659698, "fcm_dpo/q_t": 0.4944811165332794, "grad_norm": 24.519136428833008, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.7574669122695923, "logits/rejected": 0.8719462156295776, "logps/chosen": -77.45994567871094, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.58403778076172, "loss": 1.3649, "margin_dpo/margin_mean": 0.2209499329328537, "margin_dpo/margin_std": 0.46931952238082886, "step": 45 }, { "KL/chosen_KL_mean": -0.029682159423828125, "KL/mean": -0.21908630430698395, "KL/rejected_KL_mean": -0.40848541259765625, "KL/std": 0.5131125450134277, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.378805547952652, "fcm_dpo/q_t": 0.4905444383621216, "grad_norm": 33.41725540161133, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.6365070939064026, "logits/rejected": 0.8860538601875305, "logps/chosen": -90.3705062866211, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.08500671386719, "loss": 1.3496, "margin_dpo/margin_mean": 0.3788052201271057, "margin_dpo/margin_std": 0.5712540745735168, "step": 46 }, { "KL/chosen_KL_mean": 0.00205230712890625, "KL/mean": -0.2037028670310974, "KL/rejected_KL_mean": -0.4094657897949219, "KL/std": 0.5353131890296936, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.41152122616767883, "fcm_dpo/q_t": 0.4897342920303345, "grad_norm": 33.95356369018555, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.3089793622493744, "logits/rejected": 0.4829375743865967, "logps/chosen": -104.40924835205078, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.66436767578125, "loss": 1.3471, "margin_dpo/margin_mean": 0.41152074933052063, "margin_dpo/margin_std": 0.7745312452316284, "step": 47 }, { "KL/chosen_KL_mean": -0.008327484130859375, "KL/mean": -0.2850646376609802, "KL/rejected_KL_mean": -0.5618057250976562, "KL/std": 0.6658508777618408, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5534780621528625, "fcm_dpo/q_t": 0.48620203137397766, "grad_norm": 38.849205017089844, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.2079993188381195, "logits/rejected": 0.4072665870189667, "logps/chosen": -91.10859680175781, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.64237976074219, "loss": 1.3336, "margin_dpo/margin_mean": 0.5534786581993103, "margin_dpo/margin_std": 0.8667260408401489, "step": 48 }, { "KL/chosen_KL_mean": 0.06365203857421875, "KL/mean": -0.31414252519607544, "KL/rejected_KL_mean": -0.6919364929199219, "KL/std": 0.8001201152801514, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7555838823318481, "fcm_dpo/q_t": 0.48119014501571655, "grad_norm": 39.4151611328125, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.01228545606136322, "logits/rejected": 0.4740889370441437, "logps/chosen": -92.94003295898438, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.44093322753906, "loss": 1.3146, "margin_dpo/margin_mean": 0.7555838823318481, "margin_dpo/margin_std": 0.9946328401565552, "step": 49 }, { "KL/chosen_KL_mean": -0.10318756103515625, "KL/mean": -0.4414571523666382, "KL/rejected_KL_mean": -0.7797203063964844, "KL/std": 0.799066424369812, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.676538348197937, "fcm_dpo/q_t": 0.4831465482711792, "grad_norm": 33.205509185791016, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.4515685737133026, "logits/rejected": 0.6193307638168335, "logps/chosen": -94.72999572753906, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.35408020019531, "loss": 1.3222, "margin_dpo/margin_mean": 0.676538348197937, "margin_dpo/margin_std": 0.9712678790092468, "step": 50 }, { "KL/chosen_KL_mean": -0.044864654541015625, "KL/mean": -0.39222604036331177, "KL/rejected_KL_mean": -0.7395858764648438, "KL/std": 0.8703135251998901, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6947170495986938, "fcm_dpo/q_t": 0.48273295164108276, "grad_norm": 29.751827239990234, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.6615844368934631, "logits/rejected": 0.8052624464035034, "logps/chosen": -87.55213928222656, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.21194458007812, "loss": 1.3213, "margin_dpo/margin_mean": 0.6947166919708252, "margin_dpo/margin_std": 1.134157657623291, "step": 51 }, { "KL/chosen_KL_mean": -0.05788421630859375, "KL/mean": -0.48440805077552795, "KL/rejected_KL_mean": -0.9109344482421875, "KL/std": 0.7842905521392822, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8530476093292236, "fcm_dpo/q_t": 0.47875067591667175, "grad_norm": 32.1484260559082, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.3220360279083252, "logits/rejected": 0.5280216336250305, "logps/chosen": -90.68814849853516, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.94999694824219, "loss": 1.3049, "margin_dpo/margin_mean": 0.8530477285385132, "margin_dpo/margin_std": 0.9129204750061035, "step": 52 }, { "KL/chosen_KL_mean": -0.03199005126953125, "KL/mean": -0.5557708144187927, "KL/rejected_KL_mean": -1.0795516967773438, "KL/std": 1.098960041999817, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0475664138793945, "fcm_dpo/q_t": 0.4740614891052246, "grad_norm": 39.3596305847168, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.031249940395355225, "logits/rejected": 0.1429249346256256, "logps/chosen": -81.61506652832031, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.74108123779297, "loss": 1.2892, "margin_dpo/margin_mean": 1.0475660562515259, "margin_dpo/margin_std": 1.3129266500473022, "step": 53 }, { "KL/chosen_KL_mean": -0.08960723876953125, "KL/mean": -0.6380029320716858, "KL/rejected_KL_mean": -1.1863975524902344, "KL/std": 1.3109335899353027, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0967888832092285, "fcm_dpo/q_t": 0.4727639853954315, "grad_norm": 39.90190505981445, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.41930702328681946, "logits/rejected": 0.5993320345878601, "logps/chosen": -88.99977111816406, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.303955078125, "loss": 1.2838, "margin_dpo/margin_mean": 1.0967886447906494, "margin_dpo/margin_std": 1.2912437915802002, "step": 54 }, { "KL/chosen_KL_mean": 0.16521072387695312, "KL/mean": -0.45175451040267944, "KL/rejected_KL_mean": -1.0687217712402344, "KL/std": 1.2215603590011597, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.233933687210083, "fcm_dpo/q_t": 0.4695286452770233, "grad_norm": 35.0363883972168, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.14572454988956451, "logits/rejected": 0.4571814239025116, "logps/chosen": -92.29071044921875, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.03965759277344, "loss": 1.2733, "margin_dpo/margin_mean": 1.2339329719543457, "margin_dpo/margin_std": 1.5723824501037598, "step": 55 }, { "KL/chosen_KL_mean": -0.07014083862304688, "KL/mean": -0.5879875421524048, "KL/rejected_KL_mean": -1.1058311462402344, "KL/std": 1.1384570598602295, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.03568696975708, "fcm_dpo/q_t": 0.47438520193099976, "grad_norm": 33.23576736450195, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.2236095517873764, "logits/rejected": 0.5609951615333557, "logps/chosen": -87.30679321289062, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.75137329101562, "loss": 1.2914, "margin_dpo/margin_mean": 1.0356863737106323, "margin_dpo/margin_std": 1.5479753017425537, "step": 56 }, { "KL/chosen_KL_mean": -0.0172576904296875, "KL/mean": -0.5659521818161011, "KL/rejected_KL_mean": -1.1146507263183594, "KL/std": 1.1794803142547607, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0974080562591553, "fcm_dpo/q_t": 0.4728055000305176, "grad_norm": 33.79072189331055, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.4882052540779114, "logits/rejected": 0.7115618586540222, "logps/chosen": -98.16799926757812, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.16748809814453, "loss": 1.285, "margin_dpo/margin_mean": 1.097408652305603, "margin_dpo/margin_std": 1.4715585708618164, "step": 57 }, { "KL/chosen_KL_mean": 0.3107032775878906, "KL/mean": -0.4922056198120117, "KL/rejected_KL_mean": -1.2951126098632812, "KL/std": 1.4317562580108643, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.605809211730957, "fcm_dpo/q_t": 0.46048983931541443, "grad_norm": 38.008426666259766, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.2865542769432068, "logits/rejected": 0.616993248462677, "logps/chosen": -99.270263671875, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.51738739013672, "loss": 1.2411, "margin_dpo/margin_mean": 1.6058090925216675, "margin_dpo/margin_std": 1.8593244552612305, "step": 58 }, { "KL/chosen_KL_mean": 0.03055572509765625, "KL/mean": -0.5769742727279663, "KL/rejected_KL_mean": -1.18450927734375, "KL/std": 1.1787744760513306, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.215057134628296, "fcm_dpo/q_t": 0.46988144516944885, "grad_norm": 34.33761978149414, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.06326924264431, "logits/rejected": 0.24991700053215027, "logps/chosen": -89.77177429199219, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.461181640625, "loss": 1.2737, "margin_dpo/margin_mean": 1.2150566577911377, "margin_dpo/margin_std": 1.4390712976455688, "step": 59 }, { "KL/chosen_KL_mean": 0.040981292724609375, "KL/mean": -0.5741684436798096, "KL/rejected_KL_mean": -1.1893196105957031, "KL/std": 1.3340837955474854, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2303050756454468, "fcm_dpo/q_t": 0.46950262784957886, "grad_norm": 28.421905517578125, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.314486026763916, "logits/rejected": 0.5850687623023987, "logps/chosen": -95.11473083496094, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.56175231933594, "loss": 1.2725, "margin_dpo/margin_mean": 1.2303051948547363, "margin_dpo/margin_std": 1.4794132709503174, "step": 60 }, { "KL/chosen_KL_mean": -0.211822509765625, "KL/mean": -0.7876814603805542, "KL/rejected_KL_mean": -1.3635406494140625, "KL/std": 1.3975822925567627, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1517194509506226, "fcm_dpo/q_t": 0.47172319889068604, "grad_norm": 28.365310668945312, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.16400115191936493, "logits/rejected": 0.30372756719589233, "logps/chosen": -85.94412994384766, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.32929992675781, "loss": 1.2829, "margin_dpo/margin_mean": 1.1517198085784912, "margin_dpo/margin_std": 1.8181161880493164, "step": 61 }, { "KL/chosen_KL_mean": -0.14666366577148438, "KL/mean": -0.717978298664093, "KL/rejected_KL_mean": -1.2892951965332031, "KL/std": 1.3847185373306274, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1426341533660889, "fcm_dpo/q_t": 0.4717444181442261, "grad_norm": 28.671506881713867, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.5576132535934448, "logits/rejected": 0.7755956649780273, "logps/chosen": -81.78204345703125, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.32760620117188, "loss": 1.2848, "margin_dpo/margin_mean": 1.1426334381103516, "margin_dpo/margin_std": 1.818864345550537, "step": 62 }, { "KL/chosen_KL_mean": 0.005542755126953125, "KL/mean": -0.7539295554161072, "KL/rejected_KL_mean": -1.5134010314941406, "KL/std": 1.3780317306518555, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.51894211769104, "fcm_dpo/q_t": 0.4624468684196472, "grad_norm": 34.87947463989258, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.3483128249645233, "logits/rejected": 0.5583575963973999, "logps/chosen": -103.61851501464844, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.91642761230469, "loss": 1.2471, "margin_dpo/margin_mean": 1.5189417600631714, "margin_dpo/margin_std": 1.6412473917007446, "step": 63 }, { "KL/chosen_KL_mean": -0.10945510864257812, "KL/mean": -0.9727106094360352, "KL/rejected_KL_mean": -1.835968017578125, "KL/std": 1.7923610210418701, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10018503665924072, "fcm_dpo/delta": 0.09243200719356537, "fcm_dpo/margin": 1.7265081405639648, "fcm_dpo/q_t": 0.45766979455947876, "grad_norm": 34.960567474365234, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.6719385981559753, "logits/rejected": 0.8974436521530151, "logps/chosen": -87.11103820800781, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.42144012451172, "loss": 1.2323, "margin_dpo/margin_mean": 1.7265080213546753, "margin_dpo/margin_std": 2.0509119033813477, "step": 64 }, { "KL/chosen_KL_mean": -0.07342529296875, "KL/mean": -0.96152663230896, "KL/rejected_KL_mean": -1.8496360778808594, "KL/std": 1.689530611038208, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10018503665924072, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7762107849121094, "fcm_dpo/q_t": 0.4562700390815735, "grad_norm": 38.39850616455078, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.06450694799423218, "logits/rejected": 0.0797528475522995, "logps/chosen": -91.29534149169922, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.18516540527344, "loss": 1.2268, "margin_dpo/margin_mean": 1.7762103080749512, "margin_dpo/margin_std": 2.0590121746063232, "step": 65 }, { "KL/chosen_KL_mean": -0.3698234558105469, "KL/mean": -0.9966344833374023, "KL/rejected_KL_mean": -1.6234474182128906, "KL/std": 1.632431983947754, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10018503665924072, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2536189556121826, "fcm_dpo/q_t": 0.4691537320613861, "grad_norm": 25.53229522705078, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.23006492853164673, "logits/rejected": 0.4377473294734955, "logps/chosen": -84.154052734375, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.37554931640625, "loss": 1.2754, "margin_dpo/margin_mean": 1.2536194324493408, "margin_dpo/margin_std": 2.003659963607788, "step": 66 }, { "KL/chosen_KL_mean": -0.8769721984863281, "KL/mean": -1.4574064016342163, "KL/rejected_KL_mean": -2.0378341674804688, "KL/std": 2.0632896423339844, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10018503665924072, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1608614921569824, "fcm_dpo/q_t": 0.4719296097755432, "grad_norm": 24.312347412109375, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.4720457196235657, "logits/rejected": 0.759316623210907, "logps/chosen": -88.54993438720703, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.12458801269531, "loss": 1.2923, "margin_dpo/margin_mean": 1.160861849784851, "margin_dpo/margin_std": 2.7619481086730957, "step": 67 }, { "KL/chosen_KL_mean": -0.2960319519042969, "KL/mean": -1.3643372058868408, "KL/rejected_KL_mean": -2.432636260986328, "KL/std": 2.3314924240112305, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10027392208576202, "fcm_dpo/delta": 0.08864529430866241, "fcm_dpo/margin": 2.1366047859191895, "fcm_dpo/q_t": 0.4486663341522217, "grad_norm": 27.950355529785156, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.025473834946751595, "logits/rejected": 0.27600225806236267, "logps/chosen": -97.3748779296875, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.55136108398438, "loss": 1.2049, "margin_dpo/margin_mean": 2.1366047859191895, "margin_dpo/margin_std": 2.97811222076416, "step": 68 }, { "KL/chosen_KL_mean": -0.2703819274902344, "KL/mean": -1.4075684547424316, "KL/rejected_KL_mean": -2.5447540283203125, "KL/std": 2.469547748565674, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.10063183307647705, "fcm_dpo/delta": 0.17149794101715088, "fcm_dpo/margin": 2.2743735313415527, "fcm_dpo/q_t": 0.4452296495437622, "grad_norm": 30.395551681518555, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.2313685268163681, "logits/rejected": 0.34129202365875244, "logps/chosen": -85.99009704589844, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.02496337890625, "loss": 1.1937, "margin_dpo/margin_mean": 2.274374485015869, "margin_dpo/margin_std": 3.0694947242736816, "step": 69 }, { "KL/chosen_KL_mean": -0.6393280029296875, "KL/mean": -1.9120783805847168, "KL/rejected_KL_mean": -3.1848297119140625, "KL/std": 2.787428855895996, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.10092778503894806, "fcm_dpo/delta": 0.14345204830169678, "fcm_dpo/margin": 2.5455000400543213, "fcm_dpo/q_t": 0.4390297532081604, "grad_norm": 29.656530380249023, "learning_rate": 5e-07, "logits/chosen": 0.08797311782836914, "logits/rejected": 0.2549513280391693, "logps/chosen": -95.64927673339844, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.39755249023438, "loss": 1.174, "margin_dpo/margin_mean": 2.545499801635742, "margin_dpo/margin_std": 3.3174924850463867, "step": 70 }, { "KL/chosen_KL_mean": -0.07047653198242188, "KL/mean": -1.928748369216919, "KL/rejected_KL_mean": -3.7870216369628906, "KL/std": 3.605837106704712, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.10100771486759186, "fcm_dpo/delta": 0.02468518726527691, "fcm_dpo/margin": 3.716548442840576, "fcm_dpo/q_t": 0.4129292964935303, "grad_norm": 34.51301956176758, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.08633483946323395, "logits/rejected": 0.2826246917247772, "logps/chosen": -91.0478286743164, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.3780517578125, "loss": 1.0861, "margin_dpo/margin_mean": 3.716547966003418, "margin_dpo/margin_std": 4.188457012176514, "step": 71 }, { "KL/chosen_KL_mean": -0.5020866394042969, "KL/mean": -1.9630358219146729, "KL/rejected_KL_mean": -3.4239883422851562, "KL/std": 2.9500584602355957, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.10119447112083435, "fcm_dpo/delta": 0.1046375036239624, "fcm_dpo/margin": 2.921907901763916, "fcm_dpo/q_t": 0.42925816774368286, "grad_norm": 29.755680084228516, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.18364602327346802, "logits/rejected": 0.1398693472146988, "logps/chosen": -98.39361572265625, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.61570739746094, "loss": 1.1406, "margin_dpo/margin_mean": 2.921907424926758, "margin_dpo/margin_std": 3.3868348598480225, "step": 72 }, { "KL/chosen_KL_mean": -0.7555656433105469, "KL/mean": -2.29221773147583, "KL/rejected_KL_mean": -3.8288726806640625, "KL/std": 3.7820303440093994, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.10139669477939606, "fcm_dpo/delta": 0.08864926546812057, "fcm_dpo/margin": 3.073312759399414, "fcm_dpo/q_t": 0.4291788935661316, "grad_norm": 28.4433536529541, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.27178168296813965, "logits/rejected": 0.038581155240535736, "logps/chosen": -96.52458190917969, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.76130676269531, "loss": 1.1537, "margin_dpo/margin_mean": 3.073312759399414, "margin_dpo/margin_std": 4.824321746826172, "step": 73 }, { "KL/chosen_KL_mean": -1.0230751037597656, "KL/mean": -2.7614989280700684, "KL/rejected_KL_mean": -4.499919891357422, "KL/std": 3.7975244522094727, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.10149715095758438, "fcm_dpo/delta": 0.047280099242925644, "fcm_dpo/margin": 3.476841449737549, "fcm_dpo/q_t": 0.4181138277053833, "grad_norm": 31.657045364379883, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.29377904534339905, "logits/rejected": -0.15141594409942627, "logps/chosen": -79.83146667480469, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.14669036865234, "loss": 1.1123, "margin_dpo/margin_mean": 3.476841449737549, "margin_dpo/margin_std": 4.473217487335205, "step": 74 }, { "KL/chosen_KL_mean": -1.079193115234375, "KL/mean": -3.37418270111084, "KL/rejected_KL_mean": -5.669170379638672, "KL/std": 4.62941312789917, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.1014745682477951, "fcm_dpo/delta": -0.0660800114274025, "fcm_dpo/margin": 4.5899810791015625, "fcm_dpo/q_t": 0.3961049020290375, "grad_norm": 32.20328140258789, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.08339934796094894, "logits/rejected": 0.11681854724884033, "logps/chosen": -79.36104583740234, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.07228088378906, "loss": 1.0431, "margin_dpo/margin_mean": 4.589982032775879, "margin_dpo/margin_std": 5.5023579597473145, "step": 75 }, { "KL/chosen_KL_mean": -1.4165267944335938, "KL/mean": -2.999178409576416, "KL/rejected_KL_mean": -4.5818328857421875, "KL/std": 3.852570056915283, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.10152067244052887, "fcm_dpo/delta": 0.0789085328578949, "fcm_dpo/margin": 3.1653122901916504, "fcm_dpo/q_t": 0.4261290729045868, "grad_norm": 23.016029357910156, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.49073559045791626, "logits/rejected": -0.0994950458407402, "logps/chosen": -89.29367065429688, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.301513671875, "loss": 1.1418, "margin_dpo/margin_mean": 3.1653122901916504, "margin_dpo/margin_std": 4.634394645690918, "step": 76 }, { "KL/chosen_KL_mean": -1.6045989990234375, "KL/mean": -3.317474842071533, "KL/rejected_KL_mean": -5.0303497314453125, "KL/std": 4.509561538696289, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.10161743313074112, "fcm_dpo/delta": 0.052066680043935776, "fcm_dpo/margin": 3.4257547855377197, "fcm_dpo/q_t": 0.4198679029941559, "grad_norm": 27.30377197265625, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.5198691487312317, "logits/rejected": -0.18039153516292572, "logps/chosen": -101.30494689941406, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.23611450195312, "loss": 1.1325, "margin_dpo/margin_mean": 3.4257540702819824, "margin_dpo/margin_std": 5.134395599365234, "step": 77 }, { "KL/chosen_KL_mean": -0.6762123107910156, "KL/mean": -2.5207619667053223, "KL/rejected_KL_mean": -4.3653106689453125, "KL/std": 3.7102205753326416, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.10170247405767441, "fcm_dpo/delta": 0.024903487414121628, "fcm_dpo/margin": 3.6890969276428223, "fcm_dpo/q_t": 0.4125909209251404, "grad_norm": 25.507177352905273, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.032107315957546234, "logits/rejected": 0.2656756341457367, "logps/chosen": -90.9729232788086, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.50303649902344, "loss": 1.0904, "margin_dpo/margin_mean": 3.6890971660614014, "margin_dpo/margin_std": 4.329761505126953, "step": 78 }, { "KL/chosen_KL_mean": -0.9091720581054688, "KL/mean": -3.296177864074707, "KL/rejected_KL_mean": -5.683193206787109, "KL/std": 5.369949817657471, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.10160185396671295, "fcm_dpo/delta": -0.08545564860105515, "fcm_dpo/margin": 4.774021625518799, "fcm_dpo/q_t": 0.3918461799621582, "grad_norm": 26.995750427246094, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.413083553314209, "logits/rejected": -0.01541232317686081, "logps/chosen": -87.2874984741211, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.79096221923828, "loss": 1.0396, "margin_dpo/margin_mean": 4.774021625518799, "margin_dpo/margin_std": 5.964939594268799, "step": 79 }, { "KL/chosen_KL_mean": -1.5429611206054688, "KL/mean": -3.4328694343566895, "KL/rejected_KL_mean": -5.322772979736328, "KL/std": 4.724245548248291, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.10157221555709839, "fcm_dpo/delta": 0.016137033700942993, "fcm_dpo/margin": 3.779811143875122, "fcm_dpo/q_t": 0.41323745250701904, "grad_norm": 25.869234085083008, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.6771330833435059, "logits/rejected": -0.31857359409332275, "logps/chosen": -95.51327514648438, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.89718627929688, "loss": 1.1003, "margin_dpo/margin_mean": 3.779810905456543, "margin_dpo/margin_std": 5.133353233337402, "step": 80 }, { "KL/chosen_KL_mean": -1.1257057189941406, "KL/mean": -3.8120288848876953, "KL/rejected_KL_mean": -6.498355865478516, "KL/std": 5.868895053863525, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.10130524635314941, "fcm_dpo/delta": -0.1451873779296875, "fcm_dpo/margin": 5.372653007507324, "fcm_dpo/q_t": 0.3845916986465454, "grad_norm": 27.581573486328125, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5604079961776733, "logits/rejected": -0.13369396328926086, "logps/chosen": -100.95582580566406, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.1708221435547, "loss": 1.0172, "margin_dpo/margin_mean": 5.372653484344482, "margin_dpo/margin_std": 6.938673973083496, "step": 81 }, { "KL/chosen_KL_mean": -1.9872665405273438, "KL/mean": -3.816495418548584, "KL/rejected_KL_mean": -5.645729064941406, "KL/std": 4.569018363952637, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.10138373076915741, "fcm_dpo/delta": 0.029116801917552948, "fcm_dpo/margin": 3.658463954925537, "fcm_dpo/q_t": 0.4151953458786011, "grad_norm": 26.279502868652344, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.284454882144928, "logits/rejected": -0.06938290596008301, "logps/chosen": -85.03324890136719, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.17167663574219, "loss": 1.1111, "margin_dpo/margin_mean": 3.6584649085998535, "margin_dpo/margin_std": 4.9883270263671875, "step": 82 }, { "KL/chosen_KL_mean": -2.4229049682617188, "KL/mean": -4.523355960845947, "KL/rejected_KL_mean": -6.623805999755859, "KL/std": 5.197084426879883, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.10129998624324799, "fcm_dpo/delta": -0.025661559775471687, "fcm_dpo/margin": 4.200908660888672, "fcm_dpo/q_t": 0.40713298320770264, "grad_norm": 27.386615753173828, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.5685172080993652, "logits/rejected": -0.1453399360179901, "logps/chosen": -94.59912109375, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.36845397949219, "loss": 1.1054, "margin_dpo/margin_mean": 4.200908660888672, "margin_dpo/margin_std": 6.616334915161133, "step": 83 }, { "KL/chosen_KL_mean": -3.1019821166992188, "KL/mean": -4.358128547668457, "KL/rejected_KL_mean": -5.614276885986328, "KL/std": 4.691093921661377, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.10127189010381699, "fcm_dpo/delta": -0.008106674067676067, "fcm_dpo/margin": 2.5122876167297363, "fcm_dpo/q_t": 0.4427907466888428, "grad_norm": 29.347213745117188, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.2349020540714264, "logits/rejected": 0.17547503113746643, "logps/chosen": -104.62847900390625, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.04035949707031, "loss": 1.2149, "margin_dpo/margin_mean": 2.512287139892578, "margin_dpo/margin_std": 5.031121730804443, "step": 84 }, { "KL/chosen_KL_mean": -2.7023544311523438, "KL/mean": -4.430598258972168, "KL/rejected_KL_mean": -6.158847808837891, "KL/std": 5.136780738830566, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.1013912558555603, "fcm_dpo/delta": 0.0496566966176033, "fcm_dpo/margin": 3.4564950466156006, "fcm_dpo/q_t": 0.42188799381256104, "grad_norm": 27.768068313598633, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.2612093985080719, "logits/rejected": 0.08066584914922714, "logps/chosen": -98.82972717285156, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.21403503417969, "loss": 1.1605, "margin_dpo/margin_mean": 3.456495761871338, "margin_dpo/margin_std": 6.263396739959717, "step": 85 }, { "KL/chosen_KL_mean": -2.127552032470703, "KL/mean": -4.34423828125, "KL/rejected_KL_mean": -6.5609283447265625, "KL/std": 5.934553146362305, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.10128194093704224, "fcm_dpo/delta": -0.0492546409368515, "fcm_dpo/margin": 4.433376789093018, "fcm_dpo/q_t": 0.40698760747909546, "grad_norm": 22.447429656982422, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.46003708243370056, "logits/rejected": -0.2117784321308136, "logps/chosen": -83.83181762695312, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.2164306640625, "loss": 1.1023, "margin_dpo/margin_mean": 4.433377265930176, "margin_dpo/margin_std": 7.391847610473633, "step": 86 }, { "KL/chosen_KL_mean": -2.3989410400390625, "KL/mean": -4.818713188171387, "KL/rejected_KL_mean": -7.238483428955078, "KL/std": 6.537402153015137, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.10110798478126526, "fcm_dpo/delta": -0.08979697525501251, "fcm_dpo/margin": 4.83954381942749, "fcm_dpo/q_t": 0.3927527070045471, "grad_norm": 27.133623123168945, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.8483157157897949, "logits/rejected": -0.47675108909606934, "logps/chosen": -95.08490753173828, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.15666198730469, "loss": 1.0895, "margin_dpo/margin_mean": 4.839544773101807, "margin_dpo/margin_std": 7.760378837585449, "step": 87 }, { "KL/chosen_KL_mean": -3.0596542358398438, "KL/mean": -4.948164463043213, "KL/rejected_KL_mean": -6.836673736572266, "KL/std": 6.329357624053955, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.10105055570602417, "fcm_dpo/delta": 0.01832209900021553, "fcm_dpo/margin": 3.777012348175049, "fcm_dpo/q_t": 0.4167185127735138, "grad_norm": 25.128189086914062, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.47822946310043335, "logits/rejected": -0.29005515575408936, "logps/chosen": -103.8226318359375, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.98704528808594, "loss": 1.1424, "margin_dpo/margin_mean": 3.777012348175049, "margin_dpo/margin_std": 6.548985004425049, "step": 88 }, { "KL/chosen_KL_mean": -2.3613967895507812, "KL/mean": -4.5060200691223145, "KL/rejected_KL_mean": -6.650642395019531, "KL/std": 5.4340362548828125, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.10104645788669586, "fcm_dpo/delta": -0.03358633816242218, "fcm_dpo/margin": 4.2892374992370605, "fcm_dpo/q_t": 0.409179151058197, "grad_norm": 23.80463218688965, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.8802242875099182, "logits/rejected": -0.5560389757156372, "logps/chosen": -92.05247497558594, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.21897888183594, "loss": 1.1051, "margin_dpo/margin_mean": 4.289237022399902, "margin_dpo/margin_std": 7.193824768066406, "step": 89 }, { "KL/chosen_KL_mean": -2.122264862060547, "KL/mean": -4.869468688964844, "KL/rejected_KL_mean": -7.616676330566406, "KL/std": 6.274743556976318, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.10087350010871887, "fcm_dpo/delta": -0.15507148206233978, "fcm_dpo/margin": 5.494409084320068, "fcm_dpo/q_t": 0.3828698992729187, "grad_norm": 26.342453002929688, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.523920476436615, "logits/rejected": -0.39874690771102905, "logps/chosen": -83.50482177734375, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.39381408691406, "loss": 1.0261, "margin_dpo/margin_mean": 5.494409084320068, "margin_dpo/margin_std": 7.449227333068848, "step": 90 }, { "KL/chosen_KL_mean": -2.471355438232422, "KL/mean": -4.777350902557373, "KL/rejected_KL_mean": -7.083343505859375, "KL/std": 6.214813709259033, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.10065225511789322, "fcm_dpo/delta": -0.06450621038675308, "fcm_dpo/margin": 4.611999988555908, "fcm_dpo/q_t": 0.4027501940727234, "grad_norm": 26.984130859375, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.36367493867874146, "logits/rejected": 0.055834926664829254, "logps/chosen": -99.6932373046875, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.06008911132812, "loss": 1.0876, "margin_dpo/margin_mean": 4.61199951171875, "margin_dpo/margin_std": 7.270169258117676, "step": 91 }, { "KL/chosen_KL_mean": -2.6889877319335938, "KL/mean": -4.701746940612793, "KL/rejected_KL_mean": -6.714500427246094, "KL/std": 6.043922424316406, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.10063016414642334, "fcm_dpo/delta": -0.005116356536746025, "fcm_dpo/margin": 4.025515556335449, "fcm_dpo/q_t": 0.411793977022171, "grad_norm": 23.456504821777344, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8331122398376465, "logits/rejected": -0.466678261756897, "logps/chosen": -87.21926879882812, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.2158203125, "loss": 1.1145, "margin_dpo/margin_mean": 4.025515556335449, "margin_dpo/margin_std": 6.45871639251709, "step": 92 }, { "KL/chosen_KL_mean": -3.022441864013672, "KL/mean": -4.923488616943359, "KL/rejected_KL_mean": -6.8245391845703125, "KL/std": 5.788304328918457, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.10065080970525742, "fcm_dpo/delta": 0.017373407259583473, "fcm_dpo/margin": 3.8020949363708496, "fcm_dpo/q_t": 0.4169471561908722, "grad_norm": 27.72394561767578, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.9531575441360474, "logits/rejected": -0.3855959177017212, "logps/chosen": -93.27287292480469, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.91875457763672, "loss": 1.1557, "margin_dpo/margin_mean": 3.8020949363708496, "margin_dpo/margin_std": 7.085574150085449, "step": 93 }, { "KL/chosen_KL_mean": -3.098094940185547, "KL/mean": -6.001584053039551, "KL/rejected_KL_mean": -8.905071258544922, "KL/std": 6.806950569152832, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.10028410702943802, "fcm_dpo/delta": -0.18369612097740173, "fcm_dpo/margin": 5.806982040405273, "fcm_dpo/q_t": 0.37538328766822815, "grad_norm": 27.160646438598633, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.7988095879554749, "logits/rejected": -0.4992251694202423, "logps/chosen": -95.55231475830078, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.15235900878906, "loss": 1.0237, "margin_dpo/margin_mean": 5.806981086730957, "margin_dpo/margin_std": 7.8356404304504395, "step": 94 }, { "KL/chosen_KL_mean": -2.473346710205078, "KL/mean": -4.775068283081055, "KL/rejected_KL_mean": -7.076789855957031, "KL/std": 5.761507034301758, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.10022330284118652, "fcm_dpo/delta": -0.06166748329997063, "fcm_dpo/margin": 4.603446006774902, "fcm_dpo/q_t": 0.40282171964645386, "grad_norm": 22.86842918395996, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.936827540397644, "logits/rejected": -0.6059766411781311, "logps/chosen": -81.34705352783203, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.66799926757812, "loss": 1.0894, "margin_dpo/margin_mean": 4.603446006774902, "margin_dpo/margin_std": 7.132168769836426, "step": 95 }, { "KL/chosen_KL_mean": -2.137969970703125, "KL/mean": -4.9974212646484375, "KL/rejected_KL_mean": -7.85687255859375, "KL/std": 6.785944938659668, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.0998680591583252, "fcm_dpo/delta": -0.17216992378234863, "fcm_dpo/margin": 5.718906402587891, "fcm_dpo/q_t": 0.37951359152793884, "grad_norm": 23.35358428955078, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.6109691262245178, "logits/rejected": -0.15020474791526794, "logps/chosen": -108.73013305664062, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.6249008178711, "loss": 1.0145, "margin_dpo/margin_mean": 5.718906402587891, "margin_dpo/margin_std": 7.491946220397949, "step": 96 }, { "KL/chosen_KL_mean": -3.2633628845214844, "KL/mean": -5.817957878112793, "KL/rejected_KL_mean": -8.372547149658203, "KL/std": 5.888993263244629, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.0997186154127121, "fcm_dpo/delta": -0.11007820069789886, "fcm_dpo/margin": 5.109185218811035, "fcm_dpo/q_t": 0.3877449631690979, "grad_norm": 27.023897171020508, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.5577013492584229, "logits/rejected": -0.15807361900806427, "logps/chosen": -91.94324493408203, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.18484497070312, "loss": 1.0314, "margin_dpo/margin_mean": 5.109184741973877, "margin_dpo/margin_std": 6.256231307983398, "step": 97 }, { "KL/chosen_KL_mean": -2.4728622436523438, "KL/mean": -5.372150421142578, "KL/rejected_KL_mean": -8.271442413330078, "KL/std": 5.696126937866211, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.09926992654800415, "fcm_dpo/delta": -0.17678959667682648, "fcm_dpo/margin": 5.798580646514893, "fcm_dpo/q_t": 0.37431514263153076, "grad_norm": 24.46671485900879, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.49308353662490845, "logits/rejected": -0.22103042900562286, "logps/chosen": -94.71536254882812, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.79092407226562, "loss": 0.9826, "margin_dpo/margin_mean": 5.798580169677734, "margin_dpo/margin_std": 6.443201065063477, "step": 98 }, { "KL/chosen_KL_mean": -1.8428955078125, "KL/mean": -5.033215522766113, "KL/rejected_KL_mean": -8.223541259765625, "KL/std": 6.77341890335083, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0988844484090805, "fcm_dpo/delta": -0.23242518305778503, "fcm_dpo/margin": 6.380652904510498, "fcm_dpo/q_t": 0.3735881447792053, "grad_norm": 24.07392692565918, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.7299935221672058, "logits/rejected": -0.5166575908660889, "logps/chosen": -74.02754211425781, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.1048583984375, "loss": 0.9853, "margin_dpo/margin_mean": 6.3806538581848145, "margin_dpo/margin_std": 8.568216323852539, "step": 99 }, { "KL/chosen_KL_mean": -2.1033172607421875, "KL/mean": -4.422401428222656, "KL/rejected_KL_mean": -6.741481781005859, "KL/std": 5.107292175292969, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.09864535927772522, "fcm_dpo/delta": -0.057860199362039566, "fcm_dpo/margin": 4.638158798217773, "fcm_dpo/q_t": 0.39649444818496704, "grad_norm": 22.027313232421875, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6982325315475464, "logits/rejected": -0.21585571765899658, "logps/chosen": -91.61714172363281, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.95861053466797, "loss": 1.0445, "margin_dpo/margin_mean": 4.638158798217773, "margin_dpo/margin_std": 5.406659126281738, "step": 100 }, { "KL/chosen_KL_mean": -3.4191932678222656, "KL/mean": -5.496621131896973, "KL/rejected_KL_mean": -7.5740509033203125, "KL/std": 6.654545783996582, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.09859767556190491, "fcm_dpo/delta": -0.009750362485647202, "fcm_dpo/margin": 4.154858589172363, "fcm_dpo/q_t": 0.40955331921577454, "grad_norm": 27.570446014404297, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.7024272680282593, "logits/rejected": -0.4281021058559418, "logps/chosen": -97.8223876953125, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.48406982421875, "loss": 1.1311, "margin_dpo/margin_mean": 4.154858589172363, "margin_dpo/margin_std": 7.127704620361328, "step": 101 }, { "KL/chosen_KL_mean": -3.9784469604492188, "KL/mean": -5.757425785064697, "KL/rejected_KL_mean": -7.536403656005859, "KL/std": 6.349556922912598, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.09856948256492615, "fcm_dpo/delta": -0.07190993428230286, "fcm_dpo/margin": 3.5579593181610107, "fcm_dpo/q_t": 0.4225142002105713, "grad_norm": 34.277706146240234, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.8030937910079956, "logits/rejected": -0.43363526463508606, "logps/chosen": -95.58787536621094, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.30532836914062, "loss": 1.181, "margin_dpo/margin_mean": 3.5579593181610107, "margin_dpo/margin_std": 7.103670597076416, "step": 102 }, { "KL/chosen_KL_mean": -3.0359458923339844, "KL/mean": -6.052759170532227, "KL/rejected_KL_mean": -9.069572448730469, "KL/std": 6.714746952056885, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.0981946736574173, "fcm_dpo/delta": -0.19366082549095154, "fcm_dpo/margin": 6.033627510070801, "fcm_dpo/q_t": 0.3716863691806793, "grad_norm": 26.494768142700195, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.670677900314331, "logits/rejected": -0.2697671353816986, "logps/chosen": -100.4383316040039, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.05125427246094, "loss": 1.0023, "margin_dpo/margin_mean": 6.033627986907959, "margin_dpo/margin_std": 7.592037200927734, "step": 103 }, { "KL/chosen_KL_mean": -2.295135498046875, "KL/mean": -5.490629196166992, "KL/rejected_KL_mean": -8.686119079589844, "KL/std": 7.241824150085449, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.09777620434761047, "fcm_dpo/delta": -0.22632403671741486, "fcm_dpo/margin": 6.390982627868652, "fcm_dpo/q_t": 0.3726058900356293, "grad_norm": 21.81463050842285, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.948860764503479, "logits/rejected": -0.6003296971321106, "logps/chosen": -97.15149688720703, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -105.71281433105469, "loss": 0.9876, "margin_dpo/margin_mean": 6.390983581542969, "margin_dpo/margin_std": 8.538022994995117, "step": 104 }, { "KL/chosen_KL_mean": -3.039722442626953, "KL/mean": -6.244039535522461, "KL/rejected_KL_mean": -9.448360443115234, "KL/std": 6.589520454406738, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.09739828109741211, "fcm_dpo/delta": -0.22553476691246033, "fcm_dpo/margin": 6.408634185791016, "fcm_dpo/q_t": 0.3673884868621826, "grad_norm": 25.4853458404541, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.1355304718017578, "logits/rejected": -0.6774089336395264, "logps/chosen": -99.79762268066406, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.02584838867188, "loss": 0.9927, "margin_dpo/margin_mean": 6.408634185791016, "margin_dpo/margin_std": 8.216533660888672, "step": 105 }, { "KL/chosen_KL_mean": -2.928661346435547, "KL/mean": -6.019885063171387, "KL/rejected_KL_mean": -9.111103057861328, "KL/std": 6.8341450691223145, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.09697973728179932, "fcm_dpo/delta": -0.20072859525680542, "fcm_dpo/margin": 6.182445526123047, "fcm_dpo/q_t": 0.37625640630722046, "grad_norm": 24.08431053161621, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.8420675992965698, "logits/rejected": -0.419597327709198, "logps/chosen": -95.36656951904297, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.12080383300781, "loss": 1.005, "margin_dpo/margin_mean": 6.182445526123047, "margin_dpo/margin_std": 8.252553939819336, "step": 106 }, { "KL/chosen_KL_mean": -2.162109375, "KL/mean": -5.702527046203613, "KL/rejected_KL_mean": -9.242950439453125, "KL/std": 8.639034271240234, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.0964369922876358, "fcm_dpo/delta": -0.28478938341140747, "fcm_dpo/margin": 7.080842971801758, "fcm_dpo/q_t": 0.3705853819847107, "grad_norm": 24.348461151123047, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.680878758430481, "logits/rejected": -0.15938912332057953, "logps/chosen": -107.28711700439453, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -110.42544555664062, "loss": 0.9939, "margin_dpo/margin_mean": 7.080843448638916, "margin_dpo/margin_std": 10.805340766906738, "step": 107 }, { "KL/chosen_KL_mean": -3.137920379638672, "KL/mean": -6.46371603012085, "KL/rejected_KL_mean": -9.789512634277344, "KL/std": 7.831457614898682, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.09589739143848419, "fcm_dpo/delta": -0.2395905703306198, "fcm_dpo/margin": 6.651585578918457, "fcm_dpo/q_t": 0.36936914920806885, "grad_norm": 23.33451271057129, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.0543665885925293, "logits/rejected": -0.5590004920959473, "logps/chosen": -92.34728240966797, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -111.62736511230469, "loss": 0.9891, "margin_dpo/margin_mean": 6.651585102081299, "margin_dpo/margin_std": 8.607019424438477, "step": 108 }, { "KL/chosen_KL_mean": -2.5016632080078125, "KL/mean": -6.608088970184326, "KL/rejected_KL_mean": -10.71451187133789, "KL/std": 8.073616027832031, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.09525707364082336, "fcm_dpo/delta": -0.3854684829711914, "fcm_dpo/margin": 8.21284294128418, "fcm_dpo/q_t": 0.3440091609954834, "grad_norm": 22.956329345703125, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.521445095539093, "logits/rejected": -0.29038339853286743, "logps/chosen": -102.68875122070312, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -114.25190734863281, "loss": 0.9203, "margin_dpo/margin_mean": 8.212843894958496, "margin_dpo/margin_std": 9.964780807495117, "step": 109 }, { "KL/chosen_KL_mean": -3.263286590576172, "KL/mean": -5.688720703125, "KL/rejected_KL_mean": -8.114158630371094, "KL/std": 6.491455078125, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.09494858980178833, "fcm_dpo/delta": -0.06099482625722885, "fcm_dpo/margin": 4.850872039794922, "fcm_dpo/q_t": 0.4007223844528198, "grad_norm": 27.07026481628418, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.35843178629875183, "logits/rejected": 0.035159558057785034, "logps/chosen": -110.15900421142578, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -93.92536163330078, "loss": 1.0886, "margin_dpo/margin_mean": 4.8508710861206055, "margin_dpo/margin_std": 7.406844139099121, "step": 110 }, { "KL/chosen_KL_mean": -2.8618316650390625, "KL/mean": -7.550187587738037, "KL/rejected_KL_mean": -12.238544464111328, "KL/std": 9.103717803955078, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.09423038363456726, "fcm_dpo/delta": -0.4882814884185791, "fcm_dpo/margin": 9.376714706420898, "fcm_dpo/q_t": 0.32487139105796814, "grad_norm": 21.42563247680664, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.9930541515350342, "logits/rejected": -0.713172972202301, "logps/chosen": -96.19120025634766, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -116.41603088378906, "loss": 0.8635, "margin_dpo/margin_mean": 9.376714706420898, "margin_dpo/margin_std": 10.041252136230469, "step": 111 }, { "KL/chosen_KL_mean": -3.1872215270996094, "KL/mean": -5.6426591873168945, "KL/rejected_KL_mean": -8.098094940185547, "KL/std": 5.253430366516113, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.09395889937877655, "fcm_dpo/delta": -0.061718836426734924, "fcm_dpo/margin": 4.910871505737305, "fcm_dpo/q_t": 0.3986274003982544, "grad_norm": 23.777278900146484, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.2049190998077393, "logits/rejected": -0.7215330004692078, "logps/chosen": -85.93426513671875, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -79.65653991699219, "loss": 1.0754, "margin_dpo/margin_mean": 4.910872459411621, "margin_dpo/margin_std": 7.0259528160095215, "step": 112 }, { "KL/chosen_KL_mean": -4.031890869140625, "KL/mean": -6.618229866027832, "KL/rejected_KL_mean": -9.204566955566406, "KL/std": 6.841569423675537, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.09382923692464828, "fcm_dpo/delta": -0.08576002717018127, "fcm_dpo/margin": 5.172676086425781, "fcm_dpo/q_t": 0.39302459359169006, "grad_norm": 27.12205696105957, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.9474678039550781, "logits/rejected": -0.43384280800819397, "logps/chosen": -101.93138122558594, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -93.62416076660156, "loss": 1.0509, "margin_dpo/margin_mean": 5.1726765632629395, "margin_dpo/margin_std": 6.603273391723633, "step": 113 }, { "KL/chosen_KL_mean": -3.6440467834472656, "KL/mean": -7.232275009155273, "KL/rejected_KL_mean": -10.820499420166016, "KL/std": 7.942083358764648, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.09336521476507187, "fcm_dpo/delta": -0.271990031003952, "fcm_dpo/margin": 7.176455497741699, "fcm_dpo/q_t": 0.367950439453125, "grad_norm": 27.933971405029297, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.9653228521347046, "logits/rejected": -0.3922114372253418, "logps/chosen": -100.13916015625, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -102.83586120605469, "loss": 1.009, "margin_dpo/margin_mean": 7.176455974578857, "margin_dpo/margin_std": 10.260711669921875, "step": 114 }, { "KL/chosen_KL_mean": -3.608051300048828, "KL/mean": -7.298138618469238, "KL/rejected_KL_mean": -10.988227844238281, "KL/std": 8.429141998291016, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.09288573265075684, "fcm_dpo/delta": -0.28744977712631226, "fcm_dpo/margin": 7.380176544189453, "fcm_dpo/q_t": 0.3590068817138672, "grad_norm": 23.89215850830078, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.8783937692642212, "logits/rejected": -0.4778848886489868, "logps/chosen": -106.78817749023438, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -116.70418548583984, "loss": 0.9601, "margin_dpo/margin_mean": 7.380176544189453, "margin_dpo/margin_std": 9.02391529083252, "step": 115 }, { "KL/chosen_KL_mean": -3.2563133239746094, "KL/mean": -6.260927200317383, "KL/rejected_KL_mean": -9.26553726196289, "KL/std": 6.475663185119629, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.09256267547607422, "fcm_dpo/delta": -0.15707366168498993, "fcm_dpo/margin": 6.009226322174072, "fcm_dpo/q_t": 0.38000011444091797, "grad_norm": 21.038705825805664, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.0326772928237915, "logits/rejected": -0.6126086711883545, "logps/chosen": -84.41468811035156, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -95.57139587402344, "loss": 1.0104, "margin_dpo/margin_mean": 6.0092267990112305, "margin_dpo/margin_std": 7.43386173248291, "step": 116 }, { "KL/chosen_KL_mean": -3.918853759765625, "KL/mean": -6.442714214324951, "KL/rejected_KL_mean": -8.966571807861328, "KL/std": 6.613154411315918, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.0922829881310463, "fcm_dpo/delta": -0.06630825251340866, "fcm_dpo/margin": 5.04771614074707, "fcm_dpo/q_t": 0.39793938398361206, "grad_norm": 33.65953826904297, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.6724388599395752, "logits/rejected": -0.2745018005371094, "logps/chosen": -100.99980926513672, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -84.10649108886719, "loss": 1.0972, "margin_dpo/margin_mean": 5.04771614074707, "margin_dpo/margin_std": 7.955924034118652, "step": 117 }, { "KL/chosen_KL_mean": -2.7608070373535156, "KL/mean": -6.217191696166992, "KL/rejected_KL_mean": -9.673576354980469, "KL/std": 7.112594127655029, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.09198175370693207, "fcm_dpo/delta": -0.23740798234939575, "fcm_dpo/margin": 6.912773132324219, "fcm_dpo/q_t": 0.3605251610279083, "grad_norm": 20.405004501342773, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.0077803134918213, "logits/rejected": -0.6788003444671631, "logps/chosen": -89.29704284667969, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -92.60922241210938, "loss": 0.9486, "margin_dpo/margin_mean": 6.912773132324219, "margin_dpo/margin_std": 7.396409034729004, "step": 118 }, { "KL/chosen_KL_mean": -4.003276824951172, "KL/mean": -7.565979957580566, "KL/rejected_KL_mean": -11.128681182861328, "KL/std": 8.288726806640625, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.09151984751224518, "fcm_dpo/delta": -0.25383639335632324, "fcm_dpo/margin": 7.125397682189941, "fcm_dpo/q_t": 0.3639588952064514, "grad_norm": 26.02359390258789, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.8965526819229126, "logits/rejected": -0.5774672031402588, "logps/chosen": -108.61235809326172, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -120.28649139404297, "loss": 1.0176, "margin_dpo/margin_mean": 7.125397682189941, "margin_dpo/margin_std": 10.262100219726562, "step": 119 }, { "KL/chosen_KL_mean": -3.0346221923828125, "KL/mean": -7.794302940368652, "KL/rejected_KL_mean": -12.553977966308594, "KL/std": 10.127025604248047, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.09078656136989594, "fcm_dpo/delta": -0.46826791763305664, "fcm_dpo/margin": 9.519355773925781, "fcm_dpo/q_t": 0.33782660961151123, "grad_norm": 32.20235824584961, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.155075192451477, "logits/rejected": -0.623038649559021, "logps/chosen": -117.7220458984375, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -123.7398452758789, "loss": 0.946, "margin_dpo/margin_mean": 9.519355773925781, "margin_dpo/margin_std": 12.720989227294922, "step": 120 }, { "KL/chosen_KL_mean": -2.724010467529297, "KL/mean": -6.959907531738281, "KL/rejected_KL_mean": -11.195816040039062, "KL/std": 7.761944770812988, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.09003736078739166, "fcm_dpo/delta": -0.36576521396636963, "fcm_dpo/margin": 8.471811294555664, "fcm_dpo/q_t": 0.3398788571357727, "grad_norm": 23.12799644470215, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.7847450971603394, "logits/rejected": -0.4590332806110382, "logps/chosen": -94.84066772460938, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -126.16871643066406, "loss": 0.8813, "margin_dpo/margin_mean": 8.471811294555664, "margin_dpo/margin_std": 8.532042503356934, "step": 121 }, { "KL/chosen_KL_mean": -3.0685386657714844, "KL/mean": -7.823485851287842, "KL/rejected_KL_mean": -12.57843017578125, "KL/std": 8.37373161315918, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.08935344219207764, "fcm_dpo/delta": -0.4534040689468384, "fcm_dpo/margin": 9.509891510009766, "fcm_dpo/q_t": 0.324379563331604, "grad_norm": 22.816877365112305, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.8568053245544434, "logits/rejected": -0.7459336519241333, "logps/chosen": -87.94610595703125, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -123.66141510009766, "loss": 0.8592, "margin_dpo/margin_mean": 9.509889602661133, "margin_dpo/margin_std": 9.578300476074219, "step": 122 }, { "KL/chosen_KL_mean": -3.3493728637695312, "KL/mean": -6.903986930847168, "KL/rejected_KL_mean": -10.458602905273438, "KL/std": 7.7164225578308105, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.0887095183134079, "fcm_dpo/delta": -0.23249441385269165, "fcm_dpo/margin": 7.109230041503906, "fcm_dpo/q_t": 0.3668935298919678, "grad_norm": 21.65243911743164, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.8254730701446533, "logits/rejected": -0.5045914053916931, "logps/chosen": -101.828125, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -104.31277465820312, "loss": 0.9841, "margin_dpo/margin_mean": 7.10922908782959, "margin_dpo/margin_std": 8.754646301269531, "step": 123 }, { "KL/chosen_KL_mean": -3.0496482849121094, "KL/mean": -7.257837772369385, "KL/rejected_KL_mean": -11.466022491455078, "KL/std": 9.579843521118164, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.0882401317358017, "fcm_dpo/delta": -0.3451850116252899, "fcm_dpo/margin": 8.416370391845703, "fcm_dpo/q_t": 0.3634570240974426, "grad_norm": 22.970251083374023, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.7438167333602905, "logits/rejected": -0.4822293817996979, "logps/chosen": -105.81254577636719, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -134.4716339111328, "loss": 0.9878, "margin_dpo/margin_mean": 8.416369438171387, "margin_dpo/margin_std": 12.413213729858398, "step": 124 }, { "KL/chosen_KL_mean": -4.461433410644531, "KL/mean": -7.553355693817139, "KL/rejected_KL_mean": -10.645278930664062, "KL/std": 7.60814905166626, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.08788082003593445, "fcm_dpo/delta": -0.14423368871212006, "fcm_dpo/margin": 6.183838844299316, "fcm_dpo/q_t": 0.38644877076148987, "grad_norm": 25.621143341064453, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.5699500441551208, "logits/rejected": -0.3611948490142822, "logps/chosen": -94.88771057128906, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -99.89429473876953, "loss": 1.0578, "margin_dpo/margin_mean": 6.183838844299316, "margin_dpo/margin_std": 9.266258239746094, "step": 125 }, { "KL/chosen_KL_mean": -3.3916244506835938, "KL/mean": -7.840635299682617, "KL/rejected_KL_mean": -12.289653778076172, "KL/std": 8.124831199645996, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.08733153343200684, "fcm_dpo/delta": -0.38000136613845825, "fcm_dpo/margin": 8.89803695678711, "fcm_dpo/q_t": 0.33361583948135376, "grad_norm": 22.068389892578125, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.5599347949028015, "logits/rejected": -0.2803126275539398, "logps/chosen": -97.81450653076172, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -116.17121124267578, "loss": 0.8648, "margin_dpo/margin_mean": 8.89803695678711, "margin_dpo/margin_std": 8.355321884155273, "step": 126 }, { "KL/chosen_KL_mean": -3.3277664184570312, "KL/mean": -6.775944709777832, "KL/rejected_KL_mean": -10.224117279052734, "KL/std": 7.684335231781006, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.0869387835264206, "fcm_dpo/delta": -0.20071417093276978, "fcm_dpo/margin": 6.8963541984558105, "fcm_dpo/q_t": 0.37578272819519043, "grad_norm": 24.56377410888672, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.7694369554519653, "logits/rejected": -0.4444342851638794, "logps/chosen": -88.07881927490234, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -93.76934814453125, "loss": 1.0206, "margin_dpo/margin_mean": 6.8963541984558105, "margin_dpo/margin_std": 9.616605758666992, "step": 127 }, { "KL/chosen_KL_mean": -3.2622756958007812, "KL/mean": -6.745588302612305, "KL/rejected_KL_mean": -10.22889518737793, "KL/std": 9.161133766174316, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.086591437458992, "fcm_dpo/delta": -0.20443235337734222, "fcm_dpo/margin": 6.966628074645996, "fcm_dpo/q_t": 0.3707563281059265, "grad_norm": 27.25884246826172, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.40674087405204773, "logits/rejected": 0.054376840591430664, "logps/chosen": -93.88191986083984, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -93.33236694335938, "loss": 1.0447, "margin_dpo/margin_mean": 6.96662712097168, "margin_dpo/margin_std": 10.900510787963867, "step": 128 }, { "KL/chosen_KL_mean": -3.6181640625, "KL/mean": -7.237700939178467, "KL/rejected_KL_mean": -10.857246398925781, "KL/std": 7.526595115661621, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.08620011806488037, "fcm_dpo/delta": -0.22535648941993713, "fcm_dpo/margin": 7.239083766937256, "fcm_dpo/q_t": 0.3744106888771057, "grad_norm": 22.311716079711914, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.7229676842689514, "logits/rejected": -0.4347485303878784, "logps/chosen": -102.53687286376953, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -114.14144134521484, "loss": 1.0103, "margin_dpo/margin_mean": 7.239084720611572, "margin_dpo/margin_std": 9.797334671020508, "step": 129 }, { "KL/chosen_KL_mean": -2.9381103515625, "KL/mean": -7.531165599822998, "KL/rejected_KL_mean": -12.124221801757812, "KL/std": 9.055181503295898, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.08566069602966309, "fcm_dpo/delta": -0.3897724747657776, "fcm_dpo/margin": 9.18610954284668, "fcm_dpo/q_t": 0.3444591760635376, "grad_norm": 19.676156997680664, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.089937448501587, "logits/rejected": -0.8463212251663208, "logps/chosen": -76.12179565429688, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -100.14576721191406, "loss": 0.9093, "margin_dpo/margin_mean": 9.18610954284668, "margin_dpo/margin_std": 10.362530708312988, "step": 130 }, { "KL/chosen_KL_mean": -2.2840118408203125, "KL/mean": -6.2844438552856445, "KL/rejected_KL_mean": -10.28487777709961, "KL/std": 8.66209602355957, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.0851079523563385, "fcm_dpo/delta": -0.28276729583740234, "fcm_dpo/margin": 8.000861167907715, "fcm_dpo/q_t": 0.36005449295043945, "grad_norm": 20.750144958496094, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.7190581560134888, "logits/rejected": -0.500994861125946, "logps/chosen": -76.2404556274414, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -104.74070739746094, "loss": 0.9648, "margin_dpo/margin_mean": 8.000862121582031, "margin_dpo/margin_std": 9.764802932739258, "step": 131 }, { "KL/chosen_KL_mean": -3.3681182861328125, "KL/mean": -7.649886131286621, "KL/rejected_KL_mean": -11.931648254394531, "KL/std": 9.094335556030273, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.08446012437343597, "fcm_dpo/delta": -0.32584282755851746, "fcm_dpo/margin": 8.563531875610352, "fcm_dpo/q_t": 0.34734800457954407, "grad_norm": 24.996660232543945, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.0332775115966797, "logits/rejected": -0.6377445459365845, "logps/chosen": -95.80760192871094, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -106.06416320800781, "loss": 0.9288, "margin_dpo/margin_mean": 8.563531875610352, "margin_dpo/margin_std": 9.766902923583984, "step": 132 }, { "KL/chosen_KL_mean": -2.4120330810546875, "KL/mean": -6.815234184265137, "KL/rejected_KL_mean": -11.218441009521484, "KL/std": 9.192841529846191, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.08395412564277649, "fcm_dpo/delta": -0.341789186000824, "fcm_dpo/margin": 8.806411743164062, "fcm_dpo/q_t": 0.34704965353012085, "grad_norm": 21.083133697509766, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.3196005821228027, "logits/rejected": -0.9482972621917725, "logps/chosen": -95.52869415283203, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -113.3133544921875, "loss": 0.9029, "margin_dpo/margin_mean": 8.806411743164062, "margin_dpo/margin_std": 9.440458297729492, "step": 133 }, { "KL/chosen_KL_mean": -2.707653045654297, "KL/mean": -7.431875228881836, "KL/rejected_KL_mean": -12.156097412109375, "KL/std": 8.872961044311523, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.08337491750717163, "fcm_dpo/delta": -0.39065319299697876, "fcm_dpo/margin": 9.448441505432129, "fcm_dpo/q_t": 0.33600136637687683, "grad_norm": 19.582462310791016, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.579618215560913, "logits/rejected": -1.2739057540893555, "logps/chosen": -86.25778198242188, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -106.03766632080078, "loss": 0.877, "margin_dpo/margin_mean": 9.448440551757812, "margin_dpo/margin_std": 9.369983673095703, "step": 134 }, { "KL/chosen_KL_mean": -2.7595348358154297, "KL/mean": -7.027257919311523, "KL/rejected_KL_mean": -11.294990539550781, "KL/std": 9.620116233825684, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.08278828114271164, "fcm_dpo/delta": -0.3087228238582611, "fcm_dpo/margin": 8.535445213317871, "fcm_dpo/q_t": 0.35727038979530334, "grad_norm": 26.68060302734375, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.2972726821899414, "logits/rejected": -1.1382341384887695, "logps/chosen": -80.3421630859375, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -132.35147094726562, "loss": 0.9501, "margin_dpo/margin_mean": 8.535445213317871, "margin_dpo/margin_std": 10.579744338989258, "step": 135 }, { "KL/chosen_KL_mean": -2.6772384643554688, "KL/mean": -7.259044170379639, "KL/rejected_KL_mean": -11.84084701538086, "KL/std": 9.430416107177734, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.0821746289730072, "fcm_dpo/delta": -0.35574889183044434, "fcm_dpo/margin": 9.163610458374023, "fcm_dpo/q_t": 0.34021830558776855, "grad_norm": 20.144187927246094, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.188425898551941, "logits/rejected": -0.7645460367202759, "logps/chosen": -96.8499755859375, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -115.10513305664062, "loss": 0.8803, "margin_dpo/margin_mean": 9.163610458374023, "margin_dpo/margin_std": 8.986047744750977, "step": 136 }, { "KL/chosen_KL_mean": -2.1836624145507812, "KL/mean": -8.97458553314209, "KL/rejected_KL_mean": -15.7655029296875, "KL/std": 11.726319313049316, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.0811050683259964, "fcm_dpo/delta": -0.7100958824157715, "fcm_dpo/margin": 13.581846237182617, "fcm_dpo/q_t": 0.28908273577690125, "grad_norm": 20.94109535217285, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.216801404953003, "logits/rejected": -0.9871932864189148, "logps/chosen": -80.30239868164062, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -133.6878662109375, "loss": 0.7366, "margin_dpo/margin_mean": 13.58184814453125, "margin_dpo/margin_std": 12.44780158996582, "step": 137 }, { "KL/chosen_KL_mean": -4.187049865722656, "KL/mean": -8.587202072143555, "KL/rejected_KL_mean": -12.987358093261719, "KL/std": 9.042070388793945, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.0804855152964592, "fcm_dpo/delta": -0.3105945587158203, "fcm_dpo/margin": 8.80030632019043, "fcm_dpo/q_t": 0.35242828726768494, "grad_norm": 19.46355628967285, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.38615065813064575, "logits/rejected": -0.13328157365322113, "logps/chosen": -97.01699829101562, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -111.63575744628906, "loss": 0.9235, "margin_dpo/margin_mean": 8.80030632019043, "margin_dpo/margin_std": 9.706233978271484, "step": 138 }, { "KL/chosen_KL_mean": -2.6660003662109375, "KL/mean": -7.741569995880127, "KL/rejected_KL_mean": -12.817142486572266, "KL/std": 9.95403003692627, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.07985994219779968, "fcm_dpo/delta": -0.41417396068573, "fcm_dpo/margin": 10.151144027709961, "fcm_dpo/q_t": 0.3294999897480011, "grad_norm": 20.102283477783203, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.996090292930603, "logits/rejected": -0.6793619394302368, "logps/chosen": -81.66901397705078, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -98.64447021484375, "loss": 0.8951, "margin_dpo/margin_mean": 10.151145935058594, "margin_dpo/margin_std": 11.383627891540527, "step": 139 }, { "KL/chosen_KL_mean": -2.6967735290527344, "KL/mean": -7.320587158203125, "KL/rejected_KL_mean": -11.944404602050781, "KL/std": 9.526901245117188, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.07932353764772415, "fcm_dpo/delta": -0.33599919080734253, "fcm_dpo/margin": 9.247631072998047, "fcm_dpo/q_t": 0.3459613025188446, "grad_norm": 18.90019989013672, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.2433828115463257, "logits/rejected": -0.8564972877502441, "logps/chosen": -91.04922485351562, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -102.51918029785156, "loss": 0.9091, "margin_dpo/margin_mean": 9.247631072998047, "margin_dpo/margin_std": 9.851728439331055, "step": 140 }, { "KL/chosen_KL_mean": -3.4985008239746094, "KL/mean": -8.905546188354492, "KL/rejected_KL_mean": -14.31258773803711, "KL/std": 11.498414993286133, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.07865609973669052, "fcm_dpo/delta": -0.4544147253036499, "fcm_dpo/margin": 10.81409740447998, "fcm_dpo/q_t": 0.3263261318206787, "grad_norm": 23.319406509399414, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.032663345336914, "logits/rejected": -0.6798043847084045, "logps/chosen": -110.71865844726562, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -119.02481842041016, "loss": 0.8603, "margin_dpo/margin_mean": 10.814096450805664, "margin_dpo/margin_std": 11.254874229431152, "step": 141 }, { "KL/chosen_KL_mean": -4.305061340332031, "KL/mean": -9.37860107421875, "KL/rejected_KL_mean": -14.452140808105469, "KL/std": 10.866232872009277, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.07797658443450928, "fcm_dpo/delta": -0.3945319652557373, "fcm_dpo/margin": 10.147071838378906, "fcm_dpo/q_t": 0.3401244878768921, "grad_norm": 19.630823135375977, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.8691180348396301, "logits/rejected": -0.6192010045051575, "logps/chosen": -93.03401184082031, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -114.18252563476562, "loss": 0.885, "margin_dpo/margin_mean": 10.14707088470459, "margin_dpo/margin_std": 11.207542419433594, "step": 142 }, { "KL/chosen_KL_mean": -2.464527130126953, "KL/mean": -6.860726833343506, "KL/rejected_KL_mean": -11.25692367553711, "KL/std": 9.658327102661133, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.07747018337249756, "fcm_dpo/delta": -0.2833544909954071, "fcm_dpo/margin": 8.792393684387207, "fcm_dpo/q_t": 0.35788819193840027, "grad_norm": 19.73146629333496, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.2936069965362549, "logits/rejected": -0.8556095361709595, "logps/chosen": -87.96910095214844, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -97.06258392333984, "loss": 0.9605, "margin_dpo/margin_mean": 8.792393684387207, "margin_dpo/margin_std": 10.735794067382812, "step": 143 }, { "KL/chosen_KL_mean": -2.6934967041015625, "KL/mean": -7.36543083190918, "KL/rejected_KL_mean": -12.037361145019531, "KL/std": 9.948762893676758, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.0771104097366333, "fcm_dpo/delta": -0.32270902395248413, "fcm_dpo/margin": 9.343866348266602, "fcm_dpo/q_t": 0.35554131865501404, "grad_norm": 25.597143173217773, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.151316523551941, "logits/rejected": -0.6577624678611755, "logps/chosen": -107.38058471679688, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -99.75643920898438, "loss": 0.9711, "margin_dpo/margin_mean": 9.343866348266602, "margin_dpo/margin_std": 12.313922882080078, "step": 144 }, { "KL/chosen_KL_mean": -2.4618568420410156, "KL/mean": -7.692039966583252, "KL/rejected_KL_mean": -12.922222137451172, "KL/std": 9.713994979858398, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.07644686102867126, "fcm_dpo/delta": -0.4029063284397125, "fcm_dpo/margin": 10.460367202758789, "fcm_dpo/q_t": 0.3336585760116577, "grad_norm": 19.826906204223633, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.7908514142036438, "logits/rejected": -0.22048267722129822, "logps/chosen": -90.98175048828125, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -96.68736267089844, "loss": 0.8779, "margin_dpo/margin_mean": 10.460367202758789, "margin_dpo/margin_std": 10.916765213012695, "step": 145 }, { "KL/chosen_KL_mean": -2.578044891357422, "KL/mean": -8.422645568847656, "KL/rejected_KL_mean": -14.267242431640625, "KL/std": 10.380115509033203, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.07567799091339111, "fcm_dpo/delta": -0.4894665479660034, "fcm_dpo/margin": 11.689190864562988, "fcm_dpo/q_t": 0.31694358587265015, "grad_norm": 20.249095916748047, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.1171001195907593, "logits/rejected": -0.5222502946853638, "logps/chosen": -105.4839096069336, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -103.21660614013672, "loss": 0.8235, "margin_dpo/margin_mean": 11.689190864562988, "margin_dpo/margin_std": 10.792522430419922, "step": 146 }, { "KL/chosen_KL_mean": -2.8199729919433594, "KL/mean": -8.89459228515625, "KL/rejected_KL_mean": -14.96921157836914, "KL/std": 10.539289474487305, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.07497379928827286, "fcm_dpo/delta": -0.5155606269836426, "fcm_dpo/margin": 12.149245262145996, "fcm_dpo/q_t": 0.3118722438812256, "grad_norm": 18.114612579345703, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.253385305404663, "logits/rejected": -0.7075932025909424, "logps/chosen": -92.08216094970703, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -91.46932983398438, "loss": 0.8078, "margin_dpo/margin_mean": 12.14924430847168, "margin_dpo/margin_std": 11.063720703125, "step": 147 }, { "KL/chosen_KL_mean": -1.2716751098632812, "KL/mean": -6.396876335144043, "KL/rejected_KL_mean": -11.522071838378906, "KL/std": 10.100503921508789, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.07445283234119415, "fcm_dpo/delta": -0.365805983543396, "fcm_dpo/margin": 10.250398635864258, "fcm_dpo/q_t": 0.34801289439201355, "grad_norm": 20.357772827148438, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.139865756034851, "logits/rejected": -0.7476779222488403, "logps/chosen": -93.03456115722656, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -121.90760040283203, "loss": 0.9184, "margin_dpo/margin_mean": 10.250398635864258, "margin_dpo/margin_std": 11.513269424438477, "step": 148 }, { "KL/chosen_KL_mean": -3.6902503967285156, "KL/mean": -8.700223922729492, "KL/rejected_KL_mean": -13.710193634033203, "KL/std": 10.6320161819458, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.07382503151893616, "fcm_dpo/delta": -0.3424139618873596, "fcm_dpo/margin": 10.019937515258789, "fcm_dpo/q_t": 0.35529524087905884, "grad_norm": 20.416854858398438, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.879864513874054, "logits/rejected": -0.577896773815155, "logps/chosen": -97.17427062988281, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -119.35389709472656, "loss": 0.9438, "margin_dpo/margin_mean": 10.019939422607422, "margin_dpo/margin_std": 12.679777145385742, "step": 149 }, { "KL/chosen_KL_mean": -2.6922683715820312, "KL/mean": -8.171966552734375, "KL/rejected_KL_mean": -13.651664733886719, "KL/std": 11.630842208862305, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.07325714081525803, "fcm_dpo/delta": -0.40625452995300293, "fcm_dpo/margin": 10.959393501281738, "fcm_dpo/q_t": 0.33984094858169556, "grad_norm": 18.257734298706055, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.9460018873214722, "logits/rejected": -0.6737295389175415, "logps/chosen": -78.47095489501953, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -103.67842864990234, "loss": 0.8979, "margin_dpo/margin_mean": 10.959393501281738, "margin_dpo/margin_std": 12.800958633422852, "step": 150 }, { "KL/chosen_KL_mean": -4.135402679443359, "KL/mean": -9.276336669921875, "KL/rejected_KL_mean": -14.41726303100586, "KL/std": 11.812911987304688, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.07274993509054184, "fcm_dpo/delta": -0.35061249136924744, "fcm_dpo/margin": 10.281861305236816, "fcm_dpo/q_t": 0.3502689301967621, "grad_norm": 23.60828399658203, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.7054204940795898, "logits/rejected": -1.144309401512146, "logps/chosen": -106.86962890625, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -120.455078125, "loss": 0.94, "margin_dpo/margin_mean": 10.281862258911133, "margin_dpo/margin_std": 12.823887825012207, "step": 151 }, { "KL/chosen_KL_mean": -1.3869781494140625, "KL/mean": -7.570573806762695, "KL/rejected_KL_mean": -13.754165649414062, "KL/std": 11.256210327148438, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.07206709682941437, "fcm_dpo/delta": -0.49579304456710815, "fcm_dpo/margin": 12.367178916931152, "fcm_dpo/q_t": 0.31807219982147217, "grad_norm": 19.038156509399414, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.1668212413787842, "logits/rejected": -0.8041491508483887, "logps/chosen": -98.97772216796875, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -124.2532958984375, "loss": 0.828, "margin_dpo/margin_mean": 12.367179870605469, "margin_dpo/margin_std": 12.311376571655273, "step": 152 }, { "KL/chosen_KL_mean": -3.969799041748047, "KL/mean": -8.42722225189209, "KL/rejected_KL_mean": -12.884654998779297, "KL/std": 10.830718040466309, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.07163818180561066, "fcm_dpo/delta": -0.24019905924797058, "fcm_dpo/margin": 8.914863586425781, "fcm_dpo/q_t": 0.3679061532020569, "grad_norm": 19.715089797973633, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.3888646364212036, "logits/rejected": -1.0705900192260742, "logps/chosen": -90.99539947509766, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -103.95319366455078, "loss": 0.9985, "margin_dpo/margin_mean": 8.914861679077148, "margin_dpo/margin_std": 11.803839683532715, "step": 153 }, { "KL/chosen_KL_mean": -2.2713699340820312, "KL/mean": -9.475587844848633, "KL/rejected_KL_mean": -16.6798095703125, "KL/std": 12.058868408203125, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.07097921520471573, "fcm_dpo/delta": -0.6286579370498657, "fcm_dpo/margin": 14.408443450927734, "fcm_dpo/q_t": 0.2947330176830292, "grad_norm": 18.122282028198242, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.0126659870147705, "logits/rejected": -0.6860195398330688, "logps/chosen": -93.37703704833984, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -128.05218505859375, "loss": 0.7488, "margin_dpo/margin_mean": 14.408442497253418, "margin_dpo/margin_std": 11.788230895996094, "step": 154 }, { "KL/chosen_KL_mean": -3.7891693115234375, "KL/mean": -9.611421585083008, "KL/rejected_KL_mean": -15.433666229248047, "KL/std": 12.462697982788086, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.07016189396381378, "fcm_dpo/delta": -0.4209156036376953, "fcm_dpo/margin": 11.644501686096191, "fcm_dpo/q_t": 0.33443909883499146, "grad_norm": 26.1427001953125, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.8530440330505371, "logits/rejected": -0.49821460247039795, "logps/chosen": -98.35014343261719, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -100.20726013183594, "loss": 0.9427, "margin_dpo/margin_mean": 11.644500732421875, "margin_dpo/margin_std": 14.71336555480957, "step": 155 }, { "KL/chosen_KL_mean": -5.241752624511719, "KL/mean": -10.224550247192383, "KL/rejected_KL_mean": -15.207347869873047, "KL/std": 10.989608764648438, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.0697304755449295, "fcm_dpo/delta": -0.2971305847167969, "fcm_dpo/margin": 9.965591430664062, "fcm_dpo/q_t": 0.35853058099746704, "grad_norm": 23.143661499023438, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.3572912216186523, "logits/rejected": -0.8876527547836304, "logps/chosen": -99.89714050292969, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -99.62669372558594, "loss": 0.967, "margin_dpo/margin_mean": 9.965591430664062, "margin_dpo/margin_std": 12.553030014038086, "step": 156 }, { "KL/chosen_KL_mean": -4.7501983642578125, "KL/mean": -11.329426765441895, "KL/rejected_KL_mean": -17.90865707397461, "KL/std": 11.689775466918945, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.06914303451776505, "fcm_dpo/delta": -0.5143572092056274, "fcm_dpo/margin": 13.158454895019531, "fcm_dpo/q_t": 0.31289151310920715, "grad_norm": 19.658315658569336, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.565942406654358, "logits/rejected": -1.1171646118164062, "logps/chosen": -101.76258850097656, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -129.61000061035156, "loss": 0.8079, "margin_dpo/margin_mean": 13.158454895019531, "margin_dpo/margin_std": 12.033400535583496, "step": 157 }, { "KL/chosen_KL_mean": -4.261363983154297, "KL/mean": -11.099482536315918, "KL/rejected_KL_mean": -17.937599182128906, "KL/std": 11.938497543334961, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.06841342151165009, "fcm_dpo/delta": -0.5405218601226807, "fcm_dpo/margin": 13.676239013671875, "fcm_dpo/q_t": 0.308566153049469, "grad_norm": 19.358489990234375, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.23154616355896, "logits/rejected": -0.9440091848373413, "logps/chosen": -98.97432708740234, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -138.87493896484375, "loss": 0.7945, "margin_dpo/margin_mean": 13.676239013671875, "margin_dpo/margin_std": 12.284297943115234, "step": 158 }, { "KL/chosen_KL_mean": -3.463092803955078, "KL/mean": -9.8641939163208, "KL/rejected_KL_mean": -16.265300750732422, "KL/std": 12.475760459899902, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.06770861148834229, "fcm_dpo/delta": -0.47100722789764404, "fcm_dpo/margin": 12.802214622497559, "fcm_dpo/q_t": 0.32681912183761597, "grad_norm": 17.84327507019043, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.612884283065796, "logits/rejected": -1.142998218536377, "logps/chosen": -111.68133544921875, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -111.10432434082031, "loss": 0.8704, "margin_dpo/margin_mean": 12.802213668823242, "margin_dpo/margin_std": 13.998052597045898, "step": 159 }, { "KL/chosen_KL_mean": -2.6879959106445312, "KL/mean": -8.87160873413086, "KL/rejected_KL_mean": -15.055217742919922, "KL/std": 11.735422134399414, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.06711039692163467, "fcm_dpo/delta": -0.43367159366607666, "fcm_dpo/margin": 12.367219924926758, "fcm_dpo/q_t": 0.3261496424674988, "grad_norm": 17.86648178100586, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.341882586479187, "logits/rejected": -0.8081585168838501, "logps/chosen": -100.85140991210938, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -117.33619689941406, "loss": 0.8578, "margin_dpo/margin_mean": 12.367219924926758, "margin_dpo/margin_std": 12.273863792419434, "step": 160 }, { "KL/chosen_KL_mean": -4.716373443603516, "KL/mean": -10.469066619873047, "KL/rejected_KL_mean": -16.221752166748047, "KL/std": 10.995260238647461, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.06656768172979355, "fcm_dpo/delta": -0.3690239191055298, "fcm_dpo/margin": 11.505392074584961, "fcm_dpo/q_t": 0.34293293952941895, "grad_norm": 19.78435707092285, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.552584171295166, "logits/rejected": -1.0833404064178467, "logps/chosen": -91.30824279785156, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -105.23818969726562, "loss": 0.9249, "margin_dpo/margin_mean": 11.505392074584961, "margin_dpo/margin_std": 13.251201629638672, "step": 161 }, { "KL/chosen_KL_mean": -3.9810333251953125, "KL/mean": -10.070123672485352, "KL/rejected_KL_mean": -16.15921401977539, "KL/std": 12.428529739379883, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.0660661906003952, "fcm_dpo/delta": -0.40795230865478516, "fcm_dpo/margin": 12.178182601928711, "fcm_dpo/q_t": 0.3301848769187927, "grad_norm": 17.521129608154297, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.111313819885254, "logits/rejected": -0.7324024438858032, "logps/chosen": -101.38391876220703, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -112.0284423828125, "loss": 0.8816, "margin_dpo/margin_mean": 12.178182601928711, "margin_dpo/margin_std": 12.674509048461914, "step": 162 }, { "KL/chosen_KL_mean": -3.603382110595703, "KL/mean": -11.263299942016602, "KL/rejected_KL_mean": -18.923221588134766, "KL/std": 14.581140518188477, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.06533589959144592, "fcm_dpo/delta": -0.6071469783782959, "fcm_dpo/margin": 15.319843292236328, "fcm_dpo/q_t": 0.3069203495979309, "grad_norm": 15.34990406036377, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.4223655462265015, "logits/rejected": -1.0390877723693848, "logps/chosen": -85.77877807617188, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -101.7879638671875, "loss": 0.7947, "margin_dpo/margin_mean": 15.319843292236328, "margin_dpo/margin_std": 15.49990463256836, "step": 163 }, { "KL/chosen_KL_mean": -6.6843719482421875, "KL/mean": -11.78413200378418, "KL/rejected_KL_mean": -16.88388442993164, "KL/std": 11.542847633361816, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.0648147314786911, "fcm_dpo/delta": -0.26343268156051636, "fcm_dpo/margin": 10.199514389038086, "fcm_dpo/q_t": 0.3588295578956604, "grad_norm": 18.765382766723633, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.1946685314178467, "logits/rejected": -0.7626661062240601, "logps/chosen": -97.15864562988281, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -103.91197204589844, "loss": 0.9653, "margin_dpo/margin_mean": 10.199514389038086, "margin_dpo/margin_std": 11.70454216003418, "step": 164 }, { "KL/chosen_KL_mean": -3.8745880126953125, "KL/mean": -11.38048267364502, "KL/rejected_KL_mean": -18.886371612548828, "KL/std": 14.303207397460938, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.06429398059844971, "fcm_dpo/delta": -0.5704137682914734, "fcm_dpo/margin": 15.011787414550781, "fcm_dpo/q_t": 0.31333357095718384, "grad_norm": 15.998414993286133, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.3264570236206055, "logits/rejected": -0.9300845861434937, "logps/chosen": -91.18782043457031, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -100.78895568847656, "loss": 0.8095, "margin_dpo/margin_mean": 15.011787414550781, "margin_dpo/margin_std": 16.050874710083008, "step": 165 }, { "KL/chosen_KL_mean": -7.978839874267578, "KL/mean": -15.01528549194336, "KL/rejected_KL_mean": -22.051727294921875, "KL/std": 14.300453186035156, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.06359690427780151, "fcm_dpo/delta": -0.49945029616355896, "fcm_dpo/margin": 14.072884559631348, "fcm_dpo/q_t": 0.330382764339447, "grad_norm": 17.591598510742188, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.250442624092102, "logits/rejected": -1.034780502319336, "logps/chosen": -89.56491088867188, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -125.21690368652344, "loss": 0.8675, "margin_dpo/margin_mean": 14.072885513305664, "margin_dpo/margin_std": 16.631484985351562, "step": 166 }, { "KL/chosen_KL_mean": -6.604496002197266, "KL/mean": -14.352569580078125, "KL/rejected_KL_mean": -22.100643157958984, "KL/std": 13.621309280395508, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.0628373771905899, "fcm_dpo/delta": -0.5799679756164551, "fcm_dpo/margin": 15.496145248413086, "fcm_dpo/q_t": 0.30275535583496094, "grad_norm": 16.896223068237305, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.6120105981826782, "logits/rejected": -0.4125170111656189, "logps/chosen": -90.16616821289062, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -121.15369415283203, "loss": 0.7667, "margin_dpo/margin_mean": 15.496145248413086, "margin_dpo/margin_std": 13.363641738891602, "step": 167 }, { "KL/chosen_KL_mean": -6.433296203613281, "KL/mean": -12.977036476135254, "KL/rejected_KL_mean": -19.52077865600586, "KL/std": 13.564901351928711, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.0623333677649498, "fcm_dpo/delta": -0.4190866947174072, "fcm_dpo/margin": 13.087477684020996, "fcm_dpo/q_t": 0.3316546380519867, "grad_norm": 19.433401107788086, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.2393245697021484, "logits/rejected": -0.9076457023620605, "logps/chosen": -86.36453247070312, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -99.62294006347656, "loss": 0.8725, "margin_dpo/margin_mean": 13.087477684020996, "margin_dpo/margin_std": 13.60348129272461, "step": 168 }, { "KL/chosen_KL_mean": -5.038852691650391, "KL/mean": -12.402597427368164, "KL/rejected_KL_mean": -19.766334533691406, "KL/std": 14.115592956542969, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.06171972304582596, "fcm_dpo/delta": -0.5135326981544495, "fcm_dpo/margin": 14.72748851776123, "fcm_dpo/q_t": 0.31834471225738525, "grad_norm": 17.534320831298828, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.017380714416504, "logits/rejected": -0.7851332426071167, "logps/chosen": -87.94255828857422, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -109.88824462890625, "loss": 0.8282, "margin_dpo/margin_mean": 14.72748851776123, "margin_dpo/margin_std": 14.870196342468262, "step": 169 }, { "KL/chosen_KL_mean": -6.858409881591797, "KL/mean": -13.547117233276367, "KL/rejected_KL_mean": -20.235828399658203, "KL/std": 12.389293670654297, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.06111665442585945, "fcm_dpo/delta": -0.42140600085258484, "fcm_dpo/margin": 13.377418518066406, "fcm_dpo/q_t": 0.3269529938697815, "grad_norm": 18.34777069091797, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.156548023223877, "logits/rejected": -0.9291719794273376, "logps/chosen": -81.84037780761719, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -102.40142822265625, "loss": 0.8497, "margin_dpo/margin_mean": 13.377418518066406, "margin_dpo/margin_std": 12.424830436706543, "step": 170 }, { "KL/chosen_KL_mean": -8.080696105957031, "KL/mean": -15.60304069519043, "KL/rejected_KL_mean": -23.12540054321289, "KL/std": 16.468647003173828, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.06056404858827591, "fcm_dpo/delta": -0.5158318281173706, "fcm_dpo/margin": 15.044708251953125, "fcm_dpo/q_t": 0.32950982451438904, "grad_norm": 21.22678565979004, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.4303252696990967, "logits/rejected": -0.9202205538749695, "logps/chosen": -127.05921936035156, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -138.89471435546875, "loss": 0.8851, "margin_dpo/margin_mean": 15.044708251953125, "margin_dpo/margin_std": 18.650789260864258, "step": 171 }, { "KL/chosen_KL_mean": -8.961109161376953, "KL/mean": -15.930999755859375, "KL/rejected_KL_mean": -22.90088653564453, "KL/std": 12.46424388885498, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.05997470021247864, "fcm_dpo/delta": -0.439994215965271, "fcm_dpo/margin": 13.93978500366211, "fcm_dpo/q_t": 0.3169255256652832, "grad_norm": 15.16685676574707, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -1.8114268779754639, "logits/rejected": -1.09841787815094, "logps/chosen": -110.11489868164062, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -113.90657043457031, "loss": 0.8022, "margin_dpo/margin_mean": 13.939783096313477, "margin_dpo/margin_std": 10.386411666870117, "step": 172 }, { "KL/chosen_KL_mean": -8.525764465332031, "KL/mean": -14.431989669799805, "KL/rejected_KL_mean": -20.338214874267578, "KL/std": 12.005834579467773, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.05960238724946976, "fcm_dpo/delta": -0.3061917722225189, "fcm_dpo/margin": 11.812453269958496, "fcm_dpo/q_t": 0.3500869572162628, "grad_norm": 16.22954559326172, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.1605799198150635, "logits/rejected": -0.7188180685043335, "logps/chosen": -107.58245086669922, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -104.18894958496094, "loss": 0.9256, "margin_dpo/margin_mean": 11.812453269958496, "margin_dpo/margin_std": 12.788844108581543, "step": 173 }, { "KL/chosen_KL_mean": -8.273637771606445, "KL/mean": -15.832807540893555, "KL/rejected_KL_mean": -23.391979217529297, "KL/std": 15.29289436340332, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.05907499045133591, "fcm_dpo/delta": -0.49749141931533813, "fcm_dpo/margin": 15.118345260620117, "fcm_dpo/q_t": 0.31557679176330566, "grad_norm": 18.75757598876953, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.661949872970581, "logits/rejected": -1.5887726545333862, "logps/chosen": -88.68254089355469, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -131.59368896484375, "loss": 0.827, "margin_dpo/margin_mean": 15.118345260620117, "margin_dpo/margin_std": 14.610942840576172, "step": 174 }, { "KL/chosen_KL_mean": -11.333972930908203, "KL/mean": -18.96572494506836, "KL/rejected_KL_mean": -26.59747314453125, "KL/std": 14.246315002441406, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.058520421385765076, "fcm_dpo/delta": -0.4974302649497986, "fcm_dpo/margin": 15.263494491577148, "fcm_dpo/q_t": 0.3144994378089905, "grad_norm": 18.882431030273438, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.8841269016265869, "logits/rejected": -0.6191408634185791, "logps/chosen": -94.05046081542969, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -123.26460266113281, "loss": 0.808, "margin_dpo/margin_mean": 15.263494491577148, "margin_dpo/margin_std": 13.372627258300781, "step": 175 }, { "KL/chosen_KL_mean": -9.126785278320312, "KL/mean": -16.77303695678711, "KL/rejected_KL_mean": -24.419288635253906, "KL/std": 15.407899856567383, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.05790897458791733, "fcm_dpo/delta": -0.4899134039878845, "fcm_dpo/margin": 15.292500495910645, "fcm_dpo/q_t": 0.328111469745636, "grad_norm": 16.72183609008789, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.3887665271759033, "logits/rejected": -1.2041984796524048, "logps/chosen": -88.75608825683594, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -127.10787963867188, "loss": 0.876, "margin_dpo/margin_mean": 15.292501449584961, "margin_dpo/margin_std": 17.171289443969727, "step": 176 }, { "KL/chosen_KL_mean": -9.832454681396484, "KL/mean": -18.792171478271484, "KL/rejected_KL_mean": -27.751888275146484, "KL/std": 17.540138244628906, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.057273492217063904, "fcm_dpo/delta": -0.6324001550674438, "fcm_dpo/margin": 17.919429779052734, "fcm_dpo/q_t": 0.3067034184932709, "grad_norm": 15.199719429016113, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.3002066612243652, "logits/rejected": -0.950985312461853, "logps/chosen": -102.3935546875, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -131.09027099609375, "loss": 0.7838, "margin_dpo/margin_mean": 17.919429779052734, "margin_dpo/margin_std": 18.353757858276367, "step": 177 }, { "KL/chosen_KL_mean": -9.034786224365234, "KL/mean": -19.54064178466797, "KL/rejected_KL_mean": -30.046485900878906, "KL/std": 16.965171813964844, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.056364770978689194, "fcm_dpo/delta": -0.793756365776062, "fcm_dpo/margin": 21.011701583862305, "fcm_dpo/q_t": 0.2744899392127991, "grad_norm": 17.871469497680664, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.379582166671753, "logits/rejected": -1.1139165163040161, "logps/chosen": -93.30058288574219, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -140.8437957763672, "loss": 0.6922, "margin_dpo/margin_mean": 21.011701583862305, "margin_dpo/margin_std": 18.27631187438965, "step": 178 }, { "KL/chosen_KL_mean": -11.82919692993164, "KL/mean": -19.003604888916016, "KL/rejected_KL_mean": -26.17801284790039, "KL/std": 16.336036682128906, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.05576995015144348, "fcm_dpo/delta": -0.40378040075302124, "fcm_dpo/margin": 14.348817825317383, "fcm_dpo/q_t": 0.345207154750824, "grad_norm": 19.90088653564453, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.2700904607772827, "logits/rejected": -0.8378313779830933, "logps/chosen": -105.27717590332031, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -120.43342590332031, "loss": 0.9497, "margin_dpo/margin_mean": 14.348817825317383, "margin_dpo/margin_std": 18.356767654418945, "step": 179 }, { "KL/chosen_KL_mean": -9.69158935546875, "KL/mean": -18.46627426147461, "KL/rejected_KL_mean": -27.24095916748047, "KL/std": 16.19736671447754, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.05518178641796112, "fcm_dpo/delta": -0.5743665099143982, "fcm_dpo/margin": 17.549373626708984, "fcm_dpo/q_t": 0.30537116527557373, "grad_norm": 20.445066452026367, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.115468978881836, "logits/rejected": -0.8608735799789429, "logps/chosen": -98.52713012695312, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -131.14791870117188, "loss": 0.7827, "margin_dpo/margin_mean": 17.549373626708984, "margin_dpo/margin_std": 16.42658233642578, "step": 180 }, { "KL/chosen_KL_mean": -10.174213409423828, "KL/mean": -17.072263717651367, "KL/rejected_KL_mean": -23.970317840576172, "KL/std": 15.030269622802734, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.054765619337558746, "fcm_dpo/delta": -0.35822516679763794, "fcm_dpo/margin": 13.79610824584961, "fcm_dpo/q_t": 0.3458385765552521, "grad_norm": 17.777538299560547, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.268015742301941, "logits/rejected": -0.8192813396453857, "logps/chosen": -89.40817260742188, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -93.14474487304688, "loss": 0.909, "margin_dpo/margin_mean": 13.79610824584961, "margin_dpo/margin_std": 15.618301391601562, "step": 181 }, { "KL/chosen_KL_mean": -13.052658081054688, "KL/mean": -21.906391143798828, "KL/rejected_KL_mean": -30.760120391845703, "KL/std": 17.49835968017578, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.05415799468755722, "fcm_dpo/delta": -0.5649915337562561, "fcm_dpo/margin": 17.707462310791016, "fcm_dpo/q_t": 0.3148183822631836, "grad_norm": 16.362428665161133, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.2241325378417969, "logits/rejected": -0.9721157550811768, "logps/chosen": -89.13484191894531, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -125.72872924804688, "loss": 0.8443, "margin_dpo/margin_mean": 17.707462310791016, "margin_dpo/margin_std": 19.20836639404297, "step": 182 }, { "KL/chosen_KL_mean": -12.93685531616211, "KL/mean": -21.929443359375, "KL/rejected_KL_mean": -30.92203140258789, "KL/std": 16.07942008972168, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.05356167256832123, "fcm_dpo/delta": -0.5691360831260681, "fcm_dpo/margin": 17.985172271728516, "fcm_dpo/q_t": 0.3072766363620758, "grad_norm": 17.37106704711914, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.538766622543335, "logits/rejected": -0.9663299322128296, "logps/chosen": -106.84475708007812, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -114.48265075683594, "loss": 0.7939, "margin_dpo/margin_mean": 17.985172271728516, "margin_dpo/margin_std": 16.658946990966797, "step": 183 }, { "KL/chosen_KL_mean": -11.754253387451172, "KL/mean": -17.73389434814453, "KL/rejected_KL_mean": -23.713531494140625, "KL/std": 16.365581512451172, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.0532379187643528, "fcm_dpo/delta": -0.2382405698299408, "fcm_dpo/margin": 11.959274291992188, "fcm_dpo/q_t": 0.3666236996650696, "grad_norm": 18.684947967529297, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.4283318519592285, "logits/rejected": -1.185692310333252, "logps/chosen": -98.62216186523438, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -107.99809265136719, "loss": 1.0347, "margin_dpo/margin_mean": 11.959275245666504, "margin_dpo/margin_std": 17.706817626953125, "step": 184 }, { "KL/chosen_KL_mean": -12.480968475341797, "KL/mean": -19.808059692382812, "KL/rejected_KL_mean": -27.13515853881836, "KL/std": 16.126781463623047, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.052910879254341125, "fcm_dpo/delta": -0.3781513571739197, "fcm_dpo/margin": 14.654195785522461, "fcm_dpo/q_t": 0.3464380204677582, "grad_norm": 20.755199432373047, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -1.8352341651916504, "logits/rejected": -1.2437409162521362, "logps/chosen": -106.50239562988281, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -108.59184265136719, "loss": 0.9503, "margin_dpo/margin_mean": 14.654194831848145, "margin_dpo/margin_std": 18.774959564208984, "step": 185 }, { "KL/chosen_KL_mean": -14.76583480834961, "KL/mean": -22.944477081298828, "KL/rejected_KL_mean": -31.12311553955078, "KL/std": 15.554851531982422, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.052384525537490845, "fcm_dpo/delta": -0.46119022369384766, "fcm_dpo/margin": 16.357284545898438, "fcm_dpo/q_t": 0.3257066309452057, "grad_norm": 19.28972816467285, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -1.8730106353759766, "logits/rejected": -0.9941458106040955, "logps/chosen": -128.95751953125, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -124.46224212646484, "loss": 0.859, "margin_dpo/margin_mean": 16.357284545898438, "margin_dpo/margin_std": 16.80987548828125, "step": 186 }, { "KL/chosen_KL_mean": -13.275344848632812, "KL/mean": -20.50890350341797, "KL/rejected_KL_mean": -27.74245834350586, "KL/std": 16.973400115966797, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.052036769688129425, "fcm_dpo/delta": -0.3553994297981262, "fcm_dpo/margin": 14.467113494873047, "fcm_dpo/q_t": 0.3493126332759857, "grad_norm": 16.76926040649414, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.4843511581420898, "logits/rejected": -1.2443073987960815, "logps/chosen": -90.1448974609375, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -115.90740203857422, "loss": 0.953, "margin_dpo/margin_mean": 14.467113494873047, "margin_dpo/margin_std": 18.352882385253906, "step": 187 }, { "KL/chosen_KL_mean": -9.677162170410156, "KL/mean": -18.14059829711914, "KL/rejected_KL_mean": -26.604034423828125, "KL/std": 17.215335845947266, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.051517851650714874, "fcm_dpo/delta": -0.4766615629196167, "fcm_dpo/margin": 16.926870346069336, "fcm_dpo/q_t": 0.31709009408950806, "grad_norm": 16.45052146911621, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -1.8872501850128174, "logits/rejected": -1.6435751914978027, "logps/chosen": -88.87196350097656, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -123.43292999267578, "loss": 0.8477, "margin_dpo/margin_mean": 16.926868438720703, "margin_dpo/margin_std": 16.399185180664062, "step": 188 }, { "KL/chosen_KL_mean": -12.740257263183594, "KL/mean": -21.575138092041016, "KL/rejected_KL_mean": -30.410018920898438, "KL/std": 17.614673614501953, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.05109977722167969, "fcm_dpo/delta": -0.5071896314620972, "fcm_dpo/margin": 17.669761657714844, "fcm_dpo/q_t": 0.3277888000011444, "grad_norm": 14.962014198303223, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.6261205673217773, "logits/rejected": -1.0113708972930908, "logps/chosen": -96.633056640625, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -109.87591552734375, "loss": 0.8666, "margin_dpo/margin_mean": 17.669761657714844, "margin_dpo/margin_std": 20.08213233947754, "step": 189 }, { "KL/chosen_KL_mean": -9.497135162353516, "KL/mean": -19.6320743560791, "KL/rejected_KL_mean": -29.767013549804688, "KL/std": 18.601119995117188, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.050427235662937164, "fcm_dpo/delta": -0.628713846206665, "fcm_dpo/margin": 20.269882202148438, "fcm_dpo/q_t": 0.29305511713027954, "grad_norm": 16.13777732849121, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.5186882019042969, "logits/rejected": -0.9206047058105469, "logps/chosen": -112.9385986328125, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -138.98204040527344, "loss": 0.7532, "margin_dpo/margin_mean": 20.26988410949707, "margin_dpo/margin_std": 17.1693058013916, "step": 190 }, { "KL/chosen_KL_mean": -8.747406005859375, "KL/mean": -19.827409744262695, "KL/rejected_KL_mean": -30.90741729736328, "KL/std": 19.95348358154297, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.04976560175418854, "fcm_dpo/delta": -0.710400402545929, "fcm_dpo/margin": 22.160015106201172, "fcm_dpo/q_t": 0.28912556171417236, "grad_norm": 17.338224411010742, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.2283532619476318, "logits/rejected": -0.8894354701042175, "logps/chosen": -108.53292083740234, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -146.90769958496094, "loss": 0.7442, "margin_dpo/margin_mean": 22.160015106201172, "margin_dpo/margin_std": 20.417715072631836, "step": 191 }, { "KL/chosen_KL_mean": -16.27127456665039, "KL/mean": -24.089611053466797, "KL/rejected_KL_mean": -31.907943725585938, "KL/std": 16.211734771728516, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.04928049445152283, "fcm_dpo/delta": -0.3735608458518982, "fcm_dpo/margin": 15.636674880981445, "fcm_dpo/q_t": 0.34534260630607605, "grad_norm": 19.89378547668457, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.1883714199066162, "logits/rejected": -0.9291911125183105, "logps/chosen": -117.13159942626953, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -141.3529510498047, "loss": 0.9365, "margin_dpo/margin_mean": 15.636675834655762, "margin_dpo/margin_std": 19.295555114746094, "step": 192 }, { "KL/chosen_KL_mean": -12.557716369628906, "KL/mean": -19.369007110595703, "KL/rejected_KL_mean": -26.1802978515625, "KL/std": 16.61907958984375, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.049028653651475906, "fcm_dpo/delta": -0.26961052417755127, "fcm_dpo/margin": 13.622584342956543, "fcm_dpo/q_t": 0.36448633670806885, "grad_norm": 19.9300594329834, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -1.9146978855133057, "logits/rejected": -1.3636481761932373, "logps/chosen": -105.49603271484375, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -109.28947448730469, "loss": 1.0339, "margin_dpo/margin_mean": 13.622583389282227, "margin_dpo/margin_std": 19.90750503540039, "step": 193 }, { "KL/chosen_KL_mean": -13.428886413574219, "KL/mean": -21.241031646728516, "KL/rejected_KL_mean": -29.053180694580078, "KL/std": 17.76651954650879, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.04870521277189255, "fcm_dpo/delta": -0.363594114780426, "fcm_dpo/margin": 15.624295234680176, "fcm_dpo/q_t": 0.34593552350997925, "grad_norm": 15.436610221862793, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.086198329925537, "logits/rejected": -1.7153089046478271, "logps/chosen": -77.220458984375, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -97.77983093261719, "loss": 0.9346, "margin_dpo/margin_mean": 15.62429428100586, "margin_dpo/margin_std": 18.63852310180664, "step": 194 }, { "KL/chosen_KL_mean": -12.678607940673828, "KL/mean": -22.52129364013672, "KL/rejected_KL_mean": -32.36397933959961, "KL/std": 17.80502700805664, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.04820109158754349, "fcm_dpo/delta": -0.5539072751998901, "fcm_dpo/margin": 19.685379028320312, "fcm_dpo/q_t": 0.31085556745529175, "grad_norm": 15.903382301330566, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.2895069122314453, "logits/rejected": -1.1429189443588257, "logps/chosen": -96.56407165527344, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -136.365234375, "loss": 0.8411, "margin_dpo/margin_mean": 19.685379028320312, "margin_dpo/margin_std": 20.50998878479004, "step": 195 }, { "KL/chosen_KL_mean": -14.37015151977539, "KL/mean": -24.060537338256836, "KL/rejected_KL_mean": -33.75091552734375, "KL/std": 19.108898162841797, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.04766719788312912, "fcm_dpo/delta": -0.5287054181098938, "fcm_dpo/margin": 19.380767822265625, "fcm_dpo/q_t": 0.30975788831710815, "grad_norm": 13.707058906555176, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.3482537269592285, "logits/rejected": -0.9190397262573242, "logps/chosen": -103.77151489257812, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -117.0982666015625, "loss": 0.8018, "margin_dpo/margin_mean": 19.380767822265625, "margin_dpo/margin_std": 17.333938598632812, "step": 196 }, { "KL/chosen_KL_mean": -15.002140045166016, "KL/mean": -21.083717346191406, "KL/rejected_KL_mean": -27.16529083251953, "KL/std": 15.31623649597168, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.047405581921339035, "fcm_dpo/delta": -0.17766165733337402, "fcm_dpo/margin": 12.163150787353516, "fcm_dpo/q_t": 0.37291419506073, "grad_norm": 16.251699447631836, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.1287567615509033, "logits/rejected": -0.8134480714797974, "logps/chosen": -101.84284973144531, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -109.95819091796875, "loss": 1.0133, "margin_dpo/margin_mean": 12.163150787353516, "margin_dpo/margin_std": 15.834602355957031, "step": 197 }, { "KL/chosen_KL_mean": -17.666244506835938, "KL/mean": -26.821456909179688, "KL/rejected_KL_mean": -35.97667694091797, "KL/std": 17.708724975585938, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.0470503568649292, "fcm_dpo/delta": -0.465450644493103, "fcm_dpo/margin": 18.31043243408203, "fcm_dpo/q_t": 0.3240761458873749, "grad_norm": 18.666746139526367, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.4196388721466064, "logits/rejected": -1.2169227600097656, "logps/chosen": -102.71449279785156, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -140.080322265625, "loss": 0.8456, "margin_dpo/margin_mean": 18.3104305267334, "margin_dpo/margin_std": 18.420738220214844, "step": 198 }, { "KL/chosen_KL_mean": -17.229557037353516, "KL/mean": -25.14101219177246, "KL/rejected_KL_mean": -33.052467346191406, "KL/std": 17.49543571472168, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.04670250788331032, "fcm_dpo/delta": -0.34143275022506714, "fcm_dpo/margin": 15.82291030883789, "fcm_dpo/q_t": 0.3471602499485016, "grad_norm": 16.2554988861084, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.528099536895752, "logits/rejected": -1.2746338844299316, "logps/chosen": -99.01625061035156, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -123.84324645996094, "loss": 0.9277, "margin_dpo/margin_mean": 15.82291030883789, "margin_dpo/margin_std": 18.087093353271484, "step": 199 }, { "KL/chosen_KL_mean": -16.263172149658203, "KL/mean": -25.685321807861328, "KL/rejected_KL_mean": -35.10746765136719, "KL/std": 18.73578643798828, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.046251390129327774, "fcm_dpo/delta": -0.4761233329772949, "fcm_dpo/margin": 18.844295501708984, "fcm_dpo/q_t": 0.32698285579681396, "grad_norm": 15.083000183105469, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.4803893566131592, "logits/rejected": -0.9729942083358765, "logps/chosen": -100.53126525878906, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -122.07533264160156, "loss": 0.8907, "margin_dpo/margin_mean": 18.84429359436035, "margin_dpo/margin_std": 21.091991424560547, "step": 200 }, { "KL/chosen_KL_mean": -17.859981536865234, "KL/mean": -26.784791946411133, "KL/rejected_KL_mean": -35.70960235595703, "KL/std": 20.150503158569336, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.04588666185736656, "fcm_dpo/delta": -0.4224647283554077, "fcm_dpo/margin": 17.849618911743164, "fcm_dpo/q_t": 0.336277574300766, "grad_norm": 16.850452423095703, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.084730625152588, "logits/rejected": -0.6818499565124512, "logps/chosen": -125.63247680664062, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -155.5020751953125, "loss": 0.9298, "margin_dpo/margin_mean": 17.849618911743164, "margin_dpo/margin_std": 21.910778045654297, "step": 201 }, { "KL/chosen_KL_mean": -17.304706573486328, "KL/mean": -25.33537483215332, "KL/rejected_KL_mean": -33.36604690551758, "KL/std": 16.893884658813477, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.04556471109390259, "fcm_dpo/delta": -0.33421099185943604, "fcm_dpo/margin": 16.061344146728516, "fcm_dpo/q_t": 0.35097649693489075, "grad_norm": 15.665852546691895, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.5383628606796265, "logits/rejected": -1.4009127616882324, "logps/chosen": -93.27716064453125, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -127.82603454589844, "loss": 0.9402, "margin_dpo/margin_mean": 16.061342239379883, "margin_dpo/margin_std": 18.910316467285156, "step": 202 }, { "KL/chosen_KL_mean": -14.489105224609375, "KL/mean": -23.991649627685547, "KL/rejected_KL_mean": -33.494197845458984, "KL/std": 17.681072235107422, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.04516703262925148, "fcm_dpo/delta": -0.46234381198883057, "fcm_dpo/margin": 19.005088806152344, "fcm_dpo/q_t": 0.3215080499649048, "grad_norm": 13.894720077514648, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.6580169200897217, "logits/rejected": -1.214723825454712, "logps/chosen": -109.45626068115234, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -126.38186645507812, "loss": 0.8426, "margin_dpo/margin_mean": 19.005088806152344, "margin_dpo/margin_std": 18.337772369384766, "step": 203 }, { "KL/chosen_KL_mean": -19.03099822998047, "KL/mean": -27.04680633544922, "KL/rejected_KL_mean": -35.0626106262207, "KL/std": 17.082002639770508, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.04485880583524704, "fcm_dpo/delta": -0.3213825523853302, "fcm_dpo/margin": 16.0316162109375, "fcm_dpo/q_t": 0.352073609828949, "grad_norm": 18.037290573120117, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.629747986793518, "logits/rejected": -1.0003126859664917, "logps/chosen": -135.38818359375, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -128.460205078125, "loss": 0.9564, "margin_dpo/margin_mean": 16.0316162109375, "margin_dpo/margin_std": 19.64917755126953, "step": 204 }, { "KL/chosen_KL_mean": -17.23846435546875, "KL/mean": -27.648168563842773, "KL/rejected_KL_mean": -38.05787658691406, "KL/std": 19.179758071899414, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.04440478980541229, "fcm_dpo/delta": -0.5295372009277344, "fcm_dpo/margin": 20.819412231445312, "fcm_dpo/q_t": 0.3047248423099518, "grad_norm": 12.83398723602295, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -1.9590773582458496, "logits/rejected": -1.5348539352416992, "logps/chosen": -106.09780883789062, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -129.91236877441406, "loss": 0.7784, "margin_dpo/margin_mean": 20.819412231445312, "margin_dpo/margin_std": 16.999330520629883, "step": 205 }, { "KL/chosen_KL_mean": -17.61764907836914, "KL/mean": -27.2155704498291, "KL/rejected_KL_mean": -36.81350326538086, "KL/std": 18.707876205444336, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.04397869110107422, "fcm_dpo/delta": -0.44829216599464417, "fcm_dpo/margin": 19.195844650268555, "fcm_dpo/q_t": 0.319382905960083, "grad_norm": 17.516843795776367, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.5020140409469604, "logits/rejected": -1.315996527671814, "logps/chosen": -94.16400146484375, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -139.76663208007812, "loss": 0.8159, "margin_dpo/margin_mean": 19.195844650268555, "margin_dpo/margin_std": 16.077213287353516, "step": 206 }, { "KL/chosen_KL_mean": -19.259380340576172, "KL/mean": -29.460954666137695, "KL/rejected_KL_mean": -39.662532806396484, "KL/std": 21.504993438720703, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.04358307272195816, "fcm_dpo/delta": -0.49359092116355896, "fcm_dpo/margin": 20.403152465820312, "fcm_dpo/q_t": 0.332013875246048, "grad_norm": 20.082590103149414, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.8841972351074219, "logits/rejected": -1.4556334018707275, "logps/chosen": -105.49102783203125, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -130.31765747070312, "loss": 0.9073, "margin_dpo/margin_mean": 20.403152465820312, "margin_dpo/margin_std": 25.434967041015625, "step": 207 }, { "KL/chosen_KL_mean": -19.195117950439453, "KL/mean": -29.946918487548828, "KL/rejected_KL_mean": -40.69871520996094, "KL/std": 20.57773208618164, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.04315963387489319, "fcm_dpo/delta": -0.532746434211731, "fcm_dpo/margin": 21.50359344482422, "fcm_dpo/q_t": 0.30662065744400024, "grad_norm": 15.819819450378418, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.598953366279602, "logits/rejected": -1.2279999256134033, "logps/chosen": -113.35466003417969, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -141.66104125976562, "loss": 0.8041, "margin_dpo/margin_mean": 21.50359344482422, "margin_dpo/margin_std": 19.25140380859375, "step": 208 }, { "KL/chosen_KL_mean": -16.73483657836914, "KL/mean": -26.131919860839844, "KL/rejected_KL_mean": -35.52899932861328, "KL/std": 18.01914405822754, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.04271944612264633, "fcm_dpo/delta": -0.40653741359710693, "fcm_dpo/margin": 18.794166564941406, "fcm_dpo/q_t": 0.3366587162017822, "grad_norm": 14.895203590393066, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -1.991509199142456, "logits/rejected": -1.5310394763946533, "logps/chosen": -100.90540313720703, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -123.14854431152344, "loss": 0.8994, "margin_dpo/margin_mean": 18.794166564941406, "margin_dpo/margin_std": 20.60721778869629, "step": 209 }, { "KL/chosen_KL_mean": -18.699710845947266, "KL/mean": -27.795469284057617, "KL/rejected_KL_mean": -36.89122772216797, "KL/std": 19.09017562866211, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.04240277782082558, "fcm_dpo/delta": -0.374503493309021, "fcm_dpo/margin": 18.19152069091797, "fcm_dpo/q_t": 0.3423662781715393, "grad_norm": 13.88355827331543, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.8536854982376099, "logits/rejected": -1.4663913249969482, "logps/chosen": -99.87088012695312, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -121.06600952148438, "loss": 0.9011, "margin_dpo/margin_mean": 18.19152069091797, "margin_dpo/margin_std": 19.8233642578125, "step": 210 }, { "KL/chosen_KL_mean": -20.33736801147461, "KL/mean": -29.6827449798584, "KL/rejected_KL_mean": -39.02811813354492, "KL/std": 20.318092346191406, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.04210546985268593, "fcm_dpo/delta": -0.3899921178817749, "fcm_dpo/margin": 18.69075584411621, "fcm_dpo/q_t": 0.33948612213134766, "grad_norm": 15.784475326538086, "learning_rate": 4.373239415645323e-07, "logits/chosen": -1.693336844444275, "logits/rejected": -1.0729868412017822, "logps/chosen": -129.05007934570312, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -132.58377075195312, "loss": 0.9045, "margin_dpo/margin_mean": 18.690757751464844, "margin_dpo/margin_std": 21.341259002685547, "step": 211 }, { "KL/chosen_KL_mean": -13.193519592285156, "KL/mean": -25.986482620239258, "KL/rejected_KL_mean": -38.779449462890625, "KL/std": 22.293630599975586, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.04158168286085129, "fcm_dpo/delta": -0.6712970733642578, "fcm_dpo/margin": 25.585922241210938, "fcm_dpo/q_t": 0.2889118790626526, "grad_norm": 12.815353393554688, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.6053833961486816, "logits/rejected": -1.1770262718200684, "logps/chosen": -111.55546569824219, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -148.66944885253906, "loss": 0.743, "margin_dpo/margin_mean": 25.585922241210938, "margin_dpo/margin_std": 22.353567123413086, "step": 212 }, { "KL/chosen_KL_mean": -16.76946258544922, "KL/mean": -27.017419815063477, "KL/rejected_KL_mean": -37.265380859375, "KL/std": 20.57743263244629, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.041164498776197433, "fcm_dpo/delta": -0.44765597581863403, "fcm_dpo/margin": 20.49591827392578, "fcm_dpo/q_t": 0.32470637559890747, "grad_norm": 18.628131866455078, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.8949015140533447, "logits/rejected": -1.1692785024642944, "logps/chosen": -124.82476806640625, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -137.40953063964844, "loss": 0.8658, "margin_dpo/margin_mean": 20.495914459228516, "margin_dpo/margin_std": 21.094999313354492, "step": 213 }, { "KL/chosen_KL_mean": -18.19385528564453, "KL/mean": -27.13542366027832, "KL/rejected_KL_mean": -36.076995849609375, "KL/std": 21.832576751708984, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.04089023172855377, "fcm_dpo/delta": -0.3336220979690552, "fcm_dpo/margin": 17.883136749267578, "fcm_dpo/q_t": 0.3557431101799011, "grad_norm": 14.888252258300781, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.828701376914978, "logits/rejected": -1.711211085319519, "logps/chosen": -95.99859619140625, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -120.352783203125, "loss": 0.9821, "margin_dpo/margin_mean": 17.883136749267578, "margin_dpo/margin_std": 24.45751953125, "step": 214 }, { "KL/chosen_KL_mean": -15.136199951171875, "KL/mean": -26.966510772705078, "KL/rejected_KL_mean": -38.79682540893555, "KL/std": 21.77513313293457, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.04046916961669922, "fcm_dpo/delta": -0.5629321336746216, "fcm_dpo/margin": 23.660619735717773, "fcm_dpo/q_t": 0.30643904209136963, "grad_norm": 13.439775466918945, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.6722877025604248, "logits/rejected": -1.1879515647888184, "logps/chosen": -100.25009155273438, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -135.65834045410156, "loss": 0.7841, "margin_dpo/margin_mean": 23.660619735717773, "margin_dpo/margin_std": 21.457054138183594, "step": 215 }, { "KL/chosen_KL_mean": -17.24599838256836, "KL/mean": -27.301237106323242, "KL/rejected_KL_mean": -37.356475830078125, "KL/std": 18.825475692749023, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.04013023525476456, "fcm_dpo/delta": -0.4101881980895996, "fcm_dpo/margin": 20.11048126220703, "fcm_dpo/q_t": 0.33197611570358276, "grad_norm": 12.891627311706543, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.6859790086746216, "logits/rejected": -1.2812812328338623, "logps/chosen": -97.83830261230469, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -118.77631378173828, "loss": 0.8577, "margin_dpo/margin_mean": 20.11048126220703, "margin_dpo/margin_std": 19.650875091552734, "step": 216 }, { "KL/chosen_KL_mean": -20.330867767333984, "KL/mean": -29.312938690185547, "KL/rejected_KL_mean": -38.29500961303711, "KL/std": 19.069984436035156, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.03985349088907242, "fcm_dpo/delta": -0.31811103224754333, "fcm_dpo/margin": 17.964149475097656, "fcm_dpo/q_t": 0.3533533215522766, "grad_norm": 16.6281681060791, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -1.7270665168762207, "logits/rejected": -1.4132380485534668, "logps/chosen": -113.80345153808594, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -141.78353881835938, "loss": 0.9482, "margin_dpo/margin_mean": 17.964147567749023, "margin_dpo/margin_std": 21.90603256225586, "step": 217 }, { "KL/chosen_KL_mean": -19.13430404663086, "KL/mean": -29.55084228515625, "KL/rejected_KL_mean": -39.967384338378906, "KL/std": 19.972740173339844, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.03950865566730499, "fcm_dpo/delta": -0.42672330141067505, "fcm_dpo/margin": 20.83307647705078, "fcm_dpo/q_t": 0.33209094405174255, "grad_norm": 15.165288925170898, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.3099347352981567, "logits/rejected": -0.914745569229126, "logps/chosen": -108.19313049316406, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -134.27420043945312, "loss": 0.8998, "margin_dpo/margin_mean": 20.83307647705078, "margin_dpo/margin_std": 23.470312118530273, "step": 218 }, { "KL/chosen_KL_mean": -17.54022216796875, "KL/mean": -27.192874908447266, "KL/rejected_KL_mean": -36.845523834228516, "KL/std": 22.843887329101562, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.03925672173500061, "fcm_dpo/delta": -0.3604342043399811, "fcm_dpo/margin": 19.305307388305664, "fcm_dpo/q_t": 0.345703661441803, "grad_norm": 18.929990768432617, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.5744526386260986, "logits/rejected": -1.15447199344635, "logps/chosen": -118.54756164550781, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -129.31346130371094, "loss": 0.9498, "margin_dpo/margin_mean": 19.305309295654297, "margin_dpo/margin_std": 24.089712142944336, "step": 219 }, { "KL/chosen_KL_mean": -18.010032653808594, "KL/mean": -26.707988739013672, "KL/rejected_KL_mean": -35.40595245361328, "KL/std": 18.367355346679688, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.03898411989212036, "fcm_dpo/delta": -0.2800871729850769, "fcm_dpo/margin": 17.395923614501953, "fcm_dpo/q_t": 0.3537631034851074, "grad_norm": 15.382002830505371, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.3249385356903076, "logits/rejected": -0.9727566242218018, "logps/chosen": -113.54724884033203, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -129.71298217773438, "loss": 0.9321, "margin_dpo/margin_mean": 17.395923614501953, "margin_dpo/margin_std": 18.478851318359375, "step": 220 }, { "KL/chosen_KL_mean": -17.400291442871094, "KL/mean": -30.29937171936035, "KL/rejected_KL_mean": -43.19844436645508, "KL/std": 23.93475341796875, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.038622576743364334, "fcm_dpo/delta": -0.6019889712333679, "fcm_dpo/margin": 25.798152923583984, "fcm_dpo/q_t": 0.2992299795150757, "grad_norm": 14.322129249572754, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.096805453300476, "logits/rejected": -0.9367796778678894, "logps/chosen": -100.24515533447266, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -154.01025390625, "loss": 0.768, "margin_dpo/margin_mean": 25.798152923583984, "margin_dpo/margin_std": 21.679370880126953, "step": 221 }, { "KL/chosen_KL_mean": -21.903873443603516, "KL/mean": -32.721683502197266, "KL/rejected_KL_mean": -43.53949737548828, "KL/std": 22.746049880981445, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.03824714943766594, "fcm_dpo/delta": -0.4308597445487976, "fcm_dpo/margin": 21.63561248779297, "fcm_dpo/q_t": 0.33398234844207764, "grad_norm": 19.60590171813965, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.9092597961425781, "logits/rejected": -1.4849542379379272, "logps/chosen": -117.04585266113281, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -150.34390258789062, "loss": 0.9357, "margin_dpo/margin_mean": 21.63561248779297, "margin_dpo/margin_std": 26.858638763427734, "step": 222 }, { "KL/chosen_KL_mean": -20.74630355834961, "KL/mean": -29.41265106201172, "KL/rejected_KL_mean": -38.079002380371094, "KL/std": 19.86611557006836, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.03797938674688339, "fcm_dpo/delta": -0.2600671350955963, "fcm_dpo/margin": 17.33269691467285, "fcm_dpo/q_t": 0.362545371055603, "grad_norm": 14.673210144042969, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.1183130741119385, "logits/rejected": -1.7212893962860107, "logps/chosen": -106.3214111328125, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -124.53138732910156, "loss": 0.9824, "margin_dpo/margin_mean": 17.33269500732422, "margin_dpo/margin_std": 22.13071632385254, "step": 223 }, { "KL/chosen_KL_mean": -20.060691833496094, "KL/mean": -30.67736053466797, "KL/rejected_KL_mean": -41.294036865234375, "KL/std": 21.973041534423828, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.0377165712416172, "fcm_dpo/delta": -0.4040184020996094, "fcm_dpo/margin": 21.233341217041016, "fcm_dpo/q_t": 0.3342816233634949, "grad_norm": 14.532925605773926, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.7709627151489258, "logits/rejected": -1.3366880416870117, "logps/chosen": -102.78450012207031, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -123.88941955566406, "loss": 0.8828, "margin_dpo/margin_mean": 21.233341217041016, "margin_dpo/margin_std": 22.60485076904297, "step": 224 }, { "KL/chosen_KL_mean": -22.01903533935547, "KL/mean": -33.856441497802734, "KL/rejected_KL_mean": -45.69384765625, "KL/std": 23.363834381103516, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.03738105297088623, "fcm_dpo/delta": -0.4890996217727661, "fcm_dpo/margin": 23.674816131591797, "fcm_dpo/q_t": 0.3231743574142456, "grad_norm": 15.265226364135742, "learning_rate": 4.249525076191759e-07, "logits/chosen": -1.7622087001800537, "logits/rejected": -1.2816628217697144, "logps/chosen": -117.69671630859375, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -150.7907257080078, "loss": 0.8643, "margin_dpo/margin_mean": 23.67481803894043, "margin_dpo/margin_std": 25.930465698242188, "step": 225 }, { "KL/chosen_KL_mean": -18.078968048095703, "KL/mean": -28.9039306640625, "KL/rejected_KL_mean": -39.72888946533203, "KL/std": 22.519824981689453, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.03701633960008621, "fcm_dpo/delta": -0.40501219034194946, "fcm_dpo/margin": 21.649925231933594, "fcm_dpo/q_t": 0.33120912313461304, "grad_norm": 12.923030853271484, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.6414361000061035, "logits/rejected": -1.1146421432495117, "logps/chosen": -111.5398941040039, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -126.43064880371094, "loss": 0.8616, "margin_dpo/margin_mean": 21.649925231933594, "margin_dpo/margin_std": 20.7691650390625, "step": 226 }, { "KL/chosen_KL_mean": -25.35834503173828, "KL/mean": -36.26323318481445, "KL/rejected_KL_mean": -47.168121337890625, "KL/std": 21.155521392822266, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.03674504905939102, "fcm_dpo/delta": -0.40462183952331543, "fcm_dpo/margin": 21.809778213500977, "fcm_dpo/q_t": 0.3386305570602417, "grad_norm": 14.716720581054688, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.3128870725631714, "logits/rejected": -0.9736635684967041, "logps/chosen": -118.172607421875, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -151.90505981445312, "loss": 0.9157, "margin_dpo/margin_mean": 21.809778213500977, "margin_dpo/margin_std": 25.251869201660156, "step": 227 }, { "KL/chosen_KL_mean": -23.080245971679688, "KL/mean": -31.891952514648438, "KL/rejected_KL_mean": -40.70364761352539, "KL/std": 19.262298583984375, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.03653843700885773, "fcm_dpo/delta": -0.2454841136932373, "fcm_dpo/margin": 17.623401641845703, "fcm_dpo/q_t": 0.3596232533454895, "grad_norm": 14.863390922546387, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.7662415504455566, "logits/rejected": -1.3352104425430298, "logps/chosen": -117.11737060546875, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -136.72515869140625, "loss": 0.9675, "margin_dpo/margin_mean": 17.623401641845703, "margin_dpo/margin_std": 20.45838737487793, "step": 228 }, { "KL/chosen_KL_mean": -20.033267974853516, "KL/mean": -34.100975036621094, "KL/rejected_KL_mean": -48.16868209838867, "KL/std": 24.975879669189453, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.03615522384643555, "fcm_dpo/delta": -0.6234970092773438, "fcm_dpo/margin": 28.135412216186523, "fcm_dpo/q_t": 0.30081456899642944, "grad_norm": 13.05147647857666, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.5935077667236328, "logits/rejected": -0.9525737762451172, "logps/chosen": -115.61093139648438, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -137.34246826171875, "loss": 0.7806, "margin_dpo/margin_mean": 28.135414123535156, "margin_dpo/margin_std": 26.665517807006836, "step": 229 }, { "KL/chosen_KL_mean": -21.568981170654297, "KL/mean": -35.26420593261719, "KL/rejected_KL_mean": -48.95942306518555, "KL/std": 23.648513793945312, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.03572218120098114, "fcm_dpo/delta": -0.5841723084449768, "fcm_dpo/margin": 27.39043617248535, "fcm_dpo/q_t": 0.29764145612716675, "grad_norm": 14.604736328125, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.6927831172943115, "logits/rejected": -1.5939877033233643, "logps/chosen": -85.28260803222656, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -140.86813354492188, "loss": 0.7636, "margin_dpo/margin_mean": 27.390438079833984, "margin_dpo/margin_std": 22.83057403564453, "step": 230 }, { "KL/chosen_KL_mean": -24.537784576416016, "KL/mean": -35.81703186035156, "KL/rejected_KL_mean": -47.09626770019531, "KL/std": 23.355188369750977, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.03538353741168976, "fcm_dpo/delta": -0.40170812606811523, "fcm_dpo/margin": 22.558490753173828, "fcm_dpo/q_t": 0.3303024172782898, "grad_norm": 12.712801933288574, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.7209961414337158, "logits/rejected": -1.210491418838501, "logps/chosen": -119.99345397949219, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -128.05194091796875, "loss": 0.8512, "margin_dpo/margin_mean": 22.558490753173828, "margin_dpo/margin_std": 20.66366958618164, "step": 231 }, { "KL/chosen_KL_mean": -24.37189483642578, "KL/mean": -34.549171447753906, "KL/rejected_KL_mean": -44.72645950317383, "KL/std": 21.42932891845703, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.035158686339855194, "fcm_dpo/delta": -0.31797921657562256, "fcm_dpo/margin": 20.35456657409668, "fcm_dpo/q_t": 0.3499019145965576, "grad_norm": 14.189082145690918, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.2448954582214355, "logits/rejected": -0.7690390348434448, "logps/chosen": -121.26915740966797, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -134.4910888671875, "loss": 0.9367, "margin_dpo/margin_mean": 20.35456657409668, "margin_dpo/margin_std": 23.272686004638672, "step": 232 }, { "KL/chosen_KL_mean": -25.95262908935547, "KL/mean": -39.81716537475586, "KL/rejected_KL_mean": -53.68168640136719, "KL/std": 26.500343322753906, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.03480200469493866, "fcm_dpo/delta": -0.5706804990768433, "fcm_dpo/margin": 27.729066848754883, "fcm_dpo/q_t": 0.3058220148086548, "grad_norm": 13.489566802978516, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.4997715950012207, "logits/rejected": -1.252012848854065, "logps/chosen": -115.0029525756836, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -166.44085693359375, "loss": 0.7996, "margin_dpo/margin_mean": 27.729068756103516, "margin_dpo/margin_std": 25.895553588867188, "step": 233 }, { "KL/chosen_KL_mean": -24.994285583496094, "KL/mean": -39.51212692260742, "KL/rejected_KL_mean": -54.02996063232422, "KL/std": 25.90264129638672, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.03435136377811432, "fcm_dpo/delta": -0.6046146154403687, "fcm_dpo/margin": 29.03567123413086, "fcm_dpo/q_t": 0.3058413863182068, "grad_norm": 13.328933715820312, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.1172473430633545, "logits/rejected": -1.0243043899536133, "logps/chosen": -99.31265258789062, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -160.41754150390625, "loss": 0.8089, "margin_dpo/margin_mean": 29.03567123413086, "margin_dpo/margin_std": 28.139015197753906, "step": 234 }, { "KL/chosen_KL_mean": -29.79018783569336, "KL/mean": -40.78860855102539, "KL/rejected_KL_mean": -51.787025451660156, "KL/std": 22.443157196044922, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.03411596268415451, "fcm_dpo/delta": -0.353059321641922, "fcm_dpo/margin": 21.996845245361328, "fcm_dpo/q_t": 0.3396115303039551, "grad_norm": 13.191201210021973, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.5856149196624756, "logits/rejected": -1.0543267726898193, "logps/chosen": -128.00723266601562, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -149.03379821777344, "loss": 0.8849, "margin_dpo/margin_mean": 21.996845245361328, "margin_dpo/margin_std": 21.527645111083984, "step": 235 }, { "KL/chosen_KL_mean": -25.317920684814453, "KL/mean": -39.04700469970703, "KL/rejected_KL_mean": -52.77609634399414, "KL/std": 28.116500854492188, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.033795326948165894, "fcm_dpo/delta": -0.5327195525169373, "fcm_dpo/margin": 27.458171844482422, "fcm_dpo/q_t": 0.32426387071609497, "grad_norm": 13.41267204284668, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.595430612564087, "logits/rejected": -1.5648455619812012, "logps/chosen": -104.15565490722656, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -161.83953857421875, "loss": 0.8464, "margin_dpo/margin_mean": 27.458171844482422, "margin_dpo/margin_std": 31.245025634765625, "step": 236 }, { "KL/chosen_KL_mean": -29.155155181884766, "KL/mean": -40.22138214111328, "KL/rejected_KL_mean": -51.28760528564453, "KL/std": 24.667312622070312, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.033510029315948486, "fcm_dpo/delta": -0.3443029820919037, "fcm_dpo/margin": 22.132444381713867, "fcm_dpo/q_t": 0.35849785804748535, "grad_norm": 14.672990798950195, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.7351634502410889, "logits/rejected": -1.3533390760421753, "logps/chosen": -114.36644744873047, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -151.19760131835938, "loss": 1.0035, "margin_dpo/margin_mean": 22.132444381713867, "margin_dpo/margin_std": 32.296852111816406, "step": 237 }, { "KL/chosen_KL_mean": -32.20804214477539, "KL/mean": -44.275108337402344, "KL/rejected_KL_mean": -56.34217834472656, "KL/std": 25.793054580688477, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.03325130045413971, "fcm_dpo/delta": -0.40582606196403503, "fcm_dpo/margin": 24.134140014648438, "fcm_dpo/q_t": 0.33799445629119873, "grad_norm": 12.79339599609375, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.31563138961792, "logits/rejected": -1.1448123455047607, "logps/chosen": -110.93927764892578, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -151.76058959960938, "loss": 0.8982, "margin_dpo/margin_mean": 24.13414192199707, "margin_dpo/margin_std": 27.002666473388672, "step": 238 }, { "KL/chosen_KL_mean": -30.163494110107422, "KL/mean": -41.4890251159668, "KL/rejected_KL_mean": -52.814552307128906, "KL/std": 22.552127838134766, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.03298822045326233, "fcm_dpo/delta": -0.35026443004608154, "fcm_dpo/margin": 22.65105438232422, "fcm_dpo/q_t": 0.34637826681137085, "grad_norm": 13.807390213012695, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.8426134586334229, "logits/rejected": -1.3727242946624756, "logps/chosen": -122.76443481445312, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -153.33224487304688, "loss": 0.9269, "margin_dpo/margin_mean": 22.65105438232422, "margin_dpo/margin_std": 25.551124572753906, "step": 239 }, { "KL/chosen_KL_mean": -30.782257080078125, "KL/mean": -41.11897277832031, "KL/rejected_KL_mean": -51.45569610595703, "KL/std": 22.639816284179688, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.032803893089294434, "fcm_dpo/delta": -0.2802502512931824, "fcm_dpo/margin": 20.673444747924805, "fcm_dpo/q_t": 0.35541778802871704, "grad_norm": 15.92525863647461, "learning_rate": 4.106969024216348e-07, "logits/chosen": -1.84120512008667, "logits/rejected": -1.5052688121795654, "logps/chosen": -116.94203186035156, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -131.91136169433594, "loss": 0.9511, "margin_dpo/margin_mean": 20.673446655273438, "margin_dpo/margin_std": 23.572406768798828, "step": 240 }, { "KL/chosen_KL_mean": -26.538211822509766, "KL/mean": -44.27848815917969, "KL/rejected_KL_mean": -62.01876449584961, "KL/std": 27.502975463867188, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.03243744373321533, "fcm_dpo/delta": -0.7590680122375488, "fcm_dpo/margin": 35.48054885864258, "fcm_dpo/q_t": 0.2785906493663788, "grad_norm": 12.32579517364502, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.7859851121902466, "logits/rejected": -1.1722520589828491, "logps/chosen": -127.58415222167969, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -156.068115234375, "loss": 0.7122, "margin_dpo/margin_mean": 35.48054885864258, "margin_dpo/margin_std": 30.193695068359375, "step": 241 }, { "KL/chosen_KL_mean": -32.70845031738281, "KL/mean": -45.353759765625, "KL/rejected_KL_mean": -57.999061584472656, "KL/std": 22.719758987426758, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.032100629061460495, "fcm_dpo/delta": -0.41503095626831055, "fcm_dpo/margin": 25.29061508178711, "fcm_dpo/q_t": 0.32937440276145935, "grad_norm": 13.811562538146973, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.866332769393921, "logits/rejected": -1.2353755235671997, "logps/chosen": -128.38111877441406, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -148.6536102294922, "loss": 0.8655, "margin_dpo/margin_mean": 25.29061508178711, "margin_dpo/margin_std": 24.61943817138672, "step": 242 }, { "KL/chosen_KL_mean": -31.592987060546875, "KL/mean": -44.1052131652832, "KL/rejected_KL_mean": -56.61743927001953, "KL/std": 24.0534610748291, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.03186003491282463, "fcm_dpo/delta": -0.40028175711631775, "fcm_dpo/margin": 25.02444839477539, "fcm_dpo/q_t": 0.3359273374080658, "grad_norm": 13.285747528076172, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.7304954528808594, "logits/rejected": -0.9260585904121399, "logps/chosen": -129.6243896484375, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -139.80551147460938, "loss": 0.9114, "margin_dpo/margin_mean": 25.02444839477539, "margin_dpo/margin_std": 27.31308364868164, "step": 243 }, { "KL/chosen_KL_mean": -33.30897521972656, "KL/mean": -46.27266311645508, "KL/rejected_KL_mean": -59.23635482788086, "KL/std": 26.672420501708984, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.031548693776130676, "fcm_dpo/delta": -0.4215605854988098, "fcm_dpo/margin": 25.927371978759766, "fcm_dpo/q_t": 0.33372631669044495, "grad_norm": 13.17302417755127, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.060485363006592, "logits/rejected": -1.7181386947631836, "logps/chosen": -122.20289611816406, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -161.80914306640625, "loss": 0.8831, "margin_dpo/margin_mean": 25.927371978759766, "margin_dpo/margin_std": 28.01298713684082, "step": 244 }, { "KL/chosen_KL_mean": -28.269527435302734, "KL/mean": -42.675437927246094, "KL/rejected_KL_mean": -57.08135223388672, "KL/std": 25.997589111328125, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.03125441074371338, "fcm_dpo/delta": -0.5050354599952698, "fcm_dpo/margin": 28.811824798583984, "fcm_dpo/q_t": 0.31333988904953003, "grad_norm": 12.685444831848145, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.041753053665161, "logits/rejected": -1.2356386184692383, "logps/chosen": -132.46353149414062, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -149.73780822753906, "loss": 0.8081, "margin_dpo/margin_mean": 28.811824798583984, "margin_dpo/margin_std": 25.743057250976562, "step": 245 }, { "KL/chosen_KL_mean": -36.720855712890625, "KL/mean": -47.668636322021484, "KL/rejected_KL_mean": -58.61641311645508, "KL/std": 24.293378829956055, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.031030360609292984, "fcm_dpo/delta": -0.28149211406707764, "fcm_dpo/margin": 21.895565032958984, "fcm_dpo/q_t": 0.3556755781173706, "grad_norm": 13.97982406616211, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.5610425472259521, "logits/rejected": -1.0303078889846802, "logps/chosen": -139.93991088867188, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -149.60862731933594, "loss": 0.9497, "margin_dpo/margin_mean": 21.895565032958984, "margin_dpo/margin_std": 25.33405303955078, "step": 246 }, { "KL/chosen_KL_mean": -33.403236389160156, "KL/mean": -49.886375427246094, "KL/rejected_KL_mean": -66.3695068359375, "KL/std": 27.481769561767578, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.030695034191012383, "fcm_dpo/delta": -0.6186092495918274, "fcm_dpo/margin": 32.96626281738281, "fcm_dpo/q_t": 0.30009737610816956, "grad_norm": 11.870213508605957, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.122237205505371, "logits/rejected": -1.8944722414016724, "logps/chosen": -120.39761352539062, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -177.70753479003906, "loss": 0.7729, "margin_dpo/margin_mean": 32.96626281738281, "margin_dpo/margin_std": 30.354595184326172, "step": 247 }, { "KL/chosen_KL_mean": -37.872047424316406, "KL/mean": -49.50117492675781, "KL/rejected_KL_mean": -61.13029479980469, "KL/std": 24.024147033691406, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.030484354123473167, "fcm_dpo/delta": -0.31114935874938965, "fcm_dpo/margin": 23.258251190185547, "fcm_dpo/q_t": 0.3508985638618469, "grad_norm": 14.438250541687012, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.6260871887207031, "logits/rejected": -1.5633795261383057, "logps/chosen": -112.65756225585938, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -159.40719604492188, "loss": 0.9266, "margin_dpo/margin_mean": 23.258251190185547, "margin_dpo/margin_std": 25.699256896972656, "step": 248 }, { "KL/chosen_KL_mean": -40.83214569091797, "KL/mean": -51.61566162109375, "KL/rejected_KL_mean": -62.399173736572266, "KL/std": 24.821773529052734, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.03030003048479557, "fcm_dpo/delta": -0.25528502464294434, "fcm_dpo/margin": 21.567031860351562, "fcm_dpo/q_t": 0.3635568618774414, "grad_norm": 16.130489349365234, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.239929676055908, "logits/rejected": -1.7510055303573608, "logps/chosen": -136.5359344482422, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -149.545654296875, "loss": 0.9945, "margin_dpo/margin_mean": 21.567031860351562, "margin_dpo/margin_std": 28.271717071533203, "step": 249 }, { "KL/chosen_KL_mean": -37.75912857055664, "KL/mean": -49.055908203125, "KL/rejected_KL_mean": -60.35270309448242, "KL/std": 24.97289276123047, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.030119236558675766, "fcm_dpo/delta": -0.2828212380409241, "fcm_dpo/margin": 22.593570709228516, "fcm_dpo/q_t": 0.3544583320617676, "grad_norm": 17.423812866210938, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9961650371551514, "logits/rejected": -1.1623730659484863, "logps/chosen": -151.81134033203125, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -141.4403839111328, "loss": 0.9655, "margin_dpo/margin_mean": 22.593570709228516, "margin_dpo/margin_std": 26.219173431396484, "step": 250 }, { "KL/chosen_KL_mean": -36.519447326660156, "KL/mean": -47.48855209350586, "KL/rejected_KL_mean": -58.45766067504883, "KL/std": 23.787124633789062, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.029961001127958298, "fcm_dpo/delta": -0.2593482732772827, "fcm_dpo/margin": 21.938207626342773, "fcm_dpo/q_t": 0.3649369478225708, "grad_norm": 14.467461585998535, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.904625654220581, "logits/rejected": -1.2382113933563232, "logps/chosen": -134.23074340820312, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -141.98507690429688, "loss": 0.9948, "margin_dpo/margin_mean": 21.93820571899414, "margin_dpo/margin_std": 28.58325958251953, "step": 251 }, { "KL/chosen_KL_mean": -36.043418884277344, "KL/mean": -47.306236267089844, "KL/rejected_KL_mean": -58.56905746459961, "KL/std": 24.1273136138916, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.029822804033756256, "fcm_dpo/delta": -0.27363163232803345, "fcm_dpo/margin": 22.525634765625, "fcm_dpo/q_t": 0.3557099401950836, "grad_norm": 14.724553108215332, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -1.8435966968536377, "logits/rejected": -1.3402986526489258, "logps/chosen": -145.02952575683594, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -150.13330078125, "loss": 0.9528, "margin_dpo/margin_mean": 22.525634765625, "margin_dpo/margin_std": 25.735443115234375, "step": 252 }, { "KL/chosen_KL_mean": -39.574119567871094, "KL/mean": -50.32842254638672, "KL/rejected_KL_mean": -61.082733154296875, "KL/std": 26.5678653717041, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.029707318171858788, "fcm_dpo/delta": -0.24046650528907776, "fcm_dpo/margin": 21.50861358642578, "fcm_dpo/q_t": 0.3605167865753174, "grad_norm": 14.011171340942383, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -1.834316611289978, "logits/rejected": -1.6190714836120605, "logps/chosen": -139.79042053222656, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -166.75942993164062, "loss": 0.9699, "margin_dpo/margin_mean": 21.50861358642578, "margin_dpo/margin_std": 24.212142944335938, "step": 253 }, { "KL/chosen_KL_mean": -38.77762985229492, "KL/mean": -49.2047233581543, "KL/rejected_KL_mean": -59.63182067871094, "KL/std": 24.512195587158203, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.02952072210609913, "fcm_dpo/delta": -0.21726766228675842, "fcm_dpo/margin": 20.854183197021484, "fcm_dpo/q_t": 0.366879940032959, "grad_norm": 13.364747047424316, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.7615680694580078, "logits/rejected": -1.213517665863037, "logps/chosen": -137.5128173828125, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -153.3700714111328, "loss": 0.9954, "margin_dpo/margin_mean": 20.854185104370117, "margin_dpo/margin_std": 25.60382080078125, "step": 254 }, { "KL/chosen_KL_mean": -40.251346588134766, "KL/mean": -54.5115966796875, "KL/rejected_KL_mean": -68.7718505859375, "KL/std": 27.35710906982422, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.029321154579520226, "fcm_dpo/delta": -0.4398929476737976, "fcm_dpo/margin": 28.520492553710938, "fcm_dpo/q_t": 0.32468298077583313, "grad_norm": 13.420915603637695, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.9084497690200806, "logits/rejected": -1.5695667266845703, "logps/chosen": -130.6339111328125, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -165.8480987548828, "loss": 0.8506, "margin_dpo/margin_mean": 28.52049446105957, "margin_dpo/margin_std": 27.410409927368164, "step": 255 }, { "KL/chosen_KL_mean": -36.44483947753906, "KL/mean": -50.711936950683594, "KL/rejected_KL_mean": -64.97903442382812, "KL/std": 28.72021484375, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.029076021164655685, "fcm_dpo/delta": -0.4331156611442566, "fcm_dpo/margin": 28.53419303894043, "fcm_dpo/q_t": 0.3310786783695221, "grad_norm": 13.686904907226562, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.77779221534729, "logits/rejected": -1.4176926612854004, "logps/chosen": -125.19764709472656, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -163.47286987304688, "loss": 0.901, "margin_dpo/margin_mean": 28.534194946289062, "margin_dpo/margin_std": 32.195072174072266, "step": 256 }, { "KL/chosen_KL_mean": -38.83551025390625, "KL/mean": -52.361732482910156, "KL/rejected_KL_mean": -65.88795471191406, "KL/std": 28.95901107788086, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.028801001608371735, "fcm_dpo/delta": -0.38285988569259644, "fcm_dpo/margin": 27.05245590209961, "fcm_dpo/q_t": 0.3388552963733673, "grad_norm": 14.32426929473877, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.9081957340240479, "logits/rejected": -1.5410184860229492, "logps/chosen": -121.63903045654297, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -151.75575256347656, "loss": 0.9062, "margin_dpo/margin_mean": 27.05245590209961, "margin_dpo/margin_std": 28.878873825073242, "step": 257 }, { "KL/chosen_KL_mean": -38.48759841918945, "KL/mean": -51.624847412109375, "KL/rejected_KL_mean": -64.76209259033203, "KL/std": 27.56505584716797, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.028644084930419922, "fcm_dpo/delta": -0.3551442623138428, "fcm_dpo/margin": 26.27449607849121, "fcm_dpo/q_t": 0.3374693989753723, "grad_norm": 13.784102439880371, "learning_rate": 3.923409817553284e-07, "logits/chosen": -1.9661848545074463, "logits/rejected": -1.6084132194519043, "logps/chosen": -128.6753387451172, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -168.2327880859375, "loss": 0.8911, "margin_dpo/margin_mean": 26.274494171142578, "margin_dpo/margin_std": 25.77273178100586, "step": 258 }, { "KL/chosen_KL_mean": -42.38166046142578, "KL/mean": -53.28805923461914, "KL/rejected_KL_mean": -64.19447326660156, "KL/std": 26.758594512939453, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.02846161648631096, "fcm_dpo/delta": -0.22245699167251587, "fcm_dpo/margin": 21.812808990478516, "fcm_dpo/q_t": 0.36720797419548035, "grad_norm": 14.303125381469727, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.1245670318603516, "logits/rejected": -0.8692975044250488, "logps/chosen": -133.15420532226562, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -158.78262329101562, "loss": 1.0108, "margin_dpo/margin_mean": 21.812808990478516, "margin_dpo/margin_std": 29.56048583984375, "step": 259 }, { "KL/chosen_KL_mean": -42.306724548339844, "KL/mean": -52.63303756713867, "KL/rejected_KL_mean": -62.95934295654297, "KL/std": 27.543960571289062, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.028363484889268875, "fcm_dpo/delta": -0.1869281679391861, "fcm_dpo/margin": 20.65262222290039, "fcm_dpo/q_t": 0.3757591247558594, "grad_norm": 15.5162935256958, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.5463430881500244, "logits/rejected": -1.5349820852279663, "logps/chosen": -117.89942169189453, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -163.8048858642578, "loss": 1.0687, "margin_dpo/margin_mean": 20.65262222290039, "margin_dpo/margin_std": 32.011329650878906, "step": 260 }, { "KL/chosen_KL_mean": -33.94065475463867, "KL/mean": -48.86570739746094, "KL/rejected_KL_mean": -63.79075622558594, "KL/std": 28.0537109375, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.028145799413323402, "fcm_dpo/delta": -0.4440617263317108, "fcm_dpo/margin": 29.85010528564453, "fcm_dpo/q_t": 0.3237934112548828, "grad_norm": 12.307134628295898, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.8807425498962402, "logits/rejected": -1.5109401941299438, "logps/chosen": -128.195556640625, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -157.44775390625, "loss": 0.8339, "margin_dpo/margin_mean": 29.850101470947266, "margin_dpo/margin_std": 27.859710693359375, "step": 261 }, { "KL/chosen_KL_mean": -37.12225341796875, "KL/mean": -51.04755401611328, "KL/rejected_KL_mean": -64.97285461425781, "KL/std": 28.132246017456055, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.027926024049520493, "fcm_dpo/delta": -0.3808251619338989, "fcm_dpo/margin": 27.850601196289062, "fcm_dpo/q_t": 0.33597785234451294, "grad_norm": 13.036882400512695, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.6239416599273682, "logits/rejected": -1.069068193435669, "logps/chosen": -122.38955688476562, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -155.7989501953125, "loss": 0.8829, "margin_dpo/margin_mean": 27.850603103637695, "margin_dpo/margin_std": 28.43907928466797, "step": 262 }, { "KL/chosen_KL_mean": -29.898391723632812, "KL/mean": -48.37666320800781, "KL/rejected_KL_mean": -66.85493469238281, "KL/std": 31.92755889892578, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.027621671557426453, "fcm_dpo/delta": -0.6271487474441528, "fcm_dpo/margin": 36.95655059814453, "fcm_dpo/q_t": 0.2941265404224396, "grad_norm": 12.337650299072266, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.9713034629821777, "logits/rejected": -1.4197325706481934, "logps/chosen": -125.65629577636719, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -168.68870544433594, "loss": 0.7702, "margin_dpo/margin_mean": 36.95655059814453, "margin_dpo/margin_std": 33.03696823120117, "step": 263 }, { "KL/chosen_KL_mean": -32.490848541259766, "KL/mean": -47.994354248046875, "KL/rejected_KL_mean": -63.497859954833984, "KL/std": 30.50869369506836, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.027339385822415352, "fcm_dpo/delta": -0.4516720771789551, "fcm_dpo/margin": 31.00701904296875, "fcm_dpo/q_t": 0.32434895634651184, "grad_norm": 11.790332794189453, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -1.7210297584533691, "logits/rejected": -1.4319219589233398, "logps/chosen": -123.11351013183594, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -163.82339477539062, "loss": 0.8601, "margin_dpo/margin_mean": 31.007017135620117, "margin_dpo/margin_std": 31.099895477294922, "step": 264 }, { "KL/chosen_KL_mean": -35.38058853149414, "KL/mean": -50.29444885253906, "KL/rejected_KL_mean": -65.20831298828125, "KL/std": 30.471410751342773, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.027097882702946663, "fcm_dpo/delta": -0.41197702288627625, "fcm_dpo/margin": 29.82772445678711, "fcm_dpo/q_t": 0.33663177490234375, "grad_norm": 11.608314514160156, "learning_rate": 3.848656339557562e-07, "logits/chosen": -1.8284554481506348, "logits/rejected": -1.432159662246704, "logps/chosen": -127.75291442871094, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -159.83587646484375, "loss": 0.9022, "margin_dpo/margin_mean": 29.827720642089844, "margin_dpo/margin_std": 33.25975799560547, "step": 265 }, { "KL/chosen_KL_mean": -40.38309860229492, "KL/mean": -53.100589752197266, "KL/rejected_KL_mean": -65.81806945800781, "KL/std": 28.095367431640625, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.026937022805213928, "fcm_dpo/delta": -0.287203848361969, "fcm_dpo/margin": 25.43497085571289, "fcm_dpo/q_t": 0.35257548093795776, "grad_norm": 14.967239379882812, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.185138702392578, "logits/rejected": -1.4306024312973022, "logps/chosen": -142.5831298828125, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -140.1844940185547, "loss": 0.9408, "margin_dpo/margin_mean": 25.43497085571289, "margin_dpo/margin_std": 27.90793228149414, "step": 266 }, { "KL/chosen_KL_mean": -34.07396697998047, "KL/mean": -49.697269439697266, "KL/rejected_KL_mean": -65.3205795288086, "KL/std": 30.663761138916016, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.02671274170279503, "fcm_dpo/delta": -0.4386937618255615, "fcm_dpo/margin": 31.246604919433594, "fcm_dpo/q_t": 0.32329055666923523, "grad_norm": 12.612561225891113, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.7332322597503662, "logits/rejected": -1.151093602180481, "logps/chosen": -138.35997009277344, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -161.30776977539062, "loss": 0.8306, "margin_dpo/margin_mean": 31.246604919433594, "margin_dpo/margin_std": 27.79989242553711, "step": 267 }, { "KL/chosen_KL_mean": -33.5994873046875, "KL/mean": -49.958473205566406, "KL/rejected_KL_mean": -66.31745147705078, "KL/std": 29.539899826049805, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0264785997569561, "fcm_dpo/delta": -0.47049230337142944, "fcm_dpo/margin": 32.71796417236328, "fcm_dpo/q_t": 0.31792595982551575, "grad_norm": 12.528539657592773, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6352041959762573, "logits/rejected": -1.4422048330307007, "logps/chosen": -120.29570770263672, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -175.50929260253906, "loss": 0.8231, "margin_dpo/margin_mean": 32.71796417236328, "margin_dpo/margin_std": 29.25550079345703, "step": 268 }, { "KL/chosen_KL_mean": -34.93034362792969, "KL/mean": -51.78022003173828, "KL/rejected_KL_mean": -68.63008117675781, "KL/std": 32.55498504638672, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.026236586272716522, "fcm_dpo/delta": -0.4883520007133484, "fcm_dpo/margin": 33.69974899291992, "fcm_dpo/q_t": 0.3182528614997864, "grad_norm": 12.410087585449219, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.8224753141403198, "logits/rejected": -1.3549988269805908, "logps/chosen": -139.90216064453125, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -181.10655212402344, "loss": 0.8329, "margin_dpo/margin_mean": 33.69975280761719, "margin_dpo/margin_std": 32.19770812988281, "step": 269 }, { "KL/chosen_KL_mean": -38.37769317626953, "KL/mean": -53.216400146484375, "KL/rejected_KL_mean": -68.05510711669922, "KL/std": 32.14869689941406, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.026024963706731796, "fcm_dpo/delta": -0.3751603960990906, "fcm_dpo/margin": 29.67742156982422, "fcm_dpo/q_t": 0.34402206540107727, "grad_norm": 12.910859107971191, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.2842559814453125, "logits/rejected": -1.863379716873169, "logps/chosen": -139.45152282714844, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -185.80799865722656, "loss": 0.9444, "margin_dpo/margin_mean": 29.67742156982422, "margin_dpo/margin_std": 36.501502990722656, "step": 270 }, { "KL/chosen_KL_mean": -32.071346282958984, "KL/mean": -49.920066833496094, "KL/rejected_KL_mean": -67.76878356933594, "KL/std": 30.57308006286621, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.025759562849998474, "fcm_dpo/delta": -0.5244994163513184, "fcm_dpo/margin": 35.69744110107422, "fcm_dpo/q_t": 0.30819737911224365, "grad_norm": 12.215036392211914, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.3535645008087158, "logits/rejected": -0.9001301527023315, "logps/chosen": -128.7959442138672, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -166.29319763183594, "loss": 0.8385, "margin_dpo/margin_mean": 35.69744110107422, "margin_dpo/margin_std": 34.778587341308594, "step": 271 }, { "KL/chosen_KL_mean": -26.68543243408203, "KL/mean": -43.73072814941406, "KL/rejected_KL_mean": -60.776023864746094, "KL/std": 32.82990264892578, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.025516221299767494, "fcm_dpo/delta": -0.4739770293235779, "fcm_dpo/margin": 34.09059143066406, "fcm_dpo/q_t": 0.3241565227508545, "grad_norm": 13.011197090148926, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.4095349311828613, "logits/rejected": -1.2568260431289673, "logps/chosen": -107.66265106201172, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -169.33139038085938, "loss": 0.8578, "margin_dpo/margin_mean": 34.0905876159668, "margin_dpo/margin_std": 35.19842529296875, "step": 272 }, { "KL/chosen_KL_mean": -35.94442367553711, "KL/mean": -53.800601959228516, "KL/rejected_KL_mean": -71.65676879882812, "KL/std": 34.87104415893555, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.02522920072078705, "fcm_dpo/delta": -0.5065714120864868, "fcm_dpo/margin": 35.71234130859375, "fcm_dpo/q_t": 0.31881392002105713, "grad_norm": 12.30667495727539, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.560122013092041, "logits/rejected": -1.1627106666564941, "logps/chosen": -128.1690216064453, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -169.01983642578125, "loss": 0.8784, "margin_dpo/margin_mean": 35.71234130859375, "margin_dpo/margin_std": 38.546165466308594, "step": 273 }, { "KL/chosen_KL_mean": -37.33213424682617, "KL/mean": -51.40972137451172, "KL/rejected_KL_mean": -65.48731994628906, "KL/std": 31.882261276245117, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.025073044002056122, "fcm_dpo/delta": -0.30825263261795044, "fcm_dpo/margin": 28.155183792114258, "fcm_dpo/q_t": 0.3484511971473694, "grad_norm": 14.15174388885498, "learning_rate": 3.75e-07, "logits/chosen": -2.2529473304748535, "logits/rejected": -1.8704895973205566, "logps/chosen": -119.59302520751953, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -149.36431884765625, "loss": 0.9165, "margin_dpo/margin_mean": 28.155183792114258, "margin_dpo/margin_std": 29.40411949157715, "step": 274 }, { "KL/chosen_KL_mean": -31.040283203125, "KL/mean": -46.501121520996094, "KL/rejected_KL_mean": -61.96196365356445, "KL/std": 31.808469772338867, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.02492256462574005, "fcm_dpo/delta": -0.3733590841293335, "fcm_dpo/margin": 30.921674728393555, "fcm_dpo/q_t": 0.33928564190864563, "grad_norm": 13.64566421508789, "learning_rate": 3.738869612786737e-07, "logits/chosen": -1.9925382137298584, "logits/rejected": -1.65989089012146, "logps/chosen": -110.72724914550781, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -160.7129364013672, "loss": 0.9256, "margin_dpo/margin_mean": 30.921672821044922, "margin_dpo/margin_std": 34.969886779785156, "step": 275 }, { "KL/chosen_KL_mean": -39.44208526611328, "KL/mean": -54.27753829956055, "KL/rejected_KL_mean": -69.11299133300781, "KL/std": 31.616308212280273, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.02470632828772068, "fcm_dpo/delta": -0.3360960781574249, "fcm_dpo/margin": 29.670913696289062, "fcm_dpo/q_t": 0.34791409969329834, "grad_norm": 14.51209545135498, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.966467261314392, "logits/rejected": -1.4159599542617798, "logps/chosen": -125.98179626464844, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -146.96693420410156, "loss": 0.94, "margin_dpo/margin_mean": 29.670915603637695, "margin_dpo/margin_std": 34.28968811035156, "step": 276 }, { "KL/chosen_KL_mean": -39.285274505615234, "KL/mean": -52.91626739501953, "KL/rejected_KL_mean": -66.54725646972656, "KL/std": 30.263166427612305, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.024581167846918106, "fcm_dpo/delta": -0.2720816135406494, "fcm_dpo/margin": 27.261974334716797, "fcm_dpo/q_t": 0.35357367992401123, "grad_norm": 15.057683944702148, "learning_rate": 3.71651119641714e-07, "logits/chosen": -1.9856171607971191, "logits/rejected": -1.5683019161224365, "logps/chosen": -123.52938842773438, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -165.3814697265625, "loss": 0.9377, "margin_dpo/margin_mean": 27.261974334716797, "margin_dpo/margin_std": 28.891197204589844, "step": 277 }, { "KL/chosen_KL_mean": -37.610816955566406, "KL/mean": -52.608909606933594, "KL/rejected_KL_mean": -67.60700988769531, "KL/std": 33.9820671081543, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.024417694658041, "fcm_dpo/delta": -0.33519667387008667, "fcm_dpo/margin": 29.996192932128906, "fcm_dpo/q_t": 0.3438917398452759, "grad_norm": 13.089381217956543, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.117435932159424, "logits/rejected": -1.9110037088394165, "logps/chosen": -120.0427017211914, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -165.46392822265625, "loss": 0.9333, "margin_dpo/margin_mean": 29.996192932128906, "margin_dpo/margin_std": 33.46534729003906, "step": 278 }, { "KL/chosen_KL_mean": -40.28538513183594, "KL/mean": -56.44579315185547, "KL/rejected_KL_mean": -72.606201171875, "KL/std": 34.32283020019531, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.024240706115961075, "fcm_dpo/delta": -0.38672518730163574, "fcm_dpo/margin": 32.32080841064453, "fcm_dpo/q_t": 0.3365251421928406, "grad_norm": 12.916276931762695, "learning_rate": 3.6940245560867e-07, "logits/chosen": -2.0066959857940674, "logits/rejected": -1.7911162376403809, "logps/chosen": -125.45338439941406, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -166.73284912109375, "loss": 0.8929, "margin_dpo/margin_mean": 32.32080841064453, "margin_dpo/margin_std": 34.12754440307617, "step": 279 }, { "KL/chosen_KL_mean": -38.72584533691406, "KL/mean": -57.12493133544922, "KL/rejected_KL_mean": -75.52401733398438, "KL/std": 32.68305969238281, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.024033015593886375, "fcm_dpo/delta": -0.48860442638397217, "fcm_dpo/margin": 36.79816818237305, "fcm_dpo/q_t": 0.31600573658943176, "grad_norm": 12.832268714904785, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.7503564357757568, "logits/rejected": -1.535639762878418, "logps/chosen": -124.58226013183594, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -179.64260864257812, "loss": 0.827, "margin_dpo/margin_mean": 36.79816436767578, "margin_dpo/margin_std": 34.06572723388672, "step": 280 }, { "KL/chosen_KL_mean": -40.131412506103516, "KL/mean": -58.99977493286133, "KL/rejected_KL_mean": -77.8681411743164, "KL/std": 38.16148376464844, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.02378890849649906, "fcm_dpo/delta": -0.5022423267364502, "fcm_dpo/margin": 37.736732482910156, "fcm_dpo/q_t": 0.32132890820503235, "grad_norm": 11.28731918334961, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.236297845840454, "logits/rejected": -1.5351178646087646, "logps/chosen": -126.68223571777344, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -166.49681091308594, "loss": 0.8571, "margin_dpo/margin_mean": 37.736732482910156, "margin_dpo/margin_std": 40.66725158691406, "step": 281 }, { "KL/chosen_KL_mean": -38.916175842285156, "KL/mean": -61.16608428955078, "KL/rejected_KL_mean": -83.41600036621094, "KL/std": 37.975318908691406, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.02349122054874897, "fcm_dpo/delta": -0.6524159908294678, "fcm_dpo/margin": 44.49982452392578, "fcm_dpo/q_t": 0.28854602575302124, "grad_norm": 10.858722686767578, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.6010751724243164, "logits/rejected": -1.2246007919311523, "logps/chosen": -144.0212860107422, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -186.2693634033203, "loss": 0.7297, "margin_dpo/margin_mean": 44.49982452392578, "margin_dpo/margin_std": 36.32782745361328, "step": 282 }, { "KL/chosen_KL_mean": -46.11542892456055, "KL/mean": -61.38003158569336, "KL/rejected_KL_mean": -76.64464569091797, "KL/std": 36.70181655883789, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.02331283688545227, "fcm_dpo/delta": -0.31394922733306885, "fcm_dpo/margin": 30.529212951660156, "fcm_dpo/q_t": 0.35395652055740356, "grad_norm": 15.791332244873047, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.236751079559326, "logits/rejected": -1.9022769927978516, "logps/chosen": -141.16802978515625, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -180.18919372558594, "loss": 0.9921, "margin_dpo/margin_mean": 30.529212951660156, "margin_dpo/margin_std": 41.04127502441406, "step": 283 }, { "KL/chosen_KL_mean": -38.48414993286133, "KL/mean": -60.192569732666016, "KL/rejected_KL_mean": -81.90098571777344, "KL/std": 39.40203857421875, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.02305092290043831, "fcm_dpo/delta": -0.6073124408721924, "fcm_dpo/margin": 43.416831970214844, "fcm_dpo/q_t": 0.29370972514152527, "grad_norm": 10.890854835510254, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.690298318862915, "logits/rejected": -1.3783355951309204, "logps/chosen": -126.15062713623047, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -180.6520233154297, "loss": 0.7429, "margin_dpo/margin_mean": 43.416831970214844, "margin_dpo/margin_std": 35.450775146484375, "step": 284 }, { "KL/chosen_KL_mean": -45.13738250732422, "KL/mean": -60.24957275390625, "KL/rejected_KL_mean": -75.36176300048828, "KL/std": 32.83898162841797, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.02290358766913414, "fcm_dpo/delta": -0.2941926419734955, "fcm_dpo/margin": 30.224369049072266, "fcm_dpo/q_t": 0.34745365381240845, "grad_norm": 17.93300437927246, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.256251573562622, "logits/rejected": -1.9723789691925049, "logps/chosen": -139.57041931152344, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -179.4337158203125, "loss": 0.9214, "margin_dpo/margin_mean": 30.224369049072266, "margin_dpo/margin_std": 30.777359008789062, "step": 285 }, { "KL/chosen_KL_mean": -48.24298858642578, "KL/mean": -67.42131805419922, "KL/rejected_KL_mean": -86.59964752197266, "KL/std": 35.78898239135742, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.022693052887916565, "fcm_dpo/delta": -0.4746668338775635, "fcm_dpo/margin": 38.35667037963867, "fcm_dpo/q_t": 0.31789207458496094, "grad_norm": 13.485489845275879, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.799415111541748, "logits/rejected": -1.3391966819763184, "logps/chosen": -151.96337890625, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -182.85740661621094, "loss": 0.8381, "margin_dpo/margin_mean": 38.356666564941406, "margin_dpo/margin_std": 35.988525390625, "step": 286 }, { "KL/chosen_KL_mean": -57.40538024902344, "KL/mean": -69.89506530761719, "KL/rejected_KL_mean": -82.38475036621094, "KL/std": 33.675140380859375, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.022604040801525116, "fcm_dpo/delta": -0.16555531322956085, "fcm_dpo/margin": 24.979360580444336, "fcm_dpo/q_t": 0.3844905197620392, "grad_norm": 13.763415336608887, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.057457447052002, "logits/rejected": -1.479313850402832, "logps/chosen": -151.29525756835938, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -165.71841430664062, "loss": 1.0623, "margin_dpo/margin_mean": 24.979360580444336, "margin_dpo/margin_std": 37.9991340637207, "step": 287 }, { "KL/chosen_KL_mean": -54.98366165161133, "KL/mean": -75.02680206298828, "KL/rejected_KL_mean": -95.0699462890625, "KL/std": 37.7614860534668, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.022396381944417953, "fcm_dpo/delta": -0.5023754835128784, "fcm_dpo/margin": 40.08629608154297, "fcm_dpo/q_t": 0.32051169872283936, "grad_norm": 12.79360580444336, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -2.0356192588806152, "logits/rejected": -1.6519547700881958, "logps/chosen": -143.13967895507812, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -188.35189819335938, "loss": 0.8462, "margin_dpo/margin_mean": 40.08629608154297, "margin_dpo/margin_std": 41.130226135253906, "step": 288 }, { "KL/chosen_KL_mean": -54.604400634765625, "KL/mean": -75.91455078125, "KL/rejected_KL_mean": -97.22469329833984, "KL/std": 41.33440017700195, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.02216918021440506, "fcm_dpo/delta": -0.5498613119125366, "fcm_dpo/margin": 42.62029266357422, "fcm_dpo/q_t": 0.32332658767700195, "grad_norm": 11.994853019714355, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.911605715751648, "logits/rejected": -1.7074854373931885, "logps/chosen": -129.99732971191406, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -190.37896728515625, "loss": 0.8599, "margin_dpo/margin_mean": 42.62029266357422, "margin_dpo/margin_std": 48.12208557128906, "step": 289 }, { "KL/chosen_KL_mean": -60.447696685791016, "KL/mean": -81.99551391601562, "KL/rejected_KL_mean": -103.54332733154297, "KL/std": 43.04484558105469, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.02193979173898697, "fcm_dpo/delta": -0.5503648519515991, "fcm_dpo/margin": 43.095638275146484, "fcm_dpo/q_t": 0.3133336901664734, "grad_norm": 12.606535911560059, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.613610029220581, "logits/rejected": -1.622948169708252, "logps/chosen": -148.4896697998047, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -226.7554931640625, "loss": 0.8316, "margin_dpo/margin_mean": 43.09564208984375, "margin_dpo/margin_std": 44.25682830810547, "step": 290 }, { "KL/chosen_KL_mean": -58.097206115722656, "KL/mean": -75.20640563964844, "KL/rejected_KL_mean": -92.31560516357422, "KL/std": 37.43865966796875, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.02174883522093296, "fcm_dpo/delta": -0.3467201292514801, "fcm_dpo/margin": 34.21839141845703, "fcm_dpo/q_t": 0.3406580686569214, "grad_norm": 13.332621574401855, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.2086572647094727, "logits/rejected": -1.7602802515029907, "logps/chosen": -152.19244384765625, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -188.36566162109375, "loss": 0.9293, "margin_dpo/margin_mean": 34.21839141845703, "margin_dpo/margin_std": 38.572052001953125, "step": 291 }, { "KL/chosen_KL_mean": -58.23985290527344, "KL/mean": -78.39201354980469, "KL/rejected_KL_mean": -98.5441665649414, "KL/std": 37.39674377441406, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.021553009748458862, "fcm_dpo/delta": -0.47278016805648804, "fcm_dpo/margin": 40.30431365966797, "fcm_dpo/q_t": 0.31351912021636963, "grad_norm": 13.63647174835205, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -1.954956293106079, "logits/rejected": -1.5038014650344849, "logps/chosen": -146.49026489257812, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -194.96180725097656, "loss": 0.7981, "margin_dpo/margin_mean": 40.30431365966797, "margin_dpo/margin_std": 32.55630874633789, "step": 292 }, { "KL/chosen_KL_mean": -62.34445571899414, "KL/mean": -77.2676773071289, "KL/rejected_KL_mean": -92.19090270996094, "KL/std": 39.068634033203125, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.021439123898744583, "fcm_dpo/delta": -0.24135586619377136, "fcm_dpo/margin": 29.846431732177734, "fcm_dpo/q_t": 0.3701302707195282, "grad_norm": 11.806089401245117, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.488941192626953, "logits/rejected": -1.9454001188278198, "logps/chosen": -149.72100830078125, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -177.9467010498047, "loss": 1.011, "margin_dpo/margin_mean": 29.846431732177734, "margin_dpo/margin_std": 40.912296295166016, "step": 293 }, { "KL/chosen_KL_mean": -65.17967224121094, "KL/mean": -78.1676025390625, "KL/rejected_KL_mean": -91.15553283691406, "KL/std": 37.45103073120117, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.02135341987013817, "fcm_dpo/delta": -0.1555100679397583, "fcm_dpo/margin": 25.975852966308594, "fcm_dpo/q_t": 0.38634365797042847, "grad_norm": 13.062808990478516, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.6592676639556885, "logits/rejected": -1.471679449081421, "logps/chosen": -138.6876220703125, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -179.2443084716797, "loss": 1.0809, "margin_dpo/margin_mean": 25.97585678100586, "margin_dpo/margin_std": 42.524269104003906, "step": 294 }, { "KL/chosen_KL_mean": -63.26070022583008, "KL/mean": -82.21199035644531, "KL/rejected_KL_mean": -101.16326904296875, "KL/std": 38.44963073730469, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.02118988335132599, "fcm_dpo/delta": -0.4067138433456421, "fcm_dpo/margin": 37.902557373046875, "fcm_dpo/q_t": 0.3352050185203552, "grad_norm": 12.076995849609375, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.6281332969665527, "logits/rejected": -1.4856083393096924, "logps/chosen": -141.02618408203125, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -195.41053771972656, "loss": 0.893, "margin_dpo/margin_mean": 37.902557373046875, "margin_dpo/margin_std": 41.4855842590332, "step": 295 }, { "KL/chosen_KL_mean": -68.2701416015625, "KL/mean": -88.83479309082031, "KL/rejected_KL_mean": -109.39944458007812, "KL/std": 39.66255187988281, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.020999858155846596, "fcm_dpo/delta": -0.46805340051651, "fcm_dpo/margin": 41.12929916381836, "fcm_dpo/q_t": 0.32279521226882935, "grad_norm": 13.107879638671875, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.4847171306610107, "logits/rejected": -2.0265321731567383, "logps/chosen": -166.12655639648438, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -210.21575927734375, "loss": 0.8513, "margin_dpo/margin_mean": 41.129302978515625, "margin_dpo/margin_std": 41.603759765625, "step": 296 }, { "KL/chosen_KL_mean": -71.14308166503906, "KL/mean": -88.39241790771484, "KL/rejected_KL_mean": -105.64175415039062, "KL/std": 36.68788146972656, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.020864443853497505, "fcm_dpo/delta": -0.3220633268356323, "fcm_dpo/margin": 34.4986686706543, "fcm_dpo/q_t": 0.3506333827972412, "grad_norm": 13.896451950073242, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.0627163648605347, "logits/rejected": -0.735419750213623, "logps/chosen": -159.70892333984375, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -207.1983184814453, "loss": 0.9306, "margin_dpo/margin_mean": 34.4986686706543, "margin_dpo/margin_std": 39.48872375488281, "step": 297 }, { "KL/chosen_KL_mean": -63.67530059814453, "KL/mean": -84.44181823730469, "KL/rejected_KL_mean": -105.20832824707031, "KL/std": 42.993438720703125, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.020693402737379074, "fcm_dpo/delta": -0.46326181292533875, "fcm_dpo/margin": 41.53302764892578, "fcm_dpo/q_t": 0.3300492763519287, "grad_norm": 15.69887924194336, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.008817195892334, "logits/rejected": -1.9289593696594238, "logps/chosen": -158.5557403564453, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -227.51934814453125, "loss": 0.8677, "margin_dpo/margin_mean": 41.53302764892578, "margin_dpo/margin_std": 45.42005157470703, "step": 298 }, { "KL/chosen_KL_mean": -65.93927764892578, "KL/mean": -88.03410339355469, "KL/rejected_KL_mean": -110.12890625, "KL/std": 39.78760528564453, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.020471787080168724, "fcm_dpo/delta": -0.5094255805015564, "fcm_dpo/margin": 44.18962860107422, "fcm_dpo/q_t": 0.31518810987472534, "grad_norm": 11.786602973937988, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.1260977983474731, "logits/rejected": -0.8882899880409241, "logps/chosen": -146.34762573242188, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -199.66607666015625, "loss": 0.8208, "margin_dpo/margin_mean": 44.18962860107422, "margin_dpo/margin_std": 41.868587493896484, "step": 299 }, { "KL/chosen_KL_mean": -70.89126586914062, "KL/mean": -89.39435577392578, "KL/rejected_KL_mean": -107.89744567871094, "KL/std": 43.24198913574219, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.020312195643782616, "fcm_dpo/delta": -0.3544493615627289, "fcm_dpo/margin": 37.00617980957031, "fcm_dpo/q_t": 0.3469976484775543, "grad_norm": 13.139739036560059, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.7509734630584717, "logits/rejected": -1.5386929512023926, "logps/chosen": -159.0501708984375, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -208.83663940429688, "loss": 0.9433, "margin_dpo/margin_mean": 37.00617599487305, "margin_dpo/margin_std": 45.18927001953125, "step": 300 }, { "KL/chosen_KL_mean": -65.44816589355469, "KL/mean": -88.56227111816406, "KL/rejected_KL_mean": -111.6763916015625, "KL/std": 42.691322326660156, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.020099522545933723, "fcm_dpo/delta": -0.5348674654960632, "fcm_dpo/margin": 46.22822570800781, "fcm_dpo/q_t": 0.31541746854782104, "grad_norm": 12.409183502197266, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.9165620803833008, "logits/rejected": -1.490151286125183, "logps/chosen": -150.9530792236328, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -210.59323120117188, "loss": 0.8264, "margin_dpo/margin_mean": 46.22822570800781, "margin_dpo/margin_std": 45.831016540527344, "step": 301 }, { "KL/chosen_KL_mean": -67.12296295166016, "KL/mean": -85.71437072753906, "KL/rejected_KL_mean": -104.30577087402344, "KL/std": 43.79869079589844, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.019953861832618713, "fcm_dpo/delta": -0.34467992186546326, "fcm_dpo/margin": 37.18281555175781, "fcm_dpo/q_t": 0.3465021550655365, "grad_norm": 14.155974388122559, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.382387399673462, "logits/rejected": -1.1996793746948242, "logps/chosen": -146.5733642578125, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -207.3648681640625, "loss": 0.9397, "margin_dpo/margin_mean": 37.18281555175781, "margin_dpo/margin_std": 44.069847106933594, "step": 302 }, { "KL/chosen_KL_mean": -59.83890914916992, "KL/mean": -81.40945434570312, "KL/rejected_KL_mean": -102.97999572753906, "KL/std": 40.8756103515625, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.0198030024766922, "fcm_dpo/delta": -0.4580414891242981, "fcm_dpo/margin": 43.14108657836914, "fcm_dpo/q_t": 0.3229633569717407, "grad_norm": 11.884074211120605, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.7205793857574463, "logits/rejected": -1.2839865684509277, "logps/chosen": -155.43719482421875, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -195.04457092285156, "loss": 0.8506, "margin_dpo/margin_mean": 43.14108657836914, "margin_dpo/margin_std": 41.85108184814453, "step": 303 }, { "KL/chosen_KL_mean": -60.743980407714844, "KL/mean": -83.44241333007812, "KL/rejected_KL_mean": -106.14083862304688, "KL/std": 44.01972198486328, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.019605576992034912, "fcm_dpo/delta": -0.49431854486465454, "fcm_dpo/margin": 45.3968620300293, "fcm_dpo/q_t": 0.31521016359329224, "grad_norm": 12.74997329711914, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.0168466567993164, "logits/rejected": -1.5961946249008179, "logps/chosen": -160.19712829589844, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -204.2230987548828, "loss": 0.8332, "margin_dpo/margin_mean": 45.39686584472656, "margin_dpo/margin_std": 43.20547866821289, "step": 304 }, { "KL/chosen_KL_mean": -62.35185241699219, "KL/mean": -84.86117553710938, "KL/rejected_KL_mean": -107.3705062866211, "KL/std": 45.871604919433594, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.019391043111681938, "fcm_dpo/delta": -0.4778949022293091, "fcm_dpo/margin": 45.0186653137207, "fcm_dpo/q_t": 0.31993046402931213, "grad_norm": 12.025956153869629, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.7848091125488281, "logits/rejected": -1.2895686626434326, "logps/chosen": -165.61524963378906, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -205.19947814941406, "loss": 0.8442, "margin_dpo/margin_mean": 45.01866149902344, "margin_dpo/margin_std": 42.1050910949707, "step": 305 }, { "KL/chosen_KL_mean": -54.5062141418457, "KL/mean": -81.76741027832031, "KL/rejected_KL_mean": -109.02859497070312, "KL/std": 42.703269958496094, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.019169528037309647, "fcm_dpo/delta": -0.6522985696792603, "fcm_dpo/margin": 54.52239227294922, "fcm_dpo/q_t": 0.2814841866493225, "grad_norm": 11.241499900817871, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.261260509490967, "logits/rejected": -1.730342984199524, "logps/chosen": -154.81005859375, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -201.25946044921875, "loss": 0.7045, "margin_dpo/margin_mean": 54.52239227294922, "margin_dpo/margin_std": 37.88768768310547, "step": 306 }, { "KL/chosen_KL_mean": -62.59770202636719, "KL/mean": -82.10272216796875, "KL/rejected_KL_mean": -101.60774993896484, "KL/std": 43.11854553222656, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.019009657204151154, "fcm_dpo/delta": -0.34427082538604736, "fcm_dpo/margin": 39.010040283203125, "fcm_dpo/q_t": 0.34867823123931885, "grad_norm": 12.681523323059082, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.8357939720153809, "logits/rejected": -1.5076005458831787, "logps/chosen": -159.24746704101562, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -194.95635986328125, "loss": 0.9552, "margin_dpo/margin_mean": 39.010040283203125, "margin_dpo/margin_std": 48.91339874267578, "step": 307 }, { "KL/chosen_KL_mean": -64.04351806640625, "KL/mean": -83.38824462890625, "KL/rejected_KL_mean": -102.73297882080078, "KL/std": 44.19226837158203, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.01886998862028122, "fcm_dpo/delta": -0.3330450654029846, "fcm_dpo/margin": 38.689453125, "fcm_dpo/q_t": 0.35422003269195557, "grad_norm": 13.679485321044922, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.6814544200897217, "logits/rejected": -1.431399941444397, "logps/chosen": -149.7681884765625, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -189.15615844726562, "loss": 0.9655, "margin_dpo/margin_mean": 38.689453125, "margin_dpo/margin_std": 48.891483306884766, "step": 308 }, { "KL/chosen_KL_mean": -69.12553405761719, "KL/mean": -89.26737976074219, "KL/rejected_KL_mean": -109.40924072265625, "KL/std": 47.05155944824219, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.018761519342660904, "fcm_dpo/delta": -0.3584146797657013, "fcm_dpo/margin": 40.2837028503418, "fcm_dpo/q_t": 0.3427354097366333, "grad_norm": 13.580839157104492, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.6168766021728516, "logits/rejected": -1.1847994327545166, "logps/chosen": -152.27699279785156, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -195.99526977539062, "loss": 0.9216, "margin_dpo/margin_mean": 40.28370666503906, "margin_dpo/margin_std": 45.20746612548828, "step": 309 }, { "KL/chosen_KL_mean": -60.69127655029297, "KL/mean": -80.54562377929688, "KL/rejected_KL_mean": -100.39997863769531, "KL/std": 43.60961151123047, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.018630605190992355, "fcm_dpo/delta": -0.34227481484413147, "fcm_dpo/margin": 39.708702087402344, "fcm_dpo/q_t": 0.34503814578056335, "grad_norm": 12.598888397216797, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2316224575042725, "logits/rejected": -1.8297162055969238, "logps/chosen": -150.42926025390625, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -187.47080993652344, "loss": 0.9335, "margin_dpo/margin_mean": 39.708702087402344, "margin_dpo/margin_std": 45.81877899169922, "step": 310 }, { "KL/chosen_KL_mean": -75.77290344238281, "KL/mean": -100.71737670898438, "KL/rejected_KL_mean": -125.66185760498047, "KL/std": 45.415489196777344, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.01844840496778488, "fcm_dpo/delta": -0.5252406597137451, "fcm_dpo/margin": 49.888946533203125, "fcm_dpo/q_t": 0.3115510046482086, "grad_norm": 12.597408294677734, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.6193125247955322, "logits/rejected": -0.8940199613571167, "logps/chosen": -164.29934692382812, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -207.92794799804688, "loss": 0.8002, "margin_dpo/margin_mean": 49.888946533203125, "margin_dpo/margin_std": 45.32282257080078, "step": 311 }, { "KL/chosen_KL_mean": -70.67318725585938, "KL/mean": -93.5531005859375, "KL/rejected_KL_mean": -116.43299865722656, "KL/std": 49.55390167236328, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.018291080370545387, "fcm_dpo/delta": -0.44049185514450073, "fcm_dpo/margin": 45.75981903076172, "fcm_dpo/q_t": 0.32837164402008057, "grad_norm": 12.204483032226562, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.8576507568359375, "logits/rejected": -1.3526122570037842, "logps/chosen": -151.07388305664062, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -198.88015747070312, "loss": 0.8803, "margin_dpo/margin_mean": 45.75981903076172, "margin_dpo/margin_std": 48.659423828125, "step": 312 }, { "KL/chosen_KL_mean": -65.854248046875, "KL/mean": -86.69657897949219, "KL/rejected_KL_mean": -107.53892517089844, "KL/std": 49.61644744873047, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.018144069239497185, "fcm_dpo/delta": -0.3590027689933777, "fcm_dpo/margin": 41.68468475341797, "fcm_dpo/q_t": 0.3457859754562378, "grad_norm": 11.701065063476562, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.512299060821533, "logits/rejected": -2.10513973236084, "logps/chosen": -149.99615478515625, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -200.96575927734375, "loss": 0.954, "margin_dpo/margin_mean": 41.684688568115234, "margin_dpo/margin_std": 52.235923767089844, "step": 313 }, { "KL/chosen_KL_mean": -66.13716888427734, "KL/mean": -87.75468444824219, "KL/rejected_KL_mean": -109.37220764160156, "KL/std": 49.84355926513672, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.01799783855676651, "fcm_dpo/delta": -0.38131454586982727, "fcm_dpo/margin": 43.23503875732422, "fcm_dpo/q_t": 0.34038037061691284, "grad_norm": 12.680259704589844, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.9693212509155273, "logits/rejected": -1.7560160160064697, "logps/chosen": -154.9468994140625, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -213.5311279296875, "loss": 0.9114, "margin_dpo/margin_mean": 43.23503875732422, "margin_dpo/margin_std": 48.49885940551758, "step": 314 }, { "KL/chosen_KL_mean": -65.34040832519531, "KL/mean": -87.17596435546875, "KL/rejected_KL_mean": -109.01153564453125, "KL/std": 51.27227020263672, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.017866671085357666, "fcm_dpo/delta": -0.38329172134399414, "fcm_dpo/margin": 43.671119689941406, "fcm_dpo/q_t": 0.3424239158630371, "grad_norm": 13.580316543579102, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.6038403511047363, "logits/rejected": -1.422085165977478, "logps/chosen": -146.1341094970703, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -201.14865112304688, "loss": 0.9559, "margin_dpo/margin_mean": 43.671119689941406, "margin_dpo/margin_std": 55.25852584838867, "step": 315 }, { "KL/chosen_KL_mean": -70.09283447265625, "KL/mean": -92.21991729736328, "KL/rejected_KL_mean": -114.34700775146484, "KL/std": 51.13347625732422, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.017733268439769745, "fcm_dpo/delta": -0.38776788115501404, "fcm_dpo/margin": 44.25417709350586, "fcm_dpo/q_t": 0.3391193151473999, "grad_norm": 12.205835342407227, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.6658852100372314, "logits/rejected": -1.6669442653656006, "logps/chosen": -154.27708435058594, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -220.03782653808594, "loss": 0.9087, "margin_dpo/margin_mean": 44.254180908203125, "margin_dpo/margin_std": 50.493797302246094, "step": 316 }, { "KL/chosen_KL_mean": -69.64051818847656, "KL/mean": -98.7042236328125, "KL/rejected_KL_mean": -127.76790618896484, "KL/std": 58.3246955871582, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.01753898337483406, "fcm_dpo/delta": -0.6256468296051025, "fcm_dpo/margin": 58.12738037109375, "fcm_dpo/q_t": 0.3031194806098938, "grad_norm": 14.543208122253418, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -2.076808452606201, "logits/rejected": -1.6044996976852417, "logps/chosen": -157.51400756835938, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -236.03311157226562, "loss": 0.8071, "margin_dpo/margin_mean": 58.12738037109375, "margin_dpo/margin_std": 57.95631408691406, "step": 317 }, { "KL/chosen_KL_mean": -67.63369750976562, "KL/mean": -93.53376770019531, "KL/rejected_KL_mean": -119.433837890625, "KL/std": 48.13258743286133, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.01734118163585663, "fcm_dpo/delta": -0.5029230117797852, "fcm_dpo/margin": 51.80014419555664, "fcm_dpo/q_t": 0.31597164273262024, "grad_norm": 10.925496101379395, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.0392098426818848, "logits/rejected": -1.3545225858688354, "logps/chosen": -165.8392333984375, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -207.57012939453125, "loss": 0.8324, "margin_dpo/margin_mean": 51.80014419555664, "margin_dpo/margin_std": 50.394065856933594, "step": 318 }, { "KL/chosen_KL_mean": -59.10885238647461, "KL/mean": -86.15106201171875, "KL/rejected_KL_mean": -113.19326782226562, "KL/std": 50.29893112182617, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.017158569768071175, "fcm_dpo/delta": -0.5331438183784485, "fcm_dpo/margin": 54.08441162109375, "fcm_dpo/q_t": 0.31322652101516724, "grad_norm": 12.168521881103516, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.7326631546020508, "logits/rejected": -1.6424217224121094, "logps/chosen": -141.6122283935547, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -231.8439483642578, "loss": 0.8305, "margin_dpo/margin_mean": 54.08441162109375, "margin_dpo/margin_std": 53.86842346191406, "step": 319 }, { "KL/chosen_KL_mean": -62.04451370239258, "KL/mean": -85.25990295410156, "KL/rejected_KL_mean": -108.47529602050781, "KL/std": 51.84027099609375, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.017010871320962906, "fcm_dpo/delta": -0.3931153416633606, "fcm_dpo/margin": 46.4307861328125, "fcm_dpo/q_t": 0.3353227972984314, "grad_norm": 11.391035079956055, "learning_rate": 3.208807785813777e-07, "logits/chosen": -1.799586534500122, "logits/rejected": -1.6035809516906738, "logps/chosen": -149.18173217773438, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -211.6994171142578, "loss": 0.886, "margin_dpo/margin_mean": 46.4307861328125, "margin_dpo/margin_std": 49.26911163330078, "step": 320 }, { "KL/chosen_KL_mean": -69.40055084228516, "KL/mean": -95.64939880371094, "KL/rejected_KL_mean": -121.89825439453125, "KL/std": 52.72932434082031, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.016850028187036514, "fcm_dpo/delta": -0.48927056789398193, "fcm_dpo/margin": 52.497703552246094, "fcm_dpo/q_t": 0.31745126843452454, "grad_norm": 12.000794410705566, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.8558778762817383, "logits/rejected": -1.4642584323883057, "logps/chosen": -162.50521850585938, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -219.36639404296875, "loss": 0.8475, "margin_dpo/margin_mean": 52.497703552246094, "margin_dpo/margin_std": 51.89668655395508, "step": 321 }, { "KL/chosen_KL_mean": -66.15898132324219, "KL/mean": -95.1854248046875, "KL/rejected_KL_mean": -124.21188354492188, "KL/std": 55.18983459472656, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.01666080206632614, "fcm_dpo/delta": -0.573370099067688, "fcm_dpo/margin": 58.05289840698242, "fcm_dpo/q_t": 0.3090571165084839, "grad_norm": 15.196894645690918, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.8628333806991577, "logits/rejected": -1.5196549892425537, "logps/chosen": -158.35189819335938, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -225.1051025390625, "loss": 0.8338, "margin_dpo/margin_mean": 58.05289840698242, "margin_dpo/margin_std": 58.48529052734375, "step": 322 }, { "KL/chosen_KL_mean": -74.8105239868164, "KL/mean": -100.90483093261719, "KL/rejected_KL_mean": -126.99915313720703, "KL/std": 54.09056854248047, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.016504094004631042, "fcm_dpo/delta": -0.4655742347240448, "fcm_dpo/margin": 52.18861389160156, "fcm_dpo/q_t": 0.3255850076675415, "grad_norm": 11.897259712219238, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.8951036930084229, "logits/rejected": -1.5117988586425781, "logps/chosen": -155.08262634277344, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -207.302490234375, "loss": 0.8513, "margin_dpo/margin_mean": 52.18861389160156, "margin_dpo/margin_std": 52.56786346435547, "step": 323 }, { "KL/chosen_KL_mean": -76.81649780273438, "KL/mean": -104.78288269042969, "KL/rejected_KL_mean": -132.74925231933594, "KL/std": 54.5130615234375, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.016346551477909088, "fcm_dpo/delta": -0.5190116167068481, "fcm_dpo/margin": 55.93274688720703, "fcm_dpo/q_t": 0.3069418668746948, "grad_norm": 11.695333480834961, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7211729288101196, "logits/rejected": -1.068758487701416, "logps/chosen": -172.54586791992188, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -226.91848754882812, "loss": 0.795, "margin_dpo/margin_mean": 55.93274688720703, "margin_dpo/margin_std": 47.517574310302734, "step": 324 }, { "KL/chosen_KL_mean": -72.10450744628906, "KL/mean": -99.28755187988281, "KL/rejected_KL_mean": -126.4705810546875, "KL/std": 52.394012451171875, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.0161854587495327, "fcm_dpo/delta": -0.4841822385787964, "fcm_dpo/margin": 54.3660888671875, "fcm_dpo/q_t": 0.3164939284324646, "grad_norm": 10.216546058654785, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.9541754722595215, "logits/rejected": -1.269737958908081, "logps/chosen": -170.81137084960938, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -202.87869262695312, "loss": 0.8369, "margin_dpo/margin_mean": 54.366085052490234, "margin_dpo/margin_std": 51.5577392578125, "step": 325 }, { "KL/chosen_KL_mean": -82.96012878417969, "KL/mean": -107.63099670410156, "KL/rejected_KL_mean": -132.30184936523438, "KL/std": 52.620262145996094, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.016050059348344803, "fcm_dpo/delta": -0.3950706124305725, "fcm_dpo/margin": 49.34172439575195, "fcm_dpo/q_t": 0.33574122190475464, "grad_norm": 12.580079078674316, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.433692455291748, "logits/rejected": -1.801171064376831, "logps/chosen": -187.62142944335938, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -219.12722778320312, "loss": 0.9005, "margin_dpo/margin_mean": 49.34172439575195, "margin_dpo/margin_std": 53.85484313964844, "step": 326 }, { "KL/chosen_KL_mean": -79.7559585571289, "KL/mean": -105.2043685913086, "KL/rejected_KL_mean": -130.65277099609375, "KL/std": 49.101585388183594, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.015915699303150177, "fcm_dpo/delta": -0.4135178327560425, "fcm_dpo/margin": 50.89681625366211, "fcm_dpo/q_t": 0.33308354020118713, "grad_norm": 12.050386428833008, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.6177330017089844, "logits/rejected": -1.001750111579895, "logps/chosen": -187.19619750976562, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -247.7060089111328, "loss": 0.8808, "margin_dpo/margin_mean": 50.89681625366211, "margin_dpo/margin_std": 53.354637145996094, "step": 327 }, { "KL/chosen_KL_mean": -81.61115264892578, "KL/mean": -112.84171295166016, "KL/rejected_KL_mean": -144.072265625, "KL/std": 58.651611328125, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.015748076140880585, "fcm_dpo/delta": -0.5893991589546204, "fcm_dpo/margin": 62.46112823486328, "fcm_dpo/q_t": 0.30178144574165344, "grad_norm": 12.984345436096191, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.5593833923339844, "logits/rejected": -1.270606279373169, "logps/chosen": -168.2218475341797, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -241.47415161132812, "loss": 0.7867, "margin_dpo/margin_mean": 62.46112823486328, "margin_dpo/margin_std": 57.123661041259766, "step": 328 }, { "KL/chosen_KL_mean": -84.73503112792969, "KL/mean": -110.25694274902344, "KL/rejected_KL_mean": -135.77886962890625, "KL/std": 52.5092887878418, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.015597652643918991, "fcm_dpo/delta": -0.399638831615448, "fcm_dpo/margin": 51.04383087158203, "fcm_dpo/q_t": 0.33321672677993774, "grad_norm": 13.785771369934082, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.6352522373199463, "logits/rejected": -1.1404091119766235, "logps/chosen": -185.34811401367188, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -240.73257446289062, "loss": 0.8819, "margin_dpo/margin_mean": 51.04383087158203, "margin_dpo/margin_std": 52.004425048828125, "step": 329 }, { "KL/chosen_KL_mean": -75.90325164794922, "KL/mean": -103.38813781738281, "KL/rejected_KL_mean": -130.87301635742188, "KL/std": 54.68939208984375, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.015464585274457932, "fcm_dpo/delta": -0.4541179835796356, "fcm_dpo/margin": 54.96977615356445, "fcm_dpo/q_t": 0.31520846486091614, "grad_norm": 11.017935752868652, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.6464684009552002, "logits/rejected": -1.564220666885376, "logps/chosen": -152.8616943359375, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -223.68795776367188, "loss": 0.8169, "margin_dpo/margin_mean": 54.96977996826172, "margin_dpo/margin_std": 45.177452087402344, "step": 330 }, { "KL/chosen_KL_mean": -81.0125961303711, "KL/mean": -110.56180572509766, "KL/rejected_KL_mean": -140.11102294921875, "KL/std": 53.948829650878906, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.015326619148254395, "fcm_dpo/delta": -0.5101625323295593, "fcm_dpo/margin": 59.09842300415039, "fcm_dpo/q_t": 0.3098253607749939, "grad_norm": 15.320713996887207, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8518779277801514, "logits/rejected": -1.5462241172790527, "logps/chosen": -164.072021484375, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -234.24215698242188, "loss": 0.7993, "margin_dpo/margin_mean": 59.09842300415039, "margin_dpo/margin_std": 50.29570388793945, "step": 331 }, { "KL/chosen_KL_mean": -84.41827392578125, "KL/mean": -106.93721771240234, "KL/rejected_KL_mean": -129.45616149902344, "KL/std": 59.709686279296875, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.015211494639515877, "fcm_dpo/delta": -0.2870749235153198, "fcm_dpo/margin": 45.037879943847656, "fcm_dpo/q_t": 0.3546872138977051, "grad_norm": 11.595236778259277, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.146660327911377, "logits/rejected": -1.731731653213501, "logps/chosen": -166.34918212890625, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -212.30206298828125, "loss": 0.9545, "margin_dpo/margin_mean": 45.037879943847656, "margin_dpo/margin_std": 53.45393371582031, "step": 332 }, { "KL/chosen_KL_mean": -98.65228271484375, "KL/mean": -127.60285186767578, "KL/rejected_KL_mean": -156.5534210205078, "KL/std": 61.86476135253906, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.01506748702377081, "fcm_dpo/delta": -0.4772377014160156, "fcm_dpo/margin": 57.901145935058594, "fcm_dpo/q_t": 0.3265310227870941, "grad_norm": 12.205545425415039, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.6470367908477783, "logits/rejected": -1.3264291286468506, "logps/chosen": -187.1848602294922, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -259.0194396972656, "loss": 0.9041, "margin_dpo/margin_mean": 57.901145935058594, "margin_dpo/margin_std": 67.12867736816406, "step": 333 }, { "KL/chosen_KL_mean": -90.3923568725586, "KL/mean": -116.66787719726562, "KL/rejected_KL_mean": -142.94338989257812, "KL/std": 57.79753875732422, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.014949094504117966, "fcm_dpo/delta": -0.3889736533164978, "fcm_dpo/margin": 52.55103302001953, "fcm_dpo/q_t": 0.3373030126094818, "grad_norm": 13.661307334899902, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.2179940938949585, "logits/rejected": -0.9189304709434509, "logps/chosen": -188.150146484375, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -242.94921875, "loss": 0.9384, "margin_dpo/margin_mean": 52.55103302001953, "margin_dpo/margin_std": 63.35211944580078, "step": 334 }, { "KL/chosen_KL_mean": -89.7664794921875, "KL/mean": -114.16681671142578, "KL/rejected_KL_mean": -138.56716918945312, "KL/std": 56.51815414428711, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.014855434186756611, "fcm_dpo/delta": -0.3273513913154602, "fcm_dpo/margin": 48.80070114135742, "fcm_dpo/q_t": 0.34940779209136963, "grad_norm": 13.407217025756836, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.169153928756714, "logits/rejected": -1.755549430847168, "logps/chosen": -186.14251708984375, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -233.49118041992188, "loss": 0.948, "margin_dpo/margin_mean": 48.80070114135742, "margin_dpo/margin_std": 58.634071350097656, "step": 335 }, { "KL/chosen_KL_mean": -90.04165649414062, "KL/mean": -119.92893981933594, "KL/rejected_KL_mean": -149.81622314453125, "KL/std": 61.038780212402344, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.014729510992765427, "fcm_dpo/delta": -0.4845835864543915, "fcm_dpo/margin": 59.77455139160156, "fcm_dpo/q_t": 0.3189623951911926, "grad_norm": 13.285140037536621, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.581979513168335, "logits/rejected": -1.4141550064086914, "logps/chosen": -178.57118225097656, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -259.37200927734375, "loss": 0.8433, "margin_dpo/margin_mean": 59.77455139160156, "margin_dpo/margin_std": 60.94203186035156, "step": 336 }, { "KL/chosen_KL_mean": -90.26612854003906, "KL/mean": -117.63661193847656, "KL/rejected_KL_mean": -145.007080078125, "KL/std": 59.79123306274414, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.014607259072363377, "fcm_dpo/delta": -0.40269601345062256, "fcm_dpo/margin": 54.74097442626953, "fcm_dpo/q_t": 0.33810657262802124, "grad_norm": 13.568991661071777, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.2795443534851074, "logits/rejected": -1.9900648593902588, "logps/chosen": -187.57850646972656, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -255.138427734375, "loss": 0.9336, "margin_dpo/margin_mean": 54.74097442626953, "margin_dpo/margin_std": 65.89427185058594, "step": 337 }, { "KL/chosen_KL_mean": -92.84487915039062, "KL/mean": -119.37713623046875, "KL/rejected_KL_mean": -145.90939331054688, "KL/std": 61.06922912597656, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.014487138018012047, "fcm_dpo/delta": -0.3716987371444702, "fcm_dpo/margin": 53.06450653076172, "fcm_dpo/q_t": 0.33580321073532104, "grad_norm": 13.127784729003906, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.9646642208099365, "logits/rejected": -1.7999048233032227, "logps/chosen": -176.47500610351562, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -239.60174560546875, "loss": 0.8996, "margin_dpo/margin_mean": 53.06450653076172, "margin_dpo/margin_std": 55.382789611816406, "step": 338 }, { "KL/chosen_KL_mean": -95.5751953125, "KL/mean": -124.13836669921875, "KL/rejected_KL_mean": -152.7015380859375, "KL/std": 61.023860931396484, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.014381740242242813, "fcm_dpo/delta": -0.4248679280281067, "fcm_dpo/margin": 57.126338958740234, "fcm_dpo/q_t": 0.3324657082557678, "grad_norm": 12.203712463378906, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.905304193496704, "logits/rejected": -2.484269618988037, "logps/chosen": -195.42701721191406, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -261.7759094238281, "loss": 0.9098, "margin_dpo/margin_mean": 57.1263427734375, "margin_dpo/margin_std": 64.85108184814453, "step": 339 }, { "KL/chosen_KL_mean": -104.43388366699219, "KL/mean": -130.1668701171875, "KL/rejected_KL_mean": -155.89987182617188, "KL/std": 55.67139434814453, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.014271661639213562, "fcm_dpo/delta": -0.33691859245300293, "fcm_dpo/margin": 51.465999603271484, "fcm_dpo/q_t": 0.3445185422897339, "grad_norm": 15.370295524597168, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.4180450439453125, "logits/rejected": -0.9525488615036011, "logps/chosen": -189.4078369140625, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -234.70022583007812, "loss": 0.931, "margin_dpo/margin_mean": 51.465999603271484, "margin_dpo/margin_std": 60.23385238647461, "step": 340 }, { "KL/chosen_KL_mean": -111.31968688964844, "KL/mean": -135.6403045654297, "KL/rejected_KL_mean": -159.96090698242188, "KL/std": 57.769710540771484, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.014179868623614311, "fcm_dpo/delta": -0.2917969524860382, "fcm_dpo/margin": 48.64122009277344, "fcm_dpo/q_t": 0.35733211040496826, "grad_norm": 14.412906646728516, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.3966012001037598, "logits/rejected": -1.118476390838623, "logps/chosen": -196.58367919921875, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -266.046875, "loss": 0.987, "margin_dpo/margin_mean": 48.64122009277344, "margin_dpo/margin_std": 63.41450119018555, "step": 341 }, { "KL/chosen_KL_mean": -87.48794555664062, "KL/mean": -122.76753234863281, "KL/rejected_KL_mean": -158.047119140625, "KL/std": 60.80590057373047, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.014020889066159725, "fcm_dpo/delta": -0.5960535407066345, "fcm_dpo/margin": 70.55918884277344, "fcm_dpo/q_t": 0.2967090606689453, "grad_norm": 12.413627624511719, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.3023574352264404, "logits/rejected": -0.5614258646965027, "logps/chosen": -188.52655029296875, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -256.9456481933594, "loss": 0.7894, "margin_dpo/margin_mean": 70.55918884277344, "margin_dpo/margin_std": 60.8209342956543, "step": 342 }, { "KL/chosen_KL_mean": -93.99565124511719, "KL/mean": -123.74972534179688, "KL/rejected_KL_mean": -153.5037841796875, "KL/std": 61.65123748779297, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.013917284086346626, "fcm_dpo/delta": -0.43153780698776245, "fcm_dpo/margin": 59.508148193359375, "fcm_dpo/q_t": 0.32848650217056274, "grad_norm": 14.023930549621582, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1986162662506104, "logits/rejected": -2.062088966369629, "logps/chosen": -177.74301147460938, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -270.0828857421875, "loss": 0.873, "margin_dpo/margin_mean": 59.508148193359375, "margin_dpo/margin_std": 59.52987289428711, "step": 343 }, { "KL/chosen_KL_mean": -100.30621337890625, "KL/mean": -129.07240295410156, "KL/rejected_KL_mean": -157.83859252929688, "KL/std": 65.17615509033203, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.01379719004034996, "fcm_dpo/delta": -0.3968047499656677, "fcm_dpo/margin": 57.53239059448242, "fcm_dpo/q_t": 0.33770617842674255, "grad_norm": 12.379199981689453, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.6520779132843018, "logits/rejected": -1.3897595405578613, "logps/chosen": -178.5601806640625, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -253.26730346679688, "loss": 0.9126, "margin_dpo/margin_mean": 57.53239440917969, "margin_dpo/margin_std": 65.88662719726562, "step": 344 }, { "KL/chosen_KL_mean": -97.31635284423828, "KL/mean": -129.93634033203125, "KL/rejected_KL_mean": -162.55633544921875, "KL/std": 67.7120132446289, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.01365690864622593, "fcm_dpo/delta": -0.49560946226119995, "fcm_dpo/margin": 65.2399673461914, "fcm_dpo/q_t": 0.31946462392807007, "grad_norm": 13.259018898010254, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.315077781677246, "logits/rejected": -1.0059300661087036, "logps/chosen": -195.39584350585938, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -266.59112548828125, "loss": 0.8395, "margin_dpo/margin_mean": 65.23995971679688, "margin_dpo/margin_std": 64.65087890625, "step": 345 }, { "KL/chosen_KL_mean": -111.24212646484375, "KL/mean": -137.30471801757812, "KL/rejected_KL_mean": -163.36729431152344, "KL/std": 62.66020202636719, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.013566195964813232, "fcm_dpo/delta": -0.30930498242378235, "fcm_dpo/margin": 52.12515640258789, "fcm_dpo/q_t": 0.3544180989265442, "grad_norm": 11.894692420959473, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.867597222328186, "logits/rejected": -1.5578877925872803, "logps/chosen": -191.9256134033203, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -249.2598876953125, "loss": 0.9806, "margin_dpo/margin_mean": 52.125160217285156, "margin_dpo/margin_std": 68.5782470703125, "step": 346 }, { "KL/chosen_KL_mean": -105.21348571777344, "KL/mean": -130.22802734375, "KL/rejected_KL_mean": -155.2425537109375, "KL/std": 61.74641036987305, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.013491734862327576, "fcm_dpo/delta": -0.27678701281547546, "fcm_dpo/margin": 50.029052734375, "fcm_dpo/q_t": 0.36095601320266724, "grad_norm": 13.084700584411621, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.7056893110275269, "logits/rejected": -1.3619849681854248, "logps/chosen": -198.8928680419922, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -252.78506469726562, "loss": 0.9939, "margin_dpo/margin_mean": 50.029052734375, "margin_dpo/margin_std": 66.89411926269531, "step": 347 }, { "KL/chosen_KL_mean": -93.01765441894531, "KL/mean": -120.48687744140625, "KL/rejected_KL_mean": -147.95611572265625, "KL/std": 59.39344787597656, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.013386247679591179, "fcm_dpo/delta": -0.3385326862335205, "fcm_dpo/margin": 54.938453674316406, "fcm_dpo/q_t": 0.3449484705924988, "grad_norm": 11.548766136169434, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.935347557067871, "logits/rejected": -1.6482319831848145, "logps/chosen": -183.2698974609375, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -243.7263946533203, "loss": 0.935, "margin_dpo/margin_mean": 54.938453674316406, "margin_dpo/margin_std": 61.20106506347656, "step": 348 }, { "KL/chosen_KL_mean": -100.49578857421875, "KL/mean": -126.97926330566406, "KL/rejected_KL_mean": -153.46273803710938, "KL/std": 62.87340545654297, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.013326453976333141, "fcm_dpo/delta": -0.3079183101654053, "fcm_dpo/margin": 52.966949462890625, "fcm_dpo/q_t": 0.35207507014274597, "grad_norm": 12.625789642333984, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.9560048580169678, "logits/rejected": -1.7307298183441162, "logps/chosen": -192.5538787841797, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -257.4280090332031, "loss": 0.9532, "margin_dpo/margin_mean": 52.966949462890625, "margin_dpo/margin_std": 62.520816802978516, "step": 349 }, { "KL/chosen_KL_mean": -108.68162536621094, "KL/mean": -130.54421997070312, "KL/rejected_KL_mean": -152.4068145751953, "KL/std": 57.18351745605469, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.013258544728159904, "fcm_dpo/delta": -0.1808084100484848, "fcm_dpo/margin": 43.725181579589844, "fcm_dpo/q_t": 0.37552008032798767, "grad_norm": 14.151510238647461, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8824115991592407, "logits/rejected": -1.6584081649780273, "logps/chosen": -195.73582458496094, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -248.12875366210938, "loss": 1.0111, "margin_dpo/margin_mean": 43.725181579589844, "margin_dpo/margin_std": 56.49635314941406, "step": 350 }, { "KL/chosen_KL_mean": -105.19110107421875, "KL/mean": -137.53570556640625, "KL/rejected_KL_mean": -169.88031005859375, "KL/std": 66.37275695800781, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.013164312578737736, "fcm_dpo/delta": -0.45530200004577637, "fcm_dpo/margin": 64.689208984375, "fcm_dpo/q_t": 0.3237455487251282, "grad_norm": 11.655464172363281, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.4523601531982422, "logits/rejected": -1.11428964138031, "logps/chosen": -194.36013793945312, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -278.1783447265625, "loss": 0.8714, "margin_dpo/margin_mean": 64.689208984375, "margin_dpo/margin_std": 66.98726654052734, "step": 351 }, { "KL/chosen_KL_mean": -97.69917297363281, "KL/mean": -128.22659301757812, "KL/rejected_KL_mean": -158.7540283203125, "KL/std": 65.32252502441406, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.013049130327999592, "fcm_dpo/delta": -0.3999210596084595, "fcm_dpo/margin": 61.05486297607422, "fcm_dpo/q_t": 0.3364279866218567, "grad_norm": 12.049348831176758, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.8803777694702148, "logits/rejected": -1.3538835048675537, "logps/chosen": -194.51065063476562, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -243.1588592529297, "loss": 0.8877, "margin_dpo/margin_mean": 61.05486297607422, "margin_dpo/margin_std": 64.42152404785156, "step": 352 }, { "KL/chosen_KL_mean": -111.53242492675781, "KL/mean": -130.64926147460938, "KL/rejected_KL_mean": -149.76609802246094, "KL/std": 57.747703552246094, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.013000905513763428, "fcm_dpo/delta": -0.09760136902332306, "fcm_dpo/margin": 38.23368453979492, "fcm_dpo/q_t": 0.3910246193408966, "grad_norm": 13.599019050598145, "learning_rate": 2.794480701395219e-07, "logits/chosen": -1.7985155582427979, "logits/rejected": -1.4082810878753662, "logps/chosen": -196.8948974609375, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -237.1334228515625, "loss": 1.0913, "margin_dpo/margin_mean": 38.23368835449219, "margin_dpo/margin_std": 59.947628021240234, "step": 353 }, { "KL/chosen_KL_mean": -99.81256866455078, "KL/mean": -129.90045166015625, "KL/rejected_KL_mean": -159.9883575439453, "KL/std": 61.097599029541016, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.012928366661071777, "fcm_dpo/delta": -0.38083893060684204, "fcm_dpo/margin": 60.17579650878906, "fcm_dpo/q_t": 0.3319692313671112, "grad_norm": 11.13448429107666, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.856693983078003, "logits/rejected": -1.3939251899719238, "logps/chosen": -195.61373901367188, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -254.8544921875, "loss": 0.8532, "margin_dpo/margin_mean": 60.17579650878906, "margin_dpo/margin_std": 53.50132751464844, "step": 354 }, { "KL/chosen_KL_mean": -98.46890258789062, "KL/mean": -127.6426010131836, "KL/rejected_KL_mean": -156.81631469726562, "KL/std": 61.739933013916016, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.012821480631828308, "fcm_dpo/delta": -0.35114359855651855, "fcm_dpo/margin": 58.347412109375, "fcm_dpo/q_t": 0.34066009521484375, "grad_norm": 11.351016998291016, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.3363795280456543, "logits/rejected": -1.860129475593567, "logps/chosen": -184.65444946289062, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -245.2778778076172, "loss": 0.9063, "margin_dpo/margin_mean": 58.347415924072266, "margin_dpo/margin_std": 58.78508758544922, "step": 355 }, { "KL/chosen_KL_mean": -116.20709228515625, "KL/mean": -137.59483337402344, "KL/rejected_KL_mean": -158.98257446289062, "KL/std": 58.94407272338867, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.012783991172909737, "fcm_dpo/delta": -0.14762543141841888, "fcm_dpo/margin": 42.77548599243164, "fcm_dpo/q_t": 0.3819754123687744, "grad_norm": 13.83216667175293, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.7860453128814697, "logits/rejected": -1.3309009075164795, "logps/chosen": -207.02520751953125, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -241.10989379882812, "loss": 1.0698, "margin_dpo/margin_mean": 42.775489807128906, "margin_dpo/margin_std": 66.10163879394531, "step": 356 }, { "KL/chosen_KL_mean": -95.39566040039062, "KL/mean": -130.8209686279297, "KL/rejected_KL_mean": -166.24627685546875, "KL/std": 67.91034698486328, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.012660928070545197, "fcm_dpo/delta": -0.5024032592773438, "fcm_dpo/margin": 70.85061645507812, "fcm_dpo/q_t": 0.31072404980659485, "grad_norm": 10.654644012451172, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.955629587173462, "logits/rejected": -1.5959270000457764, "logps/chosen": -182.90042114257812, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -258.4956359863281, "loss": 0.803, "margin_dpo/margin_mean": 70.85061645507812, "margin_dpo/margin_std": 57.908546447753906, "step": 357 }, { "KL/chosen_KL_mean": -108.55575561523438, "KL/mean": -133.52487182617188, "KL/rejected_KL_mean": -158.49400329589844, "KL/std": 60.609336853027344, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.012600535526871681, "fcm_dpo/delta": -0.2306954264640808, "fcm_dpo/margin": 49.93824768066406, "fcm_dpo/q_t": 0.36980462074279785, "grad_norm": 11.582130432128906, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.597937822341919, "logits/rejected": -1.1125736236572266, "logps/chosen": -203.6112060546875, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -254.97012329101562, "loss": 0.9919, "margin_dpo/margin_mean": 49.93824768066406, "margin_dpo/margin_std": 64.81124114990234, "step": 358 }, { "KL/chosen_KL_mean": -119.36080169677734, "KL/mean": -142.93280029296875, "KL/rejected_KL_mean": -166.50482177734375, "KL/std": 59.65242385864258, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.012537389993667603, "fcm_dpo/delta": -0.19256789982318878, "fcm_dpo/margin": 47.14401626586914, "fcm_dpo/q_t": 0.3803368806838989, "grad_norm": 13.136520385742188, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4644304513931274, "logits/rejected": -0.9110198020935059, "logps/chosen": -206.734130859375, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -255.2766876220703, "loss": 1.0467, "margin_dpo/margin_mean": 47.144012451171875, "margin_dpo/margin_std": 69.26698303222656, "step": 359 }, { "KL/chosen_KL_mean": -113.36932373046875, "KL/mean": -143.2478485107422, "KL/rejected_KL_mean": -173.12637329101562, "KL/std": 58.6151237487793, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.012472154572606087, "fcm_dpo/delta": -0.3478919565677643, "fcm_dpo/margin": 59.757057189941406, "fcm_dpo/q_t": 0.33831655979156494, "grad_norm": 14.453511238098145, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.5752038955688477, "logits/rejected": -1.229540467262268, "logps/chosen": -188.5927734375, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -249.2203826904297, "loss": 0.8957, "margin_dpo/margin_mean": 59.757057189941406, "margin_dpo/margin_std": 60.229270935058594, "step": 360 }, { "KL/chosen_KL_mean": -114.28901672363281, "KL/mean": -140.90403747558594, "KL/rejected_KL_mean": -167.51905822753906, "KL/std": 67.31832885742188, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.012403495609760284, "fcm_dpo/delta": -0.2619330585002899, "fcm_dpo/margin": 53.23003387451172, "fcm_dpo/q_t": 0.3638260066509247, "grad_norm": 13.21297550201416, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.6042484045028687, "logits/rejected": -1.3441519737243652, "logps/chosen": -196.24234008789062, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -264.1982421875, "loss": 0.9914, "margin_dpo/margin_mean": 53.230037689208984, "margin_dpo/margin_std": 70.04544830322266, "step": 361 }, { "KL/chosen_KL_mean": -117.1817398071289, "KL/mean": -145.818359375, "KL/rejected_KL_mean": -174.45498657226562, "KL/std": 59.47690963745117, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.012325942516326904, "fcm_dpo/delta": -0.30818718671798706, "fcm_dpo/margin": 57.27322769165039, "fcm_dpo/q_t": 0.34697097539901733, "grad_norm": 12.037802696228027, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.777784824371338, "logits/rejected": -1.5577667951583862, "logps/chosen": -206.52145385742188, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -284.95697021484375, "loss": 0.9298, "margin_dpo/margin_mean": 57.273231506347656, "margin_dpo/margin_std": 61.06214904785156, "step": 362 }, { "KL/chosen_KL_mean": -109.84304809570312, "KL/mean": -147.08721923828125, "KL/rejected_KL_mean": -184.33139038085938, "KL/std": 66.91725158691406, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.012221291661262512, "fcm_dpo/delta": -0.5148134827613831, "fcm_dpo/margin": 74.48831176757812, "fcm_dpo/q_t": 0.31573694944381714, "grad_norm": 11.44267749786377, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.8852064609527588, "logits/rejected": -1.4561681747436523, "logps/chosen": -190.6180419921875, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -263.7640380859375, "loss": 0.8288, "margin_dpo/margin_mean": 74.48831176757812, "margin_dpo/margin_std": 72.4778060913086, "step": 363 }, { "KL/chosen_KL_mean": -109.38249206542969, "KL/mean": -138.94229125976562, "KL/rejected_KL_mean": -168.50210571289062, "KL/std": 61.495018005371094, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.012135181576013565, "fcm_dpo/delta": -0.3195944130420685, "fcm_dpo/margin": 59.119632720947266, "fcm_dpo/q_t": 0.3434094488620758, "grad_norm": 13.813261032104492, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -1.944597840309143, "logits/rejected": -1.7845113277435303, "logps/chosen": -185.91209411621094, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -266.0638427734375, "loss": 0.8967, "margin_dpo/margin_mean": 59.119632720947266, "margin_dpo/margin_std": 55.00733947753906, "step": 364 }, { "KL/chosen_KL_mean": -108.576904296875, "KL/mean": -140.52035522460938, "KL/rejected_KL_mean": -172.46380615234375, "KL/std": 67.08159637451172, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.012043453752994537, "fcm_dpo/delta": -0.37215960025787354, "fcm_dpo/margin": 63.88689422607422, "fcm_dpo/q_t": 0.34066158533096313, "grad_norm": 11.766288757324219, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.695927381515503, "logits/rejected": -1.5446919202804565, "logps/chosen": -189.12179565429688, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -271.7869873046875, "loss": 0.9147, "margin_dpo/margin_mean": 63.88689422607422, "margin_dpo/margin_std": 71.42550659179688, "step": 365 }, { "KL/chosen_KL_mean": -106.88079833984375, "KL/mean": -136.14096069335938, "KL/rejected_KL_mean": -165.40109252929688, "KL/std": 71.3531494140625, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.011960498988628387, "fcm_dpo/delta": -0.3020724058151245, "fcm_dpo/margin": 58.520301818847656, "fcm_dpo/q_t": 0.3559810221195221, "grad_norm": 11.077528953552246, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.9082669019699097, "logits/rejected": -1.9179580211639404, "logps/chosen": -179.11886596679688, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -257.07293701171875, "loss": 0.9299, "margin_dpo/margin_mean": 58.52029800415039, "margin_dpo/margin_std": 66.70003509521484, "step": 366 }, { "KL/chosen_KL_mean": -103.43414306640625, "KL/mean": -134.97198486328125, "KL/rejected_KL_mean": -166.50982666015625, "KL/std": 62.584632873535156, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.011880913749337196, "fcm_dpo/delta": -0.3520246744155884, "fcm_dpo/margin": 63.0756950378418, "fcm_dpo/q_t": 0.3405248522758484, "grad_norm": 11.423791885375977, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.7741427421569824, "logits/rejected": -1.4542920589447021, "logps/chosen": -188.85614013671875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -250.02761840820312, "loss": 0.889, "margin_dpo/margin_mean": 63.07569885253906, "margin_dpo/margin_std": 63.71672821044922, "step": 367 }, { "KL/chosen_KL_mean": -98.14974975585938, "KL/mean": -132.31423950195312, "KL/rejected_KL_mean": -166.47869873046875, "KL/std": 68.3837890625, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.011795086786150932, "fcm_dpo/delta": -0.40905994176864624, "fcm_dpo/margin": 68.32894897460938, "fcm_dpo/q_t": 0.3290116786956787, "grad_norm": 12.143608093261719, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.4988105297088623, "logits/rejected": -1.447813868522644, "logps/chosen": -171.99087524414062, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -271.1739807128906, "loss": 0.8807, "margin_dpo/margin_mean": 68.32894897460938, "margin_dpo/margin_std": 68.21604919433594, "step": 368 }, { "KL/chosen_KL_mean": -106.13719177246094, "KL/mean": -139.94541931152344, "KL/rejected_KL_mean": -173.75364685058594, "KL/std": 60.854774475097656, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.011696942150592804, "fcm_dpo/delta": -0.39392998814582825, "fcm_dpo/margin": 67.6164779663086, "fcm_dpo/q_t": 0.32857656478881836, "grad_norm": 11.536430358886719, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.7641355991363525, "logits/rejected": -1.4033236503601074, "logps/chosen": -204.59915161132812, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -272.9103698730469, "loss": 0.8398, "margin_dpo/margin_mean": 67.61647033691406, "margin_dpo/margin_std": 57.31822967529297, "step": 369 }, { "KL/chosen_KL_mean": -118.09854125976562, "KL/mean": -141.8253631591797, "KL/rejected_KL_mean": -165.55218505859375, "KL/std": 62.714134216308594, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.011646024882793427, "fcm_dpo/delta": -0.1534726619720459, "fcm_dpo/margin": 47.453636169433594, "fcm_dpo/q_t": 0.37958240509033203, "grad_norm": 11.972166061401367, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.884814977645874, "logits/rejected": -1.6811399459838867, "logps/chosen": -214.87428283691406, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -268.2962951660156, "loss": 1.0479, "margin_dpo/margin_mean": 47.45363998413086, "margin_dpo/margin_std": 67.50628662109375, "step": 370 }, { "KL/chosen_KL_mean": -112.61964416503906, "KL/mean": -136.99224853515625, "KL/rejected_KL_mean": -161.3648681640625, "KL/std": 63.230037689208984, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.011609761044383049, "fcm_dpo/delta": -0.1668311059474945, "fcm_dpo/margin": 48.745216369628906, "fcm_dpo/q_t": 0.3782472312450409, "grad_norm": 12.592358589172363, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.872943639755249, "logits/rejected": -1.5172017812728882, "logps/chosen": -215.60984802246094, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -266.7154846191406, "loss": 1.0318, "margin_dpo/margin_mean": 48.745216369628906, "margin_dpo/margin_std": 66.32482147216797, "step": 371 }, { "KL/chosen_KL_mean": -95.77688598632812, "KL/mean": -134.27377319335938, "KL/rejected_KL_mean": -172.77066040039062, "KL/std": 72.10191345214844, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.011512869969010353, "fcm_dpo/delta": -0.4907035827636719, "fcm_dpo/margin": 76.99376678466797, "fcm_dpo/q_t": 0.31529122591018677, "grad_norm": 10.58568286895752, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.797689437866211, "logits/rejected": -1.1786918640136719, "logps/chosen": -186.80313110351562, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -258.3808898925781, "loss": 0.819, "margin_dpo/margin_mean": 76.99376678466797, "margin_dpo/margin_std": 69.32192993164062, "step": 372 }, { "KL/chosen_KL_mean": -108.80754852294922, "KL/mean": -138.89010620117188, "KL/rejected_KL_mean": -168.97265625, "KL/std": 67.75457000732422, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.011431345716118813, "fcm_dpo/delta": -0.2898106276988983, "fcm_dpo/margin": 60.16511154174805, "fcm_dpo/q_t": 0.3533790707588196, "grad_norm": 14.421277046203613, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.061011791229248, "logits/rejected": -1.550896167755127, "logps/chosen": -208.880615234375, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -260.0889892578125, "loss": 0.946, "margin_dpo/margin_mean": 60.16511154174805, "margin_dpo/margin_std": 68.56936645507812, "step": 373 }, { "KL/chosen_KL_mean": -97.12980651855469, "KL/mean": -129.00852966308594, "KL/rejected_KL_mean": -160.8872528076172, "KL/std": 65.33059692382812, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.011355992406606674, "fcm_dpo/delta": -0.32656556367874146, "fcm_dpo/margin": 63.75745391845703, "fcm_dpo/q_t": 0.34188956022262573, "grad_norm": 10.652899742126465, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.0506491661071777, "logits/rejected": -1.6528164148330688, "logps/chosen": -195.14215087890625, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -261.6394958496094, "loss": 0.8826, "margin_dpo/margin_mean": 63.75745391845703, "margin_dpo/margin_std": 58.02842330932617, "step": 374 }, { "KL/chosen_KL_mean": -102.3846664428711, "KL/mean": -137.35189819335938, "KL/rejected_KL_mean": -172.31915283203125, "KL/std": 64.6090087890625, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.011264595203101635, "fcm_dpo/delta": -0.3914544880390167, "fcm_dpo/margin": 69.93449401855469, "fcm_dpo/q_t": 0.3333597183227539, "grad_norm": 11.821507453918457, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.7116665840148926, "logits/rejected": -1.493017554283142, "logps/chosen": -186.78094482421875, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -268.03802490234375, "loss": 0.8767, "margin_dpo/margin_mean": 69.93449401855469, "margin_dpo/margin_std": 67.66724395751953, "step": 375 }, { "KL/chosen_KL_mean": -96.7133560180664, "KL/mean": -134.773681640625, "KL/rejected_KL_mean": -172.83401489257812, "KL/std": 70.10107421875, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.011175084859132767, "fcm_dpo/delta": -0.4546225070953369, "fcm_dpo/margin": 76.12066650390625, "fcm_dpo/q_t": 0.32144492864608765, "grad_norm": 12.159934997558594, "learning_rate": 2.5e-07, "logits/chosen": -1.978079080581665, "logits/rejected": -1.6941132545471191, "logps/chosen": -192.4510498046875, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -284.80731201171875, "loss": 0.8404, "margin_dpo/margin_mean": 76.12066650390625, "margin_dpo/margin_std": 71.6180419921875, "step": 376 }, { "KL/chosen_KL_mean": -106.3299789428711, "KL/mean": -145.71218872070312, "KL/rejected_KL_mean": -185.09437561035156, "KL/std": 75.73550415039062, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.011065074242651463, "fcm_dpo/delta": -0.47609835863113403, "fcm_dpo/margin": 78.764404296875, "fcm_dpo/q_t": 0.31988275051116943, "grad_norm": 12.2509765625, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5463612079620361, "logits/rejected": -1.4189938306808472, "logps/chosen": -185.25692749023438, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -290.9932861328125, "loss": 0.8423, "margin_dpo/margin_mean": 78.764404296875, "margin_dpo/margin_std": 75.55400085449219, "step": 377 }, { "KL/chosen_KL_mean": -103.70242309570312, "KL/mean": -136.28245544433594, "KL/rejected_KL_mean": -168.86248779296875, "KL/std": 70.92340850830078, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.011002566665410995, "fcm_dpo/delta": -0.3190898895263672, "fcm_dpo/margin": 65.16006469726562, "fcm_dpo/q_t": 0.3432743549346924, "grad_norm": 12.526639938354492, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.0961241722106934, "logits/rejected": -1.8188607692718506, "logps/chosen": -179.0748291015625, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -262.56658935546875, "loss": 0.9208, "margin_dpo/margin_mean": 65.16006469726562, "margin_dpo/margin_std": 67.68412780761719, "step": 378 }, { "KL/chosen_KL_mean": -115.37296295166016, "KL/mean": -146.5482635498047, "KL/rejected_KL_mean": -177.72357177734375, "KL/std": 72.39825439453125, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.010928826406598091, "fcm_dpo/delta": -0.28336548805236816, "fcm_dpo/margin": 62.35060501098633, "fcm_dpo/q_t": 0.3595661520957947, "grad_norm": 11.653779983520508, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.870237946510315, "logits/rejected": -1.7586846351623535, "logps/chosen": -204.2286376953125, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -299.104736328125, "loss": 0.9894, "margin_dpo/margin_mean": 62.35060119628906, "margin_dpo/margin_std": 81.79911041259766, "step": 379 }, { "KL/chosen_KL_mean": -128.77821350097656, "KL/mean": -163.89501953125, "KL/rejected_KL_mean": -199.01181030273438, "KL/std": 73.96090698242188, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.01085415855050087, "fcm_dpo/delta": -0.36511653661727905, "fcm_dpo/margin": 70.23358154296875, "fcm_dpo/q_t": 0.34473705291748047, "grad_norm": 13.663859367370605, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.3081461191177368, "logits/rejected": -1.271131992340088, "logps/chosen": -208.61141967773438, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -309.7565612792969, "loss": 0.9198, "margin_dpo/margin_mean": 70.23358154296875, "margin_dpo/margin_std": 82.05387115478516, "step": 380 }, { "KL/chosen_KL_mean": -112.59991455078125, "KL/mean": -143.50674438476562, "KL/rejected_KL_mean": -174.41355895996094, "KL/std": 72.07891845703125, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.010790714994072914, "fcm_dpo/delta": -0.2688117027282715, "fcm_dpo/margin": 61.81364059448242, "fcm_dpo/q_t": 0.3555396497249603, "grad_norm": 13.396537780761719, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.8997045755386353, "logits/rejected": -1.3641526699066162, "logps/chosen": -199.166748046875, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -258.830322265625, "loss": 0.9491, "margin_dpo/margin_mean": 61.813636779785156, "margin_dpo/margin_std": 69.50436401367188, "step": 381 }, { "KL/chosen_KL_mean": -113.72232055664062, "KL/mean": -147.9296112060547, "KL/rejected_KL_mean": -182.1368865966797, "KL/std": 69.20987701416016, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.01071284431964159, "fcm_dpo/delta": -0.3358251452445984, "fcm_dpo/margin": 68.41458892822266, "fcm_dpo/q_t": 0.34401559829711914, "grad_norm": 11.406100273132324, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.855623722076416, "logits/rejected": -1.5822412967681885, "logps/chosen": -218.18411254882812, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -296.7391357421875, "loss": 0.9039, "margin_dpo/margin_mean": 68.41458129882812, "margin_dpo/margin_std": 68.54295349121094, "step": 382 }, { "KL/chosen_KL_mean": -121.91346740722656, "KL/mean": -151.57997131347656, "KL/rejected_KL_mean": -181.2464599609375, "KL/std": 69.68084716796875, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.010663392953574657, "fcm_dpo/delta": -0.23426586389541626, "fcm_dpo/margin": 59.33301544189453, "fcm_dpo/q_t": 0.3638043999671936, "grad_norm": 12.00311279296875, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.668508529663086, "logits/rejected": -1.5544838905334473, "logps/chosen": -204.029541015625, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -280.02362060546875, "loss": 0.9719, "margin_dpo/margin_mean": 59.3330078125, "margin_dpo/margin_std": 68.46599578857422, "step": 383 }, { "KL/chosen_KL_mean": -131.00173950195312, "KL/mean": -161.00645446777344, "KL/rejected_KL_mean": -191.01116943359375, "KL/std": 74.20515441894531, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.010625971481204033, "fcm_dpo/delta": -0.23913483321666718, "fcm_dpo/margin": 60.00946044921875, "fcm_dpo/q_t": 0.36409610509872437, "grad_norm": 13.205636978149414, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.813019037246704, "logits/rejected": -1.5863149166107178, "logps/chosen": -211.58697509765625, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -282.45904541015625, "loss": 1.0135, "margin_dpo/margin_mean": 60.00946044921875, "margin_dpo/margin_std": 79.29371643066406, "step": 384 }, { "KL/chosen_KL_mean": -110.9693603515625, "KL/mean": -147.33343505859375, "KL/rejected_KL_mean": -183.697509765625, "KL/std": 67.33573913574219, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.010534806177020073, "fcm_dpo/delta": -0.3695375621318817, "fcm_dpo/margin": 72.7281494140625, "fcm_dpo/q_t": 0.34164053201675415, "grad_norm": 13.414453506469727, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.2229912281036377, "logits/rejected": -1.6231918334960938, "logps/chosen": -203.72952270507812, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -268.157470703125, "loss": 0.9116, "margin_dpo/margin_mean": 72.7281494140625, "margin_dpo/margin_std": 78.42330932617188, "step": 385 }, { "KL/chosen_KL_mean": -115.40544128417969, "KL/mean": -150.70213317871094, "KL/rejected_KL_mean": -185.9988250732422, "KL/std": 68.77604675292969, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.010472984984517097, "fcm_dpo/delta": -0.3418079614639282, "fcm_dpo/margin": 70.59339904785156, "fcm_dpo/q_t": 0.33813053369522095, "grad_norm": 10.649466514587402, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.5840147733688354, "logits/rejected": -1.2997262477874756, "logps/chosen": -200.764404296875, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -282.9894714355469, "loss": 0.9101, "margin_dpo/margin_mean": 70.59339904785156, "margin_dpo/margin_std": 73.38579559326172, "step": 386 }, { "KL/chosen_KL_mean": -131.03245544433594, "KL/mean": -164.20944213867188, "KL/rejected_KL_mean": -197.38641357421875, "KL/std": 66.35701751708984, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.010411031544208527, "fcm_dpo/delta": -0.29276561737060547, "fcm_dpo/margin": 66.35398864746094, "fcm_dpo/q_t": 0.3492102026939392, "grad_norm": 11.953927993774414, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.296985387802124, "logits/rejected": -1.6115953922271729, "logps/chosen": -233.7205810546875, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -296.81500244140625, "loss": 0.916, "margin_dpo/margin_mean": 66.35398864746094, "margin_dpo/margin_std": 66.42325592041016, "step": 387 }, { "KL/chosen_KL_mean": -130.18350219726562, "KL/mean": -167.3989715576172, "KL/rejected_KL_mean": -204.61444091796875, "KL/std": 74.37055969238281, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.010334128513932228, "fcm_dpo/delta": -0.3720657229423523, "fcm_dpo/margin": 74.4309310913086, "fcm_dpo/q_t": 0.3349211513996124, "grad_norm": 11.212059020996094, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.5924067497253418, "logits/rejected": -1.4521733522415161, "logps/chosen": -213.76873779296875, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -314.7148132324219, "loss": 0.8753, "margin_dpo/margin_mean": 74.43092346191406, "margin_dpo/margin_std": 71.10804748535156, "step": 388 }, { "KL/chosen_KL_mean": -135.8497314453125, "KL/mean": -168.42466735839844, "KL/rejected_KL_mean": -200.99960327148438, "KL/std": 77.46383666992188, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.01027364656329155, "fcm_dpo/delta": -0.2711429297924042, "fcm_dpo/margin": 65.14988708496094, "fcm_dpo/q_t": 0.36037200689315796, "grad_norm": 12.868640899658203, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.434481143951416, "logits/rejected": -1.9995958805084229, "logps/chosen": -225.6719970703125, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -285.8533935546875, "loss": 0.9819, "margin_dpo/margin_mean": 65.14988708496094, "margin_dpo/margin_std": 83.72454833984375, "step": 389 }, { "KL/chosen_KL_mean": -136.73806762695312, "KL/mean": -167.03201293945312, "KL/rejected_KL_mean": -197.32598876953125, "KL/std": 74.0252914428711, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.010217750445008278, "fcm_dpo/delta": -0.22073709964752197, "fcm_dpo/margin": 60.58790588378906, "fcm_dpo/q_t": 0.37309807538986206, "grad_norm": 11.399410247802734, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.5687108039855957, "logits/rejected": -1.3519763946533203, "logps/chosen": -214.2705078125, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -289.909912109375, "loss": 1.0154, "margin_dpo/margin_mean": 60.58790588378906, "margin_dpo/margin_std": 83.50444030761719, "step": 390 }, { "KL/chosen_KL_mean": -124.19439697265625, "KL/mean": -158.1561279296875, "KL/rejected_KL_mean": -192.11785888671875, "KL/std": 71.46687316894531, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.010165644809603691, "fcm_dpo/delta": -0.2926604449748993, "fcm_dpo/margin": 67.92346954345703, "fcm_dpo/q_t": 0.35438302159309387, "grad_norm": 11.680895805358887, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.855985164642334, "logits/rejected": -1.4033786058425903, "logps/chosen": -212.4273223876953, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -285.12542724609375, "loss": 0.9508, "margin_dpo/margin_mean": 67.92347717285156, "margin_dpo/margin_std": 79.275634765625, "step": 391 }, { "KL/chosen_KL_mean": -125.63760375976562, "KL/mean": -164.07159423828125, "KL/rejected_KL_mean": -202.5055694580078, "KL/std": 78.40117645263672, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.010102061554789543, "fcm_dpo/delta": -0.37933897972106934, "fcm_dpo/margin": 76.86793518066406, "fcm_dpo/q_t": 0.33663058280944824, "grad_norm": 13.81778621673584, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.4786713123321533, "logits/rejected": -1.350109577178955, "logps/chosen": -213.38201904296875, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -311.0762939453125, "loss": 0.9072, "margin_dpo/margin_mean": 76.8679428100586, "margin_dpo/margin_std": 83.82167053222656, "step": 392 }, { "KL/chosen_KL_mean": -111.09707641601562, "KL/mean": -146.6209716796875, "KL/rejected_KL_mean": -182.1448974609375, "KL/std": 75.65280151367188, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.010029857978224754, "fcm_dpo/delta": -0.3148723840713501, "fcm_dpo/margin": 71.0478286743164, "fcm_dpo/q_t": 0.3470783829689026, "grad_norm": 10.664995193481445, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.7378004789352417, "logits/rejected": -1.423196792602539, "logps/chosen": -187.660888671875, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -261.87109375, "loss": 0.9178, "margin_dpo/margin_mean": 71.0478286743164, "margin_dpo/margin_std": 74.809326171875, "step": 393 }, { "KL/chosen_KL_mean": -115.23493194580078, "KL/mean": -152.67861938476562, "KL/rejected_KL_mean": -190.122314453125, "KL/std": 70.51298522949219, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.009961298666894436, "fcm_dpo/delta": -0.34863024950027466, "fcm_dpo/margin": 74.88738250732422, "fcm_dpo/q_t": 0.33653756976127625, "grad_norm": 12.052898406982422, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.4936456680297852, "logits/rejected": -1.2601778507232666, "logps/chosen": -199.93359375, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -287.580078125, "loss": 0.8724, "margin_dpo/margin_mean": 74.88739013671875, "margin_dpo/margin_std": 67.23429107666016, "step": 394 }, { "KL/chosen_KL_mean": -120.7069320678711, "KL/mean": -150.97406005859375, "KL/rejected_KL_mean": -181.24119567871094, "KL/std": 76.1628646850586, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.009909845888614655, "fcm_dpo/delta": -0.2012653648853302, "fcm_dpo/margin": 60.53425598144531, "fcm_dpo/q_t": 0.37979429960250854, "grad_norm": 13.156628608703613, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5743448734283447, "logits/rejected": -2.184678554534912, "logps/chosen": -209.59457397460938, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -272.2867431640625, "loss": 1.0946, "margin_dpo/margin_mean": 60.53425598144531, "margin_dpo/margin_std": 102.02790832519531, "step": 395 }, { "KL/chosen_KL_mean": -129.5296173095703, "KL/mean": -154.81976318359375, "KL/rejected_KL_mean": -180.10989379882812, "KL/std": 66.75148010253906, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.009890136308968067, "fcm_dpo/delta": -0.10075273364782333, "fcm_dpo/margin": 50.58027648925781, "fcm_dpo/q_t": 0.3904619514942169, "grad_norm": 15.835450172424316, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8066577911376953, "logits/rejected": -1.314063310623169, "logps/chosen": -232.00094604492188, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -273.34466552734375, "loss": 1.0699, "margin_dpo/margin_mean": 50.58027267456055, "margin_dpo/margin_std": 74.55264282226562, "step": 396 }, { "KL/chosen_KL_mean": -115.35830688476562, "KL/mean": -162.48358154296875, "KL/rejected_KL_mean": -209.60885620117188, "KL/std": 79.0111083984375, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.009804656729102135, "fcm_dpo/delta": -0.5291470289230347, "fcm_dpo/margin": 94.25056457519531, "fcm_dpo/q_t": 0.31330162286758423, "grad_norm": 12.434037208557129, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.3276848793029785, "logits/rejected": -1.7370299100875854, "logps/chosen": -216.40272521972656, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -318.46002197265625, "loss": 0.8418, "margin_dpo/margin_mean": 94.25056457519531, "margin_dpo/margin_std": 94.91737365722656, "step": 397 }, { "KL/chosen_KL_mean": -121.46839904785156, "KL/mean": -155.94146728515625, "KL/rejected_KL_mean": -190.41453552246094, "KL/std": 75.03477478027344, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.009739898145198822, "fcm_dpo/delta": -0.2734099328517914, "fcm_dpo/margin": 68.9461441040039, "fcm_dpo/q_t": 0.3593842387199402, "grad_norm": 12.979626655578613, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.069669246673584, "logits/rejected": -1.7791104316711426, "logps/chosen": -216.51763916015625, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -304.420166015625, "loss": 0.9827, "margin_dpo/margin_mean": 68.94615173339844, "margin_dpo/margin_std": 88.81240844726562, "step": 398 }, { "KL/chosen_KL_mean": -108.556396484375, "KL/mean": -152.8787841796875, "KL/rejected_KL_mean": -197.20114135742188, "KL/std": 70.26435089111328, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.009666120633482933, "fcm_dpo/delta": -0.46061569452285767, "fcm_dpo/margin": 88.64476013183594, "fcm_dpo/q_t": 0.3168029189109802, "grad_norm": 11.367899894714355, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.04704213142395, "logits/rejected": -1.255479097366333, "logps/chosen": -205.1949462890625, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -280.73760986328125, "loss": 0.8109, "margin_dpo/margin_mean": 88.6447525024414, "margin_dpo/margin_std": 73.20602416992188, "step": 399 }, { "KL/chosen_KL_mean": -110.76661682128906, "KL/mean": -156.48251342773438, "KL/rejected_KL_mean": -202.1983642578125, "KL/std": 79.90304565429688, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.009569292888045311, "fcm_dpo/delta": -0.4791606068611145, "fcm_dpo/margin": 91.43177032470703, "fcm_dpo/q_t": 0.3196236491203308, "grad_norm": 14.058406829833984, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.1990480422973633, "logits/rejected": -1.7159709930419922, "logps/chosen": -216.06976318359375, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -310.8539123535156, "loss": 0.8514, "margin_dpo/margin_mean": 91.43177032470703, "margin_dpo/margin_std": 90.34869384765625, "step": 400 }, { "KL/chosen_KL_mean": -124.64590454101562, "KL/mean": -153.36196899414062, "KL/rejected_KL_mean": -182.07801818847656, "KL/std": 76.6522445678711, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.009521246887743473, "fcm_dpo/delta": -0.14774754643440247, "fcm_dpo/margin": 57.43210983276367, "fcm_dpo/q_t": 0.3895654082298279, "grad_norm": 14.878680229187012, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.5103812217712402, "logits/rejected": -1.268535852432251, "logps/chosen": -208.93507385253906, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -273.0211181640625, "loss": 1.0789, "margin_dpo/margin_mean": 57.43210983276367, "margin_dpo/margin_std": 93.59178161621094, "step": 401 }, { "KL/chosen_KL_mean": -111.60310363769531, "KL/mean": -164.534423828125, "KL/rejected_KL_mean": -217.4657440185547, "KL/std": 82.62077331542969, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.009435480460524559, "fcm_dpo/delta": -0.6047588586807251, "fcm_dpo/margin": 105.86264038085938, "fcm_dpo/q_t": 0.28838658332824707, "grad_norm": 11.935503959655762, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.9700692892074585, "logits/rejected": -1.5123100280761719, "logps/chosen": -195.19622802734375, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -305.2760009765625, "loss": 0.7301, "margin_dpo/margin_mean": 105.86264038085938, "margin_dpo/margin_std": 76.14676666259766, "step": 402 }, { "KL/chosen_KL_mean": -115.45304870605469, "KL/mean": -154.77728271484375, "KL/rejected_KL_mean": -194.10153198242188, "KL/std": 79.27181243896484, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.009353186003863811, "fcm_dpo/delta": -0.3382585048675537, "fcm_dpo/margin": 78.64849853515625, "fcm_dpo/q_t": 0.34330713748931885, "grad_norm": 10.947129249572754, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.5619897842407227, "logits/rejected": -2.197150707244873, "logps/chosen": -189.2061309814453, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -278.02166748046875, "loss": 0.9114, "margin_dpo/margin_mean": 78.64849853515625, "margin_dpo/margin_std": 82.8604736328125, "step": 403 }, { "KL/chosen_KL_mean": -138.4350128173828, "KL/mean": -172.62222290039062, "KL/rejected_KL_mean": -206.80943298339844, "KL/std": 79.39253234863281, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.009304317645728588, "fcm_dpo/delta": -0.23779991269111633, "fcm_dpo/margin": 68.37442016601562, "fcm_dpo/q_t": 0.36648285388946533, "grad_norm": 16.39666175842285, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.888904094696045, "logits/rejected": -1.5481505393981934, "logps/chosen": -218.11119079589844, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -291.0899658203125, "loss": 1.0152, "margin_dpo/margin_mean": 68.37442779541016, "margin_dpo/margin_std": 95.07012939453125, "step": 404 }, { "KL/chosen_KL_mean": -139.7972869873047, "KL/mean": -174.6662139892578, "KL/rejected_KL_mean": -209.53512573242188, "KL/std": 72.25930786132812, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.009268321096897125, "fcm_dpo/delta": -0.24787940084934235, "fcm_dpo/margin": 69.73783874511719, "fcm_dpo/q_t": 0.3602423071861267, "grad_norm": 10.325687408447266, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.5393515825271606, "logits/rejected": -1.0805822610855103, "logps/chosen": -236.25924682617188, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -302.40582275390625, "loss": 0.9451, "margin_dpo/margin_mean": 69.73783874511719, "margin_dpo/margin_std": 74.12272644042969, "step": 405 }, { "KL/chosen_KL_mean": -132.8090362548828, "KL/mean": -164.03012084960938, "KL/rejected_KL_mean": -195.2511749267578, "KL/std": 77.57792663574219, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.009226994588971138, "fcm_dpo/delta": -0.17715798318386078, "fcm_dpo/margin": 62.442142486572266, "fcm_dpo/q_t": 0.376845121383667, "grad_norm": 16.60103988647461, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.9010131359100342, "logits/rejected": -1.7269821166992188, "logps/chosen": -210.9429931640625, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -293.5347900390625, "loss": 1.0434, "margin_dpo/margin_mean": 62.442138671875, "margin_dpo/margin_std": 89.61439514160156, "step": 406 }, { "KL/chosen_KL_mean": -141.2295379638672, "KL/mean": -169.35842895507812, "KL/rejected_KL_mean": -197.48731994628906, "KL/std": 76.04931640625, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.00920089054852724, "fcm_dpo/delta": -0.11824264377355576, "fcm_dpo/margin": 56.25775909423828, "fcm_dpo/q_t": 0.3873739242553711, "grad_norm": 13.410239219665527, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.2177863121032715, "logits/rejected": -1.7053488492965698, "logps/chosen": -225.24237060546875, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -280.26837158203125, "loss": 1.0583, "margin_dpo/margin_mean": 56.25775909423828, "margin_dpo/margin_std": 81.92262268066406, "step": 407 }, { "KL/chosen_KL_mean": -133.08657836914062, "KL/mean": -162.21234130859375, "KL/rejected_KL_mean": -191.33810424804688, "KL/std": 71.87669372558594, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.009174772538244724, "fcm_dpo/delta": -0.13522376120090485, "fcm_dpo/margin": 58.25154113769531, "fcm_dpo/q_t": 0.38733091950416565, "grad_norm": 12.447061538696289, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.2616610527038574, "logits/rejected": -1.660867691040039, "logps/chosen": -237.54830932617188, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -287.7102966308594, "loss": 1.0636, "margin_dpo/margin_mean": 58.25154113769531, "margin_dpo/margin_std": 88.16532897949219, "step": 408 }, { "KL/chosen_KL_mean": -143.6346893310547, "KL/mean": -172.8634796142578, "KL/rejected_KL_mean": -202.09226989746094, "KL/std": 77.04925537109375, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.00915147177875042, "fcm_dpo/delta": -0.13571856915950775, "fcm_dpo/margin": 58.45756912231445, "fcm_dpo/q_t": 0.3805464208126068, "grad_norm": 11.382444381713867, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.6058754920959473, "logits/rejected": -1.1760905981063843, "logps/chosen": -241.30300903320312, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -292.13812255859375, "loss": 1.065, "margin_dpo/margin_mean": 58.45757293701172, "margin_dpo/margin_std": 85.88185119628906, "step": 409 }, { "KL/chosen_KL_mean": -141.50262451171875, "KL/mean": -174.15255737304688, "KL/rejected_KL_mean": -206.80252075195312, "KL/std": 78.04219055175781, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.009120011702179909, "fcm_dpo/delta": -0.19669610261917114, "fcm_dpo/margin": 65.29989624023438, "fcm_dpo/q_t": 0.3673822283744812, "grad_norm": 11.21719741821289, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.6377044916152954, "logits/rejected": -1.5016980171203613, "logps/chosen": -217.97186279296875, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -295.44317626953125, "loss": 0.9909, "margin_dpo/margin_mean": 65.29989624023438, "margin_dpo/margin_std": 78.37046813964844, "step": 410 }, { "KL/chosen_KL_mean": -148.67739868164062, "KL/mean": -181.32566833496094, "KL/rejected_KL_mean": -213.97396850585938, "KL/std": 71.06066131591797, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.009080402553081512, "fcm_dpo/delta": -0.19417797029018402, "fcm_dpo/margin": 65.29655456542969, "fcm_dpo/q_t": 0.37083539366722107, "grad_norm": 11.222138404846191, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.3723888397216797, "logits/rejected": -1.979325771331787, "logps/chosen": -235.8437042236328, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -301.07000732421875, "loss": 0.982, "margin_dpo/margin_mean": 65.29656219482422, "margin_dpo/margin_std": 74.52803039550781, "step": 411 }, { "KL/chosen_KL_mean": -138.99163818359375, "KL/mean": -170.74948120117188, "KL/rejected_KL_mean": -202.50730895996094, "KL/std": 73.28567504882812, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.009047233499586582, "fcm_dpo/delta": -0.17576943337917328, "fcm_dpo/margin": 63.51569366455078, "fcm_dpo/q_t": 0.37967562675476074, "grad_norm": 11.701929092407227, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.2897019386291504, "logits/rejected": -2.065582275390625, "logps/chosen": -217.93898010253906, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -308.61285400390625, "loss": 1.0456, "margin_dpo/margin_mean": 63.51569366455078, "margin_dpo/margin_std": 92.31407165527344, "step": 412 }, { "KL/chosen_KL_mean": -150.1517791748047, "KL/mean": -185.92752075195312, "KL/rejected_KL_mean": -221.7032470703125, "KL/std": 78.02316284179688, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.009006805717945099, "fcm_dpo/delta": -0.24614912271499634, "fcm_dpo/margin": 71.55147552490234, "fcm_dpo/q_t": 0.36096835136413574, "grad_norm": 11.523744583129883, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.3940876722335815, "logits/rejected": -1.142632007598877, "logps/chosen": -245.84649658203125, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -329.66412353515625, "loss": 0.9812, "margin_dpo/margin_mean": 71.55147552490234, "margin_dpo/margin_std": 89.28776550292969, "step": 413 }, { "KL/chosen_KL_mean": -138.5650634765625, "KL/mean": -171.154541015625, "KL/rejected_KL_mean": -203.74398803710938, "KL/std": 76.22928619384766, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.008973915129899979, "fcm_dpo/delta": -0.18599680066108704, "fcm_dpo/margin": 65.17892456054688, "fcm_dpo/q_t": 0.376629501581192, "grad_norm": 16.255521774291992, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.6953978538513184, "logits/rejected": -1.3515188694000244, "logps/chosen": -226.8417510986328, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -296.614013671875, "loss": 1.0442, "margin_dpo/margin_mean": 65.17892456054688, "margin_dpo/margin_std": 95.84042358398438, "step": 414 }, { "KL/chosen_KL_mean": -147.90606689453125, "KL/mean": -176.79708862304688, "KL/rejected_KL_mean": -205.6881103515625, "KL/std": 68.33911895751953, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.008950876072049141, "fcm_dpo/delta": -0.11779701709747314, "fcm_dpo/margin": 57.782047271728516, "fcm_dpo/q_t": 0.3879052400588989, "grad_norm": 13.152162551879883, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.4886300563812256, "logits/rejected": -1.1098134517669678, "logps/chosen": -256.82196044921875, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -313.15948486328125, "loss": 1.0614, "margin_dpo/margin_mean": 57.78205108642578, "margin_dpo/margin_std": 83.7626953125, "step": 415 }, { "KL/chosen_KL_mean": -145.2874755859375, "KL/mean": -185.12709045410156, "KL/rejected_KL_mean": -224.9667205810547, "KL/std": 82.18344116210938, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.008912117220461369, "fcm_dpo/delta": -0.31222307682037354, "fcm_dpo/margin": 79.67922973632812, "fcm_dpo/q_t": 0.34930047392845154, "grad_norm": 10.609673500061035, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8273346424102783, "logits/rejected": -1.325990915298462, "logps/chosen": -238.68637084960938, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -322.63970947265625, "loss": 0.9391, "margin_dpo/margin_mean": 79.67922973632812, "margin_dpo/margin_std": 87.58592987060547, "step": 416 }, { "KL/chosen_KL_mean": -136.3604736328125, "KL/mean": -173.22604370117188, "KL/rejected_KL_mean": -210.09164428710938, "KL/std": 76.57164001464844, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.008856907486915588, "fcm_dpo/delta": -0.2546452283859253, "fcm_dpo/margin": 73.73115539550781, "fcm_dpo/q_t": 0.3565388321876526, "grad_norm": 11.105629920959473, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.3063769340515137, "logits/rejected": -1.4167399406433105, "logps/chosen": -219.89581298828125, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -284.5334777832031, "loss": 0.9486, "margin_dpo/margin_mean": 73.73115539550781, "margin_dpo/margin_std": 79.77044677734375, "step": 417 }, { "KL/chosen_KL_mean": -127.48262786865234, "KL/mean": -173.5142059326172, "KL/rejected_KL_mean": -219.54580688476562, "KL/std": 81.70866394042969, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.00880026537925005, "fcm_dpo/delta": -0.41333043575286865, "fcm_dpo/margin": 92.06315612792969, "fcm_dpo/q_t": 0.32707077264785767, "grad_norm": 13.196822166442871, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.139862537384033, "logits/rejected": -1.4856712818145752, "logps/chosen": -235.70416259765625, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -331.41046142578125, "loss": 0.8432, "margin_dpo/margin_mean": 92.06316375732422, "margin_dpo/margin_std": 77.84974670410156, "step": 418 }, { "KL/chosen_KL_mean": -135.5119171142578, "KL/mean": -175.05792236328125, "KL/rejected_KL_mean": -214.60394287109375, "KL/std": 73.49925231933594, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.008736946620047092, "fcm_dpo/delta": -0.2929493486881256, "fcm_dpo/margin": 79.09203338623047, "fcm_dpo/q_t": 0.35042113065719604, "grad_norm": 11.212968826293945, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.2997685670852661, "logits/rejected": -0.8398341536521912, "logps/chosen": -232.69520568847656, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -312.78924560546875, "loss": 0.9268, "margin_dpo/margin_mean": 79.09203338623047, "margin_dpo/margin_std": 82.77970886230469, "step": 419 }, { "KL/chosen_KL_mean": -139.24171447753906, "KL/mean": -174.99237060546875, "KL/rejected_KL_mean": -210.74301147460938, "KL/std": 78.1922607421875, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.008695402182638645, "fcm_dpo/delta": -0.22305762767791748, "fcm_dpo/margin": 71.50127410888672, "fcm_dpo/q_t": 0.36613738536834717, "grad_norm": 13.899815559387207, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.24007248878479, "logits/rejected": -1.1656806468963623, "logps/chosen": -253.55020141601562, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -286.42657470703125, "loss": 0.9822, "margin_dpo/margin_mean": 71.50127410888672, "margin_dpo/margin_std": 85.5582275390625, "step": 420 }, { "KL/chosen_KL_mean": -137.66592407226562, "KL/mean": -176.16769409179688, "KL/rejected_KL_mean": -214.66946411132812, "KL/std": 77.63030242919922, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.008643746376037598, "fcm_dpo/delta": -0.2674615979194641, "fcm_dpo/margin": 77.00355529785156, "fcm_dpo/q_t": 0.35576117038726807, "grad_norm": 10.312767028808594, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.9350297451019287, "logits/rejected": -1.5785589218139648, "logps/chosen": -223.5457763671875, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -311.0059509277344, "loss": 0.9465, "margin_dpo/margin_mean": 77.00355529785156, "margin_dpo/margin_std": 84.27220153808594, "step": 421 }, { "KL/chosen_KL_mean": -131.71817016601562, "KL/mean": -169.6834716796875, "KL/rejected_KL_mean": -207.64877319335938, "KL/std": 73.28466796875, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.008603572845458984, "fcm_dpo/delta": -0.25489023327827454, "fcm_dpo/margin": 75.93064880371094, "fcm_dpo/q_t": 0.35841065645217896, "grad_norm": 11.283828735351562, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.4036152362823486, "logits/rejected": -2.034773588180542, "logps/chosen": -232.19876098632812, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -302.0570068359375, "loss": 0.9432, "margin_dpo/margin_mean": 75.93064880371094, "margin_dpo/margin_std": 81.97745513916016, "step": 422 }, { "KL/chosen_KL_mean": -124.87263488769531, "KL/mean": -160.6893310546875, "KL/rejected_KL_mean": -196.5060577392578, "KL/std": 84.80293273925781, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.00856698863208294, "fcm_dpo/delta": -0.21494507789611816, "fcm_dpo/margin": 71.63341522216797, "fcm_dpo/q_t": 0.37150049209594727, "grad_norm": 11.851845741271973, "learning_rate": 1.902669377503756e-07, "logits/chosen": -2.1668708324432373, "logits/rejected": -2.057161331176758, "logps/chosen": -203.32257080078125, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -288.5525817871094, "loss": 1.0097, "margin_dpo/margin_mean": 71.6334228515625, "margin_dpo/margin_std": 95.13972473144531, "step": 423 }, { "KL/chosen_KL_mean": -138.8787078857422, "KL/mean": -176.73785400390625, "KL/rejected_KL_mean": -214.59698486328125, "KL/std": 77.76177215576172, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.008522449061274529, "fcm_dpo/delta": -0.24688732624053955, "fcm_dpo/margin": 75.71826934814453, "fcm_dpo/q_t": 0.35822945833206177, "grad_norm": 10.009150505065918, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.976736307144165, "logits/rejected": -1.401233196258545, "logps/chosen": -226.5210723876953, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -286.962646484375, "loss": 0.956, "margin_dpo/margin_mean": 75.71827697753906, "margin_dpo/margin_std": 84.04923248291016, "step": 424 }, { "KL/chosen_KL_mean": -142.97479248046875, "KL/mean": -168.59912109375, "KL/rejected_KL_mean": -194.2234649658203, "KL/std": 78.86929321289062, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.008504575118422508, "fcm_dpo/delta": -0.036030061542987823, "fcm_dpo/margin": 51.24867248535156, "fcm_dpo/q_t": 0.40962937474250793, "grad_norm": 12.39023208618164, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.5385820865631104, "logits/rejected": -1.4585154056549072, "logps/chosen": -221.4945831298828, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -296.97210693359375, "loss": 1.1511, "margin_dpo/margin_mean": 51.24867248535156, "margin_dpo/margin_std": 98.04598999023438, "step": 425 }, { "KL/chosen_KL_mean": -121.24417877197266, "KL/mean": -159.2057647705078, "KL/rejected_KL_mean": -197.1673583984375, "KL/std": 73.68968200683594, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.008475732058286667, "fcm_dpo/delta": -0.2450440526008606, "fcm_dpo/margin": 75.92318725585938, "fcm_dpo/q_t": 0.3577355146408081, "grad_norm": 13.10606861114502, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.6905248165130615, "logits/rejected": -0.9765642285346985, "logps/chosen": -229.75, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -285.4674072265625, "loss": 0.9477, "margin_dpo/margin_mean": 75.92318725585938, "margin_dpo/margin_std": 80.38101196289062, "step": 426 }, { "KL/chosen_KL_mean": -100.44827270507812, "KL/mean": -135.4705810546875, "KL/rejected_KL_mean": -170.4928741455078, "KL/std": 70.52210998535156, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.008440541103482246, "fcm_dpo/delta": -0.19232973456382751, "fcm_dpo/margin": 70.04460144042969, "fcm_dpo/q_t": 0.366913765668869, "grad_norm": 10.083366394042969, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.4194533824920654, "logits/rejected": -1.7975847721099854, "logps/chosen": -199.56874084472656, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -256.21746826171875, "loss": 0.9642, "margin_dpo/margin_mean": 70.04460144042969, "margin_dpo/margin_std": 72.42569732666016, "step": 427 }, { "KL/chosen_KL_mean": -143.73114013671875, "KL/mean": -172.36593627929688, "KL/rejected_KL_mean": -201.00070190429688, "KL/std": 80.86729431152344, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.008415129035711288, "fcm_dpo/delta": -0.08247723430395126, "fcm_dpo/margin": 57.269554138183594, "fcm_dpo/q_t": 0.39466556906700134, "grad_norm": 11.06969165802002, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.4141514301300049, "logits/rejected": -1.137647032737732, "logps/chosen": -249.7003936767578, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -310.1028747558594, "loss": 1.0913, "margin_dpo/margin_mean": 57.26955795288086, "margin_dpo/margin_std": 88.96458435058594, "step": 428 }, { "KL/chosen_KL_mean": -128.47010803222656, "KL/mean": -166.01136779785156, "KL/rejected_KL_mean": -203.5526123046875, "KL/std": 81.82283020019531, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.00839582085609436, "fcm_dpo/delta": -0.23179540038108826, "fcm_dpo/margin": 75.08251953125, "fcm_dpo/q_t": 0.3662126660346985, "grad_norm": 12.538039207458496, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9608471393585205, "logits/rejected": -1.2626855373382568, "logps/chosen": -242.01498413085938, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -301.79461669921875, "loss": 0.984, "margin_dpo/margin_mean": 75.08251953125, "margin_dpo/margin_std": 90.42044067382812, "step": 429 }, { "KL/chosen_KL_mean": -129.03143310546875, "KL/mean": -157.18682861328125, "KL/rejected_KL_mean": -185.34225463867188, "KL/std": 75.85673522949219, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.008371859788894653, "fcm_dpo/delta": -0.07176030427217484, "fcm_dpo/margin": 56.310791015625, "fcm_dpo/q_t": 0.39623892307281494, "grad_norm": 11.233688354492188, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.3013088703155518, "logits/rejected": -1.2566027641296387, "logps/chosen": -220.35079956054688, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -295.451904296875, "loss": 1.084, "margin_dpo/margin_mean": 56.310791015625, "margin_dpo/margin_std": 84.48430633544922, "step": 430 }, { "KL/chosen_KL_mean": -126.28607177734375, "KL/mean": -160.91712951660156, "KL/rejected_KL_mean": -195.5482177734375, "KL/std": 77.6587905883789, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.008342882618308067, "fcm_dpo/delta": -0.17890852689743042, "fcm_dpo/margin": 69.26214599609375, "fcm_dpo/q_t": 0.3733067214488983, "grad_norm": 11.068727493286133, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8883531093597412, "logits/rejected": -1.5006442070007324, "logps/chosen": -219.46728515625, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -293.68048095703125, "loss": 1.0041, "margin_dpo/margin_mean": 69.26213836669922, "margin_dpo/margin_std": 85.72547912597656, "step": 431 }, { "KL/chosen_KL_mean": -112.70266723632812, "KL/mean": -154.54556274414062, "KL/rejected_KL_mean": -196.388427734375, "KL/std": 82.61713409423828, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.008303137496113777, "fcm_dpo/delta": -0.2968272566795349, "fcm_dpo/margin": 83.68575286865234, "fcm_dpo/q_t": 0.34811413288116455, "grad_norm": 10.141386985778809, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.4455292224884033, "logits/rejected": -0.8885211944580078, "logps/chosen": -217.137451171875, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -301.47796630859375, "loss": 0.9119, "margin_dpo/margin_mean": 83.68575286865234, "margin_dpo/margin_std": 80.80133056640625, "step": 432 }, { "KL/chosen_KL_mean": -134.05465698242188, "KL/mean": -167.0984344482422, "KL/rejected_KL_mean": -200.14224243164062, "KL/std": 77.04706573486328, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.008274409919977188, "fcm_dpo/delta": -0.14763356745243073, "fcm_dpo/margin": 66.08758544921875, "fcm_dpo/q_t": 0.3806806206703186, "grad_norm": 11.14805793762207, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.7458834648132324, "logits/rejected": -1.422698974609375, "logps/chosen": -223.8978729248047, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -301.8757019042969, "loss": 1.0343, "margin_dpo/margin_mean": 66.08758544921875, "margin_dpo/margin_std": 87.60446166992188, "step": 433 }, { "KL/chosen_KL_mean": -121.58641815185547, "KL/mean": -157.2214813232422, "KL/rejected_KL_mean": -192.85653686523438, "KL/std": 84.07894897460938, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.008240592665970325, "fcm_dpo/delta": -0.18839333951473236, "fcm_dpo/margin": 71.2701416015625, "fcm_dpo/q_t": 0.3760553002357483, "grad_norm": 10.776324272155762, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.435295820236206, "logits/rejected": -1.7927484512329102, "logps/chosen": -219.27894592285156, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -276.9478454589844, "loss": 1.0164, "margin_dpo/margin_mean": 71.2701416015625, "margin_dpo/margin_std": 96.08448791503906, "step": 434 }, { "KL/chosen_KL_mean": -116.62297058105469, "KL/mean": -154.9145050048828, "KL/rejected_KL_mean": -193.20602416992188, "KL/std": 79.91590881347656, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.008204689249396324, "fcm_dpo/delta": -0.2297535538673401, "fcm_dpo/margin": 76.58306121826172, "fcm_dpo/q_t": 0.3637316823005676, "grad_norm": 11.523303985595703, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.3881068229675293, "logits/rejected": -1.938001036643982, "logps/chosen": -202.79489135742188, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -286.9577331542969, "loss": 0.9641, "margin_dpo/margin_mean": 76.58306884765625, "margin_dpo/margin_std": 87.0226821899414, "step": 435 }, { "KL/chosen_KL_mean": -96.4101791381836, "KL/mean": -137.88906860351562, "KL/rejected_KL_mean": -179.36793518066406, "KL/std": 71.00714111328125, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.008159921504557133, "fcm_dpo/delta": -0.27881288528442383, "fcm_dpo/margin": 82.957763671875, "fcm_dpo/q_t": 0.34875380992889404, "grad_norm": 11.50733470916748, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.7325283288955688, "logits/rejected": -0.8679463267326355, "logps/chosen": -202.19728088378906, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -267.9926452636719, "loss": 0.9028, "margin_dpo/margin_mean": 82.957763671875, "margin_dpo/margin_std": 74.48178100585938, "step": 436 }, { "KL/chosen_KL_mean": -114.38728332519531, "KL/mean": -145.5834503173828, "KL/rejected_KL_mean": -176.77963256835938, "KL/std": 72.74647521972656, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.008135813288390636, "fcm_dpo/delta": -0.10815472155809402, "fcm_dpo/margin": 62.392356872558594, "fcm_dpo/q_t": 0.38343602418899536, "grad_norm": 11.906576156616211, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9214478731155396, "logits/rejected": -1.2708628177642822, "logps/chosen": -210.44932556152344, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -265.7918395996094, "loss": 1.0218, "margin_dpo/margin_mean": 62.392356872558594, "margin_dpo/margin_std": 72.01850891113281, "step": 437 }, { "KL/chosen_KL_mean": -115.00796508789062, "KL/mean": -148.57476806640625, "KL/rejected_KL_mean": -182.14154052734375, "KL/std": 74.4294204711914, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.00810934230685234, "fcm_dpo/delta": -0.14531056582927704, "fcm_dpo/margin": 67.13356018066406, "fcm_dpo/q_t": 0.37752866744995117, "grad_norm": 12.509448051452637, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.207242727279663, "logits/rejected": -1.406461238861084, "logps/chosen": -216.86334228515625, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -271.5892028808594, "loss": 1.0082, "margin_dpo/margin_mean": 67.13356018066406, "margin_dpo/margin_std": 77.86924743652344, "step": 438 }, { "KL/chosen_KL_mean": -127.55235290527344, "KL/mean": -160.766357421875, "KL/rejected_KL_mean": -193.9803466796875, "KL/std": 80.85694885253906, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.00808379240334034, "fcm_dpo/delta": -0.13797986507415771, "fcm_dpo/margin": 66.4280014038086, "fcm_dpo/q_t": 0.3816812038421631, "grad_norm": 12.160362243652344, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.5283889770507812, "logits/rejected": -1.2741444110870361, "logps/chosen": -209.3079833984375, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -284.56671142578125, "loss": 1.0215, "margin_dpo/margin_mean": 66.4280014038086, "margin_dpo/margin_std": 80.98890686035156, "step": 439 }, { "KL/chosen_KL_mean": -114.029052734375, "KL/mean": -153.7742156982422, "KL/rejected_KL_mean": -193.51937866210938, "KL/std": 81.55213928222656, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.008050747215747833, "fcm_dpo/delta": -0.24172274768352509, "fcm_dpo/margin": 79.49032592773438, "fcm_dpo/q_t": 0.36113840341567993, "grad_norm": 9.673833847045898, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.413567543029785, "logits/rejected": -1.926101565361023, "logps/chosen": -219.67013549804688, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -296.92041015625, "loss": 0.9565, "margin_dpo/margin_mean": 79.49032592773438, "margin_dpo/margin_std": 87.73431396484375, "step": 440 }, { "KL/chosen_KL_mean": -138.82720947265625, "KL/mean": -178.0212860107422, "KL/rejected_KL_mean": -217.21539306640625, "KL/std": 79.1082534790039, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.008016247302293777, "fcm_dpo/delta": -0.22989168763160706, "fcm_dpo/margin": 78.3882064819336, "fcm_dpo/q_t": 0.36243897676467896, "grad_norm": 9.82705307006836, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.5924327373504639, "logits/rejected": -1.2206342220306396, "logps/chosen": -230.3567352294922, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -320.83477783203125, "loss": 0.9386, "margin_dpo/margin_mean": 78.38819885253906, "margin_dpo/margin_std": 79.6504135131836, "step": 441 }, { "KL/chosen_KL_mean": -141.11273193359375, "KL/mean": -185.4964599609375, "KL/rejected_KL_mean": -229.8802032470703, "KL/std": 82.73167419433594, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.00797208584845066, "fcm_dpo/delta": -0.30984675884246826, "fcm_dpo/margin": 88.76746368408203, "fcm_dpo/q_t": 0.3438958525657654, "grad_norm": 11.348361015319824, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.5933637619018555, "logits/rejected": -1.1260123252868652, "logps/chosen": -225.89028930664062, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -313.704345703125, "loss": 0.8894, "margin_dpo/margin_mean": 88.7674560546875, "margin_dpo/margin_std": 80.51052856445312, "step": 442 }, { "KL/chosen_KL_mean": -135.14454650878906, "KL/mean": -165.77650451660156, "KL/rejected_KL_mean": -196.40846252441406, "KL/std": 80.68560791015625, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.007943347096443176, "fcm_dpo/delta": -0.08723185956478119, "fcm_dpo/margin": 61.263912200927734, "fcm_dpo/q_t": 0.38955047726631165, "grad_norm": 10.812935829162598, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.8341758251190186, "logits/rejected": -1.2937921285629272, "logps/chosen": -237.7938232421875, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -289.446533203125, "loss": 1.0463, "margin_dpo/margin_mean": 61.263912200927734, "margin_dpo/margin_std": 75.84648132324219, "step": 443 }, { "KL/chosen_KL_mean": -142.70263671875, "KL/mean": -174.9608154296875, "KL/rejected_KL_mean": -207.21897888183594, "KL/std": 77.10272979736328, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.007933070883154869, "fcm_dpo/delta": -0.11237837374210358, "fcm_dpo/margin": 64.51634216308594, "fcm_dpo/q_t": 0.38723456859588623, "grad_norm": 11.667716979980469, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5044269561767578, "logits/rejected": -1.3317005634307861, "logps/chosen": -230.6223602294922, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -310.54241943359375, "loss": 1.0506, "margin_dpo/margin_mean": 64.51634216308594, "margin_dpo/margin_std": 88.48472595214844, "step": 444 }, { "KL/chosen_KL_mean": -125.08734130859375, "KL/mean": -162.35598754882812, "KL/rejected_KL_mean": -199.62461853027344, "KL/std": 75.5912094116211, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.007902493700385094, "fcm_dpo/delta": -0.1902395635843277, "fcm_dpo/margin": 74.53729248046875, "fcm_dpo/q_t": 0.36809253692626953, "grad_norm": 9.97458267211914, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9554861783981323, "logits/rejected": -1.5487291812896729, "logps/chosen": -226.48822021484375, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -298.66253662109375, "loss": 0.9685, "margin_dpo/margin_mean": 74.53729248046875, "margin_dpo/margin_std": 78.3582534790039, "step": 445 }, { "KL/chosen_KL_mean": -120.61907958984375, "KL/mean": -154.59213256835938, "KL/rejected_KL_mean": -188.56520080566406, "KL/std": 81.48637390136719, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.007881262339651585, "fcm_dpo/delta": -0.1362362951040268, "fcm_dpo/margin": 67.94613647460938, "fcm_dpo/q_t": 0.3794158697128296, "grad_norm": 13.469057083129883, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -2.077998161315918, "logits/rejected": -1.932283639907837, "logps/chosen": -208.04141235351562, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -295.2659606933594, "loss": 1.0047, "margin_dpo/margin_mean": 67.94613647460938, "margin_dpo/margin_std": 76.98782348632812, "step": 446 }, { "KL/chosen_KL_mean": -122.08118438720703, "KL/mean": -164.49319458007812, "KL/rejected_KL_mean": -206.90521240234375, "KL/std": 81.83554077148438, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0078440485522151, "fcm_dpo/delta": -0.26716548204421997, "fcm_dpo/margin": 84.82402038574219, "fcm_dpo/q_t": 0.35451197624206543, "grad_norm": 11.317014694213867, "learning_rate": 1.608874379754465e-07, "logits/chosen": -1.9532909393310547, "logits/rejected": -1.9056020975112915, "logps/chosen": -205.69644165039062, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -311.817626953125, "loss": 0.9378, "margin_dpo/margin_mean": 84.82402038574219, "margin_dpo/margin_std": 89.5528564453125, "step": 447 }, { "KL/chosen_KL_mean": -129.90194702148438, "KL/mean": -171.97982788085938, "KL/rejected_KL_mean": -214.05770874023438, "KL/std": 87.78751373291016, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.007808951195329428, "fcm_dpo/delta": -0.25878581404685974, "fcm_dpo/margin": 84.1557388305664, "fcm_dpo/q_t": 0.3603030741214752, "grad_norm": 11.409748077392578, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.7124378681182861, "logits/rejected": -1.4432393312454224, "logps/chosen": -222.4777374267578, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -321.74749755859375, "loss": 0.9779, "margin_dpo/margin_mean": 84.15573120117188, "margin_dpo/margin_std": 102.19550323486328, "step": 448 }, { "KL/chosen_KL_mean": -131.07418823242188, "KL/mean": -173.52825927734375, "KL/rejected_KL_mean": -215.98236083984375, "KL/std": 80.73859405517578, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.00776439905166626, "fcm_dpo/delta": -0.2609502077102661, "fcm_dpo/margin": 84.90818786621094, "fcm_dpo/q_t": 0.3552859425544739, "grad_norm": 10.43504524230957, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.516770839691162, "logits/rejected": -1.0865511894226074, "logps/chosen": -233.47311401367188, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -311.1312255859375, "loss": 0.945, "margin_dpo/margin_mean": 84.90818786621094, "margin_dpo/margin_std": 91.62651062011719, "step": 449 }, { "KL/chosen_KL_mean": -113.73502349853516, "KL/mean": -155.3515625, "KL/rejected_KL_mean": -196.96807861328125, "KL/std": 83.0919189453125, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.00772453285753727, "fcm_dpo/delta": -0.2445187270641327, "fcm_dpo/margin": 83.23306274414062, "fcm_dpo/q_t": 0.3574024438858032, "grad_norm": 10.445788383483887, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3897652626037598, "logits/rejected": -2.1910462379455566, "logps/chosen": -200.72787475585938, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -305.5001220703125, "loss": 0.9503, "margin_dpo/margin_mean": 83.23306274414062, "margin_dpo/margin_std": 89.2677230834961, "step": 450 }, { "KL/chosen_KL_mean": -128.9928436279297, "KL/mean": -163.95352172851562, "KL/rejected_KL_mean": -198.9141845703125, "KL/std": 83.92650604248047, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.007701542228460312, "fcm_dpo/delta": -0.13922837376594543, "fcm_dpo/margin": 69.92132568359375, "fcm_dpo/q_t": 0.38422900438308716, "grad_norm": 17.682947158813477, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.3624792098999023, "logits/rejected": -1.8620407581329346, "logps/chosen": -215.80413818359375, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -278.769775390625, "loss": 1.0503, "margin_dpo/margin_mean": 69.92132568359375, "margin_dpo/margin_std": 99.41156005859375, "step": 451 }, { "KL/chosen_KL_mean": -112.89006805419922, "KL/mean": -150.44190979003906, "KL/rejected_KL_mean": -187.9937744140625, "KL/std": 80.62948608398438, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0076660700142383575, "fcm_dpo/delta": -0.17712922394275665, "fcm_dpo/margin": 75.10369873046875, "fcm_dpo/q_t": 0.3697431683540344, "grad_norm": 9.845088958740234, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.9917131662368774, "logits/rejected": -1.9249992370605469, "logps/chosen": -192.26971435546875, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -291.70916748046875, "loss": 0.9645, "margin_dpo/margin_mean": 75.10369873046875, "margin_dpo/margin_std": 69.29244995117188, "step": 452 }, { "KL/chosen_KL_mean": -136.4049835205078, "KL/mean": -180.84014892578125, "KL/rejected_KL_mean": -225.2753143310547, "KL/std": 81.23770141601562, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.007635419722646475, "fcm_dpo/delta": -0.28043684363365173, "fcm_dpo/margin": 88.87035369873047, "fcm_dpo/q_t": 0.35077953338623047, "grad_norm": 11.633660316467285, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.8517749309539795, "logits/rejected": -1.380619764328003, "logps/chosen": -224.10015869140625, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -315.33355712890625, "loss": 0.9073, "margin_dpo/margin_mean": 88.87035369873047, "margin_dpo/margin_std": 82.43183898925781, "step": 453 }, { "KL/chosen_KL_mean": -145.51036071777344, "KL/mean": -181.4825439453125, "KL/rejected_KL_mean": -217.45474243164062, "KL/std": 82.36870574951172, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0076099419966340065, "fcm_dpo/delta": -0.1482771933078766, "fcm_dpo/margin": 71.94436645507812, "fcm_dpo/q_t": 0.38180363178253174, "grad_norm": 11.407761573791504, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.0113577842712402, "logits/rejected": -1.5718207359313965, "logps/chosen": -235.07659912109375, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -310.37579345703125, "loss": 1.0598, "margin_dpo/margin_mean": 71.94436645507812, "margin_dpo/margin_std": 106.10647583007812, "step": 454 }, { "KL/chosen_KL_mean": -134.03009033203125, "KL/mean": -169.3562774658203, "KL/rejected_KL_mean": -204.68246459960938, "KL/std": 81.86817932128906, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.007588631939142942, "fcm_dpo/delta": -0.13686585426330566, "fcm_dpo/margin": 70.6523666381836, "fcm_dpo/q_t": 0.38128992915153503, "grad_norm": 10.322611808776855, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8483610153198242, "logits/rejected": -1.7932649850845337, "logps/chosen": -211.6600799560547, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -323.660400390625, "loss": 1.0273, "margin_dpo/margin_mean": 70.6523666381836, "margin_dpo/margin_std": 89.57182312011719, "step": 455 }, { "KL/chosen_KL_mean": -136.1051025390625, "KL/mean": -170.47665405273438, "KL/rejected_KL_mean": -204.8482208251953, "KL/std": 83.97354125976562, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.007568529341369867, "fcm_dpo/delta": -0.12089595198631287, "fcm_dpo/margin": 68.74310302734375, "fcm_dpo/q_t": 0.3830130100250244, "grad_norm": 11.95056438446045, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -1.9113515615463257, "logits/rejected": -1.7759959697723389, "logps/chosen": -225.72198486328125, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -314.4079284667969, "loss": 1.0193, "margin_dpo/margin_mean": 68.74310302734375, "margin_dpo/margin_std": 81.3488540649414, "step": 456 }, { "KL/chosen_KL_mean": -114.45619201660156, "KL/mean": -160.59841918945312, "KL/rejected_KL_mean": -206.74063110351562, "KL/std": 82.43550109863281, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.007521755062043667, "fcm_dpo/delta": -0.2966357171535492, "fcm_dpo/margin": 92.28443908691406, "fcm_dpo/q_t": 0.3432748317718506, "grad_norm": 12.789467811584473, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.1225693225860596, "logits/rejected": -1.8149802684783936, "logps/chosen": -201.77789306640625, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -297.50726318359375, "loss": 0.8789, "margin_dpo/margin_mean": 92.28443908691406, "margin_dpo/margin_std": 71.00645446777344, "step": 457 }, { "KL/chosen_KL_mean": -129.53956604003906, "KL/mean": -163.83505249023438, "KL/rejected_KL_mean": -198.13052368164062, "KL/std": 83.22312927246094, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.007505861110985279, "fcm_dpo/delta": -0.11541382968425751, "fcm_dpo/margin": 68.59095764160156, "fcm_dpo/q_t": 0.3888845145702362, "grad_norm": 12.134658813476562, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5500223636627197, "logits/rejected": -1.3000462055206299, "logps/chosen": -223.0600128173828, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -301.49951171875, "loss": 1.0514, "margin_dpo/margin_mean": 68.59095764160156, "margin_dpo/margin_std": 96.44388580322266, "step": 458 }, { "KL/chosen_KL_mean": -123.46788024902344, "KL/mean": -165.9029541015625, "KL/rejected_KL_mean": -208.3380126953125, "KL/std": 87.311767578125, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.007469930686056614, "fcm_dpo/delta": -0.2356349527835846, "fcm_dpo/margin": 84.87013244628906, "fcm_dpo/q_t": 0.3623501658439636, "grad_norm": 8.751827239990234, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.785006046295166, "logits/rejected": -1.5321568250656128, "logps/chosen": -199.15609741210938, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -300.50848388671875, "loss": 0.9576, "margin_dpo/margin_mean": 84.87013244628906, "margin_dpo/margin_std": 93.05264282226562, "step": 459 }, { "KL/chosen_KL_mean": -130.7395477294922, "KL/mean": -168.5526580810547, "KL/rejected_KL_mean": -206.36578369140625, "KL/std": 81.99678039550781, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.007443578913807869, "fcm_dpo/delta": -0.16394852101802826, "fcm_dpo/margin": 75.62625122070312, "fcm_dpo/q_t": 0.3762071132659912, "grad_norm": 13.565555572509766, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.078439474105835, "logits/rejected": -2.0861754417419434, "logps/chosen": -211.857421875, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -316.67816162109375, "loss": 1.0058, "margin_dpo/margin_mean": 75.62625122070312, "margin_dpo/margin_std": 91.87548828125, "step": 460 }, { "KL/chosen_KL_mean": -157.86277770996094, "KL/mean": -188.33758544921875, "KL/rejected_KL_mean": -218.8123779296875, "KL/std": 73.70008850097656, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.007431529462337494, "fcm_dpo/delta": -0.0532420389354229, "fcm_dpo/margin": 60.94960021972656, "fcm_dpo/q_t": 0.3988773822784424, "grad_norm": 10.101882934570312, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.4862486124038696, "logits/rejected": -1.2879667282104492, "logps/chosen": -239.44630432128906, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -312.68951416015625, "loss": 1.0735, "margin_dpo/margin_mean": 60.94960403442383, "margin_dpo/margin_std": 83.10049438476562, "step": 461 }, { "KL/chosen_KL_mean": -117.67816162109375, "KL/mean": -159.4649658203125, "KL/rejected_KL_mean": -201.25180053710938, "KL/std": 79.46842193603516, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.00740957772359252, "fcm_dpo/delta": -0.22057300806045532, "fcm_dpo/margin": 83.57363891601562, "fcm_dpo/q_t": 0.3617488145828247, "grad_norm": 10.11267375946045, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.2520241737365723, "logits/rejected": -1.754913330078125, "logps/chosen": -211.876708984375, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -286.8834228515625, "loss": 0.9458, "margin_dpo/margin_mean": 83.57363891601562, "margin_dpo/margin_std": 83.28895568847656, "step": 462 }, { "KL/chosen_KL_mean": -115.56471252441406, "KL/mean": -163.24465942382812, "KL/rejected_KL_mean": -210.92462158203125, "KL/std": 84.76103210449219, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.007372578606009483, "fcm_dpo/delta": -0.30508142709732056, "fcm_dpo/margin": 95.35991668701172, "fcm_dpo/q_t": 0.3446401357650757, "grad_norm": 13.242420196533203, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.985978126525879, "logits/rejected": -1.5659198760986328, "logps/chosen": -201.4894561767578, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -307.82647705078125, "loss": 0.9127, "margin_dpo/margin_mean": 95.35991668701172, "margin_dpo/margin_std": 90.99501037597656, "step": 463 }, { "KL/chosen_KL_mean": -143.36549377441406, "KL/mean": -182.15283203125, "KL/rejected_KL_mean": -220.940185546875, "KL/std": 83.36231994628906, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.007334766909480095, "fcm_dpo/delta": -0.1699950098991394, "fcm_dpo/margin": 77.57469177246094, "fcm_dpo/q_t": 0.37334370613098145, "grad_norm": 13.721678733825684, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.8352253437042236, "logits/rejected": -1.9082016944885254, "logps/chosen": -223.0546875, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -328.23248291015625, "loss": 0.996, "margin_dpo/margin_mean": 77.57469177246094, "margin_dpo/margin_std": 91.39215087890625, "step": 464 }, { "KL/chosen_KL_mean": -132.98080444335938, "KL/mean": -179.56298828125, "KL/rejected_KL_mean": -226.1451416015625, "KL/std": 93.1712646484375, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.007296806201338768, "fcm_dpo/delta": -0.2818154990673065, "fcm_dpo/margin": 93.16436004638672, "fcm_dpo/q_t": 0.34993743896484375, "grad_norm": 11.570189476013184, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.4383597373962402, "logits/rejected": -1.126192569732666, "logps/chosen": -224.8410186767578, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -344.85516357421875, "loss": 0.9077, "margin_dpo/margin_mean": 93.16436767578125, "margin_dpo/margin_std": 87.36253356933594, "step": 465 }, { "KL/chosen_KL_mean": -119.66544342041016, "KL/mean": -166.90206909179688, "KL/rejected_KL_mean": -214.1387176513672, "KL/std": 89.42788696289062, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.007255447097122669, "fcm_dpo/delta": -0.2875009775161743, "fcm_dpo/margin": 94.47328186035156, "fcm_dpo/q_t": 0.3487655520439148, "grad_norm": 8.358288764953613, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.507556676864624, "logits/rejected": -1.1805169582366943, "logps/chosen": -204.36685180664062, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -310.1895751953125, "loss": 0.9104, "margin_dpo/margin_mean": 94.47328186035156, "margin_dpo/margin_std": 90.83686828613281, "step": 466 }, { "KL/chosen_KL_mean": -150.11264038085938, "KL/mean": -190.24029541015625, "KL/rejected_KL_mean": -230.36795043945312, "KL/std": 91.44695281982422, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.007222908549010754, "fcm_dpo/delta": -0.180901899933815, "fcm_dpo/margin": 80.25531005859375, "fcm_dpo/q_t": 0.37313133478164673, "grad_norm": 11.856180191040039, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.341642379760742, "logits/rejected": -1.775146245956421, "logps/chosen": -259.41094970703125, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -339.21160888671875, "loss": 1.0143, "margin_dpo/margin_mean": 80.25531005859375, "margin_dpo/margin_std": 102.56227111816406, "step": 467 }, { "KL/chosen_KL_mean": -121.07884216308594, "KL/mean": -156.02322387695312, "KL/rejected_KL_mean": -190.96762084960938, "KL/std": 89.12216186523438, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.007207885384559631, "fcm_dpo/delta": -0.10430504381656647, "fcm_dpo/margin": 69.88875579833984, "fcm_dpo/q_t": 0.3915298581123352, "grad_norm": 10.635473251342773, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.6720080375671387, "logits/rejected": -1.473069667816162, "logps/chosen": -219.40048217773438, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -297.6480712890625, "loss": 1.0609, "margin_dpo/margin_mean": 69.88874816894531, "margin_dpo/margin_std": 100.64604187011719, "step": 468 }, { "KL/chosen_KL_mean": -145.3563995361328, "KL/mean": -189.94667053222656, "KL/rejected_KL_mean": -234.53695678710938, "KL/std": 87.6291275024414, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0071758683770895, "fcm_dpo/delta": -0.24166959524154663, "fcm_dpo/margin": 89.1805419921875, "fcm_dpo/q_t": 0.35886961221694946, "grad_norm": 13.064724922180176, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.137779474258423, "logits/rejected": -1.7108285427093506, "logps/chosen": -242.12060546875, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -344.1319580078125, "loss": 0.9506, "margin_dpo/margin_mean": 89.1805419921875, "margin_dpo/margin_std": 94.84898376464844, "step": 469 }, { "KL/chosen_KL_mean": -136.33155822753906, "KL/mean": -176.9646453857422, "KL/rejected_KL_mean": -217.59774780273438, "KL/std": 84.25515747070312, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.007151451893150806, "fcm_dpo/delta": -0.18222633004188538, "fcm_dpo/margin": 81.26618957519531, "fcm_dpo/q_t": 0.36942631006240845, "grad_norm": 11.051928520202637, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.794244408607483, "logits/rejected": -1.3841426372528076, "logps/chosen": -219.82821655273438, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -306.0835266113281, "loss": 0.9691, "margin_dpo/margin_mean": 81.26618957519531, "margin_dpo/margin_std": 84.34857177734375, "step": 470 }, { "KL/chosen_KL_mean": -157.3477325439453, "KL/mean": -196.60279846191406, "KL/rejected_KL_mean": -235.85787963867188, "KL/std": 82.84588623046875, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.00712693203240633, "fcm_dpo/delta": -0.16045093536376953, "fcm_dpo/margin": 78.51013946533203, "fcm_dpo/q_t": 0.3729066848754883, "grad_norm": 10.5030517578125, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.4863860607147217, "logits/rejected": -1.0839481353759766, "logps/chosen": -245.82203674316406, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -326.339599609375, "loss": 0.9802, "margin_dpo/margin_mean": 78.51013946533203, "margin_dpo/margin_std": 81.48716735839844, "step": 471 }, { "KL/chosen_KL_mean": -142.03741455078125, "KL/mean": -190.31710815429688, "KL/rejected_KL_mean": -238.5968017578125, "KL/std": 82.59722900390625, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.007088498678058386, "fcm_dpo/delta": -0.2865696847438812, "fcm_dpo/margin": 96.55938720703125, "fcm_dpo/q_t": 0.3478410840034485, "grad_norm": 8.806059837341309, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.1201462745666504, "logits/rejected": -1.42853844165802, "logps/chosen": -267.2711181640625, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -359.6502990722656, "loss": 0.8913, "margin_dpo/margin_mean": 96.55938720703125, "margin_dpo/margin_std": 82.85914611816406, "step": 472 }, { "KL/chosen_KL_mean": -161.3885498046875, "KL/mean": -191.86605834960938, "KL/rejected_KL_mean": -222.34356689453125, "KL/std": 84.14604187011719, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.007079499773681164, "fcm_dpo/delta": -0.031671538949012756, "fcm_dpo/margin": 60.95500946044922, "fcm_dpo/q_t": 0.4058857560157776, "grad_norm": 11.269224166870117, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8089752197265625, "logits/rejected": -1.3166015148162842, "logps/chosen": -256.99993896484375, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -310.4947204589844, "loss": 1.1074, "margin_dpo/margin_mean": 60.95500946044922, "margin_dpo/margin_std": 96.92118835449219, "step": 473 }, { "KL/chosen_KL_mean": -155.46957397460938, "KL/mean": -184.4029541015625, "KL/rejected_KL_mean": -213.33633422851562, "KL/std": 77.47513580322266, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.007081366144120693, "fcm_dpo/delta": -0.009939536452293396, "fcm_dpo/margin": 57.86676025390625, "fcm_dpo/q_t": 0.40810626745224, "grad_norm": 13.828439712524414, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.5544772148132324, "logits/rejected": -1.363370418548584, "logps/chosen": -236.94932556152344, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -309.8019714355469, "loss": 1.1266, "margin_dpo/margin_mean": 57.86676025390625, "margin_dpo/margin_std": 91.8223876953125, "step": 474 }, { "KL/chosen_KL_mean": -116.15713500976562, "KL/mean": -160.96255493164062, "KL/rejected_KL_mean": -205.76797485351562, "KL/std": 86.5477066040039, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.00705123133957386, "fcm_dpo/delta": -0.23332160711288452, "fcm_dpo/margin": 89.61085510253906, "fcm_dpo/q_t": 0.3616146445274353, "grad_norm": 10.154561996459961, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.2501840591430664, "logits/rejected": -2.1952462196350098, "logps/chosen": -190.35311889648438, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -300.46038818359375, "loss": 0.9475, "margin_dpo/margin_mean": 89.61085510253906, "margin_dpo/margin_std": 94.1727294921875, "step": 475 }, { "KL/chosen_KL_mean": -138.33682250976562, "KL/mean": -177.46963500976562, "KL/rejected_KL_mean": -216.60244750976562, "KL/std": 87.25592041015625, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.007023612968623638, "fcm_dpo/delta": -0.15062889456748962, "fcm_dpo/margin": 78.26564025878906, "fcm_dpo/q_t": 0.37993067502975464, "grad_norm": 12.715035438537598, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.8527395725250244, "logits/rejected": -1.4642493724822998, "logps/chosen": -210.30792236328125, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -296.86468505859375, "loss": 1.0088, "margin_dpo/margin_mean": 78.26564025878906, "margin_dpo/margin_std": 94.58261108398438, "step": 476 }, { "KL/chosen_KL_mean": -135.68692016601562, "KL/mean": -177.51773071289062, "KL/rejected_KL_mean": -219.34857177734375, "KL/std": 79.85171508789062, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.007002205587923527, "fcm_dpo/delta": -0.18689462542533875, "fcm_dpo/margin": 83.66165161132812, "fcm_dpo/q_t": 0.3689793348312378, "grad_norm": 10.444279670715332, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.092241048812866, "logits/rejected": -1.1898974180221558, "logps/chosen": -240.69247436523438, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -301.22698974609375, "loss": 0.9875, "margin_dpo/margin_mean": 83.66165161132812, "margin_dpo/margin_std": 96.53141784667969, "step": 477 }, { "KL/chosen_KL_mean": -130.637451171875, "KL/mean": -173.43788146972656, "KL/rejected_KL_mean": -216.23834228515625, "KL/std": 85.42058563232422, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.006974611431360245, "fcm_dpo/delta": -0.1982055902481079, "fcm_dpo/margin": 85.60086822509766, "fcm_dpo/q_t": 0.3666830360889435, "grad_norm": 9.530346870422363, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.7189959287643433, "logits/rejected": -1.550855278968811, "logps/chosen": -207.4256591796875, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -306.67828369140625, "loss": 0.9695, "margin_dpo/margin_mean": 85.60087585449219, "margin_dpo/margin_std": 92.82054901123047, "step": 478 }, { "KL/chosen_KL_mean": -137.29412841796875, "KL/mean": -176.2983856201172, "KL/rejected_KL_mean": -215.30262756347656, "KL/std": 83.9625015258789, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.006955260410904884, "fcm_dpo/delta": -0.14332231879234314, "fcm_dpo/margin": 78.00848388671875, "fcm_dpo/q_t": 0.3761623501777649, "grad_norm": 11.297622680664062, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.4013054370880127, "logits/rejected": -1.2625162601470947, "logps/chosen": -218.6564483642578, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -316.3937683105469, "loss": 0.9984, "margin_dpo/margin_mean": 78.00849151611328, "margin_dpo/margin_std": 84.92399597167969, "step": 479 }, { "KL/chosen_KL_mean": -156.41488647460938, "KL/mean": -190.11593627929688, "KL/rejected_KL_mean": -223.8170166015625, "KL/std": 81.80352783203125, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.006939902901649475, "fcm_dpo/delta": -0.06808672100305557, "fcm_dpo/margin": 67.40213775634766, "fcm_dpo/q_t": 0.39514079689979553, "grad_norm": 13.047566413879395, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7191145420074463, "logits/rejected": -1.0411657094955444, "logps/chosen": -253.18878173828125, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -310.22174072265625, "loss": 1.0656, "margin_dpo/margin_mean": 67.40213775634766, "margin_dpo/margin_std": 91.67901611328125, "step": 480 }, { "KL/chosen_KL_mean": -151.95333862304688, "KL/mean": -191.85069274902344, "KL/rejected_KL_mean": -231.74806213378906, "KL/std": 84.66319274902344, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.006921843625605106, "fcm_dpo/delta": -0.1531757414340973, "fcm_dpo/margin": 79.79472351074219, "fcm_dpo/q_t": 0.37599968910217285, "grad_norm": 12.63962459564209, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.9041130542755127, "logits/rejected": -1.4598625898361206, "logps/chosen": -243.62350463867188, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -330.4429626464844, "loss": 1.0056, "margin_dpo/margin_mean": 79.79472351074219, "margin_dpo/margin_std": 94.48236083984375, "step": 481 }, { "KL/chosen_KL_mean": -146.32308959960938, "KL/mean": -185.65867614746094, "KL/rejected_KL_mean": -224.9942626953125, "KL/std": 81.89572143554688, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.006900248117744923, "fcm_dpo/delta": -0.14367234706878662, "fcm_dpo/margin": 78.67117309570312, "fcm_dpo/q_t": 0.37654411792755127, "grad_norm": 13.050161361694336, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.4306385517120361, "logits/rejected": -1.0617488622665405, "logps/chosen": -244.84320068359375, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -319.82373046875, "loss": 1.0003, "margin_dpo/margin_mean": 78.67117309570312, "margin_dpo/margin_std": 87.921630859375, "step": 482 }, { "KL/chosen_KL_mean": -151.2992401123047, "KL/mean": -192.6817626953125, "KL/rejected_KL_mean": -234.06430053710938, "KL/std": 88.33917236328125, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.006883701775223017, "fcm_dpo/delta": -0.1706765592098236, "fcm_dpo/margin": 82.76505279541016, "fcm_dpo/q_t": 0.3709464967250824, "grad_norm": 8.768048286437988, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.1146671772003174, "logits/rejected": -1.4892663955688477, "logps/chosen": -258.4178466796875, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -335.1792907714844, "loss": 0.9754, "margin_dpo/margin_mean": 82.76505279541016, "margin_dpo/margin_std": 84.42509460449219, "step": 483 }, { "KL/chosen_KL_mean": -141.72744750976562, "KL/mean": -186.28787231445312, "KL/rejected_KL_mean": -230.84832763671875, "KL/std": 91.10013580322266, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0068544019013643265, "fcm_dpo/delta": -0.21211248636245728, "fcm_dpo/margin": 89.12086486816406, "fcm_dpo/q_t": 0.36392614245414734, "grad_norm": 9.275399208068848, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.448181629180908, "logits/rejected": -1.972672462463379, "logps/chosen": -228.70736694335938, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -321.5719909667969, "loss": 0.9665, "margin_dpo/margin_mean": 89.12086486816406, "margin_dpo/margin_std": 96.59501647949219, "step": 484 }, { "KL/chosen_KL_mean": -129.29884338378906, "KL/mean": -179.83758544921875, "KL/rejected_KL_mean": -230.37632751464844, "KL/std": 92.23506164550781, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.006814903579652309, "fcm_dpo/delta": -0.29081904888153076, "fcm_dpo/margin": 101.0774917602539, "fcm_dpo/q_t": 0.34746053814888, "grad_norm": 9.8587007522583, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.627899169921875, "logits/rejected": -1.190290093421936, "logps/chosen": -223.3575897216797, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -338.9393310546875, "loss": 0.8974, "margin_dpo/margin_mean": 101.0774917602539, "margin_dpo/margin_std": 91.47610473632812, "step": 485 }, { "KL/chosen_KL_mean": -150.3681640625, "KL/mean": -194.43267822265625, "KL/rejected_KL_mean": -238.49716186523438, "KL/std": 89.05880737304688, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.006786476355046034, "fcm_dpo/delta": -0.19923335313796997, "fcm_dpo/margin": 88.12899017333984, "fcm_dpo/q_t": 0.3669247031211853, "grad_norm": 11.16351318359375, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.0224108695983887, "logits/rejected": -1.3087458610534668, "logps/chosen": -244.11404418945312, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -336.56781005859375, "loss": 0.9777, "margin_dpo/margin_mean": 88.12899017333984, "margin_dpo/margin_std": 98.00436401367188, "step": 486 }, { "KL/chosen_KL_mean": -144.99188232421875, "KL/mean": -186.141845703125, "KL/rejected_KL_mean": -227.29180908203125, "KL/std": 86.45889282226562, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0067604039795696735, "fcm_dpo/delta": -0.15728659927845, "fcm_dpo/margin": 82.29989624023438, "fcm_dpo/q_t": 0.3753056526184082, "grad_norm": 10.965670585632324, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.3086729049682617, "logits/rejected": -2.0086584091186523, "logps/chosen": -233.02545166015625, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -325.763916015625, "loss": 0.9871, "margin_dpo/margin_mean": 82.29989624023438, "margin_dpo/margin_std": 89.28451538085938, "step": 487 }, { "KL/chosen_KL_mean": -135.16064453125, "KL/mean": -178.17791748046875, "KL/rejected_KL_mean": -221.1951904296875, "KL/std": 90.89216613769531, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0067383404821157455, "fcm_dpo/delta": -0.1807672381401062, "fcm_dpo/margin": 86.03453826904297, "fcm_dpo/q_t": 0.3673364520072937, "grad_norm": 11.635273933410645, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.7192432880401611, "logits/rejected": -1.467997670173645, "logps/chosen": -219.95028686523438, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -311.4686279296875, "loss": 0.9547, "margin_dpo/margin_mean": 86.0345458984375, "margin_dpo/margin_std": 78.51849365234375, "step": 488 }, { "KL/chosen_KL_mean": -147.87237548828125, "KL/mean": -188.49642944335938, "KL/rejected_KL_mean": -229.1204833984375, "KL/std": 82.49847412109375, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0067170243710279465, "fcm_dpo/delta": -0.1465454399585724, "fcm_dpo/margin": 81.24810791015625, "fcm_dpo/q_t": 0.37761226296424866, "grad_norm": 10.044486999511719, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.652578353881836, "logits/rejected": -1.1648094654083252, "logps/chosen": -238.3416748046875, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -315.5180969238281, "loss": 0.9973, "margin_dpo/margin_mean": 81.24810791015625, "margin_dpo/margin_std": 90.80810546875, "step": 489 }, { "KL/chosen_KL_mean": -153.2631378173828, "KL/mean": -189.02333068847656, "KL/rejected_KL_mean": -224.78353881835938, "KL/std": 87.16455841064453, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.006709085311740637, "fcm_dpo/delta": -0.08027482032775879, "fcm_dpo/margin": 71.52040100097656, "fcm_dpo/q_t": 0.3960776627063751, "grad_norm": 11.66257381439209, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1802659034729004, "logits/rejected": -2.245151996612549, "logps/chosen": -238.58326721191406, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -340.77740478515625, "loss": 1.1014, "margin_dpo/margin_mean": 71.52040100097656, "margin_dpo/margin_std": 114.54646301269531, "step": 490 }, { "KL/chosen_KL_mean": -152.28561401367188, "KL/mean": -187.6307373046875, "KL/rejected_KL_mean": -222.97589111328125, "KL/std": 91.55542755126953, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.006693914067000151, "fcm_dpo/delta": -0.07354298233985901, "fcm_dpo/margin": 70.69027709960938, "fcm_dpo/q_t": 0.39261388778686523, "grad_norm": 10.027724266052246, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -2.201697826385498, "logits/rejected": -1.7349662780761719, "logps/chosen": -238.42913818359375, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -303.65533447265625, "loss": 1.0574, "margin_dpo/margin_mean": 70.69027709960938, "margin_dpo/margin_std": 92.90451049804688, "step": 491 }, { "KL/chosen_KL_mean": -139.86196899414062, "KL/mean": -168.19967651367188, "KL/rejected_KL_mean": -196.53738403320312, "KL/std": 89.28358459472656, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0066932085901498795, "fcm_dpo/delta": 0.020739004015922546, "fcm_dpo/margin": 56.675392150878906, "fcm_dpo/q_t": 0.41546761989593506, "grad_norm": 14.263859748840332, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.992531180381775, "logits/rejected": -1.6009091138839722, "logps/chosen": -220.96954345703125, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -277.28936767578125, "loss": 1.1687, "margin_dpo/margin_mean": 56.67539596557617, "margin_dpo/margin_std": 110.69635772705078, "step": 492 }, { "KL/chosen_KL_mean": -139.67819213867188, "KL/mean": -176.040771484375, "KL/rejected_KL_mean": -212.4033203125, "KL/std": 81.12615966796875, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.006682124920189381, "fcm_dpo/delta": -0.08647646009922028, "fcm_dpo/margin": 72.72510528564453, "fcm_dpo/q_t": 0.3881593942642212, "grad_norm": 13.508996963500977, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.5041449069976807, "logits/rejected": -1.7889429330825806, "logps/chosen": -251.8855438232422, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -306.010498046875, "loss": 1.0305, "margin_dpo/margin_mean": 72.72510528564453, "margin_dpo/margin_std": 81.74932861328125, "step": 493 }, { "KL/chosen_KL_mean": -134.89178466796875, "KL/mean": -184.163330078125, "KL/rejected_KL_mean": -233.43487548828125, "KL/std": 90.72200012207031, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.006653040647506714, "fcm_dpo/delta": -0.25751861929893494, "fcm_dpo/margin": 98.54306030273438, "fcm_dpo/q_t": 0.3534157872200012, "grad_norm": 10.606117248535156, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.4793682098388672, "logits/rejected": -1.3931457996368408, "logps/chosen": -208.00668334960938, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -325.5978698730469, "loss": 0.9226, "margin_dpo/margin_mean": 98.54307556152344, "margin_dpo/margin_std": 92.8673095703125, "step": 494 }, { "KL/chosen_KL_mean": -136.66085815429688, "KL/mean": -177.8853759765625, "KL/rejected_KL_mean": -219.10986328125, "KL/std": 85.92632293701172, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.006633209064602852, "fcm_dpo/delta": -0.1477200984954834, "fcm_dpo/margin": 82.44898986816406, "fcm_dpo/q_t": 0.3774929642677307, "grad_norm": 11.445216178894043, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.140122175216675, "logits/rejected": -1.677889347076416, "logps/chosen": -236.18118286132812, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -317.040771484375, "loss": 1.0125, "margin_dpo/margin_mean": 82.44898986816406, "margin_dpo/margin_std": 100.440673828125, "step": 495 }, { "KL/chosen_KL_mean": -151.80589294433594, "KL/mean": -186.89208984375, "KL/rejected_KL_mean": -221.978271484375, "KL/std": 86.43569946289062, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0066243926994502544, "fcm_dpo/delta": -0.0651518777012825, "fcm_dpo/margin": 70.17237854003906, "fcm_dpo/q_t": 0.39299312233924866, "grad_norm": 10.387298583984375, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.287910223007202, "logits/rejected": -1.6614429950714111, "logps/chosen": -259.66265869140625, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -314.74884033203125, "loss": 1.0506, "margin_dpo/margin_mean": 70.17237854003906, "margin_dpo/margin_std": 85.19818115234375, "step": 496 }, { "KL/chosen_KL_mean": -166.81719970703125, "KL/mean": -191.15188598632812, "KL/rejected_KL_mean": -215.48658752441406, "KL/std": 87.83110046386719, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.006626127287745476, "fcm_dpo/delta": 0.07776802778244019, "fcm_dpo/margin": 48.669376373291016, "fcm_dpo/q_t": 0.4272153675556183, "grad_norm": 10.402411460876465, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.9410929679870605, "logits/rejected": -1.5103304386138916, "logps/chosen": -258.90045166015625, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -297.2816162109375, "loss": 1.1795, "margin_dpo/margin_mean": 48.66937255859375, "margin_dpo/margin_std": 92.772216796875, "step": 497 }, { "KL/chosen_KL_mean": -135.3360595703125, "KL/mean": -174.9803466796875, "KL/rejected_KL_mean": -214.62461853027344, "KL/std": 83.12522888183594, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.006617994979023933, "fcm_dpo/delta": -0.12538878619670868, "fcm_dpo/margin": 79.28855895996094, "fcm_dpo/q_t": 0.3802109956741333, "grad_norm": 11.351746559143066, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.7914464473724365, "logits/rejected": -1.3017375469207764, "logps/chosen": -233.53042602539062, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -305.31207275390625, "loss": 1.0027, "margin_dpo/margin_mean": 79.28855895996094, "margin_dpo/margin_std": 85.83343505859375, "step": 498 }, { "KL/chosen_KL_mean": -146.23516845703125, "KL/mean": -186.06748962402344, "KL/rejected_KL_mean": -225.89979553222656, "KL/std": 93.4317626953125, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.006602455396205187, "fcm_dpo/delta": -0.1266319453716278, "fcm_dpo/margin": 79.66461181640625, "fcm_dpo/q_t": 0.38252419233322144, "grad_norm": 14.017510414123535, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.06657075881958, "logits/rejected": -2.0122530460357666, "logps/chosen": -227.27508544921875, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -332.8215026855469, "loss": 1.0284, "margin_dpo/margin_mean": 79.66461181640625, "margin_dpo/margin_std": 100.72923278808594, "step": 499 }, { "KL/chosen_KL_mean": -157.72921752929688, "KL/mean": -194.36068725585938, "KL/rejected_KL_mean": -230.99212646484375, "KL/std": 90.24104309082031, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.006587960757315159, "fcm_dpo/delta": -0.08306936919689178, "fcm_dpo/margin": 73.26290893554688, "fcm_dpo/q_t": 0.3940514922142029, "grad_norm": 12.319123268127441, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.6569803953170776, "logits/rejected": -1.482861042022705, "logps/chosen": -246.97727966308594, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -331.40234375, "loss": 1.0604, "margin_dpo/margin_mean": 73.26290893554688, "margin_dpo/margin_std": 100.79672241210938, "step": 500 }, { "KL/chosen_KL_mean": -138.33114624023438, "KL/mean": -176.75006103515625, "KL/rejected_KL_mean": -215.16897583007812, "KL/std": 98.68276977539062, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.006581573747098446, "fcm_dpo/delta": -0.10631810128688812, "fcm_dpo/margin": 76.83784484863281, "fcm_dpo/q_t": 0.3896797001361847, "grad_norm": 12.096955299377441, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.623960256576538, "logits/rejected": -1.1918113231658936, "logps/chosen": -228.92807006835938, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -300.46624755859375, "loss": 1.0672, "margin_dpo/margin_mean": 76.83784484863281, "margin_dpo/margin_std": 109.43427276611328, "step": 501 }, { "KL/chosen_KL_mean": -140.76052856445312, "KL/mean": -175.235595703125, "KL/rejected_KL_mean": -209.710693359375, "KL/std": 81.6031494140625, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.006570969708263874, "fcm_dpo/delta": -0.05335519462823868, "fcm_dpo/margin": 68.9501724243164, "fcm_dpo/q_t": 0.3976008892059326, "grad_norm": 10.234506607055664, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9186062812805176, "logits/rejected": -1.187234878540039, "logps/chosen": -244.23126220703125, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -298.75482177734375, "loss": 1.0889, "margin_dpo/margin_mean": 68.9501724243164, "margin_dpo/margin_std": 100.27865600585938, "step": 502 }, { "KL/chosen_KL_mean": -136.34515380859375, "KL/mean": -177.047607421875, "KL/rejected_KL_mean": -217.75009155273438, "KL/std": 86.73954772949219, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.00655602291226387, "fcm_dpo/delta": -0.1344042718410492, "fcm_dpo/margin": 81.40493774414062, "fcm_dpo/q_t": 0.37999406456947327, "grad_norm": 10.058006286621094, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.6248819828033447, "logits/rejected": -1.1121759414672852, "logps/chosen": -230.07513427734375, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -305.35906982421875, "loss": 1.0151, "margin_dpo/margin_mean": 81.4049301147461, "margin_dpo/margin_std": 94.58529663085938, "step": 503 }, { "KL/chosen_KL_mean": -137.49267578125, "KL/mean": -186.78500366210938, "KL/rejected_KL_mean": -236.07733154296875, "KL/std": 89.24425506591797, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.006524617783725262, "fcm_dpo/delta": -0.24476689100265503, "fcm_dpo/margin": 98.58464050292969, "fcm_dpo/q_t": 0.3548470139503479, "grad_norm": 13.76848316192627, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.086205005645752, "logits/rejected": -1.7504405975341797, "logps/chosen": -229.7845458984375, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -345.114990234375, "loss": 0.9248, "margin_dpo/margin_mean": 98.58464050292969, "margin_dpo/margin_std": 91.88336181640625, "step": 504 }, { "KL/chosen_KL_mean": -154.28826904296875, "KL/mean": -192.15948486328125, "KL/rejected_KL_mean": -230.03070068359375, "KL/std": 90.81521606445312, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.006506810896098614, "fcm_dpo/delta": -0.09329689294099808, "fcm_dpo/margin": 75.7424545288086, "fcm_dpo/q_t": 0.3877539038658142, "grad_norm": 9.640143394470215, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.2410342693328857, "logits/rejected": -1.6742572784423828, "logps/chosen": -260.565185546875, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -324.23681640625, "loss": 1.0278, "margin_dpo/margin_mean": 75.7424545288086, "margin_dpo/margin_std": 86.96047973632812, "step": 505 }, { "KL/chosen_KL_mean": -149.70970153808594, "KL/mean": -189.9342041015625, "KL/rejected_KL_mean": -230.15872192382812, "KL/std": 83.20291137695312, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0064896028488874435, "fcm_dpo/delta": -0.12277336418628693, "fcm_dpo/margin": 80.44900512695312, "fcm_dpo/q_t": 0.37926238775253296, "grad_norm": 10.131646156311035, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.892505168914795, "logits/rejected": -1.524788737297058, "logps/chosen": -231.15057373046875, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -315.8231201171875, "loss": 1.0024, "margin_dpo/margin_mean": 80.44900512695312, "margin_dpo/margin_std": 84.66144561767578, "step": 506 }, { "KL/chosen_KL_mean": -157.0992431640625, "KL/mean": -199.49447631835938, "KL/rejected_KL_mean": -241.88970947265625, "KL/std": 88.89933776855469, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0064739203080534935, "fcm_dpo/delta": -0.14973017573356628, "fcm_dpo/margin": 84.79048156738281, "fcm_dpo/q_t": 0.37821435928344727, "grad_norm": 9.597389221191406, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.1508703231811523, "logits/rejected": -1.7111536264419556, "logps/chosen": -256.52618408203125, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -344.1426696777344, "loss": 1.015, "margin_dpo/margin_mean": 84.79048156738281, "margin_dpo/margin_std": 103.7872314453125, "step": 507 }, { "KL/chosen_KL_mean": -154.35275268554688, "KL/mean": -188.99807739257812, "KL/rejected_KL_mean": -223.64340209960938, "KL/std": 95.42719268798828, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0064625926315784454, "fcm_dpo/delta": -0.04802127182483673, "fcm_dpo/margin": 69.29066467285156, "fcm_dpo/q_t": 0.4014105200767517, "grad_norm": 12.54348087310791, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.2095155715942383, "logits/rejected": -1.742969036102295, "logps/chosen": -252.26840209960938, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -327.44122314453125, "loss": 1.1002, "margin_dpo/margin_mean": 69.29066467285156, "margin_dpo/margin_std": 106.24467468261719, "step": 508 }, { "KL/chosen_KL_mean": -144.4061279296875, "KL/mean": -178.7329864501953, "KL/rejected_KL_mean": -213.05982971191406, "KL/std": 73.79302978515625, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.006458488758653402, "fcm_dpo/delta": -0.04359026253223419, "fcm_dpo/margin": 68.65370178222656, "fcm_dpo/q_t": 0.3973853886127472, "grad_norm": 14.090490341186523, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9896900653839111, "logits/rejected": -1.3272972106933594, "logps/chosen": -245.135986328125, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -300.003173828125, "loss": 1.0793, "margin_dpo/margin_mean": 68.65370178222656, "margin_dpo/margin_std": 94.17241668701172, "step": 509 }, { "KL/chosen_KL_mean": -170.47103881835938, "KL/mean": -210.126708984375, "KL/rejected_KL_mean": -249.78237915039062, "KL/std": 91.94354248046875, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.006446673069149256, "fcm_dpo/delta": -0.11185194551944733, "fcm_dpo/margin": 79.31134033203125, "fcm_dpo/q_t": 0.3841506242752075, "grad_norm": 11.142005920410156, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.8022522926330566, "logits/rejected": -1.6135461330413818, "logps/chosen": -259.6404113769531, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -357.7017822265625, "loss": 1.0261, "margin_dpo/margin_mean": 79.31134033203125, "margin_dpo/margin_std": 93.26494598388672, "step": 510 }, { "KL/chosen_KL_mean": -153.6510009765625, "KL/mean": -196.9484100341797, "KL/rejected_KL_mean": -240.24581909179688, "KL/std": 89.3297119140625, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.00642815139144659, "fcm_dpo/delta": -0.1574951410293579, "fcm_dpo/margin": 86.59484100341797, "fcm_dpo/q_t": 0.375995934009552, "grad_norm": 9.882804870605469, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.2009239196777344, "logits/rejected": -1.643481969833374, "logps/chosen": -243.40078735351562, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -337.97711181640625, "loss": 0.9916, "margin_dpo/margin_mean": 86.59484100341797, "margin_dpo/margin_std": 96.64553833007812, "step": 511 }, { "KL/chosen_KL_mean": -144.97528076171875, "KL/mean": -185.90545654296875, "KL/rejected_KL_mean": -226.8356170654297, "KL/std": 84.8726577758789, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.006412585731595755, "fcm_dpo/delta": -0.12558242678642273, "fcm_dpo/margin": 81.8603515625, "fcm_dpo/q_t": 0.38204169273376465, "grad_norm": 13.698766708374023, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.811490535736084, "logits/rejected": -1.3651466369628906, "logps/chosen": -230.71041870117188, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -314.46051025390625, "loss": 1.0273, "margin_dpo/margin_mean": 81.86034393310547, "margin_dpo/margin_std": 101.17031860351562, "step": 512 }, { "KL/chosen_KL_mean": -154.2011260986328, "KL/mean": -190.22744750976562, "KL/rejected_KL_mean": -226.2537841796875, "KL/std": 83.10875701904297, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.006396174430847168, "fcm_dpo/delta": -0.06123036891222, "fcm_dpo/margin": 72.05265045166016, "fcm_dpo/q_t": 0.3931971788406372, "grad_norm": 11.041118621826172, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.4192075729370117, "logits/rejected": -1.7428948879241943, "logps/chosen": -250.39163208007812, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -309.1715087890625, "loss": 1.0568, "margin_dpo/margin_mean": 72.05265045166016, "margin_dpo/margin_std": 88.92807006835938, "step": 513 }, { "KL/chosen_KL_mean": -148.34214782714844, "KL/mean": -186.1943359375, "KL/rejected_KL_mean": -224.04653930664062, "KL/std": 80.52801513671875, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.006388727575540543, "fcm_dpo/delta": -0.08407466113567352, "fcm_dpo/margin": 75.70440673828125, "fcm_dpo/q_t": 0.3910391330718994, "grad_norm": 11.425773620605469, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.7984521389007568, "logits/rejected": -1.2381038665771484, "logps/chosen": -246.16578674316406, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -309.4163818359375, "loss": 1.0403, "margin_dpo/margin_mean": 75.70440673828125, "margin_dpo/margin_std": 92.25907897949219, "step": 514 }, { "KL/chosen_KL_mean": -146.6575164794922, "KL/mean": -180.79190063476562, "KL/rejected_KL_mean": -214.92628479003906, "KL/std": 84.41987609863281, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0063829319551587105, "fcm_dpo/delta": -0.03591492027044296, "fcm_dpo/margin": 68.26876831054688, "fcm_dpo/q_t": 0.4003312587738037, "grad_norm": 15.06063175201416, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.9985483884811401, "logits/rejected": -1.774698257446289, "logps/chosen": -228.591064453125, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -308.50286865234375, "loss": 1.0791, "margin_dpo/margin_mean": 68.26876831054688, "margin_dpo/margin_std": 92.07762145996094, "step": 515 }, { "KL/chosen_KL_mean": -149.5787811279297, "KL/mean": -184.81373596191406, "KL/rejected_KL_mean": -220.04867553710938, "KL/std": 96.54652404785156, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.006378308404237032, "fcm_dpo/delta": -0.04969903826713562, "fcm_dpo/margin": 70.46989440917969, "fcm_dpo/q_t": 0.4006595015525818, "grad_norm": 10.58105182647705, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.3853468894958496, "logits/rejected": -2.191810369491577, "logps/chosen": -246.67044067382812, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -338.6053466796875, "loss": 1.1077, "margin_dpo/margin_mean": 70.46989440917969, "margin_dpo/margin_std": 113.85547637939453, "step": 516 }, { "KL/chosen_KL_mean": -159.155029296875, "KL/mean": -202.69912719726562, "KL/rejected_KL_mean": -246.2432098388672, "KL/std": 88.91107177734375, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.006356736645102501, "fcm_dpo/delta": -0.15465420484542847, "fcm_dpo/margin": 87.0881576538086, "fcm_dpo/q_t": 0.37645840644836426, "grad_norm": 10.119709968566895, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.7899665832519531, "logits/rejected": -1.3572242259979248, "logps/chosen": -252.3415985107422, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -341.83599853515625, "loss": 1.005, "margin_dpo/margin_mean": 87.08816528320312, "margin_dpo/margin_std": 100.21694946289062, "step": 517 }, { "KL/chosen_KL_mean": -164.90109252929688, "KL/mean": -197.7104034423828, "KL/rejected_KL_mean": -230.51968383789062, "KL/std": 84.3008804321289, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0063538155518472195, "fcm_dpo/delta": -0.01700415462255478, "fcm_dpo/margin": 65.61857604980469, "fcm_dpo/q_t": 0.4065180718898773, "grad_norm": 12.394344329833984, "learning_rate": 8.348171708068747e-08, "logits/chosen": -2.2657833099365234, "logits/rejected": -1.9500060081481934, "logps/chosen": -251.92247009277344, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -321.7642517089844, "loss": 1.1074, "margin_dpo/margin_mean": 65.61857604980469, "margin_dpo/margin_std": 101.30152893066406, "step": 518 }, { "KL/chosen_KL_mean": -153.4951934814453, "KL/mean": -189.5060272216797, "KL/rejected_KL_mean": -225.51687622070312, "KL/std": 79.3948974609375, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.006346012465655804, "fcm_dpo/delta": -0.057368483394384384, "fcm_dpo/margin": 72.02169799804688, "fcm_dpo/q_t": 0.3951184153556824, "grad_norm": 10.201952934265137, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.1398770809173584, "logits/rejected": -1.7754490375518799, "logps/chosen": -254.1827392578125, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -334.54730224609375, "loss": 1.0417, "margin_dpo/margin_mean": 72.02169799804688, "margin_dpo/margin_std": 79.85778045654297, "step": 519 }, { "KL/chosen_KL_mean": -139.9888916015625, "KL/mean": -182.69805908203125, "KL/rejected_KL_mean": -225.40724182128906, "KL/std": 88.86798095703125, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.006331815849989653, "fcm_dpo/delta": -0.14166322350502014, "fcm_dpo/margin": 85.41835021972656, "fcm_dpo/q_t": 0.37546706199645996, "grad_norm": 9.601655960083008, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.4856126308441162, "logits/rejected": -1.0402148962020874, "logps/chosen": -238.76651000976562, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -332.35296630859375, "loss": 0.9815, "margin_dpo/margin_mean": 85.41835021972656, "margin_dpo/margin_std": 82.72158813476562, "step": 520 }, { "KL/chosen_KL_mean": -146.56314086914062, "KL/mean": -189.05718994140625, "KL/rejected_KL_mean": -231.55125427246094, "KL/std": 84.15972900390625, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.006318532861769199, "fcm_dpo/delta": -0.13771812617778778, "fcm_dpo/margin": 84.98812103271484, "fcm_dpo/q_t": 0.37701117992401123, "grad_norm": 10.385902404785156, "learning_rate": 8.062991975753378e-08, "logits/chosen": -2.057736396789551, "logits/rejected": -1.730154037475586, "logps/chosen": -232.91421508789062, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -321.9444274902344, "loss": 0.9864, "margin_dpo/margin_mean": 84.98812103271484, "margin_dpo/margin_std": 84.03314208984375, "step": 521 }, { "KL/chosen_KL_mean": -148.78799438476562, "KL/mean": -189.3525390625, "KL/rejected_KL_mean": -229.91709899902344, "KL/std": 90.82976531982422, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.006301949732005596, "fcm_dpo/delta": -0.1118297353386879, "fcm_dpo/margin": 81.12909698486328, "fcm_dpo/q_t": 0.38521575927734375, "grad_norm": 10.004437446594238, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.6493806838989258, "logits/rejected": -1.2741186618804932, "logps/chosen": -224.79605102539062, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -304.35833740234375, "loss": 1.0249, "margin_dpo/margin_mean": 81.12909698486328, "margin_dpo/margin_std": 97.13094329833984, "step": 522 }, { "KL/chosen_KL_mean": -169.90338134765625, "KL/mean": -197.58547973632812, "KL/rejected_KL_mean": -225.26754760742188, "KL/std": 88.44792175292969, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0063004037365317345, "fcm_dpo/delta": 0.05136152356863022, "fcm_dpo/margin": 55.36416244506836, "fcm_dpo/q_t": 0.42146235704421997, "grad_norm": 11.869359970092773, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.02994966506958, "logits/rejected": -1.3283289670944214, "logps/chosen": -266.399658203125, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -316.2828369140625, "loss": 1.1647, "margin_dpo/margin_mean": 55.36416244506836, "margin_dpo/margin_std": 102.2855453491211, "step": 523 }, { "KL/chosen_KL_mean": -154.3577880859375, "KL/mean": -193.04534912109375, "KL/rejected_KL_mean": -231.73291015625, "KL/std": 89.18559265136719, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.006295011378824711, "fcm_dpo/delta": -0.08751155436038971, "fcm_dpo/margin": 77.37513732910156, "fcm_dpo/q_t": 0.3904302418231964, "grad_norm": 12.194372177124023, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.1281003952026367, "logits/rejected": -1.443786859512329, "logps/chosen": -257.7633056640625, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -319.64306640625, "loss": 1.0385, "margin_dpo/margin_mean": 77.37513732910156, "margin_dpo/margin_std": 94.79493713378906, "step": 524 }, { "KL/chosen_KL_mean": -143.31605529785156, "KL/mean": -187.4356231689453, "KL/rejected_KL_mean": -231.55520629882812, "KL/std": 89.5830078125, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.006271423771977425, "fcm_dpo/delta": -0.15463680028915405, "fcm_dpo/margin": 88.2391357421875, "fcm_dpo/q_t": 0.37914732098579407, "grad_norm": 10.171697616577148, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.7837867736816406, "logits/rejected": -1.699419379234314, "logps/chosen": -227.5225372314453, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -337.8167724609375, "loss": 1.0062, "margin_dpo/margin_mean": 88.2391357421875, "margin_dpo/margin_std": 100.85163879394531, "step": 525 }, { "KL/chosen_KL_mean": -151.7583465576172, "KL/mean": -192.83969116210938, "KL/rejected_KL_mean": -233.9210205078125, "KL/std": 86.20063018798828, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.006262991577386856, "fcm_dpo/delta": -0.11516880244016647, "fcm_dpo/margin": 82.16265869140625, "fcm_dpo/q_t": 0.3841133713722229, "grad_norm": 8.998695373535156, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.8422297239303589, "logits/rejected": -1.203418493270874, "logps/chosen": -254.61245727539062, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -327.5724792480469, "loss": 1.0267, "margin_dpo/margin_mean": 82.16266632080078, "margin_dpo/margin_std": 99.98530578613281, "step": 526 }, { "KL/chosen_KL_mean": -149.170654296875, "KL/mean": -190.43255615234375, "KL/rejected_KL_mean": -231.69442749023438, "KL/std": 89.87716674804688, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0062480284832417965, "fcm_dpo/delta": -0.11620920896530151, "fcm_dpo/margin": 82.52377319335938, "fcm_dpo/q_t": 0.3809886574745178, "grad_norm": 10.129791259765625, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.135491132736206, "logits/rejected": -2.3324389457702637, "logps/chosen": -245.93609619140625, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -317.089599609375, "loss": 0.993, "margin_dpo/margin_mean": 82.52377319335938, "margin_dpo/margin_std": 79.79975891113281, "step": 527 }, { "KL/chosen_KL_mean": -139.3464813232422, "KL/mean": -181.9528045654297, "KL/rejected_KL_mean": -224.5591278076172, "KL/std": 90.18963623046875, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.006233177147805691, "fcm_dpo/delta": -0.1318281888961792, "fcm_dpo/margin": 85.21263122558594, "fcm_dpo/q_t": 0.3797532618045807, "grad_norm": 10.565216064453125, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.159299850463867, "logits/rejected": -1.6615161895751953, "logps/chosen": -233.5113983154297, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -324.93145751953125, "loss": 1.0073, "margin_dpo/margin_mean": 85.21263885498047, "margin_dpo/margin_std": 96.52307891845703, "step": 528 }, { "KL/chosen_KL_mean": -162.84030151367188, "KL/mean": -200.28274536132812, "KL/rejected_KL_mean": -237.72518920898438, "KL/std": 89.25916290283203, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.006221970543265343, "fcm_dpo/delta": -0.06623785197734833, "fcm_dpo/margin": 74.8848876953125, "fcm_dpo/q_t": 0.3966771364212036, "grad_norm": 11.801352500915527, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8677854537963867, "logits/rejected": -1.6893212795257568, "logps/chosen": -243.61709594726562, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -334.9163513183594, "loss": 1.0714, "margin_dpo/margin_mean": 74.8848876953125, "margin_dpo/margin_std": 105.77284240722656, "step": 529 }, { "KL/chosen_KL_mean": -150.2991943359375, "KL/mean": -188.3199462890625, "KL/rejected_KL_mean": -226.34071350097656, "KL/std": 88.13179016113281, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.006210729479789734, "fcm_dpo/delta": -0.07266555726528168, "fcm_dpo/margin": 76.04150390625, "fcm_dpo/q_t": 0.39172178506851196, "grad_norm": 9.2462797164917, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.9597325325012207, "logits/rejected": -1.6502445936203003, "logps/chosen": -230.07493591308594, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -318.39459228515625, "loss": 1.028, "margin_dpo/margin_mean": 76.04150390625, "margin_dpo/margin_std": 81.4092025756836, "step": 530 }, { "KL/chosen_KL_mean": -144.49102783203125, "KL/mean": -191.6392822265625, "KL/rejected_KL_mean": -238.78753662109375, "KL/std": 87.91574096679688, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.006193527951836586, "fcm_dpo/delta": -0.1851058304309845, "fcm_dpo/margin": 94.29647827148438, "fcm_dpo/q_t": 0.3657759428024292, "grad_norm": 9.697900772094727, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.9859493970870972, "logits/rejected": -1.3976781368255615, "logps/chosen": -223.81024169921875, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -316.8958740234375, "loss": 0.9547, "margin_dpo/margin_mean": 94.29647827148438, "margin_dpo/margin_std": 88.22870635986328, "step": 531 }, { "KL/chosen_KL_mean": -154.4620361328125, "KL/mean": -205.97772216796875, "KL/rejected_KL_mean": -257.4933776855469, "KL/std": 87.36321258544922, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0061677247285842896, "fcm_dpo/delta": -0.23691965639591217, "fcm_dpo/margin": 103.0313491821289, "fcm_dpo/q_t": 0.3566772937774658, "grad_norm": 10.424992561340332, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.8513104915618896, "logits/rejected": -1.2509078979492188, "logps/chosen": -256.09893798828125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -350.8387756347656, "loss": 0.9273, "margin_dpo/margin_mean": 103.0313491821289, "margin_dpo/margin_std": 94.20802307128906, "step": 532 }, { "KL/chosen_KL_mean": -150.3850860595703, "KL/mean": -180.716552734375, "KL/rejected_KL_mean": -211.04800415039062, "KL/std": 90.50686645507812, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.006161092314869165, "fcm_dpo/delta": 0.026347316801548004, "fcm_dpo/margin": 60.66291809082031, "fcm_dpo/q_t": 0.416049063205719, "grad_norm": 10.501849174499512, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.4678897857666016, "logits/rejected": -1.8847581148147583, "logps/chosen": -241.13174438476562, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -293.9364929199219, "loss": 1.1285, "margin_dpo/margin_mean": 60.66292190551758, "margin_dpo/margin_std": 96.4880142211914, "step": 533 }, { "KL/chosen_KL_mean": -134.31671142578125, "KL/mean": -178.12841796875, "KL/rejected_KL_mean": -221.94012451171875, "KL/std": 87.16790771484375, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.006148936226963997, "fcm_dpo/delta": -0.13953959941864014, "fcm_dpo/margin": 87.6234130859375, "fcm_dpo/q_t": 0.3753499686717987, "grad_norm": 9.118853569030762, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0937538146972656, "logits/rejected": -1.7161316871643066, "logps/chosen": -220.9475555419922, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -306.81207275390625, "loss": 0.9754, "margin_dpo/margin_mean": 87.6234130859375, "margin_dpo/margin_std": 80.59527587890625, "step": 534 }, { "KL/chosen_KL_mean": -134.60784912109375, "KL/mean": -181.01278686523438, "KL/rejected_KL_mean": -227.41769409179688, "KL/std": 88.14128875732422, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.006132103968411684, "fcm_dpo/delta": -0.17005382478237152, "fcm_dpo/margin": 92.80985260009766, "fcm_dpo/q_t": 0.3714260458946228, "grad_norm": 8.486051559448242, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.783069133758545, "logits/rejected": -1.5944303274154663, "logps/chosen": -224.9532470703125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -328.1956787109375, "loss": 0.9771, "margin_dpo/margin_mean": 92.80984497070312, "margin_dpo/margin_std": 95.65823364257812, "step": 535 }, { "KL/chosen_KL_mean": -139.1716766357422, "KL/mean": -183.29454040527344, "KL/rejected_KL_mean": -227.41738891601562, "KL/std": 88.63165283203125, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.006109498906880617, "fcm_dpo/delta": -0.13993045687675476, "fcm_dpo/margin": 88.24571990966797, "fcm_dpo/q_t": 0.3766675889492035, "grad_norm": 9.183331489562988, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.8937413692474365, "logits/rejected": -1.6435590982437134, "logps/chosen": -218.13124084472656, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -320.50518798828125, "loss": 0.9928, "margin_dpo/margin_mean": 88.24571228027344, "margin_dpo/margin_std": 92.91162872314453, "step": 536 }, { "KL/chosen_KL_mean": -120.38922119140625, "KL/mean": -169.47671508789062, "KL/rejected_KL_mean": -218.56422424316406, "KL/std": 94.635498046875, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.006092669442296028, "fcm_dpo/delta": -0.19930018484592438, "fcm_dpo/margin": 98.17500305175781, "fcm_dpo/q_t": 0.3645177483558655, "grad_norm": 8.754010200500488, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.7242043018341064, "logits/rejected": -1.0439790487289429, "logps/chosen": -207.09861755371094, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -297.4718933105469, "loss": 0.9504, "margin_dpo/margin_mean": 98.17500305175781, "margin_dpo/margin_std": 91.30393981933594, "step": 537 }, { "KL/chosen_KL_mean": -161.10308837890625, "KL/mean": -201.06332397460938, "KL/rejected_KL_mean": -241.02352905273438, "KL/std": 81.28480529785156, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.00607509259134531, "fcm_dpo/delta": -0.08593405783176422, "fcm_dpo/margin": 79.92042541503906, "fcm_dpo/q_t": 0.3876693844795227, "grad_norm": 10.144905090332031, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.8551068305969238, "logits/rejected": -1.3266302347183228, "logps/chosen": -259.54522705078125, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -332.1181640625, "loss": 1.0182, "margin_dpo/margin_mean": 79.92042541503906, "margin_dpo/margin_std": 83.28892517089844, "step": 538 }, { "KL/chosen_KL_mean": -143.757080078125, "KL/mean": -184.87551879882812, "KL/rejected_KL_mean": -225.9939727783203, "KL/std": 90.60193634033203, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.006063354201614857, "fcm_dpo/delta": -0.09911798685789108, "fcm_dpo/margin": 82.23689270019531, "fcm_dpo/q_t": 0.3873369097709656, "grad_norm": 9.132953643798828, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.317896842956543, "logits/rejected": -1.836313009262085, "logps/chosen": -234.689208984375, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -323.439453125, "loss": 1.0385, "margin_dpo/margin_mean": 82.23689270019531, "margin_dpo/margin_std": 103.09193420410156, "step": 539 }, { "KL/chosen_KL_mean": -148.30477905273438, "KL/mean": -197.37709045410156, "KL/rejected_KL_mean": -246.4493865966797, "KL/std": 94.51329803466797, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.006044656038284302, "fcm_dpo/delta": -0.1943575143814087, "fcm_dpo/margin": 98.14460754394531, "fcm_dpo/q_t": 0.36817291378974915, "grad_norm": 10.805447578430176, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9322189092636108, "logits/rejected": -1.6854896545410156, "logps/chosen": -256.1746826171875, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -367.8428955078125, "loss": 0.9733, "margin_dpo/margin_mean": 98.14459991455078, "margin_dpo/margin_std": 107.11682891845703, "step": 540 }, { "KL/chosen_KL_mean": -167.4044189453125, "KL/mean": -199.7261962890625, "KL/rejected_KL_mean": -232.04794311523438, "KL/std": 82.62968444824219, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.006033450365066528, "fcm_dpo/delta": 0.009946301579475403, "fcm_dpo/margin": 64.6435317993164, "fcm_dpo/q_t": 0.4102173149585724, "grad_norm": 9.976415634155273, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.397491455078125, "logits/rejected": -1.930962324142456, "logps/chosen": -273.971923828125, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -341.1142883300781, "loss": 1.0994, "margin_dpo/margin_mean": 64.6435317993164, "margin_dpo/margin_std": 84.52337646484375, "step": 541 }, { "KL/chosen_KL_mean": -138.97036743164062, "KL/mean": -187.95782470703125, "KL/rejected_KL_mean": -236.9452667236328, "KL/std": 91.47272491455078, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.006021199747920036, "fcm_dpo/delta": -0.19102726876735687, "fcm_dpo/margin": 97.97490692138672, "fcm_dpo/q_t": 0.3651922941207886, "grad_norm": 9.57772445678711, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.156679153442383, "logits/rejected": -2.02894926071167, "logps/chosen": -226.33966064453125, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -339.765625, "loss": 0.9542, "margin_dpo/margin_mean": 97.97491455078125, "margin_dpo/margin_std": 93.16094970703125, "step": 542 }, { "KL/chosen_KL_mean": -165.06784057617188, "KL/mean": -190.35784912109375, "KL/rejected_KL_mean": -215.64785766601562, "KL/std": 95.32315063476562, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0060144527815282345, "fcm_dpo/delta": 0.0002072224742732942, "fcm_dpo/margin": 50.58001708984375, "fcm_dpo/q_t": 0.4325435757637024, "grad_norm": 11.551989555358887, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.9445209503173828, "logits/rejected": -1.9710657596588135, "logps/chosen": -252.22042846679688, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -327.1795654296875, "loss": 1.2107, "margin_dpo/margin_mean": 50.58001708984375, "margin_dpo/margin_std": 111.1041259765625, "step": 543 }, { "KL/chosen_KL_mean": -168.22836303710938, "KL/mean": -199.44766235351562, "KL/rejected_KL_mean": -230.6669464111328, "KL/std": 89.59465026855469, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.00601627491414547, "fcm_dpo/delta": 0.02444491535425186, "fcm_dpo/margin": 62.438575744628906, "fcm_dpo/q_t": 0.4147094488143921, "grad_norm": 12.36325454711914, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.7027734518051147, "logits/rejected": -1.5990586280822754, "logps/chosen": -254.37527465820312, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -337.06085205078125, "loss": 1.1136, "margin_dpo/margin_mean": 62.438575744628906, "margin_dpo/margin_std": 90.75032043457031, "step": 544 }, { "KL/chosen_KL_mean": -152.29534912109375, "KL/mean": -191.64608764648438, "KL/rejected_KL_mean": -230.99685668945312, "KL/std": 87.74798583984375, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.006011745892465115, "fcm_dpo/delta": -0.07347442209720612, "fcm_dpo/margin": 78.70149993896484, "fcm_dpo/q_t": 0.3921668529510498, "grad_norm": 10.374859809875488, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4884893894195557, "logits/rejected": -1.0617516040802002, "logps/chosen": -248.71957397460938, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -331.73089599609375, "loss": 1.0625, "margin_dpo/margin_mean": 78.70150756835938, "margin_dpo/margin_std": 105.37077331542969, "step": 545 }, { "KL/chosen_KL_mean": -156.02963256835938, "KL/mean": -192.78048706054688, "KL/rejected_KL_mean": -229.5312957763672, "KL/std": 91.23188781738281, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.006003407761454582, "fcm_dpo/delta": -0.04146437346935272, "fcm_dpo/margin": 73.50167846679688, "fcm_dpo/q_t": 0.39789077639579773, "grad_norm": 9.741209983825684, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.4354100227355957, "logits/rejected": -1.0683815479278564, "logps/chosen": -242.05076599121094, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -316.9039306640625, "loss": 1.0748, "margin_dpo/margin_mean": 73.50167846679688, "margin_dpo/margin_std": 98.17521667480469, "step": 546 }, { "KL/chosen_KL_mean": -150.91978454589844, "KL/mean": -193.0594024658203, "KL/rejected_KL_mean": -235.19903564453125, "KL/std": 89.88725280761719, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.005995899438858032, "fcm_dpo/delta": -0.10585127025842667, "fcm_dpo/margin": 84.27925109863281, "fcm_dpo/q_t": 0.3825305104255676, "grad_norm": 9.285279273986816, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.878048300743103, "logits/rejected": -1.4266390800476074, "logps/chosen": -251.57080078125, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -333.11077880859375, "loss": 0.9983, "margin_dpo/margin_mean": 84.27925109863281, "margin_dpo/margin_std": 80.22789001464844, "step": 547 }, { "KL/chosen_KL_mean": -151.33309936523438, "KL/mean": -184.1726837158203, "KL/rejected_KL_mean": -217.0122528076172, "KL/std": 89.92840576171875, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.005991922691464424, "fcm_dpo/delta": 0.006479253526777029, "fcm_dpo/margin": 65.67918395996094, "fcm_dpo/q_t": 0.41054871678352356, "grad_norm": 11.345952033996582, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.240675926208496, "logits/rejected": -1.8874354362487793, "logps/chosen": -247.36322021484375, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -315.42547607421875, "loss": 1.1321, "margin_dpo/margin_mean": 65.67918395996094, "margin_dpo/margin_std": 109.19984436035156, "step": 548 }, { "KL/chosen_KL_mean": -138.59307861328125, "KL/mean": -177.03707885742188, "KL/rejected_KL_mean": -215.4810791015625, "KL/std": 84.44085693359375, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0059885671362280846, "fcm_dpo/delta": -0.060735784471035004, "fcm_dpo/margin": 76.88800048828125, "fcm_dpo/q_t": 0.3951130211353302, "grad_norm": 11.206379890441895, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.9448320865631104, "logits/rejected": -1.529040813446045, "logps/chosen": -222.04403686523438, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -296.861328125, "loss": 1.0692, "margin_dpo/margin_mean": 76.88800048828125, "margin_dpo/margin_std": 103.35322570800781, "step": 549 }, { "KL/chosen_KL_mean": -158.15444946289062, "KL/mean": -193.93490600585938, "KL/rejected_KL_mean": -229.71533203125, "KL/std": 83.40144348144531, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.005983471404761076, "fcm_dpo/delta": -0.028310902416706085, "fcm_dpo/margin": 71.56089782714844, "fcm_dpo/q_t": 0.4012000560760498, "grad_norm": 9.731147766113281, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.661229133605957, "logits/rejected": -1.4394266605377197, "logps/chosen": -247.74879455566406, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -331.03082275390625, "loss": 1.0673, "margin_dpo/margin_mean": 71.56089782714844, "margin_dpo/margin_std": 84.92491149902344, "step": 550 }, { "KL/chosen_KL_mean": -138.51043701171875, "KL/mean": -182.20028686523438, "KL/rejected_KL_mean": -225.89016723632812, "KL/std": 93.08843994140625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.005966737400740385, "fcm_dpo/delta": -0.12213803082704544, "fcm_dpo/margin": 87.37973022460938, "fcm_dpo/q_t": 0.38276010751724243, "grad_norm": 11.03099250793457, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.6806797981262207, "logits/rejected": -1.5340566635131836, "logps/chosen": -233.59259033203125, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -343.60943603515625, "loss": 1.0029, "margin_dpo/margin_mean": 87.37972259521484, "margin_dpo/margin_std": 93.13978576660156, "step": 551 }, { "KL/chosen_KL_mean": -131.59719848632812, "KL/mean": -190.70135498046875, "KL/rejected_KL_mean": -249.80551147460938, "KL/std": 100.79647827148438, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0059346649795770645, "fcm_dpo/delta": -0.30401575565338135, "fcm_dpo/margin": 118.20832824707031, "fcm_dpo/q_t": 0.3433026075363159, "grad_norm": 8.673897743225098, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.0634140968322754, "logits/rejected": -1.697431206703186, "logps/chosen": -233.00637817382812, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -360.6439514160156, "loss": 0.8919, "margin_dpo/margin_mean": 118.20832824707031, "margin_dpo/margin_std": 103.00727844238281, "step": 552 }, { "KL/chosen_KL_mean": -145.93460083007812, "KL/mean": -182.5620880126953, "KL/rejected_KL_mean": -219.18959045410156, "KL/std": 94.07148742675781, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.005928060971200466, "fcm_dpo/delta": -0.034408796578645706, "fcm_dpo/margin": 73.25498962402344, "fcm_dpo/q_t": 0.39996761083602905, "grad_norm": 10.321332931518555, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.640963077545166, "logits/rejected": -1.3229484558105469, "logps/chosen": -243.89846801757812, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -322.76678466796875, "loss": 1.0791, "margin_dpo/margin_mean": 73.25498962402344, "margin_dpo/margin_std": 98.49639892578125, "step": 553 }, { "KL/chosen_KL_mean": -154.19277954101562, "KL/mean": -194.11260986328125, "KL/rejected_KL_mean": -234.03245544433594, "KL/std": 92.888671875, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.005922344513237476, "fcm_dpo/delta": -0.07319221645593643, "fcm_dpo/margin": 79.83969116210938, "fcm_dpo/q_t": 0.3938475549221039, "grad_norm": 9.65887451171875, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8352484703063965, "logits/rejected": -1.0640660524368286, "logps/chosen": -257.87957763671875, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -327.0245666503906, "loss": 1.0782, "margin_dpo/margin_mean": 79.83969116210938, "margin_dpo/margin_std": 115.54861450195312, "step": 554 }, { "KL/chosen_KL_mean": -161.2587890625, "KL/mean": -199.24038696289062, "KL/rejected_KL_mean": -237.22201538085938, "KL/std": 96.93866729736328, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.005912913009524345, "fcm_dpo/delta": -0.04938748851418495, "fcm_dpo/margin": 75.96319580078125, "fcm_dpo/q_t": 0.39958393573760986, "grad_norm": 10.287996292114258, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.7773603200912476, "logits/rejected": -1.6798590421676636, "logps/chosen": -247.46241760253906, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -351.76922607421875, "loss": 1.0836, "margin_dpo/margin_mean": 75.96319580078125, "margin_dpo/margin_std": 109.63931274414062, "step": 555 }, { "KL/chosen_KL_mean": -143.8109130859375, "KL/mean": -185.15109252929688, "KL/rejected_KL_mean": -226.4912567138672, "KL/std": 87.75167846679688, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.005902828648686409, "fcm_dpo/delta": -0.08849986642599106, "fcm_dpo/margin": 82.68032836914062, "fcm_dpo/q_t": 0.3889017105102539, "grad_norm": 9.755843162536621, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.3973448276519775, "logits/rejected": -1.078281283378601, "logps/chosen": -232.21441650390625, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -333.13037109375, "loss": 1.0395, "margin_dpo/margin_mean": 82.68032836914062, "margin_dpo/margin_std": 101.55421447753906, "step": 556 }, { "KL/chosen_KL_mean": -153.56674194335938, "KL/mean": -199.87185668945312, "KL/rejected_KL_mean": -246.17697143554688, "KL/std": 99.33919525146484, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.005885579623281956, "fcm_dpo/delta": -0.1459699273109436, "fcm_dpo/margin": 92.6102294921875, "fcm_dpo/q_t": 0.3785204291343689, "grad_norm": 8.310552597045898, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -2.083418607711792, "logits/rejected": -1.9133217334747314, "logps/chosen": -252.4019775390625, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -361.82586669921875, "loss": 1.005, "margin_dpo/margin_mean": 92.6102294921875, "margin_dpo/margin_std": 106.66249084472656, "step": 557 }, { "KL/chosen_KL_mean": -154.655517578125, "KL/mean": -202.59849548339844, "KL/rejected_KL_mean": -250.54147338867188, "KL/std": 86.36871337890625, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.005868903361260891, "fcm_dpo/delta": -0.16368776559829712, "fcm_dpo/margin": 95.8859634399414, "fcm_dpo/q_t": 0.3709774315357208, "grad_norm": 10.4109525680542, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3822481632232666, "logits/rejected": -1.7733087539672852, "logps/chosen": -260.6673583984375, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -358.93280029296875, "loss": 0.9626, "margin_dpo/margin_mean": 95.8859634399414, "margin_dpo/margin_std": 87.52691650390625, "step": 558 }, { "KL/chosen_KL_mean": -165.08551025390625, "KL/mean": -204.88255310058594, "KL/rejected_KL_mean": -244.6796112060547, "KL/std": 86.39302062988281, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.005859111435711384, "fcm_dpo/delta": -0.06665931642055511, "fcm_dpo/margin": 79.59407806396484, "fcm_dpo/q_t": 0.39238637685775757, "grad_norm": 11.58765983581543, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.43642520904541, "logits/rejected": -1.9625046253204346, "logps/chosen": -265.51025390625, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -345.0923156738281, "loss": 1.0508, "margin_dpo/margin_mean": 79.59408569335938, "margin_dpo/margin_std": 98.06059265136719, "step": 559 }, { "KL/chosen_KL_mean": -160.71180725097656, "KL/mean": -198.22987365722656, "KL/rejected_KL_mean": -235.74794006347656, "KL/std": 86.54643249511719, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.005853148642927408, "fcm_dpo/delta": -0.03937342017889023, "fcm_dpo/margin": 75.03611755371094, "fcm_dpo/q_t": 0.3991706371307373, "grad_norm": 13.543444633483887, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.5369473695755005, "logits/rejected": -1.096262812614441, "logps/chosen": -248.34732055664062, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -323.1278076171875, "loss": 1.0735, "margin_dpo/margin_mean": 75.03611755371094, "margin_dpo/margin_std": 99.44093322753906, "step": 560 }, { "KL/chosen_KL_mean": -166.79696655273438, "KL/mean": -203.44288635253906, "KL/rejected_KL_mean": -240.08880615234375, "KL/std": 87.30152893066406, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.005851244553923607, "fcm_dpo/delta": -0.02897777035832405, "fcm_dpo/margin": 73.29181671142578, "fcm_dpo/q_t": 0.4008789360523224, "grad_norm": 9.106813430786133, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.9748502969741821, "logits/rejected": -1.9397398233413696, "logps/chosen": -241.67837524414062, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -346.13677978515625, "loss": 1.0592, "margin_dpo/margin_mean": 73.29182434082031, "margin_dpo/margin_std": 83.9776611328125, "step": 561 }, { "KL/chosen_KL_mean": -152.643798828125, "KL/mean": -187.38555908203125, "KL/rejected_KL_mean": -222.12734985351562, "KL/std": 86.47650146484375, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.005847014486789703, "fcm_dpo/delta": -0.006309475749731064, "fcm_dpo/margin": 69.48355102539062, "fcm_dpo/q_t": 0.4077907204627991, "grad_norm": 11.408553123474121, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.29219651222229, "logits/rejected": -1.8528897762298584, "logps/chosen": -253.30702209472656, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -324.36822509765625, "loss": 1.1124, "margin_dpo/margin_mean": 69.48355102539062, "margin_dpo/margin_std": 107.87802124023438, "step": 562 }, { "KL/chosen_KL_mean": -162.655029296875, "KL/mean": -194.16624450683594, "KL/rejected_KL_mean": -225.677490234375, "KL/std": 87.84703063964844, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0058526769280433655, "fcm_dpo/delta": 0.031238321214914322, "fcm_dpo/margin": 63.02246856689453, "fcm_dpo/q_t": 0.4147476255893707, "grad_norm": 10.044753074645996, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.652646541595459, "logits/rejected": -1.312117099761963, "logps/chosen": -258.91424560546875, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -326.88250732421875, "loss": 1.1195, "margin_dpo/margin_mean": 63.0224609375, "margin_dpo/margin_std": 91.67680358886719, "step": 563 }, { "KL/chosen_KL_mean": -157.04025268554688, "KL/mean": -191.2847442626953, "KL/rejected_KL_mean": -225.52923583984375, "KL/std": 87.74859619140625, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.005853551905602217, "fcm_dpo/delta": -0.0009254850447177887, "fcm_dpo/margin": 68.48896789550781, "fcm_dpo/q_t": 0.40609878301620483, "grad_norm": 10.296096801757812, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.5614843368530273, "logits/rejected": -2.024282217025757, "logps/chosen": -270.00067138671875, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -325.0572509765625, "loss": 1.0778, "margin_dpo/margin_mean": 68.48896026611328, "margin_dpo/margin_std": 79.59944152832031, "step": 564 }, { "KL/chosen_KL_mean": -149.88003540039062, "KL/mean": -194.46685791015625, "KL/rejected_KL_mean": -239.05368041992188, "KL/std": 94.70197296142578, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.005840416997671127, "fcm_dpo/delta": -0.12146098166704178, "fcm_dpo/margin": 89.17364501953125, "fcm_dpo/q_t": 0.38151782751083374, "grad_norm": 9.38590145111084, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.713291049003601, "logits/rejected": -1.2070338726043701, "logps/chosen": -245.24560546875, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -331.27105712890625, "loss": 1.011, "margin_dpo/margin_mean": 89.17365264892578, "margin_dpo/margin_std": 100.05778503417969, "step": 565 }, { "KL/chosen_KL_mean": -157.677734375, "KL/mean": -199.67877197265625, "KL/rejected_KL_mean": -241.67982482910156, "KL/std": 97.49693298339844, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0058310385793447495, "fcm_dpo/delta": -0.09025143086910248, "fcm_dpo/margin": 84.0020980834961, "fcm_dpo/q_t": 0.38718461990356445, "grad_norm": 8.390789985656738, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -2.060070753097534, "logits/rejected": -1.6350975036621094, "logps/chosen": -256.65093994140625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -352.500732421875, "loss": 1.0314, "margin_dpo/margin_mean": 84.00210571289062, "margin_dpo/margin_std": 97.62586212158203, "step": 566 }, { "KL/chosen_KL_mean": -161.85089111328125, "KL/mean": -197.96762084960938, "KL/rejected_KL_mean": -234.08436584472656, "KL/std": 87.41484069824219, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.005822745151817799, "fcm_dpo/delta": -0.02072899043560028, "fcm_dpo/margin": 72.23348236083984, "fcm_dpo/q_t": 0.4017353653907776, "grad_norm": 10.866985321044922, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.1833746433258057, "logits/rejected": -1.509699821472168, "logps/chosen": -254.55447387695312, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -325.3086853027344, "loss": 1.0756, "margin_dpo/margin_mean": 72.23348236083984, "margin_dpo/margin_std": 89.45584106445312, "step": 567 }, { "KL/chosen_KL_mean": -152.3829345703125, "KL/mean": -196.9852294921875, "KL/rejected_KL_mean": -241.58753967285156, "KL/std": 92.30357360839844, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.005812631919980049, "fcm_dpo/delta": -0.11917871981859207, "fcm_dpo/margin": 89.20462036132812, "fcm_dpo/q_t": 0.3816778063774109, "grad_norm": 8.90457534790039, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.5898997783660889, "logits/rejected": -1.453486442565918, "logps/chosen": -225.51519775390625, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -336.5401916503906, "loss": 1.0007, "margin_dpo/margin_mean": 89.20462036132812, "margin_dpo/margin_std": 92.63197326660156, "step": 568 }, { "KL/chosen_KL_mean": -161.1562957763672, "KL/mean": -198.63134765625, "KL/rejected_KL_mean": -236.1063690185547, "KL/std": 91.06724548339844, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0058090779930353165, "fcm_dpo/delta": -0.03554902598261833, "fcm_dpo/margin": 74.95008087158203, "fcm_dpo/q_t": 0.39972829818725586, "grad_norm": 13.841301918029785, "learning_rate": 4.089328585837512e-08, "logits/chosen": -2.007678508758545, "logits/rejected": -1.6399121284484863, "logps/chosen": -250.37338256835938, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -322.5174255371094, "loss": 1.0932, "margin_dpo/margin_mean": 74.95008087158203, "margin_dpo/margin_std": 108.61431884765625, "step": 569 }, { "KL/chosen_KL_mean": -143.26266479492188, "KL/mean": -192.19671630859375, "KL/rejected_KL_mean": -241.1307830810547, "KL/std": 89.0947265625, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.005791818257421255, "fcm_dpo/delta": -0.167791947722435, "fcm_dpo/margin": 97.86812591552734, "fcm_dpo/q_t": 0.36920759081840515, "grad_norm": 9.133342742919922, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.3576016426086426, "logits/rejected": -1.6815060377120972, "logps/chosen": -238.02542114257812, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -329.97772216796875, "loss": 0.9606, "margin_dpo/margin_mean": 97.86811828613281, "margin_dpo/margin_std": 90.01815032958984, "step": 570 }, { "KL/chosen_KL_mean": -148.07801818847656, "KL/mean": -192.56890869140625, "KL/rejected_KL_mean": -237.059814453125, "KL/std": 89.13829040527344, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.00577856320887804, "fcm_dpo/delta": -0.11476118862628937, "fcm_dpo/margin": 88.98178100585938, "fcm_dpo/q_t": 0.38360437750816345, "grad_norm": 12.789885520935059, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7395436763763428, "logits/rejected": -1.580764651298523, "logps/chosen": -228.89930725097656, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -334.0751037597656, "loss": 1.0248, "margin_dpo/margin_mean": 88.9817886352539, "margin_dpo/margin_std": 106.18632507324219, "step": 571 }, { "KL/chosen_KL_mean": -150.39511108398438, "KL/mean": -194.51197814941406, "KL/rejected_KL_mean": -238.6288604736328, "KL/std": 93.26273345947266, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.005763250403106213, "fcm_dpo/delta": -0.10909955203533173, "fcm_dpo/margin": 88.23374938964844, "fcm_dpo/q_t": 0.38587209582328796, "grad_norm": 9.586052894592285, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.715162754058838, "logits/rejected": -1.5465067625045776, "logps/chosen": -227.7750244140625, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -329.19482421875, "loss": 1.0211, "margin_dpo/margin_mean": 88.23374938964844, "margin_dpo/margin_std": 102.61604309082031, "step": 572 }, { "KL/chosen_KL_mean": -146.365478515625, "KL/mean": -187.58230590820312, "KL/rejected_KL_mean": -228.7991485595703, "KL/std": 94.28816223144531, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.005758977495133877, "fcm_dpo/delta": -0.07514017075300217, "fcm_dpo/margin": 82.43367004394531, "fcm_dpo/q_t": 0.39035600423812866, "grad_norm": 9.038045883178711, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.2178993225097656, "logits/rejected": -1.9956542253494263, "logps/chosen": -230.07154846191406, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -319.4966125488281, "loss": 1.045, "margin_dpo/margin_mean": 82.43367004394531, "margin_dpo/margin_std": 97.24360656738281, "step": 573 }, { "KL/chosen_KL_mean": -155.52957153320312, "KL/mean": -198.6339569091797, "KL/rejected_KL_mean": -241.7383270263672, "KL/std": 91.97969055175781, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.005746047478169203, "fcm_dpo/delta": -0.09582333266735077, "fcm_dpo/margin": 86.20875549316406, "fcm_dpo/q_t": 0.3873439431190491, "grad_norm": 10.582090377807617, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.1105480194091797, "logits/rejected": -1.7828898429870605, "logps/chosen": -240.6590576171875, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -340.7923583984375, "loss": 1.0344, "margin_dpo/margin_mean": 86.20875549316406, "margin_dpo/margin_std": 104.07972717285156, "step": 574 }, { "KL/chosen_KL_mean": -158.27720642089844, "KL/mean": -200.81671142578125, "KL/rejected_KL_mean": -243.35623168945312, "KL/std": 94.09744262695312, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.00573329720646143, "fcm_dpo/delta": -0.08822885155677795, "fcm_dpo/margin": 85.07904052734375, "fcm_dpo/q_t": 0.3893575668334961, "grad_norm": 12.611539840698242, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.5281003713607788, "logits/rejected": -1.0802013874053955, "logps/chosen": -264.04803466796875, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -344.3056640625, "loss": 1.0439, "margin_dpo/margin_mean": 85.07904052734375, "margin_dpo/margin_std": 107.31144714355469, "step": 575 }, { "KL/chosen_KL_mean": -160.1375732421875, "KL/mean": -201.52957153320312, "KL/rejected_KL_mean": -242.92156982421875, "KL/std": 88.73370361328125, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.005725963041186333, "fcm_dpo/delta": -0.07436411082744598, "fcm_dpo/margin": 82.78400421142578, "fcm_dpo/q_t": 0.391936719417572, "grad_norm": 10.277071952819824, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.9104559421539307, "logits/rejected": -1.830960750579834, "logps/chosen": -241.34210205078125, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -348.33099365234375, "loss": 1.0542, "margin_dpo/margin_mean": 82.78399658203125, "margin_dpo/margin_std": 106.44866180419922, "step": 576 }, { "KL/chosen_KL_mean": -142.54391479492188, "KL/mean": -192.52731323242188, "KL/rejected_KL_mean": -242.51071166992188, "KL/std": 95.31307220458984, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0057063172571361065, "fcm_dpo/delta": -0.17150454223155975, "fcm_dpo/margin": 99.966796875, "fcm_dpo/q_t": 0.37079960107803345, "grad_norm": 8.248716354370117, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.025526285171509, "logits/rejected": -1.3368921279907227, "logps/chosen": -244.00811767578125, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -331.8370666503906, "loss": 0.982, "margin_dpo/margin_mean": 99.966796875, "margin_dpo/margin_std": 107.202880859375, "step": 577 }, { "KL/chosen_KL_mean": -168.56822204589844, "KL/mean": -204.3453369140625, "KL/rejected_KL_mean": -240.1224365234375, "KL/std": 89.92451477050781, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.005702869035303593, "fcm_dpo/delta": -0.008097313344478607, "fcm_dpo/margin": 71.55421447753906, "fcm_dpo/q_t": 0.4057937264442444, "grad_norm": 13.338470458984375, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.4136642217636108, "logits/rejected": -1.1782574653625488, "logps/chosen": -245.8606414794922, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -328.36944580078125, "loss": 1.0965, "margin_dpo/margin_mean": 71.55421447753906, "margin_dpo/margin_std": 100.56634521484375, "step": 578 }, { "KL/chosen_KL_mean": -150.99673461914062, "KL/mean": -188.67095947265625, "KL/rejected_KL_mean": -226.34523010253906, "KL/std": 90.7140121459961, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.005697737447917461, "fcm_dpo/delta": -0.029484355822205544, "fcm_dpo/margin": 75.34849548339844, "fcm_dpo/q_t": 0.40174075961112976, "grad_norm": 9.51835823059082, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7308001518249512, "logits/rejected": -1.4085500240325928, "logps/chosen": -238.33978271484375, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -328.46624755859375, "loss": 1.0726, "margin_dpo/margin_mean": 75.3484878540039, "margin_dpo/margin_std": 92.7239990234375, "step": 579 }, { "KL/chosen_KL_mean": -145.8365478515625, "KL/mean": -190.7360076904297, "KL/rejected_KL_mean": -235.63548278808594, "KL/std": 88.63080596923828, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.005686961114406586, "fcm_dpo/delta": -0.11134682595729828, "fcm_dpo/margin": 89.79893493652344, "fcm_dpo/q_t": 0.38484907150268555, "grad_norm": 9.82621955871582, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.4650027751922607, "logits/rejected": -1.1538257598876953, "logps/chosen": -234.7653350830078, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -324.33209228515625, "loss": 1.0199, "margin_dpo/margin_mean": 89.79893493652344, "margin_dpo/margin_std": 103.17692565917969, "step": 580 }, { "KL/chosen_KL_mean": -155.39108276367188, "KL/mean": -195.24029541015625, "KL/rejected_KL_mean": -235.08950805664062, "KL/std": 90.32875061035156, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.00568268820643425, "fcm_dpo/delta": -0.05313848704099655, "fcm_dpo/margin": 79.69843292236328, "fcm_dpo/q_t": 0.3965218663215637, "grad_norm": 10.692277908325195, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.053889036178589, "logits/rejected": -1.7118990421295166, "logps/chosen": -247.61431884765625, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -334.47894287109375, "loss": 1.0805, "margin_dpo/margin_mean": 79.69842529296875, "margin_dpo/margin_std": 112.66142272949219, "step": 581 }, { "KL/chosen_KL_mean": -134.98333740234375, "KL/mean": -172.73089599609375, "KL/rejected_KL_mean": -210.4784698486328, "KL/std": 93.73330688476562, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.005682720802724361, "fcm_dpo/delta": -0.029261551797389984, "fcm_dpo/margin": 75.49513244628906, "fcm_dpo/q_t": 0.4030170142650604, "grad_norm": 9.01851749420166, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.1517956256866455, "logits/rejected": -2.13603138923645, "logps/chosen": -222.3189697265625, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -325.63287353515625, "loss": 1.0838, "margin_dpo/margin_mean": 75.49513244628906, "margin_dpo/margin_std": 100.96768188476562, "step": 582 }, { "KL/chosen_KL_mean": -163.91610717773438, "KL/mean": -204.19178771972656, "KL/rejected_KL_mean": -244.4674530029297, "KL/std": 88.46294403076172, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.005670167040079832, "fcm_dpo/delta": -0.05704715475440025, "fcm_dpo/margin": 80.55136108398438, "fcm_dpo/q_t": 0.39688509702682495, "grad_norm": 10.195196151733398, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5030498504638672, "logits/rejected": -1.271637201309204, "logps/chosen": -247.6331787109375, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -339.0521240234375, "loss": 1.0586, "margin_dpo/margin_mean": 80.55136108398438, "margin_dpo/margin_std": 102.23233032226562, "step": 583 }, { "KL/chosen_KL_mean": -144.9049835205078, "KL/mean": -187.5401153564453, "KL/rejected_KL_mean": -230.17526245117188, "KL/std": 91.75154113769531, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.005667566787451506, "fcm_dpo/delta": -0.08372261375188828, "fcm_dpo/margin": 85.270263671875, "fcm_dpo/q_t": 0.38948220014572144, "grad_norm": 8.933947563171387, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9075154066085815, "logits/rejected": -1.4378300905227661, "logps/chosen": -239.16455078125, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -334.657958984375, "loss": 1.044, "margin_dpo/margin_mean": 85.27027130126953, "margin_dpo/margin_std": 102.54533386230469, "step": 584 }, { "KL/chosen_KL_mean": -154.052734375, "KL/mean": -203.29052734375, "KL/rejected_KL_mean": -252.5283203125, "KL/std": 104.29859924316406, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.005650227423757315, "fcm_dpo/delta": -0.15725398063659668, "fcm_dpo/margin": 98.47557067871094, "fcm_dpo/q_t": 0.3762843906879425, "grad_norm": 9.198633193969727, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.9268910884857178, "logits/rejected": -1.6468515396118164, "logps/chosen": -239.6894989013672, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -342.81378173828125, "loss": 1.0064, "margin_dpo/margin_mean": 98.47557067871094, "margin_dpo/margin_std": 117.6363525390625, "step": 585 }, { "KL/chosen_KL_mean": -170.98912048339844, "KL/mean": -204.12750244140625, "KL/rejected_KL_mean": -237.26588439941406, "KL/std": 92.45144653320312, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.005645583383738995, "fcm_dpo/delta": 0.025925740599632263, "fcm_dpo/margin": 66.27676391601562, "fcm_dpo/q_t": 0.4128888249397278, "grad_norm": 10.395636558532715, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.947370171546936, "logits/rejected": -1.6863408088684082, "logps/chosen": -254.67025756835938, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -335.62506103515625, "loss": 1.1042, "margin_dpo/margin_mean": 66.27676391601562, "margin_dpo/margin_std": 87.05513763427734, "step": 586 }, { "KL/chosen_KL_mean": -138.79788208007812, "KL/mean": -175.82028198242188, "KL/rejected_KL_mean": -212.84271240234375, "KL/std": 89.41676330566406, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.005645825527608395, "fcm_dpo/delta": -0.018123969435691833, "fcm_dpo/margin": 74.04483795166016, "fcm_dpo/q_t": 0.4035279452800751, "grad_norm": 9.377445220947266, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.3922119140625, "logits/rejected": -2.103116989135742, "logps/chosen": -216.37539672851562, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -302.16302490234375, "loss": 1.0761, "margin_dpo/margin_mean": 74.04483795166016, "margin_dpo/margin_std": 93.02389526367188, "step": 587 }, { "KL/chosen_KL_mean": -150.45281982421875, "KL/mean": -184.5447998046875, "KL/rejected_KL_mean": -218.63677978515625, "KL/std": 92.69163513183594, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.005643993616104126, "fcm_dpo/delta": 0.015218514949083328, "fcm_dpo/margin": 68.18394470214844, "fcm_dpo/q_t": 0.41279906034469604, "grad_norm": 11.891609191894531, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.7608740329742432, "logits/rejected": -1.7045562267303467, "logps/chosen": -234.257080078125, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -331.7562255859375, "loss": 1.127, "margin_dpo/margin_mean": 68.18394470214844, "margin_dpo/margin_std": 108.82221984863281, "step": 588 }, { "KL/chosen_KL_mean": -161.72592163085938, "KL/mean": -206.08740234375, "KL/rejected_KL_mean": -250.44888305664062, "KL/std": 89.96255493164062, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0056370701640844345, "fcm_dpo/delta": -0.10064809024333954, "fcm_dpo/margin": 88.72295379638672, "fcm_dpo/q_t": 0.38366132974624634, "grad_norm": 9.815714836120605, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.6614594459533691, "logits/rejected": -1.2600045204162598, "logps/chosen": -258.1909484863281, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -366.486083984375, "loss": 1.0018, "margin_dpo/margin_mean": 88.72295379638672, "margin_dpo/margin_std": 85.54708862304688, "step": 589 }, { "KL/chosen_KL_mean": -147.7459716796875, "KL/mean": -190.99415588378906, "KL/rejected_KL_mean": -234.24234008789062, "KL/std": 83.72946166992188, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.005630006082355976, "fcm_dpo/delta": -0.08740661293268204, "fcm_dpo/margin": 86.49635314941406, "fcm_dpo/q_t": 0.38741832971572876, "grad_norm": 12.46562385559082, "learning_rate": 2.736501028272095e-08, "logits/chosen": -2.03460693359375, "logits/rejected": -1.870687484741211, "logps/chosen": -236.173828125, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -344.44110107421875, "loss": 1.0137, "margin_dpo/margin_mean": 86.49635314941406, "margin_dpo/margin_std": 85.844970703125, "step": 590 }, { "KL/chosen_KL_mean": -141.7240753173828, "KL/mean": -188.77825927734375, "KL/rejected_KL_mean": -235.83242797851562, "KL/std": 92.7354736328125, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0056128064170479774, "fcm_dpo/delta": -0.1289251446723938, "fcm_dpo/margin": 94.10836791992188, "fcm_dpo/q_t": 0.3785395920276642, "grad_norm": 7.618683815002441, "learning_rate": 2.678415274939408e-08, "logits/chosen": -2.0855579376220703, "logits/rejected": -1.476128339767456, "logps/chosen": -229.57376098632812, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -325.41162109375, "loss": 0.986, "margin_dpo/margin_mean": 94.10836791992188, "margin_dpo/margin_std": 90.18539428710938, "step": 591 }, { "KL/chosen_KL_mean": -168.24090576171875, "KL/mean": -205.2711944580078, "KL/rejected_KL_mean": -242.30148315429688, "KL/std": 86.86793518066406, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.005609037820249796, "fcm_dpo/delta": -0.01547255739569664, "fcm_dpo/margin": 74.06056213378906, "fcm_dpo/q_t": 0.40460336208343506, "grad_norm": 10.641435623168945, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7284432649612427, "logits/rejected": -1.3974770307540894, "logps/chosen": -243.21664428710938, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -323.8541259765625, "loss": 1.0954, "margin_dpo/margin_mean": 74.06056213378906, "margin_dpo/margin_std": 105.46336364746094, "step": 592 }, { "KL/chosen_KL_mean": -154.64732360839844, "KL/mean": -186.40390014648438, "KL/rejected_KL_mean": -218.16046142578125, "KL/std": 92.5566177368164, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.005609815940260887, "fcm_dpo/delta": 0.04384143650531769, "fcm_dpo/margin": 63.51314163208008, "fcm_dpo/q_t": 0.41850918531417847, "grad_norm": 14.331439018249512, "learning_rate": 2.564009866938349e-08, "logits/chosen": -1.1438682079315186, "logits/rejected": -0.856263279914856, "logps/chosen": -226.8863525390625, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -288.2569885253906, "loss": 1.1372, "margin_dpo/margin_mean": 63.51313781738281, "margin_dpo/margin_std": 100.1607894897461, "step": 593 }, { "KL/chosen_KL_mean": -140.46234130859375, "KL/mean": -187.55300903320312, "KL/rejected_KL_mean": -234.64366149902344, "KL/std": 90.25090026855469, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.005604420322924852, "fcm_dpo/delta": -0.12848931550979614, "fcm_dpo/margin": 94.1812973022461, "fcm_dpo/q_t": 0.3781687021255493, "grad_norm": 9.118972778320312, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9942028522491455, "logits/rejected": -1.4801383018493652, "logps/chosen": -230.90005493164062, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -326.9112548828125, "loss": 0.9827, "margin_dpo/margin_mean": 94.1812973022461, "margin_dpo/margin_std": 86.99888610839844, "step": 594 }, { "KL/chosen_KL_mean": -148.01528930664062, "KL/mean": -187.53347778320312, "KL/rejected_KL_mean": -227.05169677734375, "KL/std": 88.38888549804688, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0055968426167964935, "fcm_dpo/delta": -0.04254409298300743, "fcm_dpo/margin": 79.03639221191406, "fcm_dpo/q_t": 0.39933356642723083, "grad_norm": 9.910359382629395, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.987053394317627, "logits/rejected": -1.5835288763046265, "logps/chosen": -229.53009033203125, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -313.5721435546875, "loss": 1.0676, "margin_dpo/margin_mean": 79.0363998413086, "margin_dpo/margin_std": 101.33411407470703, "step": 595 }, { "KL/chosen_KL_mean": -152.6013641357422, "KL/mean": -187.52407836914062, "KL/rejected_KL_mean": -222.44680786132812, "KL/std": 87.06361389160156, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.005595214664936066, "fcm_dpo/delta": 0.009235477074980736, "fcm_dpo/margin": 69.84542846679688, "fcm_dpo/q_t": 0.4108749032020569, "grad_norm": 11.324132919311523, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.53676438331604, "logits/rejected": -0.9445905685424805, "logps/chosen": -240.80941772460938, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -308.5405578613281, "loss": 1.1181, "margin_dpo/margin_mean": 69.84542846679688, "margin_dpo/margin_std": 106.52511596679688, "step": 596 }, { "KL/chosen_KL_mean": -149.72998046875, "KL/mean": -196.38160705566406, "KL/rejected_KL_mean": -243.03323364257812, "KL/std": 94.800048828125, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0055839079432189465, "fcm_dpo/delta": -0.12166604399681091, "fcm_dpo/margin": 93.30323791503906, "fcm_dpo/q_t": 0.3815397024154663, "grad_norm": 8.634089469909668, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.311307668685913, "logits/rejected": -1.1899092197418213, "logps/chosen": -233.3199462890625, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -349.9776306152344, "loss": 1.0077, "margin_dpo/margin_mean": 93.30323791503906, "margin_dpo/margin_std": 102.85150146484375, "step": 597 }, { "KL/chosen_KL_mean": -142.31948852539062, "KL/mean": -180.76425170898438, "KL/rejected_KL_mean": -219.20901489257812, "KL/std": 88.17533111572266, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.005576839204877615, "fcm_dpo/delta": -0.028975825756788254, "fcm_dpo/margin": 76.8895263671875, "fcm_dpo/q_t": 0.4003942012786865, "grad_norm": 9.093282699584961, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.9042413234710693, "logits/rejected": -1.5891954898834229, "logps/chosen": -232.50210571289062, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -314.007568359375, "loss": 1.0776, "margin_dpo/margin_mean": 76.8895263671875, "margin_dpo/margin_std": 99.524169921875, "step": 598 }, { "KL/chosen_KL_mean": -145.5168914794922, "KL/mean": -180.29183959960938, "KL/rejected_KL_mean": -215.06680297851562, "KL/std": 92.17660522460938, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.005576734431087971, "fcm_dpo/delta": 0.012142889201641083, "fcm_dpo/margin": 69.54991149902344, "fcm_dpo/q_t": 0.4105473756790161, "grad_norm": 9.303401947021484, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.7821305990219116, "logits/rejected": -1.6558418273925781, "logps/chosen": -223.35916137695312, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -304.2644348144531, "loss": 1.1103, "margin_dpo/margin_mean": 69.54991149902344, "margin_dpo/margin_std": 98.88691711425781, "step": 599 }, { "KL/chosen_KL_mean": -157.2825927734375, "KL/mean": -198.04554748535156, "KL/rejected_KL_mean": -238.80853271484375, "KL/std": 88.83648681640625, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.005575425922870636, "fcm_dpo/delta": -0.054789356887340546, "fcm_dpo/margin": 81.52593994140625, "fcm_dpo/q_t": 0.3956230878829956, "grad_norm": 8.06096363067627, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.281050682067871, "logits/rejected": -1.9286377429962158, "logps/chosen": -249.47039794921875, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -331.7628173828125, "loss": 1.0507, "margin_dpo/margin_mean": 81.52593994140625, "margin_dpo/margin_std": 96.30221557617188, "step": 600 }, { "KL/chosen_KL_mean": -161.5428924560547, "KL/mean": -203.28570556640625, "KL/rejected_KL_mean": -245.02853393554688, "KL/std": 89.50088500976562, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.005566621199250221, "fcm_dpo/delta": -0.06507260352373123, "fcm_dpo/margin": 83.48564147949219, "fcm_dpo/q_t": 0.39243084192276, "grad_norm": 8.868308067321777, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7927486896514893, "logits/rejected": -1.4706037044525146, "logps/chosen": -250.759033203125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -345.1990966796875, "loss": 1.0315, "margin_dpo/margin_mean": 83.48564147949219, "margin_dpo/margin_std": 88.22994995117188, "step": 601 }, { "KL/chosen_KL_mean": -127.85891723632812, "KL/mean": -180.60446166992188, "KL/rejected_KL_mean": -233.35000610351562, "KL/std": 96.68456268310547, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.005550889298319817, "fcm_dpo/delta": -0.18666362762451172, "fcm_dpo/margin": 105.4910888671875, "fcm_dpo/q_t": 0.368047297000885, "grad_norm": 9.599061965942383, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.0514705181121826, "logits/rejected": -1.7939108610153198, "logps/chosen": -207.91651916503906, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -326.5475158691406, "loss": 0.966, "margin_dpo/margin_mean": 105.4910888671875, "margin_dpo/margin_std": 108.6113510131836, "step": 602 }, { "KL/chosen_KL_mean": -138.97634887695312, "KL/mean": -182.89520263671875, "KL/rejected_KL_mean": -226.8140106201172, "KL/std": 88.47210693359375, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.005539283622056246, "fcm_dpo/delta": -0.08697263896465302, "fcm_dpo/margin": 87.83766174316406, "fcm_dpo/q_t": 0.38864225149154663, "grad_norm": 8.095671653747559, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1863672733306885, "logits/rejected": -1.6104544401168823, "logps/chosen": -241.28594970703125, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -334.88287353515625, "loss": 1.028, "margin_dpo/margin_mean": 87.83766174316406, "margin_dpo/margin_std": 100.45188903808594, "step": 603 }, { "KL/chosen_KL_mean": -142.6497802734375, "KL/mean": -188.3313751220703, "KL/rejected_KL_mean": -234.01296997070312, "KL/std": 91.90226745605469, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.005525755695998669, "fcm_dpo/delta": -0.10542990267276764, "fcm_dpo/margin": 91.36320495605469, "fcm_dpo/q_t": 0.38183295726776123, "grad_norm": 8.386080741882324, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.6793737411499023, "logits/rejected": -1.5195832252502441, "logps/chosen": -220.8238525390625, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -333.50909423828125, "loss": 1.0029, "margin_dpo/margin_mean": 91.36320495605469, "margin_dpo/margin_std": 90.01463317871094, "step": 604 }, { "KL/chosen_KL_mean": -154.50289916992188, "KL/mean": -192.3409423828125, "KL/rejected_KL_mean": -230.17897033691406, "KL/std": 83.51402282714844, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0055221254006028175, "fcm_dpo/delta": -0.01797487586736679, "fcm_dpo/margin": 75.67607116699219, "fcm_dpo/q_t": 0.40329304337501526, "grad_norm": 9.313089370727539, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8012206554412842, "logits/rejected": -1.2953604459762573, "logps/chosen": -249.27622985839844, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -337.48388671875, "loss": 1.0735, "margin_dpo/margin_mean": 75.67607116699219, "margin_dpo/margin_std": 92.918212890625, "step": 605 }, { "KL/chosen_KL_mean": -150.40692138671875, "KL/mean": -190.2933349609375, "KL/rejected_KL_mean": -230.17974853515625, "KL/std": 90.91091918945312, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.00552217336371541, "fcm_dpo/delta": -0.04073886573314667, "fcm_dpo/margin": 79.77281188964844, "fcm_dpo/q_t": 0.3974027931690216, "grad_norm": 9.178061485290527, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.8860158920288086, "logits/rejected": -1.4757574796676636, "logps/chosen": -238.1603240966797, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -334.42193603515625, "loss": 1.0617, "margin_dpo/margin_mean": 79.77281188964844, "margin_dpo/margin_std": 95.48795318603516, "step": 606 }, { "KL/chosen_KL_mean": -147.56829833984375, "KL/mean": -192.20751953125, "KL/rejected_KL_mean": -236.8467559814453, "KL/std": 96.47566223144531, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0055103600025177, "fcm_dpo/delta": -0.09241023659706116, "fcm_dpo/margin": 89.27845764160156, "fcm_dpo/q_t": 0.3885781764984131, "grad_norm": 11.404048919677734, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7886781692504883, "logits/rejected": -1.2584433555603027, "logps/chosen": -235.8973388671875, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -323.6148681640625, "loss": 1.0553, "margin_dpo/margin_mean": 89.27845764160156, "margin_dpo/margin_std": 120.00102233886719, "step": 607 }, { "KL/chosen_KL_mean": -133.98756408691406, "KL/mean": -180.32350158691406, "KL/rejected_KL_mean": -226.65943908691406, "KL/std": 94.84677124023438, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.005495146382600069, "fcm_dpo/delta": -0.10995884239673615, "fcm_dpo/margin": 92.671875, "fcm_dpo/q_t": 0.3819243907928467, "grad_norm": 10.343790054321289, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.8222486972808838, "logits/rejected": -1.5301284790039062, "logps/chosen": -219.75694274902344, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -326.89227294921875, "loss": 1.0166, "margin_dpo/margin_mean": 92.671875, "margin_dpo/margin_std": 99.88037109375, "step": 608 }, { "KL/chosen_KL_mean": -146.2860107421875, "KL/mean": -191.47640991210938, "KL/rejected_KL_mean": -236.66680908203125, "KL/std": 95.00874328613281, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.005485898815095425, "fcm_dpo/delta": -0.09635160863399506, "fcm_dpo/margin": 90.38076782226562, "fcm_dpo/q_t": 0.3869220018386841, "grad_norm": 10.253759384155273, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.085869550704956, "logits/rejected": -1.6509474515914917, "logps/chosen": -239.25259399414062, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -332.5849609375, "loss": 1.0239, "margin_dpo/margin_mean": 90.38076782226562, "margin_dpo/margin_std": 99.1339111328125, "step": 609 }, { "KL/chosen_KL_mean": -137.87710571289062, "KL/mean": -185.25228881835938, "KL/rejected_KL_mean": -232.62745666503906, "KL/std": 91.75267028808594, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.005474470555782318, "fcm_dpo/delta": -0.11934898793697357, "fcm_dpo/margin": 94.7503662109375, "fcm_dpo/q_t": 0.37934410572052, "grad_norm": 8.848553657531738, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.8275260925292969, "logits/rejected": -1.289609432220459, "logps/chosen": -232.577392578125, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -322.3148498535156, "loss": 0.9985, "margin_dpo/margin_mean": 94.7503662109375, "margin_dpo/margin_std": 96.245849609375, "step": 610 }, { "KL/chosen_KL_mean": -142.90713500976562, "KL/mean": -189.37770080566406, "KL/rejected_KL_mean": -235.8482666015625, "KL/std": 95.5665054321289, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.005465702153742313, "fcm_dpo/delta": -0.1085335835814476, "fcm_dpo/margin": 92.94113159179688, "fcm_dpo/q_t": 0.38424232602119446, "grad_norm": 10.111878395080566, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.3357062339782715, "logits/rejected": -2.003084421157837, "logps/chosen": -229.69046020507812, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -325.69134521484375, "loss": 1.0082, "margin_dpo/margin_mean": 92.94113159179688, "margin_dpo/margin_std": 97.2420883178711, "step": 611 }, { "KL/chosen_KL_mean": -156.47222900390625, "KL/mean": -204.021240234375, "KL/rejected_KL_mean": -251.5702667236328, "KL/std": 90.53604125976562, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.005449415184557438, "fcm_dpo/delta": -0.11887320876121521, "fcm_dpo/margin": 95.09803771972656, "fcm_dpo/q_t": 0.3803803026676178, "grad_norm": 9.003107070922852, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.0564777851104736, "logits/rejected": -1.2107850313186646, "logps/chosen": -257.49237060546875, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -345.353271484375, "loss": 0.9913, "margin_dpo/margin_mean": 95.09803771972656, "margin_dpo/margin_std": 91.73904418945312, "step": 612 }, { "KL/chosen_KL_mean": -129.09593200683594, "KL/mean": -180.96725463867188, "KL/rejected_KL_mean": -232.83859252929688, "KL/std": 92.8354721069336, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.005433023441582918, "fcm_dpo/delta": -0.1645895391702652, "fcm_dpo/margin": 103.74266052246094, "fcm_dpo/q_t": 0.36977618932724, "grad_norm": 9.097444534301758, "learning_rate": 1.551886292185553e-08, "logits/chosen": -2.0505032539367676, "logits/rejected": -1.8539490699768066, "logps/chosen": -218.0845489501953, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -342.8341064453125, "loss": 0.9525, "margin_dpo/margin_mean": 103.74266052246094, "margin_dpo/margin_std": 88.16651916503906, "step": 613 }, { "KL/chosen_KL_mean": -157.94058227539062, "KL/mean": -199.78416442871094, "KL/rejected_KL_mean": -241.6277313232422, "KL/std": 86.9818115234375, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.005424016155302525, "fcm_dpo/delta": -0.05417971312999725, "fcm_dpo/margin": 83.68716430664062, "fcm_dpo/q_t": 0.3950987458229065, "grad_norm": 10.845977783203125, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.59651780128479, "logits/rejected": -1.5820714235305786, "logps/chosen": -238.14065551757812, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -351.4901123046875, "loss": 1.0593, "margin_dpo/margin_mean": 83.68716430664062, "margin_dpo/margin_std": 103.98495483398438, "step": 614 }, { "KL/chosen_KL_mean": -148.67605590820312, "KL/mean": -185.0670928955078, "KL/rejected_KL_mean": -221.45809936523438, "KL/std": 88.46507263183594, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0054273102432489395, "fcm_dpo/delta": 0.004921458661556244, "fcm_dpo/margin": 72.78205871582031, "fcm_dpo/q_t": 0.40697234869003296, "grad_norm": 9.775019645690918, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.875748872756958, "logits/rejected": -1.4776577949523926, "logps/chosen": -249.111328125, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -322.63818359375, "loss": 1.0807, "margin_dpo/margin_mean": 72.78205871582031, "margin_dpo/margin_std": 81.77545166015625, "step": 615 }, { "KL/chosen_KL_mean": -149.9437255859375, "KL/mean": -198.2724609375, "KL/rejected_KL_mean": -246.6011962890625, "KL/std": 89.42667388916016, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.005416104570031166, "fcm_dpo/delta": -0.12413997203111649, "fcm_dpo/margin": 96.65745544433594, "fcm_dpo/q_t": 0.380195677280426, "grad_norm": 8.9322509765625, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.8695876598358154, "logits/rejected": -1.3827903270721436, "logps/chosen": -242.43666076660156, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -328.661865234375, "loss": 1.0031, "margin_dpo/margin_mean": 96.65745544433594, "margin_dpo/margin_std": 103.53277587890625, "step": 616 }, { "KL/chosen_KL_mean": -156.14907836914062, "KL/mean": -194.87754821777344, "KL/rejected_KL_mean": -233.60601806640625, "KL/std": 85.67405700683594, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.005408680997788906, "fcm_dpo/delta": -0.019020944833755493, "fcm_dpo/margin": 77.45695495605469, "fcm_dpo/q_t": 0.40236639976501465, "grad_norm": 9.074477195739746, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.729893684387207, "logits/rejected": -1.10500168800354, "logps/chosen": -247.24606323242188, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -304.01605224609375, "loss": 1.0769, "margin_dpo/margin_mean": 77.45695495605469, "margin_dpo/margin_std": 97.37188720703125, "step": 617 }, { "KL/chosen_KL_mean": -157.19696044921875, "KL/mean": -199.50308227539062, "KL/rejected_KL_mean": -241.80917358398438, "KL/std": 93.71693420410156, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.005402619019150734, "fcm_dpo/delta": -0.05741099640727043, "fcm_dpo/margin": 84.61219787597656, "fcm_dpo/q_t": 0.39335107803344727, "grad_norm": 10.38623046875, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.0187177658081055, "logits/rejected": -1.6689064502716064, "logps/chosen": -259.21759033203125, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -341.6103515625, "loss": 1.0521, "margin_dpo/margin_mean": 84.6122055053711, "margin_dpo/margin_std": 100.99073028564453, "step": 618 }, { "KL/chosen_KL_mean": -150.801513671875, "KL/mean": -189.4230194091797, "KL/rejected_KL_mean": -228.04454040527344, "KL/std": 93.22578430175781, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0054006632417440414, "fcm_dpo/delta": -0.01723603904247284, "fcm_dpo/margin": 77.24301147460938, "fcm_dpo/q_t": 0.4056333899497986, "grad_norm": 9.203324317932129, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.6345767974853516, "logits/rejected": -1.267035961151123, "logps/chosen": -240.54287719726562, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -327.9459228515625, "loss": 1.0925, "margin_dpo/margin_mean": 77.24301147460938, "margin_dpo/margin_std": 109.53182983398438, "step": 619 }, { "KL/chosen_KL_mean": -145.87649536132812, "KL/mean": -188.14205932617188, "KL/rejected_KL_mean": -230.40760803222656, "KL/std": 97.62237548828125, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0053906748071312904, "fcm_dpo/delta": -0.056169360876083374, "fcm_dpo/margin": 84.53111267089844, "fcm_dpo/q_t": 0.3960821032524109, "grad_norm": 9.058732986450195, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.239403009414673, "logits/rejected": -1.7247648239135742, "logps/chosen": -231.00079345703125, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -315.82012939453125, "loss": 1.0666, "margin_dpo/margin_mean": 84.53111267089844, "margin_dpo/margin_std": 105.92999267578125, "step": 620 }, { "KL/chosen_KL_mean": -146.53111267089844, "KL/mean": -187.91943359375, "KL/rejected_KL_mean": -229.30776977539062, "KL/std": 95.0313720703125, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.005395135842263699, "fcm_dpo/delta": -0.04690833389759064, "fcm_dpo/margin": 82.77667236328125, "fcm_dpo/q_t": 0.3984777331352234, "grad_norm": 8.751702308654785, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.9454419612884521, "logits/rejected": -1.68312668800354, "logps/chosen": -235.77952575683594, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -324.77239990234375, "loss": 1.0788, "margin_dpo/margin_mean": 82.77667236328125, "margin_dpo/margin_std": 110.76956176757812, "step": 621 }, { "KL/chosen_KL_mean": -148.42947387695312, "KL/mean": -191.67987060546875, "KL/rejected_KL_mean": -234.9302978515625, "KL/std": 84.39061737060547, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.005382763221859932, "fcm_dpo/delta": -0.06594333052635193, "fcm_dpo/margin": 86.50082397460938, "fcm_dpo/q_t": 0.3921474814414978, "grad_norm": 9.922767639160156, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.6062660217285156, "logits/rejected": -0.9510248303413391, "logps/chosen": -248.22360229492188, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -325.7585144042969, "loss": 1.0335, "margin_dpo/margin_mean": 86.50082397460938, "margin_dpo/margin_std": 92.85877990722656, "step": 622 }, { "KL/chosen_KL_mean": -152.62564086914062, "KL/mean": -183.21865844726562, "KL/rejected_KL_mean": -213.81167602539062, "KL/std": 91.23774719238281, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.005386200733482838, "fcm_dpo/delta": 0.0706758201122284, "fcm_dpo/margin": 61.186012268066406, "fcm_dpo/q_t": 0.4249955415725708, "grad_norm": 10.636822700500488, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.265012741088867, "logits/rejected": -1.830285906791687, "logps/chosen": -243.08120727539062, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -305.1344299316406, "loss": 1.1673, "margin_dpo/margin_mean": 61.186012268066406, "margin_dpo/margin_std": 110.02828979492188, "step": 623 }, { "KL/chosen_KL_mean": -156.2754364013672, "KL/mean": -203.66665649414062, "KL/rejected_KL_mean": -251.057861328125, "KL/std": 90.74809265136719, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.005380924791097641, "fcm_dpo/delta": -0.1105787456035614, "fcm_dpo/margin": 94.78240203857422, "fcm_dpo/q_t": 0.38319000601768494, "grad_norm": 10.242768287658691, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.557756185531616, "logits/rejected": -1.6318600177764893, "logps/chosen": -266.1506652832031, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -355.8310546875, "loss": 1.011, "margin_dpo/margin_mean": 94.78240203857422, "margin_dpo/margin_std": 102.9213638305664, "step": 624 }, { "KL/chosen_KL_mean": -163.4422607421875, "KL/mean": -199.44345092773438, "KL/rejected_KL_mean": -235.44464111328125, "KL/std": 91.02337646484375, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.005379818379878998, "fcm_dpo/delta": 0.012680720537900925, "fcm_dpo/margin": 72.00238037109375, "fcm_dpo/q_t": 0.41017651557922363, "grad_norm": 8.69680118560791, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.8449722528457642, "logits/rejected": -1.474625587463379, "logps/chosen": -250.6104278564453, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -327.3061218261719, "loss": 1.0955, "margin_dpo/margin_mean": 72.00237274169922, "margin_dpo/margin_std": 93.45063781738281, "step": 625 }, { "KL/chosen_KL_mean": -156.2259979248047, "KL/mean": -190.90798950195312, "KL/rejected_KL_mean": -225.58998107910156, "KL/std": 98.94080352783203, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.005382338538765907, "fcm_dpo/delta": 0.026747725903987885, "fcm_dpo/margin": 69.36399841308594, "fcm_dpo/q_t": 0.4150870442390442, "grad_norm": 10.867732048034668, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.1659107208251953, "logits/rejected": -0.714760959148407, "logps/chosen": -260.4501953125, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -329.7674560546875, "loss": 1.1278, "margin_dpo/margin_mean": 69.3639907836914, "margin_dpo/margin_std": 109.03595733642578, "step": 626 }, { "KL/chosen_KL_mean": -138.17527770996094, "KL/mean": -182.16455078125, "KL/rejected_KL_mean": -226.15383911132812, "KL/std": 102.45388793945312, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.005375940818339586, "fcm_dpo/delta": -0.07331766188144684, "fcm_dpo/margin": 87.97856140136719, "fcm_dpo/q_t": 0.3929376006126404, "grad_norm": 7.599968433380127, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.4044766426086426, "logits/rejected": -1.5472266674041748, "logps/chosen": -213.10989379882812, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -338.72674560546875, "loss": 1.0469, "margin_dpo/margin_mean": 87.97856140136719, "margin_dpo/margin_std": 109.27555084228516, "step": 627 }, { "KL/chosen_KL_mean": -143.77911376953125, "KL/mean": -190.55877685546875, "KL/rejected_KL_mean": -237.33843994140625, "KL/std": 92.15510559082031, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.005365898832678795, "fcm_dpo/delta": -0.1025453507900238, "fcm_dpo/margin": 93.559326171875, "fcm_dpo/q_t": 0.3844854235649109, "grad_norm": 10.222888946533203, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.031778335571289, "logits/rejected": -1.7660630941390991, "logps/chosen": -237.47018432617188, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -344.682373046875, "loss": 1.0104, "margin_dpo/margin_mean": 93.559326171875, "margin_dpo/margin_std": 97.49308776855469, "step": 628 }, { "KL/chosen_KL_mean": -156.3148956298828, "KL/mean": -197.74826049804688, "KL/rejected_KL_mean": -239.18162536621094, "KL/std": 92.22850036621094, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.005358301103115082, "fcm_dpo/delta": -0.044244375079870224, "fcm_dpo/margin": 82.86674499511719, "fcm_dpo/q_t": 0.396295964717865, "grad_norm": 8.826534271240234, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.1386947631835938, "logits/rejected": -1.7175967693328857, "logps/chosen": -259.5452575683594, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -336.35003662109375, "loss": 1.0555, "margin_dpo/margin_mean": 82.86674499511719, "margin_dpo/margin_std": 97.05552673339844, "step": 629 }, { "KL/chosen_KL_mean": -143.45318603515625, "KL/mean": -172.05950927734375, "KL/rejected_KL_mean": -200.6658172607422, "KL/std": 86.03264617919922, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.005368205718696117, "fcm_dpo/delta": 0.09308067709207535, "fcm_dpo/margin": 57.2126350402832, "fcm_dpo/q_t": 0.428194522857666, "grad_norm": 12.641382217407227, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.181001663208008, "logits/rejected": -1.659090280532837, "logps/chosen": -237.35073852539062, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -290.04022216796875, "loss": 1.1671, "margin_dpo/margin_mean": 57.2126350402832, "margin_dpo/margin_std": 94.77615356445312, "step": 630 }, { "KL/chosen_KL_mean": -139.53298950195312, "KL/mean": -179.96412658691406, "KL/rejected_KL_mean": -220.39524841308594, "KL/std": 85.217529296875, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.00536506250500679, "fcm_dpo/delta": -0.03398509323596954, "fcm_dpo/margin": 80.86225891113281, "fcm_dpo/q_t": 0.3987378776073456, "grad_norm": 8.410491943359375, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.570918560028076, "logits/rejected": -2.147509813308716, "logps/chosen": -227.30381774902344, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -309.0776672363281, "loss": 1.0586, "margin_dpo/margin_mean": 80.86225891113281, "margin_dpo/margin_std": 93.81573486328125, "step": 631 }, { "KL/chosen_KL_mean": -137.02052307128906, "KL/mean": -182.09490966796875, "KL/rejected_KL_mean": -227.1693115234375, "KL/std": 91.84683227539062, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.005354823544621468, "fcm_dpo/delta": -0.08325687795877457, "fcm_dpo/margin": 90.1488037109375, "fcm_dpo/q_t": 0.3902779817581177, "grad_norm": 10.255605697631836, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.0342533588409424, "logits/rejected": -1.6104099750518799, "logps/chosen": -225.6470489501953, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -318.6202392578125, "loss": 1.0442, "margin_dpo/margin_mean": 90.14879608154297, "margin_dpo/margin_std": 110.22649383544922, "step": 632 }, { "KL/chosen_KL_mean": -172.30441284179688, "KL/mean": -199.49609375, "KL/rejected_KL_mean": -226.68777465820312, "KL/std": 91.68498229980469, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.005355765577405691, "fcm_dpo/delta": -0.0014417327474802732, "fcm_dpo/margin": 54.38335037231445, "fcm_dpo/q_t": 0.43516838550567627, "grad_norm": 10.46218204498291, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.2504286766052246, "logits/rejected": -2.0603621006011963, "logps/chosen": -253.6788330078125, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -325.3134765625, "loss": 1.2154, "margin_dpo/margin_mean": 54.38335418701172, "margin_dpo/margin_std": 120.36471557617188, "step": 633 }, { "KL/chosen_KL_mean": -158.86282348632812, "KL/mean": -194.2872314453125, "KL/rejected_KL_mean": -229.71170043945312, "KL/std": 82.94100952148438, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.005361674353480339, "fcm_dpo/delta": 0.020058251917362213, "fcm_dpo/margin": 70.84889221191406, "fcm_dpo/q_t": 0.41086938977241516, "grad_norm": 9.840776443481445, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.5068415403366089, "logits/rejected": -1.3256170749664307, "logps/chosen": -254.98565673828125, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -342.55950927734375, "loss": 1.0938, "margin_dpo/margin_mean": 70.84889221191406, "margin_dpo/margin_std": 81.72084045410156, "step": 634 }, { "KL/chosen_KL_mean": -163.82418823242188, "KL/mean": -190.937744140625, "KL/rejected_KL_mean": -218.05130004882812, "KL/std": 87.08599853515625, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.005368458107113838, "fcm_dpo/delta": 0.10914070159196854, "fcm_dpo/margin": 54.227108001708984, "fcm_dpo/q_t": 0.4328666925430298, "grad_norm": 11.48480224609375, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.6709198951721191, "logits/rejected": -1.1363540887832642, "logps/chosen": -262.50830078125, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -307.95050048828125, "loss": 1.1869, "margin_dpo/margin_mean": 54.22710418701172, "margin_dpo/margin_std": 101.24923706054688, "step": 635 }, { "KL/chosen_KL_mean": -149.47381591796875, "KL/mean": -193.43487548828125, "KL/rejected_KL_mean": -237.39596557617188, "KL/std": 93.44357299804688, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.005364295095205307, "fcm_dpo/delta": -0.07197457551956177, "fcm_dpo/margin": 87.92213439941406, "fcm_dpo/q_t": 0.3908817768096924, "grad_norm": 8.90060806274414, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.4416539669036865, "logits/rejected": -0.9621982574462891, "logps/chosen": -239.8917694091797, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -325.10284423828125, "loss": 1.0281, "margin_dpo/margin_mean": 87.92213439941406, "margin_dpo/margin_std": 93.4788818359375, "step": 636 }, { "KL/chosen_KL_mean": -149.50543212890625, "KL/mean": -196.52655029296875, "KL/rejected_KL_mean": -243.54769897460938, "KL/std": 102.72685241699219, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.00535405520349741, "fcm_dpo/delta": -0.10402226448059082, "fcm_dpo/margin": 94.04226684570312, "fcm_dpo/q_t": 0.38403838872909546, "grad_norm": 8.449945449829102, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.1132912635803223, "logits/rejected": -2.010817766189575, "logps/chosen": -231.95513916015625, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -347.5762939453125, "loss": 1.0124, "margin_dpo/margin_mean": 94.0422592163086, "margin_dpo/margin_std": 99.62531280517578, "step": 637 }, { "KL/chosen_KL_mean": -139.01138305664062, "KL/mean": -185.9768829345703, "KL/rejected_KL_mean": -232.94236755371094, "KL/std": 91.41777801513672, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0053452253341674805, "fcm_dpo/delta": -0.10259798914194107, "fcm_dpo/margin": 93.93098449707031, "fcm_dpo/q_t": 0.3824614882469177, "grad_norm": 7.96666955947876, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.6202932596206665, "logits/rejected": -0.9423699378967285, "logps/chosen": -230.93637084960938, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -319.2294006347656, "loss": 0.9915, "margin_dpo/margin_mean": 93.93098449707031, "margin_dpo/margin_std": 78.63485717773438, "step": 638 }, { "KL/chosen_KL_mean": -183.52609252929688, "KL/mean": -207.45822143554688, "KL/rejected_KL_mean": -231.39036560058594, "KL/std": 86.58644104003906, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.00534199271351099, "fcm_dpo/delta": 0.04272831231355667, "fcm_dpo/margin": 47.864295959472656, "fcm_dpo/q_t": 0.4403631091117859, "grad_norm": 9.586102485656738, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3135261535644531, "logits/rejected": -0.5413253307342529, "logps/chosen": -288.05364990234375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -312.8707275390625, "loss": 1.2253, "margin_dpo/margin_mean": 47.864295959472656, "margin_dpo/margin_std": 106.79634094238281, "step": 639 }, { "KL/chosen_KL_mean": -153.46510314941406, "KL/mean": -196.47996520996094, "KL/rejected_KL_mean": -239.49484252929688, "KL/std": 93.19219970703125, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.00534411333501339, "fcm_dpo/delta": -0.06012822687625885, "fcm_dpo/margin": 86.02974700927734, "fcm_dpo/q_t": 0.3933391571044922, "grad_norm": 9.453333854675293, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8172199726104736, "logits/rejected": -1.2897007465362549, "logps/chosen": -251.35037231445312, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -352.1998596191406, "loss": 1.0416, "margin_dpo/margin_mean": 86.02974700927734, "margin_dpo/margin_std": 91.059326171875, "step": 640 }, { "KL/chosen_KL_mean": -142.22955322265625, "KL/mean": -186.45822143554688, "KL/rejected_KL_mean": -230.68692016601562, "KL/std": 105.8079833984375, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.005332523491233587, "fcm_dpo/delta": -0.07203630357980728, "fcm_dpo/margin": 88.45736694335938, "fcm_dpo/q_t": 0.39433416724205017, "grad_norm": 8.326630592346191, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.404146194458008, "logits/rejected": -1.8785754442214966, "logps/chosen": -238.67523193359375, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -325.8226013183594, "loss": 1.0526, "margin_dpo/margin_mean": 88.4573745727539, "margin_dpo/margin_std": 114.22496032714844, "step": 641 }, { "KL/chosen_KL_mean": -161.03005981445312, "KL/mean": -201.9860076904297, "KL/rejected_KL_mean": -242.94195556640625, "KL/std": 90.58287048339844, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.005328530445694923, "fcm_dpo/delta": -0.03663276135921478, "fcm_dpo/margin": 81.9118881225586, "fcm_dpo/q_t": 0.40130382776260376, "grad_norm": 9.790349960327148, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.6389844417572021, "logits/rejected": -1.3096476793289185, "logps/chosen": -261.7899169921875, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -357.64959716796875, "loss": 1.0756, "margin_dpo/margin_mean": 81.91188049316406, "margin_dpo/margin_std": 110.34489440917969, "step": 642 }, { "KL/chosen_KL_mean": -149.49200439453125, "KL/mean": -189.00543212890625, "KL/rejected_KL_mean": -228.5188446044922, "KL/std": 84.31705474853516, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.005320975556969643, "fcm_dpo/delta": -0.02069399133324623, "fcm_dpo/margin": 79.02681732177734, "fcm_dpo/q_t": 0.40197592973709106, "grad_norm": 10.719751358032227, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.2013378143310547, "logits/rejected": -1.81156587600708, "logps/chosen": -234.2356719970703, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -322.8372802734375, "loss": 1.0749, "margin_dpo/margin_mean": 79.02682495117188, "margin_dpo/margin_std": 95.5079116821289, "step": 643 }, { "KL/chosen_KL_mean": -151.99192810058594, "KL/mean": -185.01235961914062, "KL/rejected_KL_mean": -218.0327606201172, "KL/std": 84.80465698242188, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.005328072234988213, "fcm_dpo/delta": 0.048297978937625885, "fcm_dpo/margin": 66.04084014892578, "fcm_dpo/q_t": 0.41713836789131165, "grad_norm": 10.578706741333008, "learning_rate": 4.741290495811873e-09, "logits/chosen": -2.040325880050659, "logits/rejected": -1.771176815032959, "logps/chosen": -237.31466674804688, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -312.641357421875, "loss": 1.1168, "margin_dpo/margin_mean": 66.04084014892578, "margin_dpo/margin_std": 87.47483825683594, "step": 644 }, { "KL/chosen_KL_mean": -166.02587890625, "KL/mean": -188.21702575683594, "KL/rejected_KL_mean": -210.40814208984375, "KL/std": 93.88267517089844, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.005332452245056629, "fcm_dpo/delta": 0.06092112511396408, "fcm_dpo/margin": 44.382259368896484, "fcm_dpo/q_t": 0.44543784856796265, "grad_norm": 10.029483795166016, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.6333131790161133, "logits/rejected": -1.4583675861358643, "logps/chosen": -248.6161346435547, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -313.4118957519531, "loss": 1.2379, "margin_dpo/margin_mean": 44.38225555419922, "margin_dpo/margin_std": 103.79534912109375, "step": 645 }, { "KL/chosen_KL_mean": -143.40126037597656, "KL/mean": -185.58731079101562, "KL/rejected_KL_mean": -227.7733917236328, "KL/std": 82.03721618652344, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.005329630803316832, "fcm_dpo/delta": -0.04994665086269379, "fcm_dpo/margin": 84.37214660644531, "fcm_dpo/q_t": 0.39479541778564453, "grad_norm": 8.727654457092285, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.816481351852417, "logits/rejected": -1.2104263305664062, "logps/chosen": -234.59329223632812, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -311.5517272949219, "loss": 1.0366, "margin_dpo/margin_mean": 84.37214660644531, "margin_dpo/margin_std": 85.20330810546875, "step": 646 }, { "KL/chosen_KL_mean": -159.00474548339844, "KL/mean": -195.47471618652344, "KL/rejected_KL_mean": -231.94468688964844, "KL/std": 98.41416931152344, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0053308214992284775, "fcm_dpo/delta": 0.011213324032723904, "fcm_dpo/margin": 72.93994140625, "fcm_dpo/q_t": 0.41163206100463867, "grad_norm": 8.56908130645752, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.4914531707763672, "logits/rejected": -1.352895975112915, "logps/chosen": -247.84921264648438, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -331.4430236816406, "loss": 1.1142, "margin_dpo/margin_mean": 72.93994140625, "margin_dpo/margin_std": 109.40255737304688, "step": 647 }, { "KL/chosen_KL_mean": -144.34017944335938, "KL/mean": -195.9611358642578, "KL/rejected_KL_mean": -247.5820770263672, "KL/std": 99.77560424804688, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.005320893134921789, "fcm_dpo/delta": -0.1501391977071762, "fcm_dpo/margin": 103.24188232421875, "fcm_dpo/q_t": 0.3742830157279968, "grad_norm": 8.831951141357422, "learning_rate": 3.798061746947995e-09, "logits/chosen": -2.1921474933624268, "logits/rejected": -2.1142234802246094, "logps/chosen": -232.18829345703125, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -352.25213623046875, "loss": 0.9817, "margin_dpo/margin_mean": 103.24188232421875, "margin_dpo/margin_std": 104.45919799804688, "step": 648 }, { "KL/chosen_KL_mean": -132.38638305664062, "KL/mean": -174.55441284179688, "KL/rejected_KL_mean": -216.72244262695312, "KL/std": 88.32987976074219, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.005309479311108589, "fcm_dpo/delta": -0.04805298522114754, "fcm_dpo/margin": 84.3360595703125, "fcm_dpo/q_t": 0.39529532194137573, "grad_norm": 9.90986442565918, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.4922752380371094, "logits/rejected": -2.1003167629241943, "logps/chosen": -222.07888793945312, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -305.42901611328125, "loss": 1.0372, "margin_dpo/margin_mean": 84.3360595703125, "margin_dpo/margin_std": 84.09535217285156, "step": 649 }, { "KL/chosen_KL_mean": -157.30674743652344, "KL/mean": -197.49563598632812, "KL/rejected_KL_mean": -237.68450927734375, "KL/std": 93.68889617919922, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.005309120751917362, "fcm_dpo/delta": -0.026850128546357155, "fcm_dpo/margin": 80.37774658203125, "fcm_dpo/q_t": 0.4014317989349365, "grad_norm": 9.314303398132324, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.4242885112762451, "logits/rejected": -1.2332831621170044, "logps/chosen": -247.2373504638672, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -340.29730224609375, "loss": 1.0818, "margin_dpo/margin_mean": 80.37774658203125, "margin_dpo/margin_std": 108.63566589355469, "step": 650 }, { "KL/chosen_KL_mean": -132.87203979492188, "KL/mean": -181.48941040039062, "KL/rejected_KL_mean": -230.10678100585938, "KL/std": 90.98446655273438, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.005298158619552851, "fcm_dpo/delta": -0.11576782166957855, "fcm_dpo/margin": 97.2347412109375, "fcm_dpo/q_t": 0.3799465298652649, "grad_norm": 8.946884155273438, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.2710628509521484, "logits/rejected": -2.103151798248291, "logps/chosen": -212.05935668945312, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -336.04010009765625, "loss": 0.9908, "margin_dpo/margin_mean": 97.2347412109375, "margin_dpo/margin_std": 90.84093475341797, "step": 651 }, { "KL/chosen_KL_mean": -138.7995147705078, "KL/mean": -180.61846923828125, "KL/rejected_KL_mean": -222.43740844726562, "KL/std": 83.61183166503906, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.005293012596666813, "fcm_dpo/delta": -0.04288580268621445, "fcm_dpo/margin": 83.63790893554688, "fcm_dpo/q_t": 0.39596208930015564, "grad_norm": 8.28687572479248, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.4876043796539307, "logits/rejected": -2.1844263076782227, "logps/chosen": -240.58973693847656, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -338.7619934082031, "loss": 1.0337, "margin_dpo/margin_mean": 83.63790893554688, "margin_dpo/margin_std": 79.77033996582031, "step": 652 }, { "KL/chosen_KL_mean": -156.71823120117188, "KL/mean": -196.3612060546875, "KL/rejected_KL_mean": -236.0041961669922, "KL/std": 92.19729614257812, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.005290325731039047, "fcm_dpo/delta": -0.019534554332494736, "fcm_dpo/margin": 79.28594970703125, "fcm_dpo/q_t": 0.4009215235710144, "grad_norm": 8.817952156066895, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.2926526069641113, "logits/rejected": -2.0519285202026367, "logps/chosen": -250.07183837890625, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -345.12744140625, "loss": 1.0659, "margin_dpo/margin_mean": 79.28594970703125, "margin_dpo/margin_std": 90.89889526367188, "step": 653 }, { "KL/chosen_KL_mean": -158.48135375976562, "KL/mean": -192.74566650390625, "KL/rejected_KL_mean": -227.00997924804688, "KL/std": 89.95146179199219, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.005291881505399942, "fcm_dpo/delta": 0.037489235401153564, "fcm_dpo/margin": 68.52862548828125, "fcm_dpo/q_t": 0.41843053698539734, "grad_norm": 11.757072448730469, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.462388515472412, "logits/rejected": -2.2839221954345703, "logps/chosen": -247.59689331054688, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -331.929931640625, "loss": 1.1396, "margin_dpo/margin_mean": 68.52862548828125, "margin_dpo/margin_std": 113.7889404296875, "step": 654 }, { "KL/chosen_KL_mean": -145.2121124267578, "KL/mean": -186.88108825683594, "KL/rejected_KL_mean": -228.550048828125, "KL/std": 97.34956359863281, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.005290408618748188, "fcm_dpo/delta": -0.04107195883989334, "fcm_dpo/margin": 83.33793640136719, "fcm_dpo/q_t": 0.3973468840122223, "grad_norm": 8.69176197052002, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.2124996185302734, "logits/rejected": -2.2066774368286133, "logps/chosen": -226.2482147216797, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -331.3523864746094, "loss": 1.0617, "margin_dpo/margin_mean": 83.33794403076172, "margin_dpo/margin_std": 101.30123901367188, "step": 655 }, { "KL/chosen_KL_mean": -145.71780395507812, "KL/mean": -197.74896240234375, "KL/rejected_KL_mean": -249.78012084960938, "KL/std": 97.00772094726562, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.005275868810713291, "fcm_dpo/delta": -0.14986281096935272, "fcm_dpo/margin": 104.06233215332031, "fcm_dpo/q_t": 0.37413930892944336, "grad_norm": 9.873557090759277, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.717049241065979, "logits/rejected": -1.7832063436508179, "logps/chosen": -231.04315185546875, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -368.1187744140625, "loss": 0.9706, "margin_dpo/margin_mean": 104.06233215332031, "margin_dpo/margin_std": 97.14768981933594, "step": 656 }, { "KL/chosen_KL_mean": -153.4217529296875, "KL/mean": -194.24693298339844, "KL/rejected_KL_mean": -235.07208251953125, "KL/std": 89.91839599609375, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.005267918109893799, "fcm_dpo/delta": -0.030306151136755943, "fcm_dpo/margin": 81.65036010742188, "fcm_dpo/q_t": 0.4019322395324707, "grad_norm": 11.828180313110352, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9265234470367432, "logits/rejected": -1.750333309173584, "logps/chosen": -233.61947631835938, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -337.65338134765625, "loss": 1.0785, "margin_dpo/margin_mean": 81.65036010742188, "margin_dpo/margin_std": 107.33870697021484, "step": 657 }, { "KL/chosen_KL_mean": -159.58731079101562, "KL/mean": -195.67372131347656, "KL/rejected_KL_mean": -231.76011657714844, "KL/std": 84.75870513916016, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.005272488109767437, "fcm_dpo/delta": 0.019519606605172157, "fcm_dpo/margin": 72.17282104492188, "fcm_dpo/q_t": 0.4111135005950928, "grad_norm": 8.996246337890625, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.9845514297485352, "logits/rejected": -1.5129287242889404, "logps/chosen": -254.22183227539062, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -333.3917236328125, "loss": 1.0967, "margin_dpo/margin_mean": 72.17282104492188, "margin_dpo/margin_std": 90.26454162597656, "step": 658 }, { "KL/chosen_KL_mean": -140.07205200195312, "KL/mean": -184.7480926513672, "KL/rejected_KL_mean": -229.42413330078125, "KL/std": 92.0663070678711, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.005264232866466045, "fcm_dpo/delta": -0.07074190676212311, "fcm_dpo/margin": 89.35204315185547, "fcm_dpo/q_t": 0.3936379849910736, "grad_norm": 8.716135025024414, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.0899817943573, "logits/rejected": -1.125427484512329, "logps/chosen": -252.6279296875, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -323.3457336425781, "loss": 1.0652, "margin_dpo/margin_mean": 89.35204315185547, "margin_dpo/margin_std": 119.4981689453125, "step": 659 }, { "KL/chosen_KL_mean": -146.24339294433594, "KL/mean": -191.920654296875, "KL/rejected_KL_mean": -237.59791564941406, "KL/std": 90.43766021728516, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.005257292650640011, "fcm_dpo/delta": -0.08067014813423157, "fcm_dpo/margin": 91.35453796386719, "fcm_dpo/q_t": 0.38916248083114624, "grad_norm": 11.874688148498535, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8616329431533813, "logits/rejected": -1.2418535947799683, "logps/chosen": -239.62081909179688, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -318.99273681640625, "loss": 1.0257, "margin_dpo/margin_mean": 91.35453796386719, "margin_dpo/margin_std": 99.30839538574219, "step": 660 }, { "KL/chosen_KL_mean": -150.3674774169922, "KL/mean": -199.4947052001953, "KL/rejected_KL_mean": -248.6219482421875, "KL/std": 97.11583709716797, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.005247180350124836, "fcm_dpo/delta": -0.11614412814378738, "fcm_dpo/margin": 98.25446319580078, "fcm_dpo/q_t": 0.38247206807136536, "grad_norm": 8.680158615112305, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.8107914924621582, "logits/rejected": -1.2911428213119507, "logps/chosen": -238.22262573242188, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -341.0252685546875, "loss": 1.0117, "margin_dpo/margin_mean": 98.25446319580078, "margin_dpo/margin_std": 108.55375671386719, "step": 661 }, { "KL/chosen_KL_mean": -145.30499267578125, "KL/mean": -190.44012451171875, "KL/rejected_KL_mean": -235.57525634765625, "KL/std": 86.66191101074219, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.005238384939730167, "fcm_dpo/delta": -0.07321003824472427, "fcm_dpo/margin": 90.27027130126953, "fcm_dpo/q_t": 0.3897520899772644, "grad_norm": 7.968448638916016, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.9050581455230713, "logits/rejected": -1.3573627471923828, "logps/chosen": -248.0847930908203, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -330.8005676269531, "loss": 1.0199, "margin_dpo/margin_mean": 90.27027130126953, "margin_dpo/margin_std": 88.54481506347656, "step": 662 }, { "KL/chosen_KL_mean": -144.63148498535156, "KL/mean": -185.82061767578125, "KL/rejected_KL_mean": -227.00973510742188, "KL/std": 90.16123962402344, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.00523411575704813, "fcm_dpo/delta": -0.03131337836384773, "fcm_dpo/margin": 82.37825012207031, "fcm_dpo/q_t": 0.40063342452049255, "grad_norm": 8.81959056854248, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.315091609954834, "logits/rejected": -2.0917911529541016, "logps/chosen": -240.9780731201172, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -347.53619384765625, "loss": 1.0745, "margin_dpo/margin_mean": 82.37824249267578, "margin_dpo/margin_std": 106.28712463378906, "step": 663 }, { "KL/chosen_KL_mean": -159.85980224609375, "KL/mean": -197.4600067138672, "KL/rejected_KL_mean": -235.06019592285156, "KL/std": 91.21726989746094, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.005233794450759888, "fcm_dpo/delta": 0.0064338194206357, "fcm_dpo/margin": 75.20040130615234, "fcm_dpo/q_t": 0.40767672657966614, "grad_norm": 9.247379302978516, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.7757951021194458, "logits/rejected": -1.5965263843536377, "logps/chosen": -251.70220947265625, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -346.89691162109375, "loss": 1.0856, "margin_dpo/margin_mean": 75.20039367675781, "margin_dpo/margin_std": 90.36418151855469, "step": 664 }, { "KL/chosen_KL_mean": -135.67294311523438, "KL/mean": -177.85728454589844, "KL/rejected_KL_mean": -220.04165649414062, "KL/std": 90.58036041259766, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.005230742506682873, "fcm_dpo/delta": -0.04149354249238968, "fcm_dpo/margin": 84.36871337890625, "fcm_dpo/q_t": 0.39596912264823914, "grad_norm": 7.88140344619751, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.457029104232788, "logits/rejected": -2.030548572540283, "logps/chosen": -223.85911560058594, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -311.95367431640625, "loss": 1.0436, "margin_dpo/margin_mean": 84.36871337890625, "margin_dpo/margin_std": 88.364501953125, "step": 665 }, { "KL/chosen_KL_mean": -153.68197631835938, "KL/mean": -194.58631896972656, "KL/rejected_KL_mean": -235.49066162109375, "KL/std": 87.03085327148438, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.005224986467510462, "fcm_dpo/delta": -0.02759162336587906, "fcm_dpo/margin": 81.80867767333984, "fcm_dpo/q_t": 0.40182188153266907, "grad_norm": 11.631195068359375, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.7780678272247314, "logits/rejected": -1.4129550457000732, "logps/chosen": -254.6565704345703, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -336.7406005859375, "loss": 1.0744, "margin_dpo/margin_mean": 81.80867767333984, "margin_dpo/margin_std": 102.98039245605469, "step": 666 }, { "KL/chosen_KL_mean": -164.94268798828125, "KL/mean": -209.7395782470703, "KL/rejected_KL_mean": -254.53648376464844, "KL/std": 91.2158203125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.005219390150159597, "fcm_dpo/delta": -0.06796456128358841, "fcm_dpo/margin": 89.5937728881836, "fcm_dpo/q_t": 0.3903047740459442, "grad_norm": 8.649205207824707, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.5665769577026367, "logits/rejected": -0.9146552085876465, "logps/chosen": -262.51385498046875, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -348.1212463378906, "loss": 1.0208, "margin_dpo/margin_mean": 89.5937728881836, "margin_dpo/margin_std": 86.38423156738281, "step": 667 }, { "KL/chosen_KL_mean": -145.47573852539062, "KL/mean": -196.04554748535156, "KL/rejected_KL_mean": -246.61538696289062, "KL/std": 98.35223388671875, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.005209713242948055, "fcm_dpo/delta": -0.12756358087062836, "fcm_dpo/margin": 101.13964080810547, "fcm_dpo/q_t": 0.38051584362983704, "grad_norm": 12.001924514770508, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.214405059814453, "logits/rejected": -1.6799242496490479, "logps/chosen": -248.05078125, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -355.4330749511719, "loss": 0.9952, "margin_dpo/margin_mean": 101.13964080810547, "margin_dpo/margin_std": 104.99458312988281, "step": 668 }, { "KL/chosen_KL_mean": -133.6302490234375, "KL/mean": -182.58456420898438, "KL/rejected_KL_mean": -231.53887939453125, "KL/std": 94.65568542480469, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.005197003949433565, "fcm_dpo/delta": -0.10938502103090286, "fcm_dpo/margin": 97.90863037109375, "fcm_dpo/q_t": 0.38213056325912476, "grad_norm": 9.703058242797852, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.9055957794189453, "logits/rejected": -1.5146747827529907, "logps/chosen": -223.12503051757812, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -314.0583801269531, "loss": 0.9991, "margin_dpo/margin_mean": 97.90863037109375, "margin_dpo/margin_std": 96.15896606445312, "step": 669 }, { "KL/chosen_KL_mean": -132.87159729003906, "KL/mean": -186.33187866210938, "KL/rejected_KL_mean": -239.79214477539062, "KL/std": 96.022216796875, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.005182186607271433, "fcm_dpo/delta": -0.1549341380596161, "fcm_dpo/margin": 106.92054748535156, "fcm_dpo/q_t": 0.3710108995437622, "grad_norm": 8.291281700134277, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7592101097106934, "logits/rejected": -1.2231470346450806, "logps/chosen": -228.32620239257812, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -341.3250732421875, "loss": 0.968, "margin_dpo/margin_mean": 106.92054748535156, "margin_dpo/margin_std": 98.37152862548828, "step": 670 }, { "KL/chosen_KL_mean": -163.72061157226562, "KL/mean": -199.9786376953125, "KL/rejected_KL_mean": -236.2366485595703, "KL/std": 89.79009246826172, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.005182449705898762, "fcm_dpo/delta": 0.024216115474700928, "fcm_dpo/margin": 72.51602172851562, "fcm_dpo/q_t": 0.4127650260925293, "grad_norm": 9.118310928344727, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.7134103775024414, "logits/rejected": -1.4920889139175415, "logps/chosen": -245.84375, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -326.45635986328125, "loss": 1.1087, "margin_dpo/margin_mean": 72.51602172851562, "margin_dpo/margin_std": 98.08798217773438, "step": 671 }, { "KL/chosen_KL_mean": -163.1573486328125, "KL/mean": -195.48361206054688, "KL/rejected_KL_mean": -227.80990600585938, "KL/std": 87.4258041381836, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.005184961017221212, "fcm_dpo/delta": 0.06500150263309479, "fcm_dpo/margin": 64.65254211425781, "fcm_dpo/q_t": 0.4215002655982971, "grad_norm": 10.177120208740234, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.120535135269165, "logits/rejected": -1.78676438331604, "logps/chosen": -253.21925354003906, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -328.26312255859375, "loss": 1.1344, "margin_dpo/margin_mean": 64.65254211425781, "margin_dpo/margin_std": 90.80333709716797, "step": 672 }, { "KL/chosen_KL_mean": -144.51007080078125, "KL/mean": -181.201171875, "KL/rejected_KL_mean": -217.89230346679688, "KL/std": 89.8158187866211, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.005189495161175728, "fcm_dpo/delta": 0.019248606637120247, "fcm_dpo/margin": 73.38220977783203, "fcm_dpo/q_t": 0.41084617376327515, "grad_norm": 9.961067199707031, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -2.1115498542785645, "logits/rejected": -1.7503430843353271, "logps/chosen": -223.77322387695312, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -309.2415466308594, "loss": 1.094, "margin_dpo/margin_mean": 73.38220977783203, "margin_dpo/margin_std": 90.69793701171875, "step": 673 }, { "KL/chosen_KL_mean": -136.8771209716797, "KL/mean": -186.19293212890625, "KL/rejected_KL_mean": -235.50872802734375, "KL/std": 90.26255798339844, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.005179780535399914, "fcm_dpo/delta": -0.11147917807102203, "fcm_dpo/margin": 98.63160705566406, "fcm_dpo/q_t": 0.382526159286499, "grad_norm": 8.46402359008789, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.7920300960540771, "logits/rejected": -1.4054961204528809, "logps/chosen": -212.33543395996094, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -311.71234130859375, "loss": 0.9958, "margin_dpo/margin_mean": 98.63160705566406, "margin_dpo/margin_std": 96.17955017089844, "step": 674 }, { "KL/chosen_KL_mean": -152.60906982421875, "KL/mean": -196.38198852539062, "KL/rejected_KL_mean": -240.1549072265625, "KL/std": 95.71086120605469, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.005170384421944618, "fcm_dpo/delta": -0.052989933639764786, "fcm_dpo/margin": 87.54583740234375, "fcm_dpo/q_t": 0.3955810070037842, "grad_norm": 10.382542610168457, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.6420648097991943, "logits/rejected": -1.3145768642425537, "logps/chosen": -232.5186004638672, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -321.3731689453125, "loss": 1.068, "margin_dpo/margin_mean": 87.54583740234375, "margin_dpo/margin_std": 113.70344543457031, "step": 675 }, { "KL/chosen_KL_mean": -162.523681640625, "KL/mean": -206.32162475585938, "KL/rejected_KL_mean": -250.11953735351562, "KL/std": 92.76974487304688, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.005166532471776009, "fcm_dpo/delta": -0.052838124334812164, "fcm_dpo/margin": 87.59585571289062, "fcm_dpo/q_t": 0.394814670085907, "grad_norm": 8.884283065795898, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -2.1318352222442627, "logits/rejected": -1.5666425228118896, "logps/chosen": -260.69482421875, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -345.14422607421875, "loss": 1.0505, "margin_dpo/margin_mean": 87.59585571289062, "margin_dpo/margin_std": 101.04602813720703, "step": 676 }, { "KL/chosen_KL_mean": -144.884765625, "KL/mean": -189.9153594970703, "KL/rejected_KL_mean": -234.94595336914062, "KL/std": 91.31705474853516, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.005160595290362835, "fcm_dpo/delta": -0.06509288400411606, "fcm_dpo/margin": 90.06118774414062, "fcm_dpo/q_t": 0.39126884937286377, "grad_norm": 8.122529029846191, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.941672682762146, "logits/rejected": -1.2858628034591675, "logps/chosen": -236.26405334472656, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -317.82373046875, "loss": 1.0235, "margin_dpo/margin_mean": 90.06118774414062, "margin_dpo/margin_std": 87.03279876708984, "step": 677 }, { "KL/chosen_KL_mean": -137.25933837890625, "KL/mean": -190.53268432617188, "KL/rejected_KL_mean": -243.8060302734375, "KL/std": 96.7198257446289, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.005151058081537485, "fcm_dpo/delta": -0.14962589740753174, "fcm_dpo/margin": 106.54670715332031, "fcm_dpo/q_t": 0.37284839153289795, "grad_norm": 8.713842391967773, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.9124467372894287, "logits/rejected": -1.5943546295166016, "logps/chosen": -212.90521240234375, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -330.77215576171875, "loss": 0.9688, "margin_dpo/margin_mean": 106.54670715332031, "margin_dpo/margin_std": 94.70266723632812, "step": 678 }, { "KL/chosen_KL_mean": -157.34722900390625, "KL/mean": -200.56607055664062, "KL/rejected_KL_mean": -243.784912109375, "KL/std": 80.03004455566406, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.005140835419297218, "fcm_dpo/delta": -0.044562723487615585, "fcm_dpo/margin": 86.43769836425781, "fcm_dpo/q_t": 0.39592093229293823, "grad_norm": 8.565353393554688, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.4018361568450928, "logits/rejected": -1.9003572463989258, "logps/chosen": -238.12066650390625, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -326.66339111328125, "loss": 1.0333, "margin_dpo/margin_mean": 86.43769836425781, "margin_dpo/margin_std": 83.58291625976562, "step": 679 }, { "KL/chosen_KL_mean": -160.24603271484375, "KL/mean": -205.96319580078125, "KL/rejected_KL_mean": -251.6803741455078, "KL/std": 96.45393371582031, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.00513845169916749, "fcm_dpo/delta": -0.07022334635257721, "fcm_dpo/margin": 91.43434143066406, "fcm_dpo/q_t": 0.39208996295928955, "grad_norm": 7.4343414306640625, "learning_rate": 1.31753782067201e-11, "logits/chosen": -2.0971198081970215, "logits/rejected": -1.7863624095916748, "logps/chosen": -267.928955078125, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -367.7752380371094, "loss": 1.0372, "margin_dpo/margin_mean": 91.43434143066406, "margin_dpo/margin_std": 100.01477813720703, "step": 680 }, { "KL/chosen_KL_mean": -143.53192138671875, "KL/mean": -184.2694091796875, "KL/rejected_KL_mean": -225.0069122314453, "KL/std": 90.86480712890625, "epoch": 1.0, "fcm_dpo/beta": 0.005132156424224377, "fcm_dpo/delta": -0.018220867961645126, "fcm_dpo/margin": 81.4749984741211, "fcm_dpo/q_t": 0.4028432071208954, "grad_norm": 8.839402198791504, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.468421459197998, "logits/rejected": -2.171750783920288, "logps/chosen": -236.54298400878906, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -319.8291015625, "loss": 1.0704, "margin_dpo/margin_mean": 81.4749984741211, "margin_dpo/margin_std": 97.62229919433594, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0084157071449684, "train_runtime": 1976.6765, "train_samples_per_second": 22.056, "train_steps_per_second": 0.345 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }