{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.81736373901367, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.75570297241211, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.0429840087890625, "KL/mean": 0.0239119753241539, "KL/rejected_KL_mean": 0.004840850830078125, "KL/std": 0.3051183223724365, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03814421594142914, "fcm_dpo/q_t": 0.4990467131137848, "grad_norm": 25.995201110839844, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4268626570701599, "logits/rejected": 0.6501672267913818, "logps/chosen": -93.76800537109375, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.22466278076172, "loss": 1.383, "margin_dpo/margin_mean": 0.038144245743751526, "margin_dpo/margin_std": 0.45532819628715515, "step": 3 }, { "KL/chosen_KL_mean": -0.00168609619140625, "KL/mean": 0.021700620651245117, "KL/rejected_KL_mean": 0.04508209228515625, "KL/std": 0.31362205743789673, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04676848649978638, "fcm_dpo/q_t": 0.5011677742004395, "grad_norm": 32.865665435791016, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6467341780662537, "logits/rejected": 0.7781357765197754, "logps/chosen": -87.29415893554688, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.7791748046875, "loss": 1.3915, "margin_dpo/margin_mean": -0.04676815867424011, "margin_dpo/margin_std": 0.4420514702796936, "step": 4 }, { "KL/chosen_KL_mean": -0.021228790283203125, "KL/mean": 0.009770162403583527, "KL/rejected_KL_mean": 0.040767669677734375, "KL/std": 0.3097767233848572, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06199650466442108, "fcm_dpo/q_t": 0.5015479326248169, "grad_norm": 35.3111572265625, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4460833668708801, "logits/rejected": 0.6570177674293518, "logps/chosen": -89.35798645019531, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.70706176757812, "loss": 1.393, "margin_dpo/margin_mean": -0.061996251344680786, "margin_dpo/margin_std": 0.4374711811542511, "step": 5 }, { "KL/chosen_KL_mean": -0.046764373779296875, "KL/mean": -0.024320386350154877, "KL/rejected_KL_mean": -0.001873016357421875, "KL/std": 0.27654772996902466, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04489682614803314, "fcm_dpo/q_t": 0.5011206865310669, "grad_norm": 36.23640441894531, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.48675912618637085, "logits/rejected": 0.8019292950630188, "logps/chosen": -97.36824035644531, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.88532257080078, "loss": 1.3913, "margin_dpo/margin_mean": -0.044897615909576416, "margin_dpo/margin_std": 0.4138798117637634, "step": 6 }, { "KL/chosen_KL_mean": -0.03824615478515625, "KL/mean": -0.052672579884529114, "KL/rejected_KL_mean": -0.06710052490234375, "KL/std": 0.3281650245189667, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028851181268692017, "fcm_dpo/q_t": 0.4992789626121521, "grad_norm": 37.90618896484375, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4989933371543884, "logits/rejected": 0.5468893051147461, "logps/chosen": -86.68678283691406, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.68328094482422, "loss": 1.3839, "margin_dpo/margin_mean": 0.028850793838500977, "margin_dpo/margin_std": 0.4455782175064087, "step": 7 }, { "KL/chosen_KL_mean": -0.0029754638671875, "KL/mean": 0.027411311864852905, "KL/rejected_KL_mean": 0.05780029296875, "KL/std": 0.2895718216896057, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06077638268470764, "fcm_dpo/q_t": 0.5015197992324829, "grad_norm": 32.848602294921875, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4461521506309509, "logits/rejected": 0.7616853713989258, "logps/chosen": -89.9266586303711, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.17024230957031, "loss": 1.3928, "margin_dpo/margin_mean": -0.06077580153942108, "margin_dpo/margin_std": 0.40099892020225525, "step": 8 }, { "KL/chosen_KL_mean": 0.030345916748046875, "KL/mean": 0.030311882495880127, "KL/rejected_KL_mean": 0.03028106689453125, "KL/std": 0.3189079165458679, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.710124969482422e-05, "fcm_dpo/q_t": 0.4999968409538269, "grad_norm": 36.10388946533203, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7690897583961487, "logits/rejected": 0.9557182788848877, "logps/chosen": -103.8267822265625, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.2890396118164, "loss": 1.3869, "margin_dpo/margin_mean": 5.620718002319336e-05, "margin_dpo/margin_std": 0.48112696409225464, "step": 9 }, { "KL/chosen_KL_mean": -0.06256103515625, "KL/mean": 0.03435363620519638, "KL/rejected_KL_mean": 0.13126754760742188, "KL/std": 0.2609013617038727, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.19382847845554352, "fcm_dpo/q_t": 0.5048441886901855, "grad_norm": 33.84069061279297, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5644654631614685, "logits/rejected": 0.7793796062469482, "logps/chosen": -76.26692199707031, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.02083587646484, "loss": 1.4061, "margin_dpo/margin_mean": -0.19382765889167786, "margin_dpo/margin_std": 0.35793429613113403, "step": 10 }, { "KL/chosen_KL_mean": 0.052547454833984375, "KL/mean": -0.007161453366279602, "KL/rejected_KL_mean": -0.06687164306640625, "KL/std": 0.32794466614723206, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11941578984260559, "fcm_dpo/q_t": 0.497016966342926, "grad_norm": 36.622154235839844, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.13964328169822693, "logits/rejected": 0.3489128649234772, "logps/chosen": -82.31394958496094, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.33149719238281, "loss": 1.3749, "margin_dpo/margin_mean": 0.11941659450531006, "margin_dpo/margin_std": 0.4265800714492798, "step": 11 }, { "KL/chosen_KL_mean": 0.05767059326171875, "KL/mean": 0.039729565382003784, "KL/rejected_KL_mean": 0.021785736083984375, "KL/std": 0.35822319984436035, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03587988018989563, "fcm_dpo/q_t": 0.4991023540496826, "grad_norm": 40.1848258972168, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.2868014872074127, "logits/rejected": 0.4106314182281494, "logps/chosen": -99.0478286743164, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.24962615966797, "loss": 1.3834, "margin_dpo/margin_mean": 0.035880327224731445, "margin_dpo/margin_std": 0.5032839179039001, "step": 12 }, { "KL/chosen_KL_mean": 0.038806915283203125, "KL/mean": 0.024868011474609375, "KL/rejected_KL_mean": 0.010929107666015625, "KL/std": 0.36879175901412964, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027884602546691895, "fcm_dpo/q_t": 0.49930402636528015, "grad_norm": 41.7713508605957, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.4415253698825836, "logits/rejected": 0.6924610733985901, "logps/chosen": -90.52091979980469, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.68016815185547, "loss": 1.3842, "margin_dpo/margin_mean": 0.02788454294204712, "margin_dpo/margin_std": 0.5317202806472778, "step": 13 }, { "KL/chosen_KL_mean": -0.05811309814453125, "KL/mean": -0.009903013706207275, "KL/rejected_KL_mean": 0.03830718994140625, "KL/std": 0.3361932337284088, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09642308950424194, "fcm_dpo/q_t": 0.502408504486084, "grad_norm": 35.96969985961914, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.766148567199707, "logits/rejected": 0.8497700691223145, "logps/chosen": -99.88528442382812, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.90369415283203, "loss": 1.3965, "margin_dpo/margin_mean": -0.09642329812049866, "margin_dpo/margin_std": 0.4542115032672882, "step": 14 }, { "KL/chosen_KL_mean": 0.0135040283203125, "KL/mean": 0.016941040754318237, "KL/rejected_KL_mean": 0.02037811279296875, "KL/std": 0.3204549551010132, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006874561309814453, "fcm_dpo/q_t": 0.5001724362373352, "grad_norm": 31.861989974975586, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.4086008369922638, "logits/rejected": 0.5759817361831665, "logps/chosen": -78.89646911621094, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.04196166992188, "loss": 1.3875, "margin_dpo/margin_mean": -0.006874829530715942, "margin_dpo/margin_std": 0.43014830350875854, "step": 15 }, { "KL/chosen_KL_mean": 0.008098602294921875, "KL/mean": 0.0059181153774261475, "KL/rejected_KL_mean": 0.003734588623046875, "KL/std": 0.3458266258239746, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.004362642765045166, "fcm_dpo/q_t": 0.4998897910118103, "grad_norm": 34.587684631347656, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.5397837162017822, "logits/rejected": 0.7955995798110962, "logps/chosen": -97.41517639160156, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.5957260131836, "loss": 1.3864, "margin_dpo/margin_mean": 0.0043624043464660645, "margin_dpo/margin_std": 0.4671025276184082, "step": 16 }, { "KL/chosen_KL_mean": 0.0416259765625, "KL/mean": -0.010942116379737854, "KL/rejected_KL_mean": -0.06351852416992188, "KL/std": 0.2965068817138672, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10514697432518005, "fcm_dpo/q_t": 0.49737274646759033, "grad_norm": 35.329994201660156, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.2885022461414337, "logits/rejected": 0.4947429299354553, "logps/chosen": -104.32267761230469, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.53124237060547, "loss": 1.3763, "margin_dpo/margin_mean": 0.10514608025550842, "margin_dpo/margin_std": 0.4310953617095947, "step": 17 }, { "KL/chosen_KL_mean": 0.00424957275390625, "KL/mean": 0.022642701864242554, "KL/rejected_KL_mean": 0.04103851318359375, "KL/std": 0.31233111023902893, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03679424524307251, "fcm_dpo/q_t": 0.5009199380874634, "grad_norm": 41.812625885009766, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.45096883177757263, "logits/rejected": 0.8025933504104614, "logps/chosen": -87.08770751953125, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.80967712402344, "loss": 1.3905, "margin_dpo/margin_mean": -0.036793842911720276, "margin_dpo/margin_std": 0.4743492007255554, "step": 18 }, { "KL/chosen_KL_mean": -0.01335906982421875, "KL/mean": -0.0025848224759101868, "KL/rejected_KL_mean": 0.00818634033203125, "KL/std": 0.3201755881309509, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02155621349811554, "fcm_dpo/q_t": 0.5005382299423218, "grad_norm": 31.48802375793457, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.7050544023513794, "logits/rejected": 0.9873103499412537, "logps/chosen": -105.88690185546875, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.92204284667969, "loss": 1.389, "margin_dpo/margin_mean": -0.021555960178375244, "margin_dpo/margin_std": 0.470305860042572, "step": 19 }, { "KL/chosen_KL_mean": -0.0823974609375, "KL/mean": -0.046094685792922974, "KL/rejected_KL_mean": -0.009799957275390625, "KL/std": 0.3365275263786316, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07259362936019897, "fcm_dpo/q_t": 0.5018151998519897, "grad_norm": 32.87249755859375, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.36925578117370605, "logits/rejected": 0.6579221487045288, "logps/chosen": -90.84051513671875, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.922119140625, "loss": 1.3942, "margin_dpo/margin_mean": -0.07259383797645569, "margin_dpo/margin_std": 0.49379056692123413, "step": 20 }, { "KL/chosen_KL_mean": 0.06891250610351562, "KL/mean": 0.032166868448257446, "KL/rejected_KL_mean": -0.00457763671875, "KL/std": 0.24268293380737305, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07349240779876709, "fcm_dpo/q_t": 0.49816370010375977, "grad_norm": 31.46790885925293, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.7366076707839966, "logits/rejected": 0.9307318925857544, "logps/chosen": -80.26454162597656, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.93830871582031, "loss": 1.3793, "margin_dpo/margin_mean": 0.07349258661270142, "margin_dpo/margin_std": 0.3390304446220398, "step": 21 }, { "KL/chosen_KL_mean": -0.007232666015625, "KL/mean": -0.019053831696510315, "KL/rejected_KL_mean": -0.0308685302734375, "KL/std": 0.359155535697937, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02362540364265442, "fcm_dpo/q_t": 0.49941056966781616, "grad_norm": 42.98631286621094, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.8122717142105103, "logits/rejected": 0.9415315389633179, "logps/chosen": -95.40253448486328, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.50438690185547, "loss": 1.3844, "margin_dpo/margin_mean": 0.023625105619430542, "margin_dpo/margin_std": 0.4225511848926544, "step": 22 }, { "KL/chosen_KL_mean": -0.0012035369873046875, "KL/mean": 0.007942400872707367, "KL/rejected_KL_mean": 0.01709747314453125, "KL/std": 0.3183085322380066, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.018298223614692688, "fcm_dpo/q_t": 0.5004568099975586, "grad_norm": 31.47071075439453, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7392282485961914, "logits/rejected": 0.9183744192123413, "logps/chosen": -90.63871765136719, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.57716369628906, "loss": 1.3887, "margin_dpo/margin_mean": -0.018298476934432983, "margin_dpo/margin_std": 0.458950400352478, "step": 23 }, { "KL/chosen_KL_mean": -0.012697219848632812, "KL/mean": -0.017536692321300507, "KL/rejected_KL_mean": -0.02237701416015625, "KL/std": 0.32559359073638916, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.009682074189186096, "fcm_dpo/q_t": 0.4997590482234955, "grad_norm": 44.25326156616211, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.4274885058403015, "logits/rejected": 0.4615425169467926, "logps/chosen": -69.92999267578125, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.63227844238281, "loss": 1.386, "margin_dpo/margin_mean": 0.00968228280544281, "margin_dpo/margin_std": 0.519757091999054, "step": 24 }, { "KL/chosen_KL_mean": 0.0026092529296875, "KL/mean": -0.021665342152118683, "KL/rejected_KL_mean": -0.045948028564453125, "KL/std": 0.3184720277786255, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04854951798915863, "fcm_dpo/q_t": 0.49878713488578796, "grad_norm": 36.896034240722656, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.5257510542869568, "logits/rejected": 0.6927392482757568, "logps/chosen": -80.8228759765625, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.0030517578125, "loss": 1.3819, "margin_dpo/margin_mean": 0.048549264669418335, "margin_dpo/margin_std": 0.4143008887767792, "step": 25 }, { "KL/chosen_KL_mean": -0.012699127197265625, "KL/mean": -0.027202725410461426, "KL/rejected_KL_mean": -0.041698455810546875, "KL/std": 0.33099454641342163, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028999924659729004, "fcm_dpo/q_t": 0.49927517771720886, "grad_norm": 41.04677200317383, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.26985764503479004, "logits/rejected": 0.47715699672698975, "logps/chosen": -88.91387176513672, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.86988830566406, "loss": 1.3839, "margin_dpo/margin_mean": 0.02899998426437378, "margin_dpo/margin_std": 0.41472673416137695, "step": 26 }, { "KL/chosen_KL_mean": 0.023021697998046875, "KL/mean": -0.02137547731399536, "KL/rejected_KL_mean": -0.06577301025390625, "KL/std": 0.3412773013114929, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08879166841506958, "fcm_dpo/q_t": 0.4977822005748749, "grad_norm": 44.487369537353516, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.1419305056333542, "logits/rejected": 0.3295147120952606, "logps/chosen": -77.57299041748047, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.00428009033203, "loss": 1.378, "margin_dpo/margin_mean": 0.0887913703918457, "margin_dpo/margin_std": 0.4912106394767761, "step": 27 }, { "KL/chosen_KL_mean": 0.08417510986328125, "KL/mean": 0.02214685082435608, "KL/rejected_KL_mean": -0.039875030517578125, "KL/std": 0.3780503273010254, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12404787540435791, "fcm_dpo/q_t": 0.49689769744873047, "grad_norm": 36.640167236328125, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.7655789852142334, "logits/rejected": 1.0042157173156738, "logps/chosen": -102.14439392089844, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -96.99937438964844, "loss": 1.3746, "margin_dpo/margin_mean": 0.12404829263687134, "margin_dpo/margin_std": 0.532873272895813, "step": 28 }, { "KL/chosen_KL_mean": -0.006946563720703125, "KL/mean": -0.0919378399848938, "KL/rejected_KL_mean": -0.1769256591796875, "KL/std": 0.35224515199661255, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16998618841171265, "fcm_dpo/q_t": 0.495754599571228, "grad_norm": 42.139488220214844, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.47809112071990967, "logits/rejected": 0.7488316893577576, "logps/chosen": -88.65399932861328, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.13703918457031, "loss": 1.3699, "margin_dpo/margin_mean": 0.16998699307441711, "margin_dpo/margin_std": 0.46211105585098267, "step": 29 }, { "KL/chosen_KL_mean": 0.017486572265625, "KL/mean": -0.028979197144508362, "KL/rejected_KL_mean": -0.0754547119140625, "KL/std": 0.3443613648414612, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09294340014457703, "fcm_dpo/q_t": 0.4976781904697418, "grad_norm": 39.20701217651367, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.8141601085662842, "logits/rejected": 0.9385364055633545, "logps/chosen": -88.37089538574219, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.39434814453125, "loss": 1.3775, "margin_dpo/margin_mean": 0.09294372797012329, "margin_dpo/margin_std": 0.4078952670097351, "step": 30 }, { "KL/chosen_KL_mean": -0.020111083984375, "KL/mean": -0.08165115118026733, "KL/rejected_KL_mean": -0.14319229125976562, "KL/std": 0.34569865465164185, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12308007478713989, "fcm_dpo/q_t": 0.49692898988723755, "grad_norm": 31.15327262878418, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.6746906042098999, "logits/rejected": 0.9148104190826416, "logps/chosen": -101.14576721191406, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -80.00161743164062, "loss": 1.3747, "margin_dpo/margin_mean": 0.12308052182197571, "margin_dpo/margin_std": 0.5164879560470581, "step": 31 }, { "KL/chosen_KL_mean": -0.045734405517578125, "KL/mean": -0.0766584575176239, "KL/rejected_KL_mean": -0.10758590698242188, "KL/std": 0.3385852575302124, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06184875965118408, "fcm_dpo/q_t": 0.49845433235168457, "grad_norm": 35.19805908203125, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.4724675416946411, "logits/rejected": 0.616531491279602, "logps/chosen": -96.66276550292969, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.33195495605469, "loss": 1.3807, "margin_dpo/margin_mean": 0.06184963881969452, "margin_dpo/margin_std": 0.4785849153995514, "step": 32 }, { "KL/chosen_KL_mean": -0.024204254150390625, "KL/mean": -0.043385498225688934, "KL/rejected_KL_mean": -0.06256484985351562, "KL/std": 0.3216189444065094, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.038360610604286194, "fcm_dpo/q_t": 0.49904122948646545, "grad_norm": 33.91378402709961, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6332091689109802, "logits/rejected": 0.814473032951355, "logps/chosen": -81.54521942138672, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.86851501464844, "loss": 1.383, "margin_dpo/margin_mean": 0.03836128115653992, "margin_dpo/margin_std": 0.45518195629119873, "step": 33 }, { "KL/chosen_KL_mean": 0.013614654541015625, "KL/mean": -0.13107460737228394, "KL/rejected_KL_mean": -0.27576446533203125, "KL/std": 0.4426405727863312, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2893804907798767, "fcm_dpo/q_t": 0.49278783798217773, "grad_norm": 40.270137786865234, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5893830060958862, "logits/rejected": 0.762258768081665, "logps/chosen": -77.18842315673828, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.9945297241211, "loss": 1.3586, "margin_dpo/margin_mean": 0.28938043117523193, "margin_dpo/margin_std": 0.6266696453094482, "step": 34 }, { "KL/chosen_KL_mean": 0.010181427001953125, "KL/mean": -0.09747739136219025, "KL/rejected_KL_mean": -0.20513534545898438, "KL/std": 0.38029801845550537, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21531453728675842, "fcm_dpo/q_t": 0.4946216344833374, "grad_norm": 42.67320251464844, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.1892535537481308, "logits/rejected": 0.3457608222961426, "logps/chosen": -77.56016540527344, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.39369201660156, "loss": 1.3656, "margin_dpo/margin_mean": 0.21531441807746887, "margin_dpo/margin_std": 0.5249334573745728, "step": 35 }, { "KL/chosen_KL_mean": 0.0181884765625, "KL/mean": -0.08579163998365402, "KL/rejected_KL_mean": -0.1897735595703125, "KL/std": 0.33762121200561523, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20796172320842743, "fcm_dpo/q_t": 0.49480631947517395, "grad_norm": 32.49176788330078, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.3053165376186371, "logits/rejected": 0.4886474907398224, "logps/chosen": -83.27005004882812, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.94718933105469, "loss": 1.3662, "margin_dpo/margin_mean": 0.20796170830726624, "margin_dpo/margin_std": 0.4839822053909302, "step": 36 }, { "KL/chosen_KL_mean": -0.05966949462890625, "KL/mean": -0.17964279651641846, "KL/rejected_KL_mean": -0.29962158203125, "KL/std": 0.47617682814598083, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23994314670562744, "fcm_dpo/q_t": 0.49401938915252686, "grad_norm": 35.85492706298828, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.7049973011016846, "logits/rejected": 0.8692312240600586, "logps/chosen": -94.83076477050781, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01689147949219, "loss": 1.3634, "margin_dpo/margin_mean": 0.23994341492652893, "margin_dpo/margin_std": 0.6225153207778931, "step": 37 }, { "KL/chosen_KL_mean": 0.05214691162109375, "KL/mean": -0.061003245413303375, "KL/rejected_KL_mean": -0.17415237426757812, "KL/std": 0.40525829792022705, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22629781067371368, "fcm_dpo/q_t": 0.49435412883758545, "grad_norm": 36.98949432373047, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.4012713134288788, "logits/rejected": 0.5389770269393921, "logps/chosen": -75.92636108398438, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.27816772460938, "loss": 1.3647, "margin_dpo/margin_mean": 0.22629870474338531, "margin_dpo/margin_std": 0.5794901251792908, "step": 38 }, { "KL/chosen_KL_mean": -0.011280059814453125, "KL/mean": -0.15123844146728516, "KL/rejected_KL_mean": -0.29119110107421875, "KL/std": 0.3706660270690918, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2799084782600403, "fcm_dpo/q_t": 0.4930092692375183, "grad_norm": 36.20969772338867, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.36109668016433716, "logits/rejected": 0.5412145256996155, "logps/chosen": -81.19705963134766, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.48710632324219, "loss": 1.3591, "margin_dpo/margin_mean": 0.2799084186553955, "margin_dpo/margin_std": 0.49610650539398193, "step": 39 }, { "KL/chosen_KL_mean": -0.006927490234375, "KL/mean": -0.12307606637477875, "KL/rejected_KL_mean": -0.23921966552734375, "KL/std": 0.3317256271839142, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2322949469089508, "fcm_dpo/q_t": 0.4941971004009247, "grad_norm": 29.058786392211914, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.7195208072662354, "logits/rejected": 0.9235665202140808, "logps/chosen": -83.3395004272461, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.49513244628906, "loss": 1.3637, "margin_dpo/margin_mean": 0.23229458928108215, "margin_dpo/margin_std": 0.44466906785964966, "step": 40 }, { "KL/chosen_KL_mean": -0.0541839599609375, "KL/mean": -0.16184620559215546, "KL/rejected_KL_mean": -0.2695121765136719, "KL/std": 0.4260156750679016, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21533432602882385, "fcm_dpo/q_t": 0.49462252855300903, "grad_norm": 33.50916290283203, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6032605767250061, "logits/rejected": 0.7886573076248169, "logps/chosen": -93.20285034179688, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.34872436523438, "loss": 1.3658, "margin_dpo/margin_mean": 0.21533501148223877, "margin_dpo/margin_std": 0.5831628441810608, "step": 41 }, { "KL/chosen_KL_mean": 0.022747039794921875, "KL/mean": -0.18902161717414856, "KL/rejected_KL_mean": -0.4007911682128906, "KL/std": 0.5274932384490967, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.42353391647338867, "fcm_dpo/q_t": 0.4894259572029114, "grad_norm": 40.75303268432617, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.37981969118118286, "logits/rejected": 0.5216679573059082, "logps/chosen": -90.67887878417969, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.26544189453125, "loss": 1.3454, "margin_dpo/margin_mean": 0.4235345721244812, "margin_dpo/margin_std": 0.6362380981445312, "step": 42 }, { "KL/chosen_KL_mean": -0.013942718505859375, "KL/mean": -0.21507157385349274, "KL/rejected_KL_mean": -0.41620635986328125, "KL/std": 0.5018473863601685, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4022635519504547, "fcm_dpo/q_t": 0.48996204137802124, "grad_norm": 37.96099090576172, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.6489501595497131, "logits/rejected": 0.8432712554931641, "logps/chosen": -89.65797424316406, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.36805725097656, "loss": 1.3478, "margin_dpo/margin_mean": 0.40226420760154724, "margin_dpo/margin_std": 0.7206612229347229, "step": 43 }, { "KL/chosen_KL_mean": -0.033969879150390625, "KL/mean": -0.24864976108074188, "KL/rejected_KL_mean": -0.4633331298828125, "KL/std": 0.4498182535171509, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4293631613254547, "fcm_dpo/q_t": 0.4892776310443878, "grad_norm": 34.41104507446289, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.9320164918899536, "logits/rejected": 1.0505315065383911, "logps/chosen": -81.91229248046875, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.88496398925781, "loss": 1.3446, "margin_dpo/margin_mean": 0.42936423420906067, "margin_dpo/margin_std": 0.538060188293457, "step": 44 }, { "KL/chosen_KL_mean": -0.12507247924804688, "KL/mean": -0.22626550495624542, "KL/rejected_KL_mean": -0.32745361328125, "KL/std": 0.42587023973464966, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20238527655601501, "fcm_dpo/q_t": 0.4949449896812439, "grad_norm": 24.484243392944336, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.7184796929359436, "logits/rejected": 0.8319056630134583, "logps/chosen": -77.46966552734375, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.5751953125, "loss": 1.3669, "margin_dpo/margin_mean": 0.20238599181175232, "margin_dpo/margin_std": 0.5583761930465698, "step": 45 }, { "KL/chosen_KL_mean": -0.015460968017578125, "KL/mean": -0.21331781148910522, "KL/rejected_KL_mean": -0.4111747741699219, "KL/std": 0.5433715581893921, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3957180678844452, "fcm_dpo/q_t": 0.4901181757450104, "grad_norm": 33.413387298583984, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.4370655417442322, "logits/rejected": 0.6759687662124634, "logps/chosen": -90.35628509521484, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.08769226074219, "loss": 1.3479, "margin_dpo/margin_mean": 0.39571839570999146, "margin_dpo/margin_std": 0.549108624458313, "step": 46 }, { "KL/chosen_KL_mean": -0.018047332763671875, "KL/mean": -0.19171085953712463, "KL/rejected_KL_mean": -0.3653755187988281, "KL/std": 0.5312111377716064, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.34733086824417114, "fcm_dpo/q_t": 0.4913371503353119, "grad_norm": 34.13525390625, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.37992942333221436, "logits/rejected": 0.5573457479476929, "logps/chosen": -104.42935180664062, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.62026977539062, "loss": 1.3534, "margin_dpo/margin_mean": 0.3473300337791443, "margin_dpo/margin_std": 0.7703601717948914, "step": 47 }, { "KL/chosen_KL_mean": 6.866455078125e-05, "KL/mean": -0.2880919277667999, "KL/rejected_KL_mean": -0.5762481689453125, "KL/std": 0.6750929355621338, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5763165950775146, "fcm_dpo/q_t": 0.485620379447937, "grad_norm": 38.69538879394531, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.24516700208187103, "logits/rejected": 0.44690626859664917, "logps/chosen": -91.10020446777344, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.65682983398438, "loss": 1.3313, "margin_dpo/margin_mean": 0.576317310333252, "margin_dpo/margin_std": 0.8368958234786987, "step": 48 }, { "KL/chosen_KL_mean": 0.043918609619140625, "KL/mean": -0.3341946005821228, "KL/rejected_KL_mean": -0.712310791015625, "KL/std": 0.7903217673301697, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7562255859375, "fcm_dpo/q_t": 0.4811681807041168, "grad_norm": 39.50700759887695, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.03635607659816742, "logits/rejected": 0.49953192472457886, "logps/chosen": -92.95976257324219, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.4613037109375, "loss": 1.3145, "margin_dpo/margin_mean": 0.7562252879142761, "margin_dpo/margin_std": 0.9816439151763916, "step": 49 }, { "KL/chosen_KL_mean": -0.10453033447265625, "KL/mean": -0.4131060242652893, "KL/rejected_KL_mean": -0.7216720581054688, "KL/std": 0.761412501335144, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6171351671218872, "fcm_dpo/q_t": 0.4846228361129761, "grad_norm": 33.235469818115234, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.4436914622783661, "logits/rejected": 0.6105895638465881, "logps/chosen": -94.73133850097656, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.29603576660156, "loss": 1.3278, "margin_dpo/margin_mean": 0.617134690284729, "margin_dpo/margin_std": 0.9403774738311768, "step": 50 }, { "KL/chosen_KL_mean": -0.09041976928710938, "KL/mean": -0.40026333928108215, "KL/rejected_KL_mean": -0.7101020812988281, "KL/std": 0.8579465746879578, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6196815371513367, "fcm_dpo/q_t": 0.484605073928833, "grad_norm": 29.98044776916504, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5268702507019043, "logits/rejected": 0.6630069613456726, "logps/chosen": -87.59770202636719, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.18245697021484, "loss": 1.3283, "margin_dpo/margin_mean": 0.619681715965271, "margin_dpo/margin_std": 1.07266104221344, "step": 51 }, { "KL/chosen_KL_mean": -0.025142669677734375, "KL/mean": -0.4526369869709015, "KL/rejected_KL_mean": -0.8801345825195312, "KL/std": 0.7843031883239746, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8549854755401611, "fcm_dpo/q_t": 0.478704571723938, "grad_norm": 32.1417350769043, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.2880971431732178, "logits/rejected": 0.4911080598831177, "logps/chosen": -90.65541076660156, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.91919708251953, "loss": 1.3048, "margin_dpo/margin_mean": 0.8549858331680298, "margin_dpo/margin_std": 0.932863712310791, "step": 52 }, { "KL/chosen_KL_mean": -0.00116729736328125, "KL/mean": -0.5573905110359192, "KL/rejected_KL_mean": -1.1136131286621094, "KL/std": 1.153260588645935, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1124446392059326, "fcm_dpo/q_t": 0.47248655557632446, "grad_norm": 39.273155212402344, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.09592023491859436, "logits/rejected": 0.20840616524219513, "logps/chosen": -81.58424377441406, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.775146484375, "loss": 1.2836, "margin_dpo/margin_mean": 1.1124446392059326, "margin_dpo/margin_std": 1.3767818212509155, "step": 53 }, { "KL/chosen_KL_mean": -0.02259063720703125, "KL/mean": -0.5828275680541992, "KL/rejected_KL_mean": -1.14306640625, "KL/std": 1.2611310482025146, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1204702854156494, "fcm_dpo/q_t": 0.47215768694877625, "grad_norm": 39.822105407714844, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.24937774240970612, "logits/rejected": 0.4222053289413452, "logps/chosen": -88.93275451660156, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.2606201171875, "loss": 1.2814, "margin_dpo/margin_mean": 1.1204699277877808, "margin_dpo/margin_std": 1.2740484476089478, "step": 54 }, { "KL/chosen_KL_mean": 0.15224838256835938, "KL/mean": -0.44210296869277954, "KL/rejected_KL_mean": -1.0364494323730469, "KL/std": 1.1725882291793823, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1886900663375854, "fcm_dpo/q_t": 0.47060227394104004, "grad_norm": 35.23947525024414, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.07326973974704742, "logits/rejected": 0.37910196185112, "logps/chosen": -92.30367279052734, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.00738525390625, "loss": 1.2772, "margin_dpo/margin_mean": 1.1886895895004272, "margin_dpo/margin_std": 1.5347590446472168, "step": 55 }, { "KL/chosen_KL_mean": -0.08604049682617188, "KL/mean": -0.6608006954193115, "KL/rejected_KL_mean": -1.2355575561523438, "KL/std": 1.215212345123291, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.149519443511963, "fcm_dpo/q_t": 0.47162503004074097, "grad_norm": 33.11716842651367, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.14235596358776093, "logits/rejected": 0.4788201153278351, "logps/chosen": -87.32269287109375, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.881103515625, "loss": 1.2813, "margin_dpo/margin_mean": 1.1495189666748047, "margin_dpo/margin_std": 1.6299397945404053, "step": 56 }, { "KL/chosen_KL_mean": -0.052654266357421875, "KL/mean": -0.6003687381744385, "KL/rejected_KL_mean": -1.1480941772460938, "KL/std": 1.2077171802520752, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0954478979110718, "fcm_dpo/q_t": 0.4728599190711975, "grad_norm": 33.8088264465332, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.3862971067428589, "logits/rejected": 0.6066336035728455, "logps/chosen": -98.20339965820312, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.200927734375, "loss": 1.2852, "margin_dpo/margin_mean": 1.0954480171203613, "margin_dpo/margin_std": 1.4872894287109375, "step": 57 }, { "KL/chosen_KL_mean": 0.300750732421875, "KL/mean": -0.4764409065246582, "KL/rejected_KL_mean": -1.2536239624023438, "KL/std": 1.3965684175491333, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5543750524520874, "fcm_dpo/q_t": 0.46174049377441406, "grad_norm": 38.12613296508789, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.24242793023586273, "logits/rejected": 0.5693404078483582, "logps/chosen": -99.28022003173828, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.47589874267578, "loss": 1.2454, "margin_dpo/margin_mean": 1.5543756484985352, "margin_dpo/margin_std": 1.7737044095993042, "step": 58 }, { "KL/chosen_KL_mean": 0.06934356689453125, "KL/mean": -0.5653474926948547, "KL/rejected_KL_mean": -1.2000389099121094, "KL/std": 1.1898784637451172, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2693814039230347, "fcm_dpo/q_t": 0.46854496002197266, "grad_norm": 34.22159194946289, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.21858379244804382, "logits/rejected": 0.41595345735549927, "logps/chosen": -89.73298645019531, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.47671508789062, "loss": 1.2689, "margin_dpo/margin_mean": 1.2693805694580078, "margin_dpo/margin_std": 1.4770762920379639, "step": 59 }, { "KL/chosen_KL_mean": 0.07920455932617188, "KL/mean": -0.5176018476486206, "KL/rejected_KL_mean": -1.1144142150878906, "KL/std": 1.2306692600250244, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1936142444610596, "fcm_dpo/q_t": 0.47037240862846375, "grad_norm": 28.754013061523438, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.28216612339019775, "logits/rejected": 0.5433273911476135, "logps/chosen": -95.07650756835938, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.48684692382812, "loss": 1.2752, "margin_dpo/margin_mean": 1.1936142444610596, "margin_dpo/margin_std": 1.3801355361938477, "step": 60 }, { "KL/chosen_KL_mean": -0.221832275390625, "KL/mean": -0.7823347449302673, "KL/rejected_KL_mean": -1.34283447265625, "KL/std": 1.404515266418457, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.12100350856781, "fcm_dpo/q_t": 0.47246548533439636, "grad_norm": 28.303503036499023, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.26033177971839905, "logits/rejected": 0.4018956422805786, "logps/chosen": -85.95413970947266, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.30859375, "loss": 1.2858, "margin_dpo/margin_mean": 1.1210036277770996, "margin_dpo/margin_std": 1.797379493713379, "step": 61 }, { "KL/chosen_KL_mean": -0.10095977783203125, "KL/mean": -0.7198488712310791, "KL/rejected_KL_mean": -1.3387374877929688, "KL/std": 1.3630318641662598, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2377817630767822, "fcm_dpo/q_t": 0.4693794250488281, "grad_norm": 28.724411010742188, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.3473777770996094, "logits/rejected": 0.5519990921020508, "logps/chosen": -81.73634338378906, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.37704467773438, "loss": 1.2747, "margin_dpo/margin_mean": 1.2377815246582031, "margin_dpo/margin_std": 1.7339961528778076, "step": 62 }, { "KL/chosen_KL_mean": 0.006641387939453125, "KL/mean": -0.7737717628479004, "KL/rejected_KL_mean": -1.5541877746582031, "KL/std": 1.3789281845092773, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5608335733413696, "fcm_dpo/q_t": 0.46134722232818604, "grad_norm": 34.91961669921875, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.3699162006378174, "logits/rejected": 0.5832129716873169, "logps/chosen": -103.61741638183594, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.95721435546875, "loss": 1.2425, "margin_dpo/margin_mean": 1.5608333349227905, "margin_dpo/margin_std": 1.5519497394561768, "step": 63 }, { "KL/chosen_KL_mean": -0.12594223022460938, "KL/mean": -0.9803224802017212, "KL/rejected_KL_mean": -1.8347129821777344, "KL/std": 1.7988821268081665, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10398910194635391, "fcm_dpo/delta": 0.19557936489582062, "fcm_dpo/margin": 1.7087697982788086, "fcm_dpo/q_t": 0.45746004581451416, "grad_norm": 35.88642120361328, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.40213543176651, "logits/rejected": 0.6141066551208496, "logps/chosen": -87.12753295898438, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.42018127441406, "loss": 1.2289, "margin_dpo/margin_mean": 1.708769679069519, "margin_dpo/margin_std": 2.093371868133545, "step": 64 }, { "KL/chosen_KL_mean": -0.07381057739257812, "KL/mean": -0.974731981754303, "KL/rejected_KL_mean": -1.8756599426269531, "KL/std": 1.6817071437835693, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10398910194635391, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8018431663513184, "fcm_dpo/q_t": 0.45400112867355347, "grad_norm": 39.34226608276367, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.11763587594032288, "logits/rejected": 0.263107568025589, "logps/chosen": -91.29573059082031, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.21119689941406, "loss": 1.2193, "margin_dpo/margin_mean": 1.8018436431884766, "margin_dpo/margin_std": 2.081486225128174, "step": 65 }, { "KL/chosen_KL_mean": -0.3283958435058594, "KL/mean": -0.9527988433837891, "KL/rejected_KL_mean": -1.5772018432617188, "KL/std": 1.584284782409668, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10398910194635391, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2488038539886475, "fcm_dpo/q_t": 0.4681243896484375, "grad_norm": 26.40554428100586, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.017767319455742836, "logits/rejected": 0.2138039916753769, "logps/chosen": -84.11262512207031, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.32929992675781, "loss": 1.2712, "margin_dpo/margin_mean": 1.2488038539886475, "margin_dpo/margin_std": 1.9072579145431519, "step": 66 }, { "KL/chosen_KL_mean": -0.8110733032226562, "KL/mean": -1.384751796722412, "KL/rejected_KL_mean": -1.9584197998046875, "KL/std": 1.978896141052246, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10398910194635391, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1473488807678223, "fcm_dpo/q_t": 0.4711531400680542, "grad_norm": 25.214763641357422, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.4560564458370209, "logits/rejected": 0.7426882982254028, "logps/chosen": -88.48403930664062, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.045166015625, "loss": 1.2896, "margin_dpo/margin_mean": 1.1473491191864014, "margin_dpo/margin_std": 2.676466941833496, "step": 67 }, { "KL/chosen_KL_mean": -0.23154830932617188, "KL/mean": -1.319340705871582, "KL/rejected_KL_mean": -2.407123565673828, "KL/std": 2.228159189224243, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10998094081878662, "fcm_dpo/delta": 0.36930739879608154, "fcm_dpo/margin": 2.1755712032318115, "fcm_dpo/q_t": 0.44473013281822205, "grad_norm": 29.940662384033203, "learning_rate": 4.855072463768116e-07, "logits/chosen": 0.02824719436466694, "logits/rejected": 0.33314260840415955, "logps/chosen": -97.31039428710938, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.52584838867188, "loss": 1.1849, "margin_dpo/margin_mean": 2.1755709648132324, "margin_dpo/margin_std": 2.81801700592041, "step": 68 }, { "KL/chosen_KL_mean": -0.26697540283203125, "KL/mean": -1.4056921005249023, "KL/rejected_KL_mean": -2.5443992614746094, "KL/std": 2.498429298400879, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.11794446408748627, "fcm_dpo/delta": 0.3401183784008026, "fcm_dpo/margin": 2.2774252891540527, "fcm_dpo/q_t": 0.4384741187095642, "grad_norm": 33.70655822753906, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.3490777611732483, "logits/rejected": 0.4557574987411499, "logps/chosen": -85.98668670654297, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.02460479736328, "loss": 1.1687, "margin_dpo/margin_mean": 2.277426242828369, "margin_dpo/margin_std": 3.1731104850769043, "step": 69 }, { "KL/chosen_KL_mean": -0.6223678588867188, "KL/mean": -1.9097142219543457, "KL/rejected_KL_mean": -3.1970596313476562, "KL/std": 2.769139289855957, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.12519848346710205, "fcm_dpo/delta": 0.2866342067718506, "fcm_dpo/margin": 2.5746874809265137, "fcm_dpo/q_t": 0.4271574020385742, "grad_norm": 33.49689483642578, "learning_rate": 5e-07, "logits/chosen": 0.28328144550323486, "logits/rejected": 0.4618927240371704, "logps/chosen": -95.63231658935547, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.40978240966797, "loss": 1.1341, "margin_dpo/margin_mean": 2.5746872425079346, "margin_dpo/margin_std": 3.3985962867736816, "step": 70 }, { "KL/chosen_KL_mean": -0.08945083618164062, "KL/mean": -1.9268879890441895, "KL/rejected_KL_mean": -3.7643280029296875, "KL/std": 3.566713571548462, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.12898707389831543, "fcm_dpo/delta": 0.13246965408325195, "fcm_dpo/margin": 3.6748743057250977, "fcm_dpo/q_t": 0.39444321393966675, "grad_norm": 39.32881546020508, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.2629149258136749, "logits/rejected": 0.09621065855026245, "logps/chosen": -91.06680297851562, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.35536193847656, "loss": 1.0277, "margin_dpo/margin_mean": 3.6748738288879395, "margin_dpo/margin_std": 4.068259239196777, "step": 71 }, { "KL/chosen_KL_mean": -0.5234718322753906, "KL/mean": -1.9917746782302856, "KL/rejected_KL_mean": -3.4600830078125, "KL/std": 2.9352807998657227, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.13441890478134155, "fcm_dpo/delta": 0.2138877660036087, "fcm_dpo/margin": 2.9366073608398438, "fcm_dpo/q_t": 0.409924179315567, "grad_norm": 35.326316833496094, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.16893988847732544, "logits/rejected": 0.15466877818107605, "logps/chosen": -98.41499328613281, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.65179443359375, "loss": 1.077, "margin_dpo/margin_mean": 2.9366073608398438, "margin_dpo/margin_std": 3.3446874618530273, "step": 72 }, { "KL/chosen_KL_mean": -0.7640151977539062, "KL/mean": -2.3243000507354736, "KL/rejected_KL_mean": -3.88458251953125, "KL/std": 3.839069128036499, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.13968615233898163, "fcm_dpo/delta": 0.17136873304843903, "fcm_dpo/margin": 3.1205687522888184, "fcm_dpo/q_t": 0.40909385681152344, "grad_norm": 33.78772735595703, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.38651883602142334, "logits/rejected": -0.08620986342430115, "logps/chosen": -96.53303527832031, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.8170166015625, "loss": 1.0992, "margin_dpo/margin_mean": 3.12056827545166, "margin_dpo/margin_std": 4.958006858825684, "step": 73 }, { "KL/chosen_KL_mean": -1.001495361328125, "KL/mean": -2.796499729156494, "KL/rejected_KL_mean": -4.591503143310547, "KL/std": 3.8519368171691895, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.14246401190757751, "fcm_dpo/delta": 0.09342867136001587, "fcm_dpo/margin": 3.590010166168213, "fcm_dpo/q_t": 0.38916540145874023, "grad_norm": 35.971378326416016, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.11857746541500092, "logits/rejected": 0.024847006425261497, "logps/chosen": -79.80989074707031, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.23827362060547, "loss": 1.0288, "margin_dpo/margin_mean": 3.590010166168213, "margin_dpo/margin_std": 4.50029182434082, "step": 74 }, { "KL/chosen_KL_mean": -1.0830345153808594, "KL/mean": -3.3089380264282227, "KL/rejected_KL_mean": -5.534839630126953, "KL/std": 4.546594142913818, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.14337146282196045, "fcm_dpo/delta": -0.0413295142352581, "fcm_dpo/margin": 4.451813220977783, "fcm_dpo/q_t": 0.36762166023254395, "grad_norm": 36.954776763916016, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.2983923554420471, "logits/rejected": -0.09576811641454697, "logps/chosen": -79.36488342285156, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -120.93795776367188, "loss": 0.9665, "margin_dpo/margin_mean": 4.451813220977783, "margin_dpo/margin_std": 5.378294944763184, "step": 75 }, { "KL/chosen_KL_mean": -1.3871116638183594, "KL/mean": -2.960355758666992, "KL/rejected_KL_mean": -4.533603668212891, "KL/std": 3.8272085189819336, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.1455375999212265, "fcm_dpo/delta": 0.14875048398971558, "fcm_dpo/margin": 3.1464953422546387, "fcm_dpo/q_t": 0.40202730894088745, "grad_norm": 30.647092819213867, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6299934387207031, "logits/rejected": -0.2529221773147583, "logps/chosen": -89.26425170898438, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.25328063964844, "loss": 1.0767, "margin_dpo/margin_mean": 3.146495819091797, "margin_dpo/margin_std": 4.627011299133301, "step": 76 }, { "KL/chosen_KL_mean": -1.5728874206542969, "KL/mean": -3.2881603240966797, "KL/rejected_KL_mean": -5.003437042236328, "KL/std": 4.425921440124512, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.14829838275909424, "fcm_dpo/delta": 0.09625890105962753, "fcm_dpo/margin": 3.430551528930664, "fcm_dpo/q_t": 0.391254186630249, "grad_norm": 36.80995559692383, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.16492493450641632, "logits/rejected": 0.20722337067127228, "logps/chosen": -101.27323913574219, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.20919799804688, "loss": 1.0617, "margin_dpo/margin_mean": 3.4305505752563477, "margin_dpo/margin_std": 4.9861249923706055, "step": 77 }, { "KL/chosen_KL_mean": -0.6675376892089844, "KL/mean": -2.490302085876465, "KL/rejected_KL_mean": -4.3130645751953125, "KL/std": 3.663149833679199, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.15049782395362854, "fcm_dpo/delta": 0.054363761097192764, "fcm_dpo/margin": 3.645519256591797, "fcm_dpo/q_t": 0.3804710805416107, "grad_norm": 31.010622024536133, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.07048391550779343, "logits/rejected": 0.3044285178184509, "logps/chosen": -90.96424865722656, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.45079040527344, "loss": 0.9993, "margin_dpo/margin_mean": 3.645519256591797, "margin_dpo/margin_std": 4.293097496032715, "step": 78 }, { "KL/chosen_KL_mean": -0.8865165710449219, "KL/mean": -3.2527010440826416, "KL/rejected_KL_mean": -5.618888854980469, "KL/std": 5.321778297424316, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.14897465705871582, "fcm_dpo/delta": -0.11281772702932358, "fcm_dpo/margin": 4.732373237609863, "fcm_dpo/q_t": 0.35572195053100586, "grad_norm": 32.278053283691406, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6592733860015869, "logits/rejected": -0.2814685106277466, "logps/chosen": -87.26484680175781, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.72665405273438, "loss": 0.9501, "margin_dpo/margin_mean": 4.732372760772705, "margin_dpo/margin_std": 5.879494667053223, "step": 79 }, { "KL/chosen_KL_mean": -1.4747200012207031, "KL/mean": -3.3643813133239746, "KL/rejected_KL_mean": -5.254039764404297, "KL/std": 4.66335916519165, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.14855645596981049, "fcm_dpo/delta": 0.04084630310535431, "fcm_dpo/margin": 3.779317617416382, "fcm_dpo/q_t": 0.3833255469799042, "grad_norm": 33.24898147583008, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.5862987041473389, "logits/rejected": -0.22371122241020203, "logps/chosen": -95.44503784179688, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.82846069335938, "loss": 1.0183, "margin_dpo/margin_mean": 3.779317617416382, "margin_dpo/margin_std": 5.084364891052246, "step": 80 }, { "KL/chosen_KL_mean": -1.0434532165527344, "KL/mean": -3.6851096153259277, "KL/rejected_KL_mean": -6.326755523681641, "KL/std": 5.751766681671143, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.14384877681732178, "fcm_dpo/delta": -0.1759658008813858, "fcm_dpo/margin": 5.2833147048950195, "fcm_dpo/q_t": 0.3511362671852112, "grad_norm": 32.3696403503418, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5794361233711243, "logits/rejected": -0.1566147357225418, "logps/chosen": -100.87358093261719, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -139.9992218017578, "loss": 0.9362, "margin_dpo/margin_mean": 5.283315658569336, "margin_dpo/margin_std": 6.765234470367432, "step": 81 }, { "KL/chosen_KL_mean": -1.8787498474121094, "KL/mean": -3.652263641357422, "KL/rejected_KL_mean": -5.425777435302734, "KL/std": 4.429215431213379, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.14674559235572815, "fcm_dpo/delta": 0.08153703808784485, "fcm_dpo/margin": 3.547027587890625, "fcm_dpo/q_t": 0.3889966309070587, "grad_norm": 33.595848083496094, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.35512375831604004, "logits/rejected": -0.1449110507965088, "logps/chosen": -84.92472839355469, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -99.95172882080078, "loss": 1.0466, "margin_dpo/margin_mean": 3.547027349472046, "margin_dpo/margin_std": 4.853558540344238, "step": 82 }, { "KL/chosen_KL_mean": -2.202728271484375, "KL/mean": -4.258989334106445, "KL/rejected_KL_mean": -6.315250396728516, "KL/std": 4.985145568847656, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.1460040956735611, "fcm_dpo/delta": -0.000477764755487442, "fcm_dpo/margin": 4.112531661987305, "fcm_dpo/q_t": 0.3798926770687103, "grad_norm": 39.30076217651367, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6640176773071289, "logits/rejected": -0.2515318989753723, "logps/chosen": -94.37894439697266, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.05989074707031, "loss": 1.0436, "margin_dpo/margin_mean": 4.112531661987305, "margin_dpo/margin_std": 6.234769344329834, "step": 83 }, { "KL/chosen_KL_mean": -2.7712554931640625, "KL/mean": -4.069101810455322, "KL/rejected_KL_mean": -5.366954803466797, "KL/std": 4.4830522537231445, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.14601179957389832, "fcm_dpo/delta": -0.0013735861284658313, "fcm_dpo/margin": 2.5956978797912598, "fcm_dpo/q_t": 0.42114800214767456, "grad_norm": 42.22005844116211, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.4905933737754822, "logits/rejected": -0.11474094539880753, "logps/chosen": -104.29774475097656, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.79304504394531, "loss": 1.1627, "margin_dpo/margin_mean": 2.5956976413726807, "margin_dpo/margin_std": 4.861915588378906, "step": 84 }, { "KL/chosen_KL_mean": -2.394451141357422, "KL/mean": -4.0943098068237305, "KL/rejected_KL_mean": -5.794170379638672, "KL/std": 4.760000705718994, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.14898881316184998, "fcm_dpo/delta": 0.09723717719316483, "fcm_dpo/margin": 3.399724006652832, "fcm_dpo/q_t": 0.3960205912590027, "grad_norm": 41.77076721191406, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.2942071855068207, "logits/rejected": 0.0344044528901577, "logps/chosen": -98.5218276977539, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -90.84935760498047, "loss": 1.108, "margin_dpo/margin_mean": 3.3997249603271484, "margin_dpo/margin_std": 5.840973854064941, "step": 85 }, { "KL/chosen_KL_mean": -1.9143333435058594, "KL/mean": -4.051482200622559, "KL/rejected_KL_mean": -6.188636779785156, "KL/std": 5.652594089508057, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.14784696698188782, "fcm_dpo/delta": -0.034234561026096344, "fcm_dpo/margin": 4.2743024826049805, "fcm_dpo/q_t": 0.38175708055496216, "grad_norm": 35.01856231689453, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.4509730935096741, "logits/rejected": -0.20765888690948486, "logps/chosen": -83.61859130859375, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -99.84413146972656, "loss": 1.0411, "margin_dpo/margin_mean": 4.274302959442139, "margin_dpo/margin_std": 6.953662872314453, "step": 86 }, { "KL/chosen_KL_mean": -2.17059326171875, "KL/mean": -4.488838195800781, "KL/rejected_KL_mean": -6.807079315185547, "KL/std": 6.174598217010498, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.14556774497032166, "fcm_dpo/delta": -0.08091689646244049, "fcm_dpo/margin": 4.636486053466797, "fcm_dpo/q_t": 0.36331605911254883, "grad_norm": 38.63408279418945, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.7568728923797607, "logits/rejected": -0.38827812671661377, "logps/chosen": -94.85655975341797, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -109.72525787353516, "loss": 1.0408, "margin_dpo/margin_mean": 4.636485576629639, "margin_dpo/margin_std": 7.2978715896606445, "step": 87 }, { "KL/chosen_KL_mean": -2.600189208984375, "KL/mean": -4.453143119812012, "KL/rejected_KL_mean": -6.306098937988281, "KL/std": 5.853734016418457, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.1455577164888382, "fcm_dpo/delta": 0.0629434809088707, "fcm_dpo/margin": 3.705901622772217, "fcm_dpo/q_t": 0.39113444089889526, "grad_norm": 35.67405700683594, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.31905582547187805, "logits/rejected": -0.1384592354297638, "logps/chosen": -103.36317443847656, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.45646667480469, "loss": 1.088, "margin_dpo/margin_mean": 3.7059011459350586, "margin_dpo/margin_std": 6.097817897796631, "step": 88 }, { "KL/chosen_KL_mean": -2.1470680236816406, "KL/mean": -4.114473819732666, "KL/rejected_KL_mean": -6.081878662109375, "KL/std": 4.958282470703125, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.14729197323322296, "fcm_dpo/delta": 0.02151663228869438, "fcm_dpo/margin": 3.9348068237304688, "fcm_dpo/q_t": 0.38829731941223145, "grad_norm": 30.834142684936523, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7676856517791748, "logits/rejected": -0.445925235748291, "logps/chosen": -91.83815002441406, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -94.65020751953125, "loss": 1.056, "margin_dpo/margin_mean": 3.9348068237304688, "margin_dpo/margin_std": 6.584025859832764, "step": 89 }, { "KL/chosen_KL_mean": -1.8543319702148438, "KL/mean": -4.38532018661499, "KL/rejected_KL_mean": -6.916313171386719, "KL/std": 5.726137161254883, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.14557525515556335, "fcm_dpo/delta": -0.1476038694381714, "fcm_dpo/margin": 5.061980724334717, "fcm_dpo/q_t": 0.3551107347011566, "grad_norm": 32.797122955322266, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.04606659337878227, "logits/rejected": 0.06433600187301636, "logps/chosen": -83.23689270019531, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -124.69345092773438, "loss": 0.9655, "margin_dpo/margin_mean": 5.061980724334717, "margin_dpo/margin_std": 6.787753105163574, "step": 90 }, { "KL/chosen_KL_mean": -2.0451126098632812, "KL/mean": -4.273472785949707, "KL/rejected_KL_mean": -6.5018310546875, "KL/std": 5.67744255065918, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.1430130898952484, "fcm_dpo/delta": -0.03989646956324577, "fcm_dpo/margin": 4.456713676452637, "fcm_dpo/q_t": 0.3754676580429077, "grad_norm": 36.30229949951172, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.2980725169181824, "logits/rejected": 0.10939715802669525, "logps/chosen": -99.26699829101562, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.47857666015625, "loss": 1.0153, "margin_dpo/margin_mean": 4.456713676452637, "margin_dpo/margin_std": 6.539417743682861, "step": 91 }, { "KL/chosen_KL_mean": -2.394519805908203, "KL/mean": -4.274661540985107, "KL/rejected_KL_mean": -6.1548004150390625, "KL/std": 5.552699565887451, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.14409781992435455, "fcm_dpo/delta": 0.06129460409283638, "fcm_dpo/margin": 3.7602791786193848, "fcm_dpo/q_t": 0.3905436396598816, "grad_norm": 30.68474578857422, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8639460802078247, "logits/rejected": -0.514735221862793, "logps/chosen": -86.92479705810547, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -90.65612030029297, "loss": 1.0662, "margin_dpo/margin_mean": 3.760279893875122, "margin_dpo/margin_std": 5.965705871582031, "step": 92 }, { "KL/chosen_KL_mean": -2.4781723022460938, "KL/mean": -4.365277290344238, "KL/rejected_KL_mean": -6.252391815185547, "KL/std": 5.254633903503418, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.14575953781604767, "fcm_dpo/delta": 0.052529968321323395, "fcm_dpo/margin": 3.7742233276367188, "fcm_dpo/q_t": 0.39038142561912537, "grad_norm": 38.95704650878906, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.9231927394866943, "logits/rejected": -0.37570643424987793, "logps/chosen": -92.72860717773438, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.34660339355469, "loss": 1.0974, "margin_dpo/margin_mean": 3.7742233276367188, "margin_dpo/margin_std": 6.463961601257324, "step": 93 }, { "KL/chosen_KL_mean": -2.5483627319335938, "KL/mean": -5.254777908325195, "KL/rejected_KL_mean": -7.961193084716797, "KL/std": 6.014457702636719, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.14034628868103027, "fcm_dpo/delta": -0.17781278491020203, "fcm_dpo/margin": 5.412829399108887, "fcm_dpo/q_t": 0.3461531400680542, "grad_norm": 34.74405288696289, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.5805813074111938, "logits/rejected": -0.28810930252075195, "logps/chosen": -95.00257873535156, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -113.20848083496094, "loss": 0.9571, "margin_dpo/margin_mean": 5.412829399108887, "margin_dpo/margin_std": 6.8721818923950195, "step": 94 }, { "KL/chosen_KL_mean": -2.086383819580078, "KL/mean": -4.222187519073486, "KL/rejected_KL_mean": -6.357982635498047, "KL/std": 5.063711166381836, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.14150778949260712, "fcm_dpo/delta": -0.0055768415331840515, "fcm_dpo/margin": 4.271595001220703, "fcm_dpo/q_t": 0.3803215026855469, "grad_norm": 30.210119247436523, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.7056533098220825, "logits/rejected": -0.3756963312625885, "logps/chosen": -80.96009063720703, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -89.9491958618164, "loss": 1.0303, "margin_dpo/margin_mean": 4.271595478057861, "margin_dpo/margin_std": 6.217037200927734, "step": 95 }, { "KL/chosen_KL_mean": -1.5491294860839844, "KL/mean": -4.247531890869141, "KL/rejected_KL_mean": -6.9459381103515625, "KL/std": 6.148512840270996, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.1370888501405716, "fcm_dpo/delta": -0.1515318751335144, "fcm_dpo/margin": 5.39681339263916, "fcm_dpo/q_t": 0.3518095910549164, "grad_norm": 29.040117263793945, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.48820096254348755, "logits/rejected": -0.0480177067220211, "logps/chosen": -108.14129638671875, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -108.7139663696289, "loss": 0.9484, "margin_dpo/margin_mean": 5.396812915802002, "margin_dpo/margin_std": 6.800342082977295, "step": 96 }, { "KL/chosen_KL_mean": -2.6099014282226562, "KL/mean": -5.025163650512695, "KL/rejected_KL_mean": -7.440425872802734, "KL/std": 5.185455322265625, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.1363273710012436, "fcm_dpo/delta": -0.06381455063819885, "fcm_dpo/margin": 4.830530166625977, "fcm_dpo/q_t": 0.3612979054450989, "grad_norm": 32.90639877319336, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.07437529414892197, "logits/rejected": 0.3483273684978485, "logps/chosen": -91.28977966308594, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -92.25271606445312, "loss": 0.9552, "margin_dpo/margin_mean": 4.830530166625977, "margin_dpo/margin_std": 5.4903364181518555, "step": 97 }, { "KL/chosen_KL_mean": -1.8503799438476562, "KL/mean": -4.536694049835205, "KL/rejected_KL_mean": -7.2230072021484375, "KL/std": 5.083706855773926, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.13142412900924683, "fcm_dpo/delta": -0.11594255268573761, "fcm_dpo/margin": 5.372622489929199, "fcm_dpo/q_t": 0.3508375883102417, "grad_norm": 29.940095901489258, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.42178407311439514, "logits/rejected": -0.16275888681411743, "logps/chosen": -94.09288024902344, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -108.74249267578125, "loss": 0.9183, "margin_dpo/margin_mean": 5.372622489929199, "margin_dpo/margin_std": 5.780214309692383, "step": 98 }, { "KL/chosen_KL_mean": -1.4149742126464844, "KL/mean": -4.2843708992004395, "KL/rejected_KL_mean": -7.153766632080078, "KL/std": 5.895498275756836, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.12860321998596191, "fcm_dpo/delta": -0.14911258220672607, "fcm_dpo/margin": 5.738797664642334, "fcm_dpo/q_t": 0.35599392652511597, "grad_norm": 27.8778076171875, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.44168800115585327, "logits/rejected": -0.2307979017496109, "logps/chosen": -73.59962463378906, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -99.03508758544922, "loss": 0.9387, "margin_dpo/margin_mean": 5.738797187805176, "margin_dpo/margin_std": 7.4234113693237305, "step": 99 }, { "KL/chosen_KL_mean": -1.636871337890625, "KL/mean": -3.8397412300109863, "KL/rejected_KL_mean": -6.042610168457031, "KL/std": 4.669720649719238, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.1278403401374817, "fcm_dpo/delta": 0.03872145712375641, "fcm_dpo/margin": 4.405738353729248, "fcm_dpo/q_t": 0.37727057933807373, "grad_norm": 26.422075271606445, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.5715891122817993, "logits/rejected": -0.10611826926469803, "logps/chosen": -91.15069580078125, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.25973510742188, "loss": 0.9879, "margin_dpo/margin_mean": 4.40573787689209, "margin_dpo/margin_std": 4.997339725494385, "step": 100 }, { "KL/chosen_KL_mean": -2.70819091796875, "KL/mean": -4.613160610198975, "KL/rejected_KL_mean": -6.51812744140625, "KL/std": 5.655867576599121, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.1301465630531311, "fcm_dpo/delta": 0.10962408781051636, "fcm_dpo/margin": 3.809932231903076, "fcm_dpo/q_t": 0.39603933691978455, "grad_norm": 34.171875, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.4689900577068329, "logits/rejected": -0.2007327675819397, "logps/chosen": -97.11138916015625, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -96.42814636230469, "loss": 1.097, "margin_dpo/margin_mean": 3.8099327087402344, "margin_dpo/margin_std": 6.267298221588135, "step": 101 }, { "KL/chosen_KL_mean": -3.2166175842285156, "KL/mean": -4.862425804138184, "KL/rejected_KL_mean": -6.508235931396484, "KL/std": 5.459533214569092, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.13622941076755524, "fcm_dpo/delta": 0.15372905135154724, "fcm_dpo/margin": 3.2916202545166016, "fcm_dpo/q_t": 0.40477102994918823, "grad_norm": 44.855220794677734, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.6039717793464661, "logits/rejected": -0.24568969011306763, "logps/chosen": -94.82604217529297, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -82.27716064453125, "loss": 1.1424, "margin_dpo/margin_mean": 3.2916207313537598, "margin_dpo/margin_std": 6.101104736328125, "step": 102 }, { "KL/chosen_KL_mean": -2.246448516845703, "KL/mean": -4.989956855773926, "KL/rejected_KL_mean": -7.733463287353516, "KL/std": 5.707414627075195, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.1326180398464203, "fcm_dpo/delta": -0.13805840909481049, "fcm_dpo/margin": 5.487015247344971, "fcm_dpo/q_t": 0.3472788333892822, "grad_norm": 32.21012496948242, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6339147090911865, "logits/rejected": -0.26732516288757324, "logps/chosen": -99.64883422851562, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -106.71514892578125, "loss": 0.9375, "margin_dpo/margin_mean": 5.487016201019287, "margin_dpo/margin_std": 6.3958659172058105, "step": 103 }, { "KL/chosen_KL_mean": -1.6147270202636719, "KL/mean": -4.529810428619385, "KL/rejected_KL_mean": -7.444889068603516, "KL/std": 6.292789459228516, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.12865546345710754, "fcm_dpo/delta": -0.1624649465084076, "fcm_dpo/margin": 5.83016300201416, "fcm_dpo/q_t": 0.3512665927410126, "grad_norm": 25.952098846435547, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.83022141456604, "logits/rejected": -0.5089714527130127, "logps/chosen": -96.47108459472656, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -104.47158813476562, "loss": 0.929, "margin_dpo/margin_mean": 5.83016300201416, "margin_dpo/margin_std": 7.357025146484375, "step": 104 }, { "KL/chosen_KL_mean": -2.191974639892578, "KL/mean": -5.047431945800781, "KL/rejected_KL_mean": -7.902889251708984, "KL/std": 5.6106157302856445, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.12602867186069489, "fcm_dpo/delta": -0.12896713614463806, "fcm_dpo/margin": 5.710913181304932, "fcm_dpo/q_t": 0.35062211751937866, "grad_norm": 30.053913116455078, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.5322980880737305, "logits/rejected": -0.06503552943468094, "logps/chosen": -98.94987487792969, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -109.48037719726562, "loss": 0.9494, "margin_dpo/margin_mean": 5.710913181304932, "margin_dpo/margin_std": 6.986258506774902, "step": 105 }, { "KL/chosen_KL_mean": -2.204883575439453, "KL/mean": -4.995815277099609, "KL/rejected_KL_mean": -7.786746978759766, "KL/std": 5.861738204956055, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.1233520358800888, "fcm_dpo/delta": -0.09513126313686371, "fcm_dpo/margin": 5.581860065460205, "fcm_dpo/q_t": 0.35984283685684204, "grad_norm": 27.433122634887695, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.6240434646606445, "logits/rejected": -0.21578462421894073, "logps/chosen": -94.64279174804688, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -98.79644775390625, "loss": 0.9556, "margin_dpo/margin_mean": 5.581859588623047, "margin_dpo/margin_std": 6.928107261657715, "step": 106 }, { "KL/chosen_KL_mean": -1.361480712890625, "KL/mean": -4.547911643981934, "KL/rejected_KL_mean": -7.734344482421875, "KL/std": 7.367633819580078, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.1190643459558487, "fcm_dpo/delta": -0.17180728912353516, "fcm_dpo/margin": 6.372864723205566, "fcm_dpo/q_t": 0.3556305170059204, "grad_norm": 27.170469284057617, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.3930664360523224, "logits/rejected": 0.11366605013608932, "logps/chosen": -106.48648834228516, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -108.91683959960938, "loss": 0.9478, "margin_dpo/margin_mean": 6.372865200042725, "margin_dpo/margin_std": 9.144984245300293, "step": 107 }, { "KL/chosen_KL_mean": -2.3557205200195312, "KL/mean": -5.231471538543701, "KL/rejected_KL_mean": -8.107227325439453, "KL/std": 6.428241729736328, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.1160096824169159, "fcm_dpo/delta": -0.07325372099876404, "fcm_dpo/margin": 5.75150203704834, "fcm_dpo/q_t": 0.3620080351829529, "grad_norm": 25.65009307861328, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.7896138429641724, "logits/rejected": -0.2946704626083374, "logps/chosen": -91.56507873535156, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -109.94507598876953, "loss": 0.9643, "margin_dpo/margin_mean": 5.751502513885498, "margin_dpo/margin_std": 7.012262344360352, "step": 108 }, { "KL/chosen_KL_mean": -1.6381912231445312, "KL/mean": -5.269526481628418, "KL/rejected_KL_mean": -8.900867462158203, "KL/std": 6.81550931930542, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.11202439665794373, "fcm_dpo/delta": -0.23333078622817993, "fcm_dpo/margin": 7.262670040130615, "fcm_dpo/q_t": 0.33495068550109863, "grad_norm": 25.731616973876953, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.4038838744163513, "logits/rejected": -0.19465038180351257, "logps/chosen": -101.82527923583984, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -112.43826293945312, "loss": 0.8913, "margin_dpo/margin_mean": 7.262670516967773, "margin_dpo/margin_std": 8.243940353393555, "step": 109 }, { "KL/chosen_KL_mean": -2.2047767639160156, "KL/mean": -4.480343818664551, "KL/rejected_KL_mean": -6.755912780761719, "KL/std": 5.437847137451172, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.11195176839828491, "fcm_dpo/delta": 0.09525752067565918, "fcm_dpo/margin": 4.551135540008545, "fcm_dpo/q_t": 0.39022907614707947, "grad_norm": 26.867969512939453, "learning_rate": 4.947482930773511e-07, "logits/chosen": 0.02003449946641922, "logits/rejected": 0.4021705687046051, "logps/chosen": -109.10049438476562, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -92.5671157836914, "loss": 1.0479, "margin_dpo/margin_mean": 4.551135063171387, "margin_dpo/margin_std": 6.17197322845459, "step": 110 }, { "KL/chosen_KL_mean": -1.9212875366210938, "KL/mean": -6.034536361694336, "KL/rejected_KL_mean": -10.147785186767578, "KL/std": 7.634692192077637, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.10707098245620728, "fcm_dpo/delta": -0.31229496002197266, "fcm_dpo/margin": 8.226493835449219, "fcm_dpo/q_t": 0.31856870651245117, "grad_norm": 23.36945152282715, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.8248056173324585, "logits/rejected": -0.556470513343811, "logps/chosen": -95.25065612792969, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -114.32527160644531, "loss": 0.8392, "margin_dpo/margin_mean": 8.226493835449219, "margin_dpo/margin_std": 8.070582389831543, "step": 111 }, { "KL/chosen_KL_mean": -2.1901168823242188, "KL/mean": -4.424112319946289, "KL/rejected_KL_mean": -6.658107757568359, "KL/std": 4.442264080047607, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.10814262926578522, "fcm_dpo/delta": 0.12283194065093994, "fcm_dpo/margin": 4.467988967895508, "fcm_dpo/q_t": 0.3939743638038635, "grad_norm": 24.414445877075195, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.6989029049873352, "logits/rejected": -0.22060739994049072, "logps/chosen": -84.93716430664062, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -78.216552734375, "loss": 1.0485, "margin_dpo/margin_mean": 4.467988967895508, "margin_dpo/margin_std": 5.79315710067749, "step": 112 }, { "KL/chosen_KL_mean": -2.810749053955078, "KL/mean": -5.170855522155762, "KL/rejected_KL_mean": -7.530963897705078, "KL/std": 5.744589805603027, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.11032646149396896, "fcm_dpo/delta": 0.08353646099567413, "fcm_dpo/margin": 4.720218658447266, "fcm_dpo/q_t": 0.3856438100337982, "grad_norm": 28.410852432250977, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.44468188285827637, "logits/rejected": 0.07534168660640717, "logps/chosen": -100.71024322509766, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -91.9505615234375, "loss": 1.0228, "margin_dpo/margin_mean": 4.720218658447266, "margin_dpo/margin_std": 5.579396724700928, "step": 113 }, { "KL/chosen_KL_mean": -2.540294647216797, "KL/mean": -5.728448390960693, "KL/rejected_KL_mean": -8.916603088378906, "KL/std": 6.713683128356934, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.10852333903312683, "fcm_dpo/delta": -0.0999608114361763, "fcm_dpo/margin": 6.376307010650635, "fcm_dpo/q_t": 0.3622761368751526, "grad_norm": 28.150697708129883, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.6871543526649475, "logits/rejected": -0.1303718388080597, "logps/chosen": -99.03541564941406, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -100.93196105957031, "loss": 0.9768, "margin_dpo/margin_mean": 6.376308441162109, "margin_dpo/margin_std": 8.409533500671387, "step": 114 }, { "KL/chosen_KL_mean": -2.4351577758789062, "KL/mean": -5.6857500076293945, "KL/rejected_KL_mean": -8.93634033203125, "KL/std": 7.035308837890625, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.10698038339614868, "fcm_dpo/delta": -0.10257962346076965, "fcm_dpo/margin": 6.501177787780762, "fcm_dpo/q_t": 0.35427141189575195, "grad_norm": 24.070714950561523, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.6683632135391235, "logits/rejected": -0.288242906332016, "logps/chosen": -105.61528015136719, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -114.65229797363281, "loss": 0.9372, "margin_dpo/margin_mean": 6.501177787780762, "margin_dpo/margin_std": 7.43740177154541, "step": 115 }, { "KL/chosen_KL_mean": -2.2283477783203125, "KL/mean": -4.902841091156006, "KL/rejected_KL_mean": -7.577335357666016, "KL/std": 5.408741474151611, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.10711782425642014, "fcm_dpo/delta": 0.028394395485520363, "fcm_dpo/margin": 5.3489885330200195, "fcm_dpo/q_t": 0.376577228307724, "grad_norm": 23.035079956054688, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.7781617641448975, "logits/rejected": -0.3684685528278351, "logps/chosen": -83.38671875, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -93.88319396972656, "loss": 0.9944, "margin_dpo/margin_mean": 5.3489885330200195, "margin_dpo/margin_std": 6.345819473266602, "step": 116 }, { "KL/chosen_KL_mean": -2.7900657653808594, "KL/mean": -5.169136047363281, "KL/rejected_KL_mean": -7.548206329345703, "KL/std": 5.520646572113037, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.10758466273546219, "fcm_dpo/delta": 0.09230232238769531, "fcm_dpo/margin": 4.758138656616211, "fcm_dpo/q_t": 0.3887106478214264, "grad_norm": 31.57113265991211, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.35456207394599915, "logits/rejected": 0.02986607700586319, "logps/chosen": -99.87101745605469, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -82.68812561035156, "loss": 1.047, "margin_dpo/margin_mean": 4.758138179779053, "margin_dpo/margin_std": 6.36179780960083, "step": 117 }, { "KL/chosen_KL_mean": -1.8193817138671875, "KL/mean": -4.924945831298828, "KL/rejected_KL_mean": -8.030502319335938, "KL/std": 6.023618698120117, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.10761499404907227, "fcm_dpo/delta": -0.07344862818717957, "fcm_dpo/margin": 6.211120128631592, "fcm_dpo/q_t": 0.35277289152145386, "grad_norm": 22.828210830688477, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.7473567128181458, "logits/rejected": -0.42740458250045776, "logps/chosen": -88.35562133789062, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -90.96614074707031, "loss": 0.9252, "margin_dpo/margin_mean": 6.21112060546875, "margin_dpo/margin_std": 6.302579879760742, "step": 118 }, { "KL/chosen_KL_mean": -2.7917823791503906, "KL/mean": -5.968376159667969, "KL/rejected_KL_mean": -9.144966125488281, "KL/std": 6.907724380493164, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.1058032438158989, "fcm_dpo/delta": -0.07781042158603668, "fcm_dpo/margin": 6.353182315826416, "fcm_dpo/q_t": 0.35819727182388306, "grad_norm": 27.12972068786621, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.6882138848304749, "logits/rejected": -0.3870362639427185, "logps/chosen": -107.40086364746094, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -118.30278015136719, "loss": 0.9884, "margin_dpo/margin_mean": 6.353182792663574, "margin_dpo/margin_std": 8.452669143676758, "step": 119 }, { "KL/chosen_KL_mean": -1.8634719848632812, "KL/mean": -6.185707092285156, "KL/rejected_KL_mean": -10.507949829101562, "KL/std": 8.631650924682617, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.10100238025188446, "fcm_dpo/delta": -0.29964786767959595, "fcm_dpo/margin": 8.644479751586914, "fcm_dpo/q_t": 0.3305753469467163, "grad_norm": 30.252201080322266, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.8595898747444153, "logits/rejected": -0.33414605259895325, "logps/chosen": -116.5509033203125, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -121.69381713867188, "loss": 0.911, "margin_dpo/margin_mean": 8.644479751586914, "margin_dpo/margin_std": 10.678121566772461, "step": 120 }, { "KL/chosen_KL_mean": -1.7956924438476562, "KL/mean": -5.532514572143555, "KL/rejected_KL_mean": -9.269340515136719, "KL/std": 6.729030609130859, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.09713802486658096, "fcm_dpo/delta": -0.13679131865501404, "fcm_dpo/margin": 7.4736552238464355, "fcm_dpo/q_t": 0.34480053186416626, "grad_norm": 25.91554832458496, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.7682168483734131, "logits/rejected": -0.4566764831542969, "logps/chosen": -93.912353515625, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -124.24224090576172, "loss": 0.895, "margin_dpo/margin_mean": 7.473655700683594, "margin_dpo/margin_std": 7.514223098754883, "step": 121 }, { "KL/chosen_KL_mean": -2.160747528076172, "KL/mean": -6.320587158203125, "KL/rejected_KL_mean": -10.480419158935547, "KL/std": 7.204160690307617, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.09443425387144089, "fcm_dpo/delta": -0.2010134607553482, "fcm_dpo/margin": 8.319670677185059, "fcm_dpo/q_t": 0.33161669969558716, "grad_norm": 26.28276824951172, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.5749606490135193, "logits/rejected": -0.4523996114730835, "logps/chosen": -87.03831481933594, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -121.56340026855469, "loss": 0.8731, "margin_dpo/margin_mean": 8.319669723510742, "margin_dpo/margin_std": 8.144250869750977, "step": 122 }, { "KL/chosen_KL_mean": -2.340831756591797, "KL/mean": -5.544158935546875, "KL/rejected_KL_mean": -8.747486114501953, "KL/std": 6.654641151428223, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.09238104522228241, "fcm_dpo/delta": 0.007309213280677795, "fcm_dpo/margin": 6.406643867492676, "fcm_dpo/q_t": 0.37154078483581543, "grad_norm": 22.474430084228516, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.9172804355621338, "logits/rejected": -0.6286522150039673, "logps/chosen": -100.819580078125, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -102.6016616821289, "loss": 0.9842, "margin_dpo/margin_mean": 6.406643867492676, "margin_dpo/margin_std": 7.36826229095459, "step": 123 }, { "KL/chosen_KL_mean": -2.1043243408203125, "KL/mean": -5.859222412109375, "KL/rejected_KL_mean": -9.614116668701172, "KL/std": 8.371244430541992, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.09195709973573685, "fcm_dpo/delta": -0.09721198678016663, "fcm_dpo/margin": 7.509791374206543, "fcm_dpo/q_t": 0.3666444420814514, "grad_norm": 22.93729019165039, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.6400359869003296, "logits/rejected": -0.38579949736595154, "logps/chosen": -104.86722564697266, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -132.61973571777344, "loss": 0.9899, "margin_dpo/margin_mean": 7.509791374206543, "margin_dpo/margin_std": 10.680034637451172, "step": 124 }, { "KL/chosen_KL_mean": -3.4460372924804688, "KL/mean": -6.390720367431641, "KL/rejected_KL_mean": -9.335411071777344, "KL/std": 6.773427963256836, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.09219115972518921, "fcm_dpo/delta": 0.06033901125192642, "fcm_dpo/margin": 5.88936710357666, "fcm_dpo/q_t": 0.3844181299209595, "grad_norm": 24.04238510131836, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.5515244603157043, "logits/rejected": -0.35779067873954773, "logps/chosen": -93.872314453125, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -98.58442687988281, "loss": 1.0352, "margin_dpo/margin_mean": 5.889368057250977, "margin_dpo/margin_std": 8.061115264892578, "step": 125 }, { "KL/chosen_KL_mean": -2.4354095458984375, "KL/mean": -6.522261619567871, "KL/rejected_KL_mean": -10.609119415283203, "KL/std": 7.057982444763184, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.09053629636764526, "fcm_dpo/delta": -0.15108484029769897, "fcm_dpo/margin": 8.173712730407715, "fcm_dpo/q_t": 0.33646827936172485, "grad_norm": 22.900821685791016, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.44258996844291687, "logits/rejected": -0.17565295100212097, "logps/chosen": -96.85829162597656, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -114.49067687988281, "loss": 0.8677, "margin_dpo/margin_mean": 8.173712730407715, "margin_dpo/margin_std": 7.159849166870117, "step": 126 }, { "KL/chosen_KL_mean": -2.452777862548828, "KL/mean": -5.685831069946289, "KL/rejected_KL_mean": -8.918888092041016, "KL/std": 6.876796722412109, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.09043458104133606, "fcm_dpo/delta": 0.01550811156630516, "fcm_dpo/margin": 6.466109275817871, "fcm_dpo/q_t": 0.3760107159614563, "grad_norm": 23.794109344482422, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.5466611385345459, "logits/rejected": -0.22713801264762878, "logps/chosen": -87.20382690429688, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -92.46412658691406, "loss": 1.0103, "margin_dpo/margin_mean": 6.466108798980713, "margin_dpo/margin_std": 8.457048416137695, "step": 127 }, { "KL/chosen_KL_mean": -2.4083290100097656, "KL/mean": -5.691444396972656, "KL/rejected_KL_mean": -8.974557876586914, "KL/std": 8.0487642288208, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.09047839045524597, "fcm_dpo/delta": 0.005884464830160141, "fcm_dpo/margin": 6.566227436065674, "fcm_dpo/q_t": 0.3707578182220459, "grad_norm": 25.711618423461914, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.46138137578964233, "logits/rejected": -0.037178028374910355, "logps/chosen": -93.02796936035156, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -92.07803344726562, "loss": 1.0225, "margin_dpo/margin_mean": 6.566226959228516, "margin_dpo/margin_std": 9.264544486999512, "step": 128 }, { "KL/chosen_KL_mean": -2.8266448974609375, "KL/mean": -6.170069217681885, "KL/rejected_KL_mean": -9.513504028320312, "KL/std": 6.743484973907471, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.09047134220600128, "fcm_dpo/delta": -0.005708225071430206, "fcm_dpo/margin": 6.686853885650635, "fcm_dpo/q_t": 0.3746364116668701, "grad_norm": 21.522624969482422, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.33037829399108887, "logits/rejected": -0.03833659365773201, "logps/chosen": -101.74535369873047, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -112.79769897460938, "loss": 0.9987, "margin_dpo/margin_mean": 6.686853408813477, "margin_dpo/margin_std": 8.436392784118652, "step": 129 }, { "KL/chosen_KL_mean": -2.317657470703125, "KL/mean": -6.590223789215088, "KL/rejected_KL_mean": -10.86279296875, "KL/std": 8.309723854064941, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.08868275582790375, "fcm_dpo/delta": -0.17114917933940887, "fcm_dpo/margin": 8.545134544372559, "fcm_dpo/q_t": 0.345961332321167, "grad_norm": 20.89288330078125, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.669031023979187, "logits/rejected": -0.4136059284210205, "logps/chosen": -75.5013427734375, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -98.88433837890625, "loss": 0.9132, "margin_dpo/margin_mean": 8.545133590698242, "margin_dpo/margin_std": 9.452320098876953, "step": 130 }, { "KL/chosen_KL_mean": -1.7040824890136719, "KL/mean": -5.403424263000488, "KL/rejected_KL_mean": -9.102771759033203, "KL/std": 7.875365257263184, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.08706669509410858, "fcm_dpo/delta": -0.04813521355390549, "fcm_dpo/margin": 7.398685932159424, "fcm_dpo/q_t": 0.36472994089126587, "grad_norm": 21.178720474243164, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.7194492816925049, "logits/rejected": -0.5025196671485901, "logps/chosen": -75.6605224609375, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -103.55859375, "loss": 0.9727, "margin_dpo/margin_mean": 7.398685455322266, "margin_dpo/margin_std": 8.878637313842773, "step": 131 }, { "KL/chosen_KL_mean": -2.5819854736328125, "KL/mean": -6.63177490234375, "KL/rejected_KL_mean": -10.681564331054688, "KL/std": 8.334089279174805, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.08460044860839844, "fcm_dpo/delta": -0.09235803782939911, "fcm_dpo/margin": 8.09957218170166, "fcm_dpo/q_t": 0.35041141510009766, "grad_norm": 23.12295150756836, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.8946075439453125, "logits/rejected": -0.5056662559509277, "logps/chosen": -95.02146911621094, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -104.8140869140625, "loss": 0.9301, "margin_dpo/margin_mean": 8.099573135375977, "margin_dpo/margin_std": 8.7253999710083, "step": 132 }, { "KL/chosen_KL_mean": -1.79156494140625, "KL/mean": -5.888368129730225, "KL/rejected_KL_mean": -9.985172271728516, "KL/std": 8.482995986938477, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.08364309370517731, "fcm_dpo/delta": -0.09151685982942581, "fcm_dpo/margin": 8.193607330322266, "fcm_dpo/q_t": 0.35416483879089355, "grad_norm": 21.170167922973633, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.2558485269546509, "logits/rejected": -0.8962242603302002, "logps/chosen": -94.9082260131836, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -112.08008575439453, "loss": 0.9236, "margin_dpo/margin_mean": 8.19360637664795, "margin_dpo/margin_std": 8.694095611572266, "step": 133 }, { "KL/chosen_KL_mean": -2.221874237060547, "KL/mean": -6.650839805603027, "KL/rejected_KL_mean": -11.079811096191406, "KL/std": 8.208121299743652, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.0819946676492691, "fcm_dpo/delta": -0.13630571961402893, "fcm_dpo/margin": 8.857930183410645, "fcm_dpo/q_t": 0.3429855704307556, "grad_norm": 19.65479850769043, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.3803662061691284, "logits/rejected": -1.0772172212600708, "logps/chosen": -85.77200317382812, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -104.96138000488281, "loss": 0.8949, "margin_dpo/margin_mean": 8.857930183410645, "margin_dpo/margin_std": 8.541658401489258, "step": 134 }, { "KL/chosen_KL_mean": -2.1728744506835938, "KL/mean": -6.185899257659912, "KL/rejected_KL_mean": -10.198928833007812, "KL/std": 8.996078491210938, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.08056032657623291, "fcm_dpo/delta": -0.049957554787397385, "fcm_dpo/margin": 8.026050567626953, "fcm_dpo/q_t": 0.36572954058647156, "grad_norm": 26.501583099365234, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.8580453395843506, "logits/rejected": -0.7076246738433838, "logps/chosen": -79.75550079345703, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -131.25540161132812, "loss": 0.9673, "margin_dpo/margin_mean": 8.026052474975586, "margin_dpo/margin_std": 9.836691856384277, "step": 135 }, { "KL/chosen_KL_mean": -2.0341949462890625, "KL/mean": -6.42809534072876, "KL/rejected_KL_mean": -10.821998596191406, "KL/std": 8.800153732299805, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.07868927717208862, "fcm_dpo/delta": -0.09859780967235565, "fcm_dpo/margin": 8.787801742553711, "fcm_dpo/q_t": 0.3476041853427887, "grad_norm": 19.209884643554688, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.8712599277496338, "logits/rejected": -0.4387863874435425, "logps/chosen": -96.20692443847656, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -114.08628845214844, "loss": 0.8988, "margin_dpo/margin_mean": 8.787799835205078, "margin_dpo/margin_std": 8.1824951171875, "step": 136 }, { "KL/chosen_KL_mean": -1.7102699279785156, "KL/mean": -8.150555610656738, "KL/rejected_KL_mean": -14.590843200683594, "KL/std": 11.001092910766602, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.07344755530357361, "fcm_dpo/delta": -0.38726353645324707, "fcm_dpo/margin": 12.880568504333496, "fcm_dpo/q_t": 0.3042087256908417, "grad_norm": 21.523439407348633, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.9501944780349731, "logits/rejected": -0.7177352905273438, "logps/chosen": -79.82901000976562, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -132.51321411132812, "loss": 0.7852, "margin_dpo/margin_mean": 12.880569458007812, "margin_dpo/margin_std": 11.676231384277344, "step": 137 }, { "KL/chosen_KL_mean": -3.6505508422851562, "KL/mean": -7.861407279968262, "KL/rejected_KL_mean": -12.072265625, "KL/std": 8.484853744506836, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.07235515862703323, "fcm_dpo/delta": -0.010059346444904804, "fcm_dpo/margin": 8.421720504760742, "fcm_dpo/q_t": 0.36709389090538025, "grad_norm": 18.365293502807617, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.5306152701377869, "logits/rejected": -0.29715001583099365, "logps/chosen": -96.48049926757812, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -110.72067260742188, "loss": 0.9576, "margin_dpo/margin_mean": 8.421720504760742, "margin_dpo/margin_std": 8.967493057250977, "step": 138 }, { "KL/chosen_KL_mean": -2.2138938903808594, "KL/mean": -7.081016540527344, "KL/rejected_KL_mean": -11.948139190673828, "KL/std": 9.447371482849121, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.07120867073535919, "fcm_dpo/delta": -0.10048343241214752, "fcm_dpo/margin": 9.734247207641602, "fcm_dpo/q_t": 0.3474549949169159, "grad_norm": 18.42364501953125, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.7812180519104004, "logits/rejected": -0.4569655954837799, "logps/chosen": -81.21690368652344, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -97.77546691894531, "loss": 0.9305, "margin_dpo/margin_mean": 9.734247207641602, "margin_dpo/margin_std": 10.70001220703125, "step": 139 }, { "KL/chosen_KL_mean": -2.3506546020507812, "KL/mean": -6.879921913146973, "KL/rejected_KL_mean": -11.409187316894531, "KL/std": 9.233577728271484, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.07060247659683228, "fcm_dpo/delta": -0.04224336892366409, "fcm_dpo/margin": 9.058530807495117, "fcm_dpo/q_t": 0.36018168926239014, "grad_norm": 17.6881046295166, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.1774414777755737, "logits/rejected": -0.7951514720916748, "logps/chosen": -90.7031021118164, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -101.98396301269531, "loss": 0.9431, "margin_dpo/margin_mean": 9.058530807495117, "margin_dpo/margin_std": 9.486725807189941, "step": 140 }, { "KL/chosen_KL_mean": -3.0654373168945312, "KL/mean": -8.358266830444336, "KL/rejected_KL_mean": -13.65108871459961, "KL/std": 11.159049987792969, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.0690082460641861, "fcm_dpo/delta": -0.140707865357399, "fcm_dpo/margin": 10.585655212402344, "fcm_dpo/q_t": 0.3436414897441864, "grad_norm": 19.400352478027344, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.0334872007369995, "logits/rejected": -0.6897342801094055, "logps/chosen": -110.28559875488281, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -118.36331939697266, "loss": 0.9015, "margin_dpo/margin_mean": 10.585653305053711, "margin_dpo/margin_std": 10.984273910522461, "step": 141 }, { "KL/chosen_KL_mean": -3.971393585205078, "KL/mean": -8.931512832641602, "KL/rejected_KL_mean": -13.891639709472656, "KL/std": 10.6544189453125, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.06744419038295746, "fcm_dpo/delta": -0.07463416457176208, "fcm_dpo/margin": 9.920244216918945, "fcm_dpo/q_t": 0.35816168785095215, "grad_norm": 18.50437355041504, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.9203962087631226, "logits/rejected": -0.6759716272354126, "logps/chosen": -92.7003402709961, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -113.62202453613281, "loss": 0.9331, "margin_dpo/margin_mean": 9.920244216918945, "margin_dpo/margin_std": 10.974271774291992, "step": 142 }, { "KL/chosen_KL_mean": -2.2159652709960938, "KL/mean": -6.513680458068848, "KL/rejected_KL_mean": -10.811386108398438, "KL/std": 9.348644256591797, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.06729145348072052, "fcm_dpo/delta": 0.022383786737918854, "fcm_dpo/margin": 8.595416069030762, "fcm_dpo/q_t": 0.37461599707603455, "grad_norm": 17.615032196044922, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.1435155868530273, "logits/rejected": -0.6993631720542908, "logps/chosen": -87.72054290771484, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -96.61705017089844, "loss": 0.9968, "margin_dpo/margin_mean": 8.595415115356445, "margin_dpo/margin_std": 10.398183822631836, "step": 143 }, { "KL/chosen_KL_mean": -2.5314598083496094, "KL/mean": -7.094165802001953, "KL/rejected_KL_mean": -11.656875610351562, "KL/std": 9.763832092285156, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.06808388233184814, "fcm_dpo/delta": -0.02395346760749817, "fcm_dpo/margin": 9.125414848327637, "fcm_dpo/q_t": 0.3700926899909973, "grad_norm": 22.62956428527832, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.0307559967041016, "logits/rejected": -0.5282217860221863, "logps/chosen": -107.21855163574219, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -99.37596130371094, "loss": 1.0022, "margin_dpo/margin_mean": 9.125414848327637, "margin_dpo/margin_std": 12.090323448181152, "step": 144 }, { "KL/chosen_KL_mean": -2.4559478759765625, "KL/mean": -7.561844825744629, "KL/rejected_KL_mean": -12.667736053466797, "KL/std": 9.600131034851074, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.0665380209684372, "fcm_dpo/delta": -0.08533187210559845, "fcm_dpo/margin": 10.2117919921875, "fcm_dpo/q_t": 0.3527753949165344, "grad_norm": 18.214406967163086, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.6675140857696533, "logits/rejected": -0.08715488761663437, "logps/chosen": -90.97583770751953, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -96.43287658691406, "loss": 0.9289, "margin_dpo/margin_mean": 10.2117919921875, "margin_dpo/margin_std": 10.930527687072754, "step": 145 }, { "KL/chosen_KL_mean": -2.573863983154297, "KL/mean": -8.279586791992188, "KL/rejected_KL_mean": -13.985305786132812, "KL/std": 10.249943733215332, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.06435221433639526, "fcm_dpo/delta": -0.14741867780685425, "fcm_dpo/margin": 11.411436080932617, "fcm_dpo/q_t": 0.3395462930202484, "grad_norm": 17.635154724121094, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.3040237426757812, "logits/rejected": -0.7391974925994873, "logps/chosen": -105.47972869873047, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -102.9346694946289, "loss": 0.8856, "margin_dpo/margin_mean": 11.411437034606934, "margin_dpo/margin_std": 10.735688209533691, "step": 146 }, { "KL/chosen_KL_mean": -2.650615692138672, "KL/mean": -8.698829650878906, "KL/rejected_KL_mean": -14.74704360961914, "KL/std": 10.391220092773438, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.06275095790624619, "fcm_dpo/delta": -0.17211401462554932, "fcm_dpo/margin": 12.096426010131836, "fcm_dpo/q_t": 0.3334345817565918, "grad_norm": 15.978226661682129, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.1471443176269531, "logits/rejected": -0.5894607305526733, "logps/chosen": -91.91280364990234, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -91.24716186523438, "loss": 0.8635, "margin_dpo/margin_mean": 12.096426010131836, "margin_dpo/margin_std": 10.855104446411133, "step": 147 }, { "KL/chosen_KL_mean": -1.3414344787597656, "KL/mean": -6.440007209777832, "KL/rejected_KL_mean": -11.538578033447266, "KL/std": 10.117034912109375, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.062437139451503754, "fcm_dpo/delta": -0.04084836691617966, "fcm_dpo/margin": 10.197147369384766, "fcm_dpo/q_t": 0.36587202548980713, "grad_norm": 18.654260635375977, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.2066044807434082, "logits/rejected": -0.8170025944709778, "logps/chosen": -93.10431671142578, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -121.92410278320312, "loss": 0.965, "margin_dpo/margin_mean": 10.197147369384766, "margin_dpo/margin_std": 11.435999870300293, "step": 148 }, { "KL/chosen_KL_mean": -3.7430343627929688, "KL/mean": -8.75997543334961, "KL/rejected_KL_mean": -13.776912689208984, "KL/std": 10.749823570251465, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.061309218406677246, "fcm_dpo/delta": -0.016485057771205902, "fcm_dpo/margin": 10.033880233764648, "fcm_dpo/q_t": 0.37327155470848083, "grad_norm": 17.588134765625, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.1539486646652222, "logits/rejected": -0.8646851778030396, "logps/chosen": -97.22705078125, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -119.42060852050781, "loss": 0.9894, "margin_dpo/margin_mean": 10.033881187438965, "margin_dpo/margin_std": 12.870650291442871, "step": 149 }, { "KL/chosen_KL_mean": -2.8939666748046875, "KL/mean": -8.346768379211426, "KL/rejected_KL_mean": -13.799568176269531, "KL/std": 11.718517303466797, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.06071936711668968, "fcm_dpo/delta": -0.06688056141138077, "fcm_dpo/margin": 10.905596733093262, "fcm_dpo/q_t": 0.3602423071861267, "grad_norm": 20.69290542602539, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.9698970317840576, "logits/rejected": -0.6968490481376648, "logps/chosen": -78.67265319824219, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -103.82633209228516, "loss": 0.95, "margin_dpo/margin_mean": 10.905596733093262, "margin_dpo/margin_std": 12.825929641723633, "step": 150 }, { "KL/chosen_KL_mean": -4.320850372314453, "KL/mean": -9.516517639160156, "KL/rejected_KL_mean": -14.712181091308594, "KL/std": 11.832818984985352, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.06042507663369179, "fcm_dpo/delta": -0.029755940660834312, "fcm_dpo/margin": 10.391332626342773, "fcm_dpo/q_t": 0.3672083020210266, "grad_norm": 19.85477066040039, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.6965043544769287, "logits/rejected": -1.131915807723999, "logps/chosen": -107.05506896972656, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -120.74999237060547, "loss": 0.9784, "margin_dpo/margin_mean": 10.391332626342773, "margin_dpo/margin_std": 12.890874862670898, "step": 151 }, { "KL/chosen_KL_mean": -1.7671165466308594, "KL/mean": -7.904864311218262, "KL/rejected_KL_mean": -14.042617797851562, "KL/std": 11.265260696411133, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.05898825079202652, "fcm_dpo/delta": -0.13417460024356842, "fcm_dpo/margin": 12.275493621826172, "fcm_dpo/q_t": 0.3431406021118164, "grad_norm": 17.1513729095459, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.1985526084899902, "logits/rejected": -0.8334140777587891, "logps/chosen": -99.35786437988281, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -124.541748046875, "loss": 0.8965, "margin_dpo/margin_mean": 12.275493621826172, "margin_dpo/margin_std": 12.437931060791016, "step": 152 }, { "KL/chosen_KL_mean": -4.230350494384766, "KL/mean": -8.69035530090332, "KL/rejected_KL_mean": -13.150360107421875, "KL/std": 11.012676239013672, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.059346746653318405, "fcm_dpo/delta": 0.07458621263504028, "fcm_dpo/margin": 8.920018196105957, "fcm_dpo/q_t": 0.3863345980644226, "grad_norm": 17.12094497680664, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.6723289489746094, "logits/rejected": -1.369149088859558, "logps/chosen": -91.25595092773438, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -104.21890258789062, "loss": 1.04, "margin_dpo/margin_mean": 8.92001724243164, "margin_dpo/margin_std": 12.098730087280273, "step": 153 }, { "KL/chosen_KL_mean": -2.668445587158203, "KL/mean": -9.890018463134766, "KL/rejected_KL_mean": -17.111591339111328, "KL/std": 12.249594688415527, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.057942233979701996, "fcm_dpo/delta": -0.2579953968524933, "fcm_dpo/margin": 14.443153381347656, "fcm_dpo/q_t": 0.3202112317085266, "grad_norm": 16.801677703857422, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.0122629404067993, "logits/rejected": -0.6831006407737732, "logps/chosen": -93.77410888671875, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -128.48397827148438, "loss": 0.8267, "margin_dpo/margin_mean": 14.443153381347656, "margin_dpo/margin_std": 12.224981307983398, "step": 154 }, { "KL/chosen_KL_mean": -4.098133087158203, "KL/mean": -9.94772720336914, "KL/rejected_KL_mean": -15.797317504882812, "KL/std": 12.642512321472168, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.05574240908026695, "fcm_dpo/delta": -0.05705678090453148, "fcm_dpo/margin": 11.699189186096191, "fcm_dpo/q_t": 0.35826075077056885, "grad_norm": 20.616060256958008, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.05380380153656, "logits/rejected": -0.7070010900497437, "logps/chosen": -98.65910339355469, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -100.57090759277344, "loss": 0.9868, "margin_dpo/margin_mean": 11.699188232421875, "margin_dpo/margin_std": 14.93471908569336, "step": 155 }, { "KL/chosen_KL_mean": -5.769916534423828, "KL/mean": -10.770740509033203, "KL/rejected_KL_mean": -15.771560668945312, "KL/std": 11.218917846679688, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.05608939379453659, "fcm_dpo/delta": 0.04121623933315277, "fcm_dpo/margin": 10.001644134521484, "fcm_dpo/q_t": 0.38021421432495117, "grad_norm": 18.874570846557617, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.5230157375335693, "logits/rejected": -1.0597717761993408, "logps/chosen": -100.42530822753906, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -100.19090270996094, "loss": 1.0193, "margin_dpo/margin_mean": 10.001644134521484, "margin_dpo/margin_std": 12.983247756958008, "step": 156 }, { "KL/chosen_KL_mean": -5.201072692871094, "KL/mean": -11.900674819946289, "KL/rejected_KL_mean": -18.60028076171875, "KL/std": 12.00521469116211, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.055277954787015915, "fcm_dpo/delta": -0.15170317888259888, "fcm_dpo/margin": 13.399202346801758, "fcm_dpo/q_t": 0.33810240030288696, "grad_norm": 17.695276260375977, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.6895902156829834, "logits/rejected": -1.2398557662963867, "logps/chosen": -102.21346282958984, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -130.30162048339844, "loss": 0.8766, "margin_dpo/margin_mean": 13.399202346801758, "margin_dpo/margin_std": 12.388044357299805, "step": 157 }, { "KL/chosen_KL_mean": -4.859405517578125, "KL/mean": -11.886869430541992, "KL/rejected_KL_mean": -18.914337158203125, "KL/std": 12.47777271270752, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.05353248864412308, "fcm_dpo/delta": -0.16452531516551971, "fcm_dpo/margin": 14.054920196533203, "fcm_dpo/q_t": 0.33601605892181396, "grad_norm": 17.173831939697266, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.191420078277588, "logits/rejected": -0.9043850898742676, "logps/chosen": -99.57237243652344, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -139.8516845703125, "loss": 0.8703, "margin_dpo/margin_mean": 14.054920196533203, "margin_dpo/margin_std": 12.94216537475586, "step": 158 }, { "KL/chosen_KL_mean": -4.071403503417969, "KL/mean": -10.683317184448242, "KL/rejected_KL_mean": -17.29523468017578, "KL/std": 12.857282638549805, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.05213826149702072, "fcm_dpo/delta": -0.09624259173870087, "fcm_dpo/margin": 13.223838806152344, "fcm_dpo/q_t": 0.35206788778305054, "grad_norm": 14.840791702270508, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.7676692008972168, "logits/rejected": -1.3038177490234375, "logps/chosen": -112.28964233398438, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -112.13426208496094, "loss": 0.93, "margin_dpo/margin_mean": 13.223838806152344, "margin_dpo/margin_std": 14.42451000213623, "step": 159 }, { "KL/chosen_KL_mean": -3.3457260131835938, "KL/mean": -9.626791954040527, "KL/rejected_KL_mean": -15.907859802246094, "KL/std": 12.029001235961914, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.05145548656582832, "fcm_dpo/delta": -0.04978324472904205, "fcm_dpo/margin": 12.562129974365234, "fcm_dpo/q_t": 0.35626184940338135, "grad_norm": 15.67940616607666, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.3832693099975586, "logits/rejected": -0.8401064872741699, "logps/chosen": -101.50914001464844, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -118.1888427734375, "loss": 0.9341, "margin_dpo/margin_mean": 12.562129974365234, "margin_dpo/margin_std": 12.676630973815918, "step": 160 }, { "KL/chosen_KL_mean": -5.434299468994141, "KL/mean": -11.346993446350098, "KL/rejected_KL_mean": -17.259685516357422, "KL/std": 11.362947463989258, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.051115673035383224, "fcm_dpo/delta": -0.00535949319601059, "fcm_dpo/margin": 11.825395584106445, "fcm_dpo/q_t": 0.3689253330230713, "grad_norm": 16.081960678100586, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.5971035957336426, "logits/rejected": -1.1189405918121338, "logps/chosen": -92.02616882324219, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -106.276123046875, "loss": 0.9867, "margin_dpo/margin_mean": 11.825395584106445, "margin_dpo/margin_std": 14.062060356140137, "step": 161 }, { "KL/chosen_KL_mean": -4.692085266113281, "KL/mean": -11.00771713256836, "KL/rejected_KL_mean": -17.323341369628906, "KL/std": 12.898788452148438, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.05091368407011032, "fcm_dpo/delta": -0.04625662788748741, "fcm_dpo/margin": 12.631258010864258, "fcm_dpo/q_t": 0.35702845454216003, "grad_norm": 14.797911643981934, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.09999680519104, "logits/rejected": -0.7103325128555298, "logps/chosen": -102.094970703125, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -113.19257354736328, "loss": 0.9446, "margin_dpo/margin_mean": 12.63125991821289, "margin_dpo/margin_std": 13.31132698059082, "step": 162 }, { "KL/chosen_KL_mean": -4.324550628662109, "KL/mean": -12.208023071289062, "KL/rejected_KL_mean": -20.09149932861328, "KL/std": 15.170753479003906, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.049338605254888535, "fcm_dpo/delta": -0.19355183839797974, "fcm_dpo/margin": 15.766950607299805, "fcm_dpo/q_t": 0.33658450841903687, "grad_norm": 13.12332534790039, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.2965056896209717, "logits/rejected": -0.8939577341079712, "logps/chosen": -86.49993896484375, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -102.95623779296875, "loss": 0.8763, "margin_dpo/margin_mean": 15.766950607299805, "margin_dpo/margin_std": 16.2303524017334, "step": 163 }, { "KL/chosen_KL_mean": -7.618114471435547, "KL/mean": -12.799482345581055, "KL/rejected_KL_mean": -17.980846405029297, "KL/std": 11.835062980651855, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.04918459802865982, "fcm_dpo/delta": 0.09415318071842194, "fcm_dpo/margin": 10.36273193359375, "fcm_dpo/q_t": 0.3863057792186737, "grad_norm": 15.660857200622559, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.1088857650756836, "logits/rejected": -0.6557607650756836, "logps/chosen": -98.0923843383789, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -105.00894165039062, "loss": 1.033, "margin_dpo/margin_mean": 10.36273193359375, "margin_dpo/margin_std": 12.298709869384766, "step": 164 }, { "KL/chosen_KL_mean": -4.741447448730469, "KL/mean": -12.443620681762695, "KL/rejected_KL_mean": -20.145793914794922, "KL/std": 14.754302978515625, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.04888567700982094, "fcm_dpo/delta": -0.1651880443096161, "fcm_dpo/margin": 15.404348373413086, "fcm_dpo/q_t": 0.3425597548484802, "grad_norm": 13.414756774902344, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.1108825206756592, "logits/rejected": -0.6758971214294434, "logps/chosen": -92.05467224121094, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -102.04837799072266, "loss": 0.89, "margin_dpo/margin_mean": 15.40434741973877, "margin_dpo/margin_std": 16.635162353515625, "step": 165 }, { "KL/chosen_KL_mean": -8.943981170654297, "KL/mean": -16.308679580688477, "KL/rejected_KL_mean": -23.673381805419922, "KL/std": 15.132356643676758, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.04751087725162506, "fcm_dpo/delta": -0.10732638835906982, "fcm_dpo/margin": 14.729394912719727, "fcm_dpo/q_t": 0.35661882162094116, "grad_norm": 14.68244743347168, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.324919581413269, "logits/rejected": -1.1114251613616943, "logps/chosen": -90.53004455566406, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -126.83856201171875, "loss": 0.9354, "margin_dpo/margin_mean": 14.729395866394043, "margin_dpo/margin_std": 17.675304412841797, "step": 166 }, { "KL/chosen_KL_mean": -7.744514465332031, "KL/mean": -15.768632888793945, "KL/rejected_KL_mean": -23.792755126953125, "KL/std": 14.169123649597168, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.04588838666677475, "fcm_dpo/delta": -0.14939774572849274, "fcm_dpo/margin": 16.048233032226562, "fcm_dpo/q_t": 0.33840012550354004, "grad_norm": 14.13165283203125, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.8881231546401978, "logits/rejected": -0.6953379511833191, "logps/chosen": -91.30618286132812, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -122.84580993652344, "loss": 0.8687, "margin_dpo/margin_mean": 16.048233032226562, "margin_dpo/margin_std": 14.080564498901367, "step": 167 }, { "KL/chosen_KL_mean": -7.522312164306641, "KL/mean": -14.389703750610352, "KL/rejected_KL_mean": -21.257099151611328, "KL/std": 14.217185020446777, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.04576166719198227, "fcm_dpo/delta": -0.030483687296509743, "fcm_dpo/margin": 13.734785079956055, "fcm_dpo/q_t": 0.3620396852493286, "grad_norm": 15.666106224060059, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.4890189170837402, "logits/rejected": -1.160757303237915, "logps/chosen": -87.45355224609375, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -101.35926818847656, "loss": 0.9475, "margin_dpo/margin_mean": 13.734785079956055, "margin_dpo/margin_std": 14.34788703918457, "step": 168 }, { "KL/chosen_KL_mean": -6.0166778564453125, "KL/mean": -13.805206298828125, "KL/rejected_KL_mean": -21.593730926513672, "KL/std": 14.898513793945312, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.04491722211241722, "fcm_dpo/delta": -0.10708025097846985, "fcm_dpo/margin": 15.577059745788574, "fcm_dpo/q_t": 0.348813533782959, "grad_norm": 14.80154800415039, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.1357991695404053, "logits/rejected": -0.901911735534668, "logps/chosen": -88.92037963867188, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -111.71563720703125, "loss": 0.9066, "margin_dpo/margin_mean": 15.57706069946289, "margin_dpo/margin_std": 15.667001724243164, "step": 169 }, { "KL/chosen_KL_mean": -7.923549652099609, "KL/mean": -14.81826400756836, "KL/rejected_KL_mean": -21.712982177734375, "KL/std": 12.858461380004883, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.0442856028676033, "fcm_dpo/delta": -0.012092437595129013, "fcm_dpo/margin": 13.789435386657715, "fcm_dpo/q_t": 0.36232686042785645, "grad_norm": 15.224604606628418, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.1954232454299927, "logits/rejected": -0.9652056097984314, "logps/chosen": -82.905517578125, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -103.87858581542969, "loss": 0.9423, "margin_dpo/margin_mean": 13.789436340332031, "margin_dpo/margin_std": 12.928497314453125, "step": 170 }, { "KL/chosen_KL_mean": -9.573169708251953, "KL/mean": -17.30157470703125, "KL/rejected_KL_mean": -25.029979705810547, "KL/std": 17.566835403442383, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.043968573212623596, "fcm_dpo/delta": -0.08538427948951721, "fcm_dpo/margin": 15.45681095123291, "fcm_dpo/q_t": 0.362047016620636, "grad_norm": 16.491281509399414, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.316501498222351, "logits/rejected": -0.7713274955749512, "logps/chosen": -128.55169677734375, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -140.79928588867188, "loss": 0.9678, "margin_dpo/margin_mean": 15.456811904907227, "margin_dpo/margin_std": 20.221752166748047, "step": 171 }, { "KL/chosen_KL_mean": -10.344955444335938, "KL/mean": -17.530147552490234, "KL/rejected_KL_mean": -24.71533966064453, "KL/std": 13.064635276794434, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.04341629147529602, "fcm_dpo/delta": -0.025916704908013344, "fcm_dpo/margin": 14.370386123657227, "fcm_dpo/q_t": 0.356118768453598, "grad_norm": 13.00464916229248, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -1.9741084575653076, "logits/rejected": -1.2488579750061035, "logps/chosen": -111.49874877929688, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -115.72102355957031, "loss": 0.9093, "margin_dpo/margin_mean": 14.370386123657227, "margin_dpo/margin_std": 10.842920303344727, "step": 172 }, { "KL/chosen_KL_mean": -9.621604919433594, "KL/mean": -15.753321647644043, "KL/rejected_KL_mean": -21.88503646850586, "KL/std": 12.557456016540527, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.043999142944812775, "fcm_dpo/delta": 0.0637490451335907, "fcm_dpo/margin": 12.263426780700684, "fcm_dpo/q_t": 0.3790205121040344, "grad_norm": 13.350013732910156, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.2277157306671143, "logits/rejected": -0.7679325342178345, "logps/chosen": -108.67829132080078, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -105.73576354980469, "loss": 0.9954, "margin_dpo/margin_mean": 12.263426780700684, "margin_dpo/margin_std": 13.305219650268555, "step": 173 }, { "KL/chosen_KL_mean": -9.373804092407227, "KL/mean": -17.449310302734375, "KL/rejected_KL_mean": -25.524818420410156, "KL/std": 16.535051345825195, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.043432459235191345, "fcm_dpo/delta": -0.10903347283601761, "fcm_dpo/margin": 16.151016235351562, "fcm_dpo/q_t": 0.34625035524368286, "grad_norm": 15.608811378479004, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.7371737957000732, "logits/rejected": -1.6713142395019531, "logps/chosen": -89.78271484375, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -133.72653198242188, "loss": 0.9072, "margin_dpo/margin_mean": 16.151016235351562, "margin_dpo/margin_std": 16.181791305541992, "step": 174 }, { "KL/chosen_KL_mean": -12.682926177978516, "KL/mean": -20.721057891845703, "KL/rejected_KL_mean": -28.759193420410156, "KL/std": 15.148465156555176, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.04274030774831772, "fcm_dpo/delta": -0.09354110807180405, "fcm_dpo/margin": 16.07626724243164, "fcm_dpo/q_t": 0.3470172584056854, "grad_norm": 15.31360149383545, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.8849430084228516, "logits/rejected": -0.6129493713378906, "logps/chosen": -95.3994140625, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -125.42632293701172, "loss": 0.8977, "margin_dpo/margin_mean": 16.07626724243164, "margin_dpo/margin_std": 14.321072578430176, "step": 175 }, { "KL/chosen_KL_mean": -10.200420379638672, "KL/mean": -18.373668670654297, "KL/rejected_KL_mean": -26.546916961669922, "KL/std": 16.459619522094727, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.04177354648709297, "fcm_dpo/delta": -0.08894173800945282, "fcm_dpo/margin": 16.34649658203125, "fcm_dpo/q_t": 0.35506555438041687, "grad_norm": 14.038694381713867, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.4817163944244385, "logits/rejected": -1.3022964000701904, "logps/chosen": -89.82972717285156, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -129.23550415039062, "loss": 0.941, "margin_dpo/margin_mean": 16.34649658203125, "margin_dpo/margin_std": 18.434185028076172, "step": 176 }, { "KL/chosen_KL_mean": -11.114383697509766, "KL/mean": -20.638145446777344, "KL/rejected_KL_mean": -30.16191864013672, "KL/std": 18.7408447265625, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.0406847819685936, "fcm_dpo/delta": -0.18924787640571594, "fcm_dpo/margin": 19.047531127929688, "fcm_dpo/q_t": 0.33938488364219666, "grad_norm": 12.640216827392578, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.4807155132293701, "logits/rejected": -1.1383321285247803, "logps/chosen": -103.67548370361328, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -133.50030517578125, "loss": 0.8767, "margin_dpo/margin_mean": 19.047531127929688, "margin_dpo/margin_std": 19.709243774414062, "step": 177 }, { "KL/chosen_KL_mean": -10.321640014648438, "KL/mean": -21.506256103515625, "KL/rejected_KL_mean": -32.69085693359375, "KL/std": 18.14514923095703, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.038352929055690765, "fcm_dpo/delta": -0.28259849548339844, "fcm_dpo/margin": 22.369213104248047, "fcm_dpo/q_t": 0.316496878862381, "grad_norm": 14.480009078979492, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.5161950588226318, "logits/rejected": -1.2613036632537842, "logps/chosen": -94.58743286132812, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -143.4881591796875, "loss": 0.811, "margin_dpo/margin_mean": 22.369213104248047, "margin_dpo/margin_std": 19.61292266845703, "step": 178 }, { "KL/chosen_KL_mean": -13.550373077392578, "KL/mean": -21.045757293701172, "KL/rejected_KL_mean": -28.541133880615234, "KL/std": 17.69839859008789, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.037864312529563904, "fcm_dpo/delta": 0.03406227380037308, "fcm_dpo/margin": 14.990765571594238, "fcm_dpo/q_t": 0.3795953392982483, "grad_norm": 14.29662799835205, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.238407850265503, "logits/rejected": -0.7975925803184509, "logps/chosen": -106.99835205078125, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -122.79654693603516, "loss": 1.0265, "margin_dpo/margin_mean": 14.990764617919922, "margin_dpo/margin_std": 20.155555725097656, "step": 179 }, { "KL/chosen_KL_mean": -11.252616882324219, "KL/mean": -20.556602478027344, "KL/rejected_KL_mean": -29.86058807373047, "KL/std": 17.25983428955078, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.03737686201930046, "fcm_dpo/delta": -0.10337096452713013, "fcm_dpo/margin": 18.60797119140625, "fcm_dpo/q_t": 0.3469793200492859, "grad_norm": 16.525339126586914, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.208809494972229, "logits/rejected": -0.9572755694389343, "logps/chosen": -100.0881576538086, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -133.76754760742188, "loss": 0.894, "margin_dpo/margin_mean": 18.607973098754883, "margin_dpo/margin_std": 17.512096405029297, "step": 180 }, { "KL/chosen_KL_mean": -11.421504974365234, "KL/mean": -18.796409606933594, "KL/rejected_KL_mean": -26.171310424804688, "KL/std": 16.262466430664062, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.03763171285390854, "fcm_dpo/delta": 0.04749670624732971, "fcm_dpo/margin": 14.749801635742188, "fcm_dpo/q_t": 0.3790430426597595, "grad_norm": 13.302220344543457, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.4882631301879883, "logits/rejected": -1.0439276695251465, "logps/chosen": -90.65547180175781, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -95.34573364257812, "loss": 0.9951, "margin_dpo/margin_mean": 14.749801635742188, "margin_dpo/margin_std": 17.170066833496094, "step": 181 }, { "KL/chosen_KL_mean": -14.695587158203125, "KL/mean": -24.081369400024414, "KL/rejected_KL_mean": -33.46714782714844, "KL/std": 18.841421127319336, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.03691215440630913, "fcm_dpo/delta": -0.10103189200162888, "fcm_dpo/margin": 18.771564483642578, "fcm_dpo/q_t": 0.3521079421043396, "grad_norm": 13.173837661743164, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.1005642414093018, "logits/rejected": -0.8543146848678589, "logps/chosen": -90.77777099609375, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -128.43576049804688, "loss": 0.9333, "margin_dpo/margin_mean": 18.771564483642578, "margin_dpo/margin_std": 20.875591278076172, "step": 182 }, { "KL/chosen_KL_mean": -14.858020782470703, "KL/mean": -24.235387802124023, "KL/rejected_KL_mean": -33.612754821777344, "KL/std": 17.11053466796875, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.03628591075539589, "fcm_dpo/delta": -0.08723864704370499, "fcm_dpo/margin": 18.754728317260742, "fcm_dpo/q_t": 0.34982597827911377, "grad_norm": 13.765605926513672, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.466099739074707, "logits/rejected": -0.870518147945404, "logps/chosen": -108.76593017578125, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -117.17337036132812, "loss": 0.9079, "margin_dpo/margin_mean": 18.754728317260742, "margin_dpo/margin_std": 17.644973754882812, "step": 183 }, { "KL/chosen_KL_mean": -13.419410705566406, "KL/mean": -19.71581268310547, "KL/rejected_KL_mean": -26.01221466064453, "KL/std": 17.74881362915039, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.037061769515275955, "fcm_dpo/delta": 0.13976962864398956, "fcm_dpo/margin": 12.592802047729492, "fcm_dpo/q_t": 0.3965834677219391, "grad_norm": 13.942252159118652, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.377471923828125, "logits/rejected": -1.1157610416412354, "logps/chosen": -100.28732299804688, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -110.29676818847656, "loss": 1.089, "margin_dpo/margin_mean": 12.59280014038086, "margin_dpo/margin_std": 19.38140106201172, "step": 184 }, { "KL/chosen_KL_mean": -14.362480163574219, "KL/mean": -21.968490600585938, "KL/rejected_KL_mean": -29.57449722290039, "KL/std": 17.373355865478516, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.0376364067196846, "fcm_dpo/delta": 0.02863926626741886, "fcm_dpo/margin": 15.212013244628906, "fcm_dpo/q_t": 0.3778904676437378, "grad_norm": 15.905083656311035, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -1.8063063621520996, "logits/rejected": -1.1862945556640625, "logps/chosen": -108.38390350341797, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -111.03117370605469, "loss": 1.0196, "margin_dpo/margin_mean": 15.212013244628906, "margin_dpo/margin_std": 20.29389762878418, "step": 185 }, { "KL/chosen_KL_mean": -17.485218048095703, "KL/mean": -25.979835510253906, "KL/rejected_KL_mean": -34.474449157714844, "KL/std": 16.858356475830078, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.03717076778411865, "fcm_dpo/delta": -0.034209974110126495, "fcm_dpo/margin": 16.989234924316406, "fcm_dpo/q_t": 0.36145520210266113, "grad_norm": 15.6112060546875, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.095181703567505, "logits/rejected": -1.2176064252853394, "logps/chosen": -131.67691040039062, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -127.8135757446289, "loss": 0.9535, "margin_dpo/margin_mean": 16.989234924316406, "margin_dpo/margin_std": 18.291961669921875, "step": 186 }, { "KL/chosen_KL_mean": -15.010005950927734, "KL/mean": -22.80898094177246, "KL/rejected_KL_mean": -30.607952117919922, "KL/std": 18.596141815185547, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.03749664127826691, "fcm_dpo/delta": 0.01574406772851944, "fcm_dpo/margin": 15.597947120666504, "fcm_dpo/q_t": 0.37496089935302734, "grad_norm": 13.674775123596191, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.599191665649414, "logits/rejected": -1.3596041202545166, "logps/chosen": -91.87955474853516, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -118.77289581298828, "loss": 1.0091, "margin_dpo/margin_mean": 15.597947120666504, "margin_dpo/margin_std": 20.15418815612793, "step": 187 }, { "KL/chosen_KL_mean": -11.299446105957031, "KL/mean": -20.273855209350586, "KL/rejected_KL_mean": -29.248268127441406, "KL/std": 18.663179397583008, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.03681416064500809, "fcm_dpo/delta": -0.06613938510417938, "fcm_dpo/margin": 17.948822021484375, "fcm_dpo/q_t": 0.3522428870201111, "grad_norm": 13.654606819152832, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.072474479675293, "logits/rejected": -1.8311965465545654, "logps/chosen": -90.49424743652344, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -126.07716369628906, "loss": 0.9315, "margin_dpo/margin_mean": 17.948823928833008, "margin_dpo/margin_std": 17.823427200317383, "step": 188 }, { "KL/chosen_KL_mean": -14.92544937133789, "KL/mean": -24.293060302734375, "KL/rejected_KL_mean": -33.660675048828125, "KL/std": 18.944881439208984, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.03669549897313118, "fcm_dpo/delta": -0.09461771696805954, "fcm_dpo/margin": 18.735225677490234, "fcm_dpo/q_t": 0.3564212918281555, "grad_norm": 12.300936698913574, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.3281278610229492, "logits/rejected": -0.6604208946228027, "logps/chosen": -98.81825256347656, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -113.12657165527344, "loss": 0.9405, "margin_dpo/margin_mean": 18.735225677490234, "margin_dpo/margin_std": 21.458765029907227, "step": 189 }, { "KL/chosen_KL_mean": -11.670337677001953, "KL/mean": -22.591691970825195, "KL/rejected_KL_mean": -33.5130500793457, "KL/std": 20.50206756591797, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.03518030047416687, "fcm_dpo/delta": -0.18284493684768677, "fcm_dpo/margin": 21.842710494995117, "fcm_dpo/q_t": 0.33117419481277466, "grad_norm": 13.637959480285645, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.6019375324249268, "logits/rejected": -0.9986509680747986, "logps/chosen": -115.11180114746094, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -142.7280731201172, "loss": 0.8565, "margin_dpo/margin_mean": 21.842710494995117, "margin_dpo/margin_std": 19.24144744873047, "step": 190 }, { "KL/chosen_KL_mean": -10.860038757324219, "KL/mean": -22.93557357788086, "KL/rejected_KL_mean": -35.0111083984375, "KL/std": 21.98157501220703, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.033797960728406906, "fcm_dpo/delta": -0.23504412174224854, "fcm_dpo/margin": 24.151073455810547, "fcm_dpo/q_t": 0.3260577917098999, "grad_norm": 14.197420120239258, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.386749505996704, "logits/rejected": -1.0572245121002197, "logps/chosen": -110.64555358886719, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -151.01138305664062, "loss": 0.844, "margin_dpo/margin_mean": 24.15107536315918, "margin_dpo/margin_std": 22.414432525634766, "step": 191 }, { "KL/chosen_KL_mean": -19.231945037841797, "KL/mean": -27.709735870361328, "KL/rejected_KL_mean": -36.18751907348633, "KL/std": 17.783672332763672, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.033426057547330856, "fcm_dpo/delta": 0.0352415107190609, "fcm_dpo/margin": 16.955589294433594, "fcm_dpo/q_t": 0.37734997272491455, "grad_norm": 15.315131187438965, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.4420750141143799, "logits/rejected": -1.185784101486206, "logps/chosen": -120.09226989746094, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -145.6325225830078, "loss": 1.005, "margin_dpo/margin_mean": 16.955589294433594, "margin_dpo/margin_std": 21.042728424072266, "step": 192 }, { "KL/chosen_KL_mean": -15.265388488769531, "KL/mean": -22.532798767089844, "KL/rejected_KL_mean": -29.800212860107422, "KL/std": 18.551286697387695, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.034226201474666595, "fcm_dpo/delta": 0.10705623030662537, "fcm_dpo/margin": 14.534821510314941, "fcm_dpo/q_t": 0.3921981453895569, "grad_norm": 14.897360801696777, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -1.7292828559875488, "logits/rejected": -1.135681390762329, "logps/chosen": -108.20370483398438, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -112.90939331054688, "loss": 1.0843, "margin_dpo/margin_mean": 14.534821510314941, "margin_dpo/margin_std": 22.342926025390625, "step": 193 }, { "KL/chosen_KL_mean": -15.936695098876953, "KL/mean": -24.273998260498047, "KL/rejected_KL_mean": -32.611305236816406, "KL/std": 19.379459381103516, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.03457144647836685, "fcm_dpo/delta": 0.024282660335302353, "fcm_dpo/margin": 16.674617767333984, "fcm_dpo/q_t": 0.375179260969162, "grad_norm": 12.222079277038574, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.2975940704345703, "logits/rejected": -1.9307920932769775, "logps/chosen": -79.728271484375, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -101.33795928955078, "loss": 1.0018, "margin_dpo/margin_mean": 16.67461585998535, "margin_dpo/margin_std": 20.451744079589844, "step": 194 }, { "KL/chosen_KL_mean": -15.218944549560547, "KL/mean": -26.021759033203125, "KL/rejected_KL_mean": -36.82456588745117, "KL/std": 19.79120445251465, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0338343009352684, "fcm_dpo/delta": -0.1411038637161255, "fcm_dpo/margin": 21.60561752319336, "fcm_dpo/q_t": 0.3418634533882141, "grad_norm": 12.764904975891113, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.0151879787445068, "logits/rejected": -0.878206729888916, "logps/chosen": -99.10440063476562, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -140.82582092285156, "loss": 0.9111, "margin_dpo/margin_mean": 21.60561752319336, "margin_dpo/margin_std": 22.748077392578125, "step": 195 }, { "KL/chosen_KL_mean": -17.630184173583984, "KL/mean": -28.04671287536621, "KL/rejected_KL_mean": -38.46324157714844, "KL/std": 21.226993560791016, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.033002208918333054, "fcm_dpo/delta": -0.09403122961521149, "fcm_dpo/margin": 20.833051681518555, "fcm_dpo/q_t": 0.3468974828720093, "grad_norm": 11.346840858459473, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.1221222877502441, "logits/rejected": -0.6674832105636597, "logps/chosen": -107.03154754638672, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -121.81059265136719, "loss": 0.9014, "margin_dpo/margin_mean": 20.833053588867188, "margin_dpo/margin_std": 19.17495346069336, "step": 196 }, { "KL/chosen_KL_mean": -18.278839111328125, "KL/mean": -24.70205307006836, "KL/rejected_KL_mean": -31.125263214111328, "KL/std": 17.04050064086914, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.03370559960603714, "fcm_dpo/delta": 0.1746271550655365, "fcm_dpo/margin": 12.846421241760254, "fcm_dpo/q_t": 0.402008593082428, "grad_norm": 13.04179573059082, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.1146445274353027, "logits/rejected": -0.7851884365081787, "logps/chosen": -105.11954498291016, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -113.91815948486328, "loss": 1.0819, "margin_dpo/margin_mean": 12.846421241760254, "margin_dpo/margin_std": 17.848594665527344, "step": 197 }, { "KL/chosen_KL_mean": -21.25279998779297, "KL/mean": -31.270069122314453, "KL/rejected_KL_mean": -41.28734588623047, "KL/std": 19.747154235839844, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.03365708142518997, "fcm_dpo/delta": -0.07961506396532059, "fcm_dpo/margin": 20.034542083740234, "fcm_dpo/q_t": 0.3532215356826782, "grad_norm": 15.415843963623047, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.6046730279922485, "logits/rejected": -1.4118740558624268, "logps/chosen": -106.30104064941406, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -145.3909912109375, "loss": 0.9215, "margin_dpo/margin_mean": 20.034542083740234, "margin_dpo/margin_std": 20.482391357421875, "step": 198 }, { "KL/chosen_KL_mean": -20.60059356689453, "KL/mean": -29.197650909423828, "KL/rejected_KL_mean": -37.794715881347656, "KL/std": 19.496292114257812, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.03365474194288254, "fcm_dpo/delta": 0.02253994159400463, "fcm_dpo/margin": 17.194124221801758, "fcm_dpo/q_t": 0.37366393208503723, "grad_norm": 13.2687406539917, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.5075774192810059, "logits/rejected": -1.2465811967849731, "logps/chosen": -102.38729095458984, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -128.5854949951172, "loss": 0.994, "margin_dpo/margin_mean": 17.194124221801758, "margin_dpo/margin_std": 20.391551971435547, "step": 199 }, { "KL/chosen_KL_mean": -19.885452270507812, "KL/mean": -30.026321411132812, "KL/rejected_KL_mean": -40.16718292236328, "KL/std": 20.82263946533203, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.03306082636117935, "fcm_dpo/delta": -0.07674050331115723, "fcm_dpo/margin": 20.281726837158203, "fcm_dpo/q_t": 0.35675719380378723, "grad_norm": 12.427189826965332, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.7273075580596924, "logits/rejected": -1.2242531776428223, "logps/chosen": -104.15354919433594, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -127.13505554199219, "loss": 0.9599, "margin_dpo/margin_mean": 20.281726837158203, "margin_dpo/margin_std": 23.667316436767578, "step": 200 }, { "KL/chosen_KL_mean": -21.689258575439453, "KL/mean": -31.436580657958984, "KL/rejected_KL_mean": -41.183902740478516, "KL/std": 22.61981201171875, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.03296644240617752, "fcm_dpo/delta": -0.04559415951371193, "fcm_dpo/margin": 19.494653701782227, "fcm_dpo/q_t": 0.3626253008842468, "grad_norm": 13.58653736114502, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.1615136861801147, "logits/rejected": -0.7577431201934814, "logps/chosen": -129.46176147460938, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -160.97637939453125, "loss": 0.9839, "margin_dpo/margin_mean": 19.494653701782227, "margin_dpo/margin_std": 24.554241180419922, "step": 201 }, { "KL/chosen_KL_mean": -20.65963363647461, "KL/mean": -29.557512283325195, "KL/rejected_KL_mean": -38.455387115478516, "KL/std": 18.78018569946289, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.03295065462589264, "fcm_dpo/delta": 0.014371881261467934, "fcm_dpo/margin": 17.795753479003906, "fcm_dpo/q_t": 0.3732941746711731, "grad_norm": 13.184235572814941, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.6903612613677979, "logits/rejected": -1.5571693181991577, "logps/chosen": -96.63208770751953, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -132.91537475585938, "loss": 0.9924, "margin_dpo/margin_mean": 17.795753479003906, "margin_dpo/margin_std": 21.043575286865234, "step": 202 }, { "KL/chosen_KL_mean": -18.072601318359375, "KL/mean": -28.36652374267578, "KL/rejected_KL_mean": -38.66044616699219, "KL/std": 19.70428466796875, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.03257571905851364, "fcm_dpo/delta": -0.07569441199302673, "fcm_dpo/margin": 20.587848663330078, "fcm_dpo/q_t": 0.3516688942909241, "grad_norm": 11.869770050048828, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.754921555519104, "logits/rejected": -1.3047497272491455, "logps/chosen": -113.03975677490234, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -131.54812622070312, "loss": 0.9199, "margin_dpo/margin_mean": 20.587848663330078, "margin_dpo/margin_std": 20.293766021728516, "step": 203 }, { "KL/chosen_KL_mean": -23.10736083984375, "KL/mean": -31.889026641845703, "KL/rejected_KL_mean": -40.67068862915039, "KL/std": 19.31324577331543, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.03266973793506622, "fcm_dpo/delta": 0.027443725615739822, "fcm_dpo/margin": 17.56332778930664, "fcm_dpo/q_t": 0.37612950801849365, "grad_norm": 14.53193187713623, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.6107479333877563, "logits/rejected": -0.9505150318145752, "logps/chosen": -139.4645538330078, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -134.0682830810547, "loss": 1.0141, "margin_dpo/margin_mean": 17.563325881958008, "margin_dpo/margin_std": 22.444055557250977, "step": 204 }, { "KL/chosen_KL_mean": -20.891101837158203, "KL/mean": -32.28116226196289, "KL/rejected_KL_mean": -43.67122268676758, "KL/std": 21.40902328491211, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.03186136111617088, "fcm_dpo/delta": -0.1361762434244156, "fcm_dpo/margin": 22.780120849609375, "fcm_dpo/q_t": 0.3368755578994751, "grad_norm": 11.15788459777832, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.0121028423309326, "logits/rejected": -1.5787427425384521, "logps/chosen": -109.75045013427734, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -135.5257110595703, "loss": 0.8665, "margin_dpo/margin_mean": 22.780120849609375, "margin_dpo/margin_std": 18.934959411621094, "step": 205 }, { "KL/chosen_KL_mean": -21.514278411865234, "KL/mean": -31.96915626525879, "KL/rejected_KL_mean": -42.42404556274414, "KL/std": 20.98330307006836, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.031307466328144073, "fcm_dpo/delta": -0.059123918414115906, "fcm_dpo/margin": 20.909759521484375, "fcm_dpo/q_t": 0.3521230220794678, "grad_norm": 14.418815612792969, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.7123664617538452, "logits/rejected": -1.5412907600402832, "logps/chosen": -98.06062316894531, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -145.37718200683594, "loss": 0.9075, "margin_dpo/margin_mean": 20.909757614135742, "margin_dpo/margin_std": 18.053428649902344, "step": 206 }, { "KL/chosen_KL_mean": -22.97022247314453, "KL/mean": -34.31324005126953, "KL/rejected_KL_mean": -45.6562614440918, "KL/std": 24.72857666015625, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.030863430351018906, "fcm_dpo/delta": -0.10768073797225952, "fcm_dpo/margin": 22.686038970947266, "fcm_dpo/q_t": 0.3563573360443115, "grad_norm": 15.195684432983398, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.8134467601776123, "logits/rejected": -1.3730719089508057, "logps/chosen": -109.20187377929688, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -136.31137084960938, "loss": 0.964, "margin_dpo/margin_mean": 22.686038970947266, "margin_dpo/margin_std": 28.860347747802734, "step": 207 }, { "KL/chosen_KL_mean": -23.196895599365234, "KL/mean": -35.008750915527344, "KL/rejected_KL_mean": -46.82060241699219, "KL/std": 23.192136764526367, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.03029707446694374, "fcm_dpo/delta": -0.12462113797664642, "fcm_dpo/margin": 23.62370491027832, "fcm_dpo/q_t": 0.34029072523117065, "grad_norm": 13.2672119140625, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.575986385345459, "logits/rejected": -1.2050753831863403, "logps/chosen": -117.3564453125, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -147.78292846679688, "loss": 0.8896, "margin_dpo/margin_mean": 23.623706817626953, "margin_dpo/margin_std": 21.5144100189209, "step": 208 }, { "KL/chosen_KL_mean": -19.803203582763672, "KL/mean": -30.20620346069336, "KL/rejected_KL_mean": -40.60919952392578, "KL/std": 20.466522216796875, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.029676878824830055, "fcm_dpo/delta": -0.019344590604305267, "fcm_dpo/margin": 20.805999755859375, "fcm_dpo/q_t": 0.36541199684143066, "grad_norm": 11.971503257751465, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.1772620677948, "logits/rejected": -1.7222741842269897, "logps/chosen": -103.97377014160156, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -128.22874450683594, "loss": 0.9687, "margin_dpo/margin_mean": 20.805999755859375, "margin_dpo/margin_std": 23.24773597717285, "step": 209 }, { "KL/chosen_KL_mean": -22.447582244873047, "KL/mean": -32.43077850341797, "KL/rejected_KL_mean": -42.41397476196289, "KL/std": 21.832761764526367, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.029744751751422882, "fcm_dpo/delta": 0.006301777437329292, "fcm_dpo/margin": 19.966388702392578, "fcm_dpo/q_t": 0.3706839680671692, "grad_norm": 11.197772979736328, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.8653286695480347, "logits/rejected": -1.4755324125289917, "logps/chosen": -103.61875915527344, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -126.58876037597656, "loss": 0.9762, "margin_dpo/margin_mean": 19.966388702392578, "margin_dpo/margin_std": 22.45195198059082, "step": 210 }, { "KL/chosen_KL_mean": -24.68486785888672, "KL/mean": -34.727622985839844, "KL/rejected_KL_mean": -44.770381927490234, "KL/std": 23.01360511779785, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.029908742755651474, "fcm_dpo/delta": -0.0008707772940397263, "fcm_dpo/margin": 20.08551025390625, "fcm_dpo/q_t": 0.36986246705055237, "grad_norm": 13.3218412399292, "learning_rate": 4.373239415645323e-07, "logits/chosen": -1.907552719116211, "logits/rejected": -1.2907359600067139, "logps/chosen": -133.3975830078125, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -138.32603454589844, "loss": 0.9817, "margin_dpo/margin_mean": 20.08551025390625, "margin_dpo/margin_std": 23.827468872070312, "step": 211 }, { "KL/chosen_KL_mean": -16.22506332397461, "KL/mean": -30.399871826171875, "KL/rejected_KL_mean": -44.574676513671875, "KL/std": 25.519960403442383, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.02872364968061447, "fcm_dpo/delta": -0.2344253659248352, "fcm_dpo/margin": 28.349609375, "fcm_dpo/q_t": 0.32393723726272583, "grad_norm": 11.074641227722168, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.5024776458740234, "logits/rejected": -1.0631954669952393, "logps/chosen": -114.5870132446289, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -154.4646759033203, "loss": 0.8383, "margin_dpo/margin_mean": 28.349609375, "margin_dpo/margin_std": 25.395172119140625, "step": 212 }, { "KL/chosen_KL_mean": -20.115703582763672, "KL/mean": -31.16232681274414, "KL/rejected_KL_mean": -42.208946228027344, "KL/std": 23.67479705810547, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.028282275423407555, "fcm_dpo/delta": -0.026752831414341927, "fcm_dpo/margin": 22.09324836730957, "fcm_dpo/q_t": 0.3622516989707947, "grad_norm": 14.950519561767578, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.5826215744018555, "logits/rejected": -0.785760760307312, "logps/chosen": -128.1710205078125, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -142.35308837890625, "loss": 0.9666, "margin_dpo/margin_mean": 22.09324836730957, "margin_dpo/margin_std": 25.019672393798828, "step": 213 }, { "KL/chosen_KL_mean": -21.616046905517578, "KL/mean": -31.63296127319336, "KL/rejected_KL_mean": -41.64988327026367, "KL/std": 25.268375396728516, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.02851586416363716, "fcm_dpo/delta": 0.03028884530067444, "fcm_dpo/margin": 20.033836364746094, "fcm_dpo/q_t": 0.3795204162597656, "grad_norm": 11.925753593444824, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.7268590927124023, "logits/rejected": -1.6037342548370361, "logps/chosen": -99.42078399658203, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -125.92567443847656, "loss": 1.0315, "margin_dpo/margin_mean": 20.033836364746094, "margin_dpo/margin_std": 27.95937728881836, "step": 214 }, { "KL/chosen_KL_mean": -18.16773223876953, "KL/mean": -31.403461456298828, "KL/rejected_KL_mean": -44.639183044433594, "KL/std": 24.960371017456055, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.027859974652528763, "fcm_dpo/delta": -0.14854371547698975, "fcm_dpo/margin": 26.471446990966797, "fcm_dpo/q_t": 0.3390721380710602, "grad_norm": 11.233661651611328, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.8102452754974365, "logits/rejected": -1.3372290134429932, "logps/chosen": -103.28163146972656, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -141.50070190429688, "loss": 0.8738, "margin_dpo/margin_mean": 26.471446990966797, "margin_dpo/margin_std": 24.33329963684082, "step": 215 }, { "KL/chosen_KL_mean": -20.190799713134766, "KL/mean": -31.618907928466797, "KL/rejected_KL_mean": -43.04701614379883, "KL/std": 22.050291061401367, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.02766409143805504, "fcm_dpo/delta": -0.03485286980867386, "fcm_dpo/margin": 22.856216430664062, "fcm_dpo/q_t": 0.36073732376098633, "grad_norm": 10.69111442565918, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.5344929695129395, "logits/rejected": -1.1117100715637207, "logps/chosen": -100.7831039428711, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -124.46685791015625, "loss": 0.9384, "margin_dpo/margin_mean": 22.856216430664062, "margin_dpo/margin_std": 22.82801055908203, "step": 216 }, { "KL/chosen_KL_mean": -23.97481918334961, "KL/mean": -34.015663146972656, "KL/rejected_KL_mean": -44.0565071105957, "KL/std": 22.7479190826416, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.027738582342863083, "fcm_dpo/delta": 0.045177195221185684, "fcm_dpo/margin": 20.081680297851562, "fcm_dpo/q_t": 0.38014328479766846, "grad_norm": 13.503419876098633, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -1.9388713836669922, "logits/rejected": -1.6298065185546875, "logps/chosen": -117.44740295410156, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -147.54502868652344, "loss": 1.0164, "margin_dpo/margin_mean": 20.081680297851562, "margin_dpo/margin_std": 25.85262107849121, "step": 217 }, { "KL/chosen_KL_mean": -23.205524444580078, "KL/mean": -34.79460144042969, "KL/rejected_KL_mean": -46.38368225097656, "KL/std": 23.289047241210938, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.027473315596580505, "fcm_dpo/delta": -0.039597928524017334, "fcm_dpo/margin": 23.17816162109375, "fcm_dpo/q_t": 0.3618423640727997, "grad_norm": 11.800704002380371, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.537433385848999, "logits/rejected": -1.154493808746338, "logps/chosen": -112.26435852050781, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -140.69049072265625, "loss": 0.9735, "margin_dpo/margin_mean": 23.17816162109375, "margin_dpo/margin_std": 27.571006774902344, "step": 218 }, { "KL/chosen_KL_mean": -20.883586883544922, "KL/mean": -31.445755004882812, "KL/rejected_KL_mean": -42.0079345703125, "KL/std": 26.703296661376953, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.02776181697845459, "fcm_dpo/delta": 0.013150937855243683, "fcm_dpo/margin": 21.12434196472168, "fcm_dpo/q_t": 0.3743200898170471, "grad_norm": 14.606476783752441, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.3784209489822388, "logits/rejected": -0.9233542680740356, "logps/chosen": -121.89093017578125, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -134.4758758544922, "loss": 1.0208, "margin_dpo/margin_mean": 21.124343872070312, "margin_dpo/margin_std": 28.46731948852539, "step": 219 }, { "KL/chosen_KL_mean": -21.17791748046875, "KL/mean": -30.952255249023438, "KL/rejected_KL_mean": -40.726593017578125, "KL/std": 21.85750961303711, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.02782927080988884, "fcm_dpo/delta": 0.05916573852300644, "fcm_dpo/margin": 19.54867935180664, "fcm_dpo/q_t": 0.3788089454174042, "grad_norm": 13.04234790802002, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.3891525268554688, "logits/rejected": -1.0343806743621826, "logps/chosen": -116.71513366699219, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -135.03363037109375, "loss": 0.9988, "margin_dpo/margin_mean": 19.548681259155273, "margin_dpo/margin_std": 21.85271644592285, "step": 220 }, { "KL/chosen_KL_mean": -20.259136199951172, "KL/mean": -34.90754699707031, "KL/rejected_KL_mean": -49.555965423583984, "KL/std": 27.771577835083008, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.027259493246674538, "fcm_dpo/delta": -0.21551363170146942, "fcm_dpo/margin": 29.296817779541016, "fcm_dpo/q_t": 0.3270253837108612, "grad_norm": 12.406855583190918, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.278171181678772, "logits/rejected": -1.1273565292358398, "logps/chosen": -103.10400390625, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -160.36776733398438, "loss": 0.8448, "margin_dpo/margin_mean": 29.29681968688965, "margin_dpo/margin_std": 24.922855377197266, "step": 221 }, { "KL/chosen_KL_mean": -26.222381591796875, "KL/mean": -38.26365661621094, "KL/rejected_KL_mean": -50.304931640625, "KL/std": 26.604204177856445, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.02669001743197441, "fcm_dpo/delta": -0.0460730716586113, "fcm_dpo/margin": 24.082550048828125, "fcm_dpo/q_t": 0.36245620250701904, "grad_norm": 14.988308906555176, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.8423449993133545, "logits/rejected": -1.4073848724365234, "logps/chosen": -121.36436462402344, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -157.10934448242188, "loss": 1.0002, "margin_dpo/margin_mean": 24.082550048828125, "margin_dpo/margin_std": 31.63479232788086, "step": 222 }, { "KL/chosen_KL_mean": -25.043113708496094, "KL/mean": -34.12859344482422, "KL/rejected_KL_mean": -43.214080810546875, "KL/std": 23.463045120239258, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.026909802109003067, "fcm_dpo/delta": 0.11684775352478027, "fcm_dpo/margin": 18.17096710205078, "fcm_dpo/q_t": 0.39333581924438477, "grad_norm": 13.454719543457031, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.0925252437591553, "logits/rejected": -1.6949093341827393, "logps/chosen": -110.61822509765625, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -129.6664581298828, "loss": 1.0636, "margin_dpo/margin_mean": 18.17096710205078, "margin_dpo/margin_std": 25.680675506591797, "step": 223 }, { "KL/chosen_KL_mean": -23.33550262451172, "KL/mean": -35.37522506713867, "KL/rejected_KL_mean": -47.414955139160156, "KL/std": 25.854228973388672, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.026914894580841064, "fcm_dpo/delta": -0.05140702798962593, "fcm_dpo/margin": 24.079452514648438, "fcm_dpo/q_t": 0.35937803983688354, "grad_norm": 12.670136451721191, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.8183977603912354, "logits/rejected": -1.3764841556549072, "logps/chosen": -106.05931091308594, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -130.01034545898438, "loss": 0.9533, "margin_dpo/margin_mean": 24.079452514648438, "margin_dpo/margin_std": 27.024581909179688, "step": 224 }, { "KL/chosen_KL_mean": -26.135478973388672, "KL/mean": -39.1927375793457, "KL/rejected_KL_mean": -52.24998474121094, "KL/std": 27.39688491821289, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.026545995846390724, "fcm_dpo/delta": -0.1001259982585907, "fcm_dpo/margin": 26.114503860473633, "fcm_dpo/q_t": 0.35271474719047546, "grad_norm": 13.096248626708984, "learning_rate": 4.249525076191759e-07, "logits/chosen": -1.9108197689056396, "logits/rejected": -1.4369807243347168, "logps/chosen": -121.81316375732422, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -157.34686279296875, "loss": 0.9443, "margin_dpo/margin_mean": 26.114501953125, "margin_dpo/margin_std": 30.540508270263672, "step": 225 }, { "KL/chosen_KL_mean": -21.453899383544922, "KL/mean": -33.56843566894531, "KL/rejected_KL_mean": -45.68296813964844, "KL/std": 26.415496826171875, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.02600889652967453, "fcm_dpo/delta": -0.03316984325647354, "fcm_dpo/margin": 24.229066848754883, "fcm_dpo/q_t": 0.3606122136116028, "grad_norm": 10.949889183044434, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.7831740379333496, "logits/rejected": -1.2580734491348267, "logps/chosen": -114.91482543945312, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -132.38473510742188, "loss": 0.9399, "margin_dpo/margin_mean": 24.229068756103516, "margin_dpo/margin_std": 23.999238967895508, "step": 226 }, { "KL/chosen_KL_mean": -29.91840362548828, "KL/mean": -42.30659484863281, "KL/rejected_KL_mean": -54.694786071777344, "KL/std": 24.9915771484375, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.025956418365240097, "fcm_dpo/delta": -0.046052783727645874, "fcm_dpo/margin": 24.776386260986328, "fcm_dpo/q_t": 0.3622854948043823, "grad_norm": 12.189157485961914, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.6310513019561768, "logits/rejected": -1.296666145324707, "logps/chosen": -122.732666015625, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -159.4317169189453, "loss": 0.9733, "margin_dpo/margin_mean": 24.776386260986328, "margin_dpo/margin_std": 29.621814727783203, "step": 227 }, { "KL/chosen_KL_mean": -28.03594207763672, "KL/mean": -37.81810760498047, "KL/rejected_KL_mean": -47.60026931762695, "KL/std": 23.396774291992188, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.026238415390253067, "fcm_dpo/delta": 0.09134963154792786, "fcm_dpo/margin": 19.564321517944336, "fcm_dpo/q_t": 0.3859117031097412, "grad_norm": 12.82266902923584, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.776263952255249, "logits/rejected": -1.3417589664459229, "logps/chosen": -122.07306671142578, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -143.62179565429688, "loss": 1.0421, "margin_dpo/margin_mean": 19.56432342529297, "margin_dpo/margin_std": 25.713115692138672, "step": 228 }, { "KL/chosen_KL_mean": -24.58877182006836, "KL/mean": -40.48166275024414, "KL/rejected_KL_mean": -56.37455368041992, "KL/std": 29.276599884033203, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.025502249598503113, "fcm_dpo/delta": -0.22903801500797272, "fcm_dpo/margin": 31.785781860351562, "fcm_dpo/q_t": 0.3288545608520508, "grad_norm": 11.366392135620117, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.7387281656265259, "logits/rejected": -1.106353759765625, "logps/chosen": -120.16644287109375, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -145.54833984375, "loss": 0.8601, "margin_dpo/margin_mean": 31.785781860351562, "margin_dpo/margin_std": 31.34054183959961, "step": 229 }, { "KL/chosen_KL_mean": -26.441959381103516, "KL/mean": -42.20930480957031, "KL/rejected_KL_mean": -57.97665023803711, "KL/std": 28.289073944091797, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0244866032153368, "fcm_dpo/delta": -0.1866535097360611, "fcm_dpo/margin": 31.534687042236328, "fcm_dpo/q_t": 0.32977449893951416, "grad_norm": 12.627509117126465, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.6247515678405762, "logits/rejected": -1.541365623474121, "logps/chosen": -90.15558624267578, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -149.88536071777344, "loss": 0.855, "margin_dpo/margin_mean": 31.534687042236328, "margin_dpo/margin_std": 27.700279235839844, "step": 230 }, { "KL/chosen_KL_mean": -30.17333984375, "KL/mean": -43.489891052246094, "KL/rejected_KL_mean": -56.806427001953125, "KL/std": 27.605449676513672, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.02397286705672741, "fcm_dpo/delta": -0.04197520762681961, "fcm_dpo/margin": 26.63309097290039, "fcm_dpo/q_t": 0.35667672753334045, "grad_norm": 10.620189666748047, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.6398537158966064, "logits/rejected": -1.0962942838668823, "logps/chosen": -125.62901306152344, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -137.76211547851562, "loss": 0.9216, "margin_dpo/margin_mean": 26.63309097290039, "margin_dpo/margin_std": 23.966245651245117, "step": 231 }, { "KL/chosen_KL_mean": -31.059097290039062, "KL/mean": -42.54182815551758, "KL/rejected_KL_mean": -54.02456283569336, "KL/std": 25.231170654296875, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.024187028408050537, "fcm_dpo/delta": 0.047176677733659744, "fcm_dpo/margin": 22.965465545654297, "fcm_dpo/q_t": 0.3770817518234253, "grad_norm": 11.561541557312012, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.4742908477783203, "logits/rejected": -1.001031756401062, "logps/chosen": -127.95635986328125, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -143.7891845703125, "loss": 1.0002, "margin_dpo/margin_mean": 22.965465545654297, "margin_dpo/margin_std": 26.589725494384766, "step": 232 }, { "KL/chosen_KL_mean": -33.11127471923828, "KL/mean": -48.822288513183594, "KL/rejected_KL_mean": -64.53329467773438, "KL/std": 31.394725799560547, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.023670662194490433, "fcm_dpo/delta": -0.15584608912467957, "fcm_dpo/margin": 31.42202377319336, "fcm_dpo/q_t": 0.3382393717765808, "grad_norm": 11.705220222473145, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.5907435417175293, "logits/rejected": -1.3590840101242065, "logps/chosen": -122.1615982055664, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -177.29246520996094, "loss": 0.8866, "margin_dpo/margin_mean": 31.42202377319336, "margin_dpo/margin_std": 30.440975189208984, "step": 233 }, { "KL/chosen_KL_mean": -31.81253433227539, "KL/mean": -48.56425476074219, "KL/rejected_KL_mean": -65.31597900390625, "KL/std": 30.707019805908203, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.02268315851688385, "fcm_dpo/delta": -0.17741169035434723, "fcm_dpo/margin": 33.503440856933594, "fcm_dpo/q_t": 0.3375135064125061, "grad_norm": 11.004179954528809, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.4095287322998047, "logits/rejected": -1.325160026550293, "logps/chosen": -106.13089752197266, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -171.7035675048828, "loss": 0.8903, "margin_dpo/margin_mean": 33.503440856933594, "margin_dpo/margin_std": 33.39353942871094, "step": 234 }, { "KL/chosen_KL_mean": -38.14078140258789, "KL/mean": -50.820003509521484, "KL/rejected_KL_mean": -63.499229431152344, "KL/std": 26.94939422607422, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.02282383292913437, "fcm_dpo/delta": 0.022458747029304504, "fcm_dpo/margin": 25.358455657958984, "fcm_dpo/q_t": 0.3702664375305176, "grad_norm": 11.009840965270996, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.42415452003479, "logits/rejected": -0.8660916090011597, "logps/chosen": -136.35781860351562, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -160.74600219726562, "loss": 0.9665, "margin_dpo/margin_mean": 25.358455657958984, "margin_dpo/margin_std": 25.70137596130371, "step": 235 }, { "KL/chosen_KL_mean": -32.49833297729492, "KL/mean": -48.46290588378906, "KL/rejected_KL_mean": -64.427490234375, "KL/std": 33.75263977050781, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.022469956427812576, "fcm_dpo/delta": -0.12633229792118073, "fcm_dpo/margin": 31.92914390563965, "fcm_dpo/q_t": 0.35326099395751953, "grad_norm": 11.388341903686523, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.6224640607833862, "logits/rejected": -1.6050695180892944, "logps/chosen": -111.3360595703125, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -173.4909210205078, "loss": 0.9255, "margin_dpo/margin_mean": 31.92914581298828, "margin_dpo/margin_std": 37.370079040527344, "step": 236 }, { "KL/chosen_KL_mean": -37.7026252746582, "KL/mean": -50.13774108886719, "KL/rejected_KL_mean": -62.572853088378906, "KL/std": 29.683502197265625, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.022383861243724823, "fcm_dpo/delta": 0.04588513821363449, "fcm_dpo/margin": 24.870227813720703, "fcm_dpo/q_t": 0.3859967589378357, "grad_norm": 11.582831382751465, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.5812885761260986, "logits/rejected": -1.2084081172943115, "logps/chosen": -122.9139175415039, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -162.48284912109375, "loss": 1.0641, "margin_dpo/margin_mean": 24.870229721069336, "margin_dpo/margin_std": 38.79283142089844, "step": 237 }, { "KL/chosen_KL_mean": -41.3282470703125, "KL/mean": -55.280601501464844, "KL/rejected_KL_mean": -69.23295593261719, "KL/std": 31.155271530151367, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.022342519834637642, "fcm_dpo/delta": -0.025089513510465622, "fcm_dpo/margin": 27.904705047607422, "fcm_dpo/q_t": 0.36615878343582153, "grad_norm": 10.668248176574707, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.0831797122955322, "logits/rejected": -0.9190908670425415, "logps/chosen": -120.05947875976562, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -164.6513671875, "loss": 0.9705, "margin_dpo/margin_mean": 27.904706954956055, "margin_dpo/margin_std": 32.556976318359375, "step": 238 }, { "KL/chosen_KL_mean": -37.94101333618164, "KL/mean": -51.004249572753906, "KL/rejected_KL_mean": -64.0674819946289, "KL/std": 27.307636260986328, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.022240478545427322, "fcm_dpo/delta": 0.01909753680229187, "fcm_dpo/margin": 26.12647247314453, "fcm_dpo/q_t": 0.3735753893852234, "grad_norm": 11.204874038696289, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.9833416938781738, "logits/rejected": -1.5207741260528564, "logps/chosen": -130.54196166992188, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -164.58517456054688, "loss": 0.997, "margin_dpo/margin_mean": 26.126468658447266, "margin_dpo/margin_std": 30.715160369873047, "step": 239 }, { "KL/chosen_KL_mean": -39.478023529052734, "KL/mean": -51.249290466308594, "KL/rejected_KL_mean": -63.02055740356445, "KL/std": 27.33614158630371, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.022596552968025208, "fcm_dpo/delta": 0.07185894250869751, "fcm_dpo/margin": 23.542545318603516, "fcm_dpo/q_t": 0.38220706582069397, "grad_norm": 12.88066291809082, "learning_rate": 4.106969024216348e-07, "logits/chosen": -1.6996513605117798, "logits/rejected": -1.3365478515625, "logps/chosen": -125.6377944946289, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -143.47622680664062, "loss": 1.0198, "margin_dpo/margin_mean": 23.542545318603516, "margin_dpo/margin_std": 28.594619750976562, "step": 240 }, { "KL/chosen_KL_mean": -34.47979736328125, "KL/mean": -55.17344665527344, "KL/rejected_KL_mean": -75.86708068847656, "KL/std": 32.756134033203125, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.02182016521692276, "fcm_dpo/delta": -0.3318721354007721, "fcm_dpo/margin": 41.38729476928711, "fcm_dpo/q_t": 0.30897897481918335, "grad_norm": 10.333210945129395, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.5284082889556885, "logits/rejected": -0.8569146990776062, "logps/chosen": -135.52574157714844, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -169.91644287109375, "loss": 0.8022, "margin_dpo/margin_mean": 41.387290954589844, "margin_dpo/margin_std": 36.1435546875, "step": 241 }, { "KL/chosen_KL_mean": -42.058143615722656, "KL/mean": -56.6781005859375, "KL/rejected_KL_mean": -71.29805755615234, "KL/std": 27.774934768676758, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.021315952762961388, "fcm_dpo/delta": -0.025188326835632324, "fcm_dpo/margin": 29.23990821838379, "fcm_dpo/q_t": 0.361325204372406, "grad_norm": 11.223978996276855, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.6468056440353394, "logits/rejected": -0.976578950881958, "logps/chosen": -137.73080444335938, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -161.95260620117188, "loss": 0.9528, "margin_dpo/margin_mean": 29.239906311035156, "margin_dpo/margin_std": 30.308300018310547, "step": 242 }, { "KL/chosen_KL_mean": -40.7613525390625, "KL/mean": -55.106101989746094, "KL/rejected_KL_mean": -69.45085144042969, "KL/std": 29.41200828552246, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.021357949823141098, "fcm_dpo/delta": -0.015652522444725037, "fcm_dpo/margin": 28.689498901367188, "fcm_dpo/q_t": 0.3670157194137573, "grad_norm": 11.067669868469238, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.8208260536193848, "logits/rejected": -1.009543538093567, "logps/chosen": -138.79275512695312, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -152.638916015625, "loss": 0.9909, "margin_dpo/margin_mean": 28.689498901367188, "margin_dpo/margin_std": 33.96335220336914, "step": 243 }, { "KL/chosen_KL_mean": -42.472312927246094, "KL/mean": -57.63190460205078, "KL/rejected_KL_mean": -72.79148864746094, "KL/std": 32.77070999145508, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.02094869688153267, "fcm_dpo/delta": -0.037751004099845886, "fcm_dpo/margin": 30.319171905517578, "fcm_dpo/q_t": 0.36187177896499634, "grad_norm": 10.972872734069824, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.1047635078430176, "logits/rejected": -1.7642847299575806, "logps/chosen": -131.36624145507812, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -175.36427307128906, "loss": 0.9557, "margin_dpo/margin_mean": 30.319169998168945, "margin_dpo/margin_std": 33.51457977294922, "step": 244 }, { "KL/chosen_KL_mean": -36.13850784301758, "KL/mean": -52.648311614990234, "KL/rejected_KL_mean": -69.15811157226562, "KL/std": 31.265708923339844, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.02069661021232605, "fcm_dpo/delta": -0.0894637331366539, "fcm_dpo/margin": 33.01960372924805, "fcm_dpo/q_t": 0.34890544414520264, "grad_norm": 10.407121658325195, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.110779285430908, "logits/rejected": -1.284542202949524, "logps/chosen": -140.33250427246094, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -161.8145751953125, "loss": 0.9076, "margin_dpo/margin_mean": 33.01960372924805, "margin_dpo/margin_std": 31.342018127441406, "step": 245 }, { "KL/chosen_KL_mean": -46.44175720214844, "KL/mean": -59.198211669921875, "KL/rejected_KL_mean": -71.95466613769531, "KL/std": 29.568824768066406, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.020739587023854256, "fcm_dpo/delta": 0.07481832057237625, "fcm_dpo/margin": 25.512916564941406, "fcm_dpo/q_t": 0.38311484456062317, "grad_norm": 10.978471755981445, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.6378669738769531, "logits/rejected": -1.0933966636657715, "logps/chosen": -149.66079711914062, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -162.94688415527344, "loss": 1.0197, "margin_dpo/margin_mean": 25.512916564941406, "margin_dpo/margin_std": 31.039066314697266, "step": 246 }, { "KL/chosen_KL_mean": -41.98240661621094, "KL/mean": -61.99413299560547, "KL/rejected_KL_mean": -82.00584411621094, "KL/std": 33.70969772338867, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.020096857100725174, "fcm_dpo/delta": -0.2242925465106964, "fcm_dpo/margin": 40.02344512939453, "fcm_dpo/q_t": 0.3280399441719055, "grad_norm": 10.023884773254395, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.1454501152038574, "logits/rejected": -1.9373878240585327, "logps/chosen": -128.97677612304688, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -193.3438720703125, "loss": 0.8525, "margin_dpo/margin_mean": 40.0234489440918, "margin_dpo/margin_std": 37.45133972167969, "step": 247 }, { "KL/chosen_KL_mean": -47.79905700683594, "KL/mean": -61.439239501953125, "KL/rejected_KL_mean": -75.07942962646484, "KL/std": 29.55758285522461, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.020200185477733612, "fcm_dpo/delta": 0.05140257999300957, "fcm_dpo/margin": 27.280372619628906, "fcm_dpo/q_t": 0.3780377507209778, "grad_norm": 11.732197761535645, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -2.136711835861206, "logits/rejected": -2.0853514671325684, "logps/chosen": -122.58457946777344, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -173.3563232421875, "loss": 0.9978, "margin_dpo/margin_mean": 27.280372619628906, "margin_dpo/margin_std": 31.06748390197754, "step": 248 }, { "KL/chosen_KL_mean": -52.07401657104492, "KL/mean": -64.79183197021484, "KL/rejected_KL_mean": -77.5096435546875, "KL/std": 31.157882690429688, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.02037045732140541, "fcm_dpo/delta": 0.08636321127414703, "fcm_dpo/margin": 25.43564224243164, "fcm_dpo/q_t": 0.38753050565719604, "grad_norm": 12.965365409851074, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -1.9121811389923096, "logits/rejected": -1.3700027465820312, "logps/chosen": -147.77780151367188, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -164.65611267089844, "loss": 1.0523, "margin_dpo/margin_mean": 25.43564224243164, "margin_dpo/margin_std": 35.34349822998047, "step": 249 }, { "KL/chosen_KL_mean": -47.28365707397461, "KL/mean": -60.79273986816406, "KL/rejected_KL_mean": -74.30183410644531, "KL/std": 31.433671951293945, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.020542338490486145, "fcm_dpo/delta": 0.046970319002866745, "fcm_dpo/margin": 27.01817512512207, "fcm_dpo/q_t": 0.37707072496414185, "grad_norm": 13.092029571533203, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.0167007446289062, "logits/rejected": -1.141892671585083, "logps/chosen": -161.33587646484375, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -155.38951110839844, "loss": 1.0203, "margin_dpo/margin_mean": 27.01817512512207, "margin_dpo/margin_std": 33.11515808105469, "step": 250 }, { "KL/chosen_KL_mean": -46.30205535888672, "KL/mean": -59.02735900878906, "KL/rejected_KL_mean": -71.75267028808594, "KL/std": 29.224882125854492, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.02080637589097023, "fcm_dpo/delta": 0.0737934410572052, "fcm_dpo/margin": 25.45061492919922, "fcm_dpo/q_t": 0.3869265913963318, "grad_norm": 12.45988655090332, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.896494746208191, "logits/rejected": -1.212632656097412, "logps/chosen": -144.01333618164062, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -155.28009033203125, "loss": 1.0511, "margin_dpo/margin_mean": 25.45061683654785, "margin_dpo/margin_std": 35.14149475097656, "step": 251 }, { "KL/chosen_KL_mean": -45.81730651855469, "KL/mean": -58.80479049682617, "KL/rejected_KL_mean": -71.79226684570312, "KL/std": 30.376096725463867, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.021217800676822662, "fcm_dpo/delta": 0.05167728289961815, "fcm_dpo/margin": 25.974964141845703, "fcm_dpo/q_t": 0.37831413745880127, "grad_norm": 13.034337043762207, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -1.86942458152771, "logits/rejected": -1.3415379524230957, "logps/chosen": -154.80340576171875, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -163.35650634765625, "loss": 1.0171, "margin_dpo/margin_mean": 25.974964141845703, "margin_dpo/margin_std": 32.366241455078125, "step": 252 }, { "KL/chosen_KL_mean": -50.23309326171875, "KL/mean": -62.431304931640625, "KL/rejected_KL_mean": -74.62950897216797, "KL/std": 33.45896911621094, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.021693840622901917, "fcm_dpo/delta": 0.07219475507736206, "fcm_dpo/margin": 24.39640998840332, "fcm_dpo/q_t": 0.3814728856086731, "grad_norm": 12.625275611877441, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.020409107208252, "logits/rejected": -1.8101792335510254, "logps/chosen": -150.44940185546875, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -180.30621337890625, "loss": 1.0268, "margin_dpo/margin_mean": 24.39640998840332, "margin_dpo/margin_std": 29.5477352142334, "step": 253 }, { "KL/chosen_KL_mean": -49.2394905090332, "KL/mean": -61.46051025390625, "KL/rejected_KL_mean": -73.68153381347656, "KL/std": 30.929838180541992, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.021669182926416397, "fcm_dpo/delta": 0.07391853630542755, "fcm_dpo/margin": 24.442039489746094, "fcm_dpo/q_t": 0.3832131624221802, "grad_norm": 12.242829322814941, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.0314650535583496, "logits/rejected": -1.49741792678833, "logps/chosen": -147.97467041015625, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -167.41978454589844, "loss": 1.0421, "margin_dpo/margin_mean": 24.442039489746094, "margin_dpo/margin_std": 32.395362854003906, "step": 254 }, { "KL/chosen_KL_mean": -50.163997650146484, "KL/mean": -67.56504821777344, "KL/rejected_KL_mean": -84.96609497070312, "KL/std": 34.72138214111328, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.021407779306173325, "fcm_dpo/delta": -0.15659618377685547, "fcm_dpo/margin": 34.80209732055664, "fcm_dpo/q_t": 0.33824843168258667, "grad_norm": 12.097771644592285, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.770204782485962, "logits/rejected": -1.4222946166992188, "logps/chosen": -140.54656982421875, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -182.04234313964844, "loss": 0.8931, "margin_dpo/margin_mean": 34.802101135253906, "margin_dpo/margin_std": 34.6953239440918, "step": 255 }, { "KL/chosen_KL_mean": -47.04194259643555, "KL/mean": -63.48567199707031, "KL/rejected_KL_mean": -79.92939758300781, "KL/std": 36.164527893066406, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.020983921363949776, "fcm_dpo/delta": -0.0966934934258461, "fcm_dpo/margin": 32.887454986572266, "fcm_dpo/q_t": 0.3537742495536804, "grad_norm": 12.469510078430176, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.886181116104126, "logits/rejected": -1.5361230373382568, "logps/chosen": -135.7947540283203, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -178.42323303222656, "loss": 0.9714, "margin_dpo/margin_mean": 32.887454986572266, "margin_dpo/margin_std": 41.105716705322266, "step": 256 }, { "KL/chosen_KL_mean": -49.223548889160156, "KL/mean": -65.31234741210938, "KL/rejected_KL_mean": -81.40113830566406, "KL/std": 36.88652038574219, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.020384207367897034, "fcm_dpo/delta": -0.06320657581090927, "fcm_dpo/margin": 32.17759704589844, "fcm_dpo/q_t": 0.358612060546875, "grad_norm": 11.699127197265625, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.7901161909103394, "logits/rejected": -1.399821400642395, "logps/chosen": -132.02706909179688, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -167.26893615722656, "loss": 0.9621, "margin_dpo/margin_mean": 32.17759704589844, "margin_dpo/margin_std": 36.10734558105469, "step": 257 }, { "KL/chosen_KL_mean": -50.38591766357422, "KL/mean": -65.64173889160156, "KL/rejected_KL_mean": -80.89756774902344, "KL/std": 34.939796447753906, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.02059350535273552, "fcm_dpo/delta": -0.030976958572864532, "fcm_dpo/margin": 30.511667251586914, "fcm_dpo/q_t": 0.3607671558856964, "grad_norm": 12.132224082946777, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.0627102851867676, "logits/rejected": -1.7050683498382568, "logps/chosen": -140.57366943359375, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -184.36825561523438, "loss": 0.9609, "margin_dpo/margin_mean": 30.51166534423828, "margin_dpo/margin_std": 32.8673095703125, "step": 258 }, { "KL/chosen_KL_mean": -55.00556182861328, "KL/mean": -67.77142333984375, "KL/rejected_KL_mean": -80.53729248046875, "KL/std": 34.363250732421875, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.02053195610642433, "fcm_dpo/delta": 0.07962623983621597, "fcm_dpo/margin": 25.531726837158203, "fcm_dpo/q_t": 0.38557636737823486, "grad_norm": 12.572392463684082, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.3159854412078857, "logits/rejected": -1.0666191577911377, "logps/chosen": -145.77810668945312, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -175.12545776367188, "loss": 1.0663, "margin_dpo/margin_mean": 25.531728744506836, "margin_dpo/margin_std": 38.01142120361328, "step": 259 }, { "KL/chosen_KL_mean": -55.62242889404297, "KL/mean": -67.23291015625, "KL/rejected_KL_mean": -78.84339141845703, "KL/std": 34.96982192993164, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.02108220010995865, "fcm_dpo/delta": 0.11625918745994568, "fcm_dpo/margin": 23.220951080322266, "fcm_dpo/q_t": 0.3942277729511261, "grad_norm": 14.051042556762695, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.764245867729187, "logits/rejected": -1.7682816982269287, "logps/chosen": -131.21511840820312, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -179.68893432617188, "loss": 1.1152, "margin_dpo/margin_mean": 23.220951080322266, "margin_dpo/margin_std": 39.71216583251953, "step": 260 }, { "KL/chosen_KL_mean": -43.93332290649414, "KL/mean": -62.08281707763672, "KL/rejected_KL_mean": -80.23231506347656, "KL/std": 35.70299530029297, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.02061646617949009, "fcm_dpo/delta": -0.16195423901081085, "fcm_dpo/margin": 36.298988342285156, "fcm_dpo/q_t": 0.33775997161865234, "grad_norm": 11.157303810119629, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.8307852745056152, "logits/rejected": -1.455869197845459, "logps/chosen": -138.188232421875, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -173.88931274414062, "loss": 0.8793, "margin_dpo/margin_mean": 36.298988342285156, "margin_dpo/margin_std": 34.87677764892578, "step": 261 }, { "KL/chosen_KL_mean": -47.66316223144531, "KL/mean": -64.61615753173828, "KL/rejected_KL_mean": -81.56915283203125, "KL/std": 36.20745086669922, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.02022511512041092, "fcm_dpo/delta": -0.0924750417470932, "fcm_dpo/margin": 33.90599060058594, "fcm_dpo/q_t": 0.35102277994155884, "grad_norm": 11.408646583557129, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.8961164951324463, "logits/rejected": -1.3651156425476074, "logps/chosen": -132.9304656982422, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -172.39524841308594, "loss": 0.9258, "margin_dpo/margin_mean": 33.9059944152832, "margin_dpo/margin_std": 35.8066520690918, "step": 262 }, { "KL/chosen_KL_mean": -38.69598388671875, "KL/mean": -61.192665100097656, "KL/rejected_KL_mean": -83.68936157226562, "KL/std": 40.57989501953125, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.01933453045785427, "fcm_dpo/delta": -0.29548877477645874, "fcm_dpo/margin": 44.993377685546875, "fcm_dpo/q_t": 0.31437236070632935, "grad_norm": 10.634689331054688, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.1852564811706543, "logits/rejected": -1.6536786556243896, "logps/chosen": -134.45388793945312, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -185.52313232421875, "loss": 0.8335, "margin_dpo/margin_mean": 44.993377685546875, "margin_dpo/margin_std": 42.28729248046875, "step": 263 }, { "KL/chosen_KL_mean": -43.51506423950195, "KL/mean": -62.352027893066406, "KL/rejected_KL_mean": -81.18898010253906, "KL/std": 38.409820556640625, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.01866101287305355, "fcm_dpo/delta": -0.11131103336811066, "fcm_dpo/margin": 37.67393112182617, "fcm_dpo/q_t": 0.34703633189201355, "grad_norm": 10.06856632232666, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.0372724533081055, "logits/rejected": -1.7543597221374512, "logps/chosen": -134.13772583007812, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -181.5145263671875, "loss": 0.9176, "margin_dpo/margin_mean": 37.673927307128906, "margin_dpo/margin_std": 38.82753372192383, "step": 264 }, { "KL/chosen_KL_mean": -45.83721923828125, "KL/mean": -63.69260025024414, "KL/rejected_KL_mean": -81.54798889160156, "KL/std": 38.51799392700195, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.01833738200366497, "fcm_dpo/delta": -0.059704020619392395, "fcm_dpo/margin": 35.71075439453125, "fcm_dpo/q_t": 0.3602437376976013, "grad_norm": 10.095213890075684, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.0608723163604736, "logits/rejected": -1.6708756685256958, "logps/chosen": -138.2095489501953, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -176.17556762695312, "loss": 0.9636, "margin_dpo/margin_mean": 35.710758209228516, "margin_dpo/margin_std": 41.63832092285156, "step": 265 }, { "KL/chosen_KL_mean": -53.26393508911133, "KL/mean": -68.68094635009766, "KL/rejected_KL_mean": -84.09796142578125, "KL/std": 36.36904525756836, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.018431194126605988, "fcm_dpo/delta": 0.03355543315410614, "fcm_dpo/margin": 30.834012985229492, "fcm_dpo/q_t": 0.3754233419895172, "grad_norm": 12.797066688537598, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.3223564624786377, "logits/rejected": -1.5511927604675293, "logps/chosen": -155.46395874023438, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -158.46438598632812, "loss": 1.0032, "margin_dpo/margin_mean": 30.834012985229492, "margin_dpo/margin_std": 36.906044006347656, "step": 266 }, { "KL/chosen_KL_mean": -45.26059341430664, "KL/mean": -63.78740692138672, "KL/rejected_KL_mean": -82.31421661376953, "KL/std": 38.15686798095703, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.01818893291056156, "fcm_dpo/delta": -0.08039677143096924, "fcm_dpo/margin": 37.05363082885742, "fcm_dpo/q_t": 0.3506849408149719, "grad_norm": 10.846717834472656, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.678443193435669, "logits/rejected": -1.069075107574463, "logps/chosen": -149.54660034179688, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -178.30140686035156, "loss": 0.9098, "margin_dpo/margin_mean": 37.05363082885742, "margin_dpo/margin_std": 34.530548095703125, "step": 267 }, { "KL/chosen_KL_mean": -45.045166015625, "KL/mean": -65.12213897705078, "KL/rejected_KL_mean": -85.19911193847656, "KL/std": 37.38362121582031, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.01781470701098442, "fcm_dpo/delta": -0.1249554455280304, "fcm_dpo/margin": 40.15393829345703, "fcm_dpo/q_t": 0.34126344323158264, "grad_norm": 10.987445831298828, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6620032787322998, "logits/rejected": -1.4856903553009033, "logps/chosen": -131.74139404296875, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -194.39096069335938, "loss": 0.887, "margin_dpo/margin_mean": 40.15393829345703, "margin_dpo/margin_std": 36.388572692871094, "step": 268 }, { "KL/chosen_KL_mean": -46.77866744995117, "KL/mean": -67.34500122070312, "KL/rejected_KL_mean": -87.91133117675781, "KL/std": 41.212059020996094, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.017480023205280304, "fcm_dpo/delta": -0.12802359461784363, "fcm_dpo/margin": 41.132667541503906, "fcm_dpo/q_t": 0.3428999185562134, "grad_norm": 10.6517333984375, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.9230566024780273, "logits/rejected": -1.4659779071807861, "logps/chosen": -151.75048828125, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -200.38780212402344, "loss": 0.8975, "margin_dpo/margin_mean": 41.132667541503906, "margin_dpo/margin_std": 39.8093147277832, "step": 269 }, { "KL/chosen_KL_mean": -51.59083938598633, "KL/mean": -69.45450592041016, "KL/rejected_KL_mean": -87.31818389892578, "KL/std": 40.703269958496094, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0173244159668684, "fcm_dpo/delta": -0.020390255376696587, "fcm_dpo/margin": 35.727333068847656, "fcm_dpo/q_t": 0.3671169579029083, "grad_norm": 10.368598937988281, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.385664463043213, "logits/rejected": -1.968564748764038, "logps/chosen": -152.6646728515625, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -205.07107543945312, "loss": 1.0012, "margin_dpo/margin_mean": 35.727333068847656, "margin_dpo/margin_std": 45.855979919433594, "step": 270 }, { "KL/chosen_KL_mean": -42.9041633605957, "KL/mean": -64.91822052001953, "KL/rejected_KL_mean": -86.93228912353516, "KL/std": 38.52204895019531, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.016827944666147232, "fcm_dpo/delta": -0.15254725515842438, "fcm_dpo/margin": 44.02812957763672, "fcm_dpo/q_t": 0.3356161117553711, "grad_norm": 10.270514488220215, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.6432631015777588, "logits/rejected": -1.2085604667663574, "logps/chosen": -139.62875366210938, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -185.45669555664062, "loss": 0.8989, "margin_dpo/margin_mean": 44.02812957763672, "margin_dpo/margin_std": 43.58592987060547, "step": 271 }, { "KL/chosen_KL_mean": -36.34591293334961, "KL/mean": -56.95027542114258, "KL/rejected_KL_mean": -77.55464172363281, "KL/std": 42.07804870605469, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.01650695502758026, "fcm_dpo/delta": -0.0861324816942215, "fcm_dpo/margin": 41.208717346191406, "fcm_dpo/q_t": 0.35350972414016724, "grad_norm": 10.774606704711914, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.387490153312683, "logits/rejected": -1.251193881034851, "logps/chosen": -117.32313537597656, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -186.11000061035156, "loss": 0.9369, "margin_dpo/margin_mean": 41.208717346191406, "margin_dpo/margin_std": 44.89857864379883, "step": 272 }, { "KL/chosen_KL_mean": -50.203277587890625, "KL/mean": -71.89898681640625, "KL/rejected_KL_mean": -93.59468841552734, "KL/std": 44.23356628417969, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.01601198874413967, "fcm_dpo/delta": -0.10544279217720032, "fcm_dpo/margin": 43.39139938354492, "fcm_dpo/q_t": 0.3493504524230957, "grad_norm": 9.783014297485352, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.5958242416381836, "logits/rejected": -1.1936789751052856, "logps/chosen": -142.42788696289062, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -190.95774841308594, "loss": 0.9489, "margin_dpo/margin_mean": 43.39139938354492, "margin_dpo/margin_std": 48.88934326171875, "step": 273 }, { "KL/chosen_KL_mean": -51.62882614135742, "KL/mean": -68.78704833984375, "KL/rejected_KL_mean": -85.94526672363281, "KL/std": 41.66260528564453, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.016169056296348572, "fcm_dpo/delta": 0.047764576971530914, "fcm_dpo/margin": 34.31644821166992, "fcm_dpo/q_t": 0.37644654512405396, "grad_norm": 12.44456672668457, "learning_rate": 3.75e-07, "logits/chosen": -2.107360601425171, "logits/rejected": -1.7155675888061523, "logps/chosen": -133.88970947265625, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -169.822265625, "loss": 0.9922, "margin_dpo/margin_mean": 34.31644821166992, "margin_dpo/margin_std": 38.06298828125, "step": 274 }, { "KL/chosen_KL_mean": -43.23433303833008, "KL/mean": -62.11888122558594, "KL/rejected_KL_mean": -81.00343322753906, "KL/std": 41.71607208251953, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.016315527260303497, "fcm_dpo/delta": -0.01804516837000847, "fcm_dpo/margin": 37.76909637451172, "fcm_dpo/q_t": 0.3657800257205963, "grad_norm": 11.616429328918457, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.2329463958740234, "logits/rejected": -1.9122369289398193, "logps/chosen": -122.92129516601562, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -179.75439453125, "loss": 0.9993, "margin_dpo/margin_mean": 37.76909637451172, "margin_dpo/margin_std": 47.188934326171875, "step": 275 }, { "KL/chosen_KL_mean": -54.007205963134766, "KL/mean": -72.87872314453125, "KL/rejected_KL_mean": -91.75023651123047, "KL/std": 41.656272888183594, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.01603788137435913, "fcm_dpo/delta": -0.007269665598869324, "fcm_dpo/margin": 37.74303436279297, "fcm_dpo/q_t": 0.3687402904033661, "grad_norm": 12.015377044677734, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.9135591983795166, "logits/rejected": -1.340560793876648, "logps/chosen": -140.54690551757812, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -169.60418701171875, "loss": 0.9913, "margin_dpo/margin_mean": 37.74303436279297, "margin_dpo/margin_std": 45.23383331298828, "step": 276 }, { "KL/chosen_KL_mean": -56.7051887512207, "KL/mean": -72.35625457763672, "KL/rejected_KL_mean": -88.00731658935547, "KL/std": 40.11278533935547, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.016341477632522583, "fcm_dpo/delta": 0.09332635253667831, "fcm_dpo/margin": 31.302133560180664, "fcm_dpo/q_t": 0.3858821392059326, "grad_norm": 13.176647186279297, "learning_rate": 3.71651119641714e-07, "logits/chosen": -1.8759377002716064, "logits/rejected": -1.4501951932907104, "logps/chosen": -140.94931030273438, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -186.84152221679688, "loss": 1.0329, "margin_dpo/margin_mean": 31.302133560180664, "margin_dpo/margin_std": 38.85381317138672, "step": 277 }, { "KL/chosen_KL_mean": -54.521949768066406, "KL/mean": -72.78646850585938, "KL/rejected_KL_mean": -91.05099487304688, "KL/std": 43.93386459350586, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.016401417553424835, "fcm_dpo/delta": 0.0004968829452991486, "fcm_dpo/margin": 36.52904510498047, "fcm_dpo/q_t": 0.367180198431015, "grad_norm": 12.187837600708008, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.071317195892334, "logits/rejected": -1.8599553108215332, "logps/chosen": -136.95382690429688, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -188.9079132080078, "loss": 0.9916, "margin_dpo/margin_mean": 36.52904510498047, "margin_dpo/margin_std": 43.21200942993164, "step": 278 }, { "KL/chosen_KL_mean": -57.36602020263672, "KL/mean": -78.47025299072266, "KL/rejected_KL_mean": -99.57447814941406, "KL/std": 45.344722747802734, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.01619250327348709, "fcm_dpo/delta": -0.09028400480747223, "fcm_dpo/margin": 42.208457946777344, "fcm_dpo/q_t": 0.3511165976524353, "grad_norm": 11.429241180419922, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.9995089769363403, "logits/rejected": -1.788658618927002, "logps/chosen": -142.53402709960938, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -193.7011260986328, "loss": 0.9291, "margin_dpo/margin_mean": 42.208457946777344, "margin_dpo/margin_std": 44.56724166870117, "step": 279 }, { "KL/chosen_KL_mean": -57.516815185546875, "KL/mean": -80.04637145996094, "KL/rejected_KL_mean": -102.57593536376953, "KL/std": 42.293373107910156, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.01591867208480835, "fcm_dpo/delta": -0.12616533041000366, "fcm_dpo/margin": 45.059120178222656, "fcm_dpo/q_t": 0.34263402223587036, "grad_norm": 11.256477355957031, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.8938719034194946, "logits/rejected": -1.6887825727462769, "logps/chosen": -143.37322998046875, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -206.69451904296875, "loss": 0.9, "margin_dpo/margin_mean": 45.059120178222656, "margin_dpo/margin_std": 43.69725799560547, "step": 280 }, { "KL/chosen_KL_mean": -60.7031135559082, "KL/mean": -83.85073852539062, "KL/rejected_KL_mean": -106.99837493896484, "KL/std": 50.309837341308594, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.015465808100998402, "fcm_dpo/delta": -0.12517547607421875, "fcm_dpo/margin": 46.29527282714844, "fcm_dpo/q_t": 0.34831324219703674, "grad_norm": 9.73710823059082, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.182039499282837, "logits/rejected": -1.4621529579162598, "logps/chosen": -147.25393676757812, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -195.62704467773438, "loss": 0.9296, "margin_dpo/margin_mean": 46.29527282714844, "margin_dpo/margin_std": 52.41375732421875, "step": 281 }, { "KL/chosen_KL_mean": -58.55364227294922, "KL/mean": -86.87506103515625, "KL/rejected_KL_mean": -115.19648742675781, "KL/std": 49.25910568237305, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.014769460074603558, "fcm_dpo/delta": -0.2590848207473755, "fcm_dpo/margin": 56.642845153808594, "fcm_dpo/q_t": 0.3178322911262512, "grad_norm": 9.170939445495605, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.749898910522461, "logits/rejected": -1.3752869367599487, "logps/chosen": -163.65875244140625, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -218.0498504638672, "loss": 0.8193, "margin_dpo/margin_mean": 56.64284896850586, "margin_dpo/margin_std": 48.13591766357422, "step": 282 }, { "KL/chosen_KL_mean": -69.62220764160156, "KL/mean": -89.18400573730469, "KL/rejected_KL_mean": -108.74581146240234, "KL/std": 48.27381134033203, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.01466631330549717, "fcm_dpo/delta": 0.027803786098957062, "fcm_dpo/margin": 39.12359619140625, "fcm_dpo/q_t": 0.37650713324546814, "grad_norm": 12.607582092285156, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.3578944206237793, "logits/rejected": -2.0300755500793457, "logps/chosen": -164.6748046875, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -212.2903594970703, "loss": 1.0291, "margin_dpo/margin_mean": 39.12359619140625, "margin_dpo/margin_std": 53.22113800048828, "step": 283 }, { "KL/chosen_KL_mean": -60.701107025146484, "KL/mean": -87.35679626464844, "KL/rejected_KL_mean": -114.0124740600586, "KL/std": 49.817684173583984, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.014236303977668285, "fcm_dpo/delta": -0.1737486720085144, "fcm_dpo/margin": 53.311363220214844, "fcm_dpo/q_t": 0.33088716864585876, "grad_norm": 9.93166446685791, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.6404573917388916, "logits/rejected": -1.31294584274292, "logps/chosen": -148.36758422851562, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -212.76351928710938, "loss": 0.8494, "margin_dpo/margin_mean": 53.311363220214844, "margin_dpo/margin_std": 44.20699691772461, "step": 284 }, { "KL/chosen_KL_mean": -69.76573944091797, "KL/mean": -88.70942687988281, "KL/rejected_KL_mean": -107.65312957763672, "KL/std": 43.708499908447266, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.01436782255768776, "fcm_dpo/delta": 0.05834877863526344, "fcm_dpo/margin": 37.88738250732422, "fcm_dpo/q_t": 0.37628209590911865, "grad_norm": 14.151948928833008, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.261826276779175, "logits/rejected": -1.9773106575012207, "logps/chosen": -164.1987762451172, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -211.72506713867188, "loss": 0.9946, "margin_dpo/margin_mean": 37.88738250732422, "margin_dpo/margin_std": 40.811885833740234, "step": 285 }, { "KL/chosen_KL_mean": -74.2843246459961, "KL/mean": -98.87948608398438, "KL/rejected_KL_mean": -123.47463989257812, "KL/std": 47.30650329589844, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.01411445252597332, "fcm_dpo/delta": -0.10162456333637238, "fcm_dpo/margin": 49.190330505371094, "fcm_dpo/q_t": 0.3465542495250702, "grad_norm": 10.794189453125, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.0061943531036377, "logits/rejected": -1.5523154735565186, "logps/chosen": -178.00473022460938, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -219.73239135742188, "loss": 0.9134, "margin_dpo/margin_mean": 49.190330505371094, "margin_dpo/margin_std": 48.11893081665039, "step": 286 }, { "KL/chosen_KL_mean": -87.43531036376953, "KL/mean": -102.48440551757812, "KL/rejected_KL_mean": -117.53351593017578, "KL/std": 44.313270568847656, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.014520399272441864, "fcm_dpo/delta": 0.1684514731168747, "fcm_dpo/margin": 30.098209381103516, "fcm_dpo/q_t": 0.40680208802223206, "grad_norm": 11.094439506530762, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.02724289894104, "logits/rejected": -1.4271173477172852, "logps/chosen": -181.3251953125, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -200.86717224121094, "loss": 1.119, "margin_dpo/margin_mean": 30.098209381103516, "margin_dpo/margin_std": 49.68162536621094, "step": 287 }, { "KL/chosen_KL_mean": -84.33525085449219, "KL/mean": -110.00700378417969, "KL/rejected_KL_mean": -135.67877197265625, "KL/std": 50.01521301269531, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.014221672900021076, "fcm_dpo/delta": -0.1408231258392334, "fcm_dpo/margin": 51.343505859375, "fcm_dpo/q_t": 0.3452059328556061, "grad_norm": 10.378539085388184, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.819796085357666, "logits/rejected": -1.4145545959472656, "logps/chosen": -172.49127197265625, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -228.96072387695312, "loss": 0.9112, "margin_dpo/margin_mean": 51.343505859375, "margin_dpo/margin_std": 54.815372467041016, "step": 288 }, { "KL/chosen_KL_mean": -83.39556121826172, "KL/mean": -109.99638366699219, "KL/rejected_KL_mean": -136.59719848632812, "KL/std": 53.36102294921875, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.013876164332032204, "fcm_dpo/delta": -0.14900289475917816, "fcm_dpo/margin": 53.20164489746094, "fcm_dpo/q_t": 0.34905049204826355, "grad_norm": 9.901618003845215, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.8854670524597168, "logits/rejected": -1.681816577911377, "logps/chosen": -158.78848266601562, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -229.75149536132812, "loss": 0.9288, "margin_dpo/margin_mean": 53.20165252685547, "margin_dpo/margin_std": 61.816131591796875, "step": 289 }, { "KL/chosen_KL_mean": -91.90423583984375, "KL/mean": -119.09119415283203, "KL/rejected_KL_mean": -146.2781524658203, "KL/std": 55.680118560791016, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.013551105745136738, "fcm_dpo/delta": -0.14796362817287445, "fcm_dpo/margin": 54.3739128112793, "fcm_dpo/q_t": 0.342935174703598, "grad_norm": 10.326698303222656, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.2297048568725586, "logits/rejected": -1.2574832439422607, "logps/chosen": -179.9462127685547, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -269.49029541015625, "loss": 0.9081, "margin_dpo/margin_mean": 54.3739128112793, "margin_dpo/margin_std": 57.06800842285156, "step": 290 }, { "KL/chosen_KL_mean": -87.56625366210938, "KL/mean": -110.08284759521484, "KL/rejected_KL_mean": -132.5994415283203, "KL/std": 51.113677978515625, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.013350771740078926, "fcm_dpo/delta": -0.0013612844049930573, "fcm_dpo/margin": 45.033180236816406, "fcm_dpo/q_t": 0.36593347787857056, "grad_norm": 10.895191192626953, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.3271851539611816, "logits/rejected": -1.8727360963821411, "logps/chosen": -181.6614990234375, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -228.64950561523438, "loss": 0.9796, "margin_dpo/margin_mean": 45.03318405151367, "margin_dpo/margin_std": 51.313812255859375, "step": 291 }, { "KL/chosen_KL_mean": -90.02174377441406, "KL/mean": -114.76429748535156, "KL/rejected_KL_mean": -139.50686645507812, "KL/std": 49.921836853027344, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.013205183669924736, "fcm_dpo/delta": -0.05725127458572388, "fcm_dpo/margin": 49.4851188659668, "fcm_dpo/q_t": 0.3533305525779724, "grad_norm": 10.695060729980469, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0066745281219482, "logits/rejected": -1.564594030380249, "logps/chosen": -178.27215576171875, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -235.92449951171875, "loss": 0.913, "margin_dpo/margin_mean": 49.4851188659668, "margin_dpo/margin_std": 44.94405746459961, "step": 292 }, { "KL/chosen_KL_mean": -92.79507446289062, "KL/mean": -112.09004974365234, "KL/rejected_KL_mean": -131.385009765625, "KL/std": 53.226295471191406, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.013427002355456352, "fcm_dpo/delta": 0.0849938914179802, "fcm_dpo/margin": 38.589942932128906, "fcm_dpo/q_t": 0.3897326588630676, "grad_norm": 9.799050331115723, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.4161601066589355, "logits/rejected": -1.8580584526062012, "logps/chosen": -180.171630859375, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -217.1408233642578, "loss": 1.0572, "margin_dpo/margin_mean": 38.58993911743164, "margin_dpo/margin_std": 55.06817626953125, "step": 293 }, { "KL/chosen_KL_mean": -97.41282653808594, "KL/mean": -113.52125549316406, "KL/rejected_KL_mean": -129.62969970703125, "KL/std": 49.8419189453125, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.013779654167592525, "fcm_dpo/delta": 0.16245834529399872, "fcm_dpo/margin": 32.216880798339844, "fcm_dpo/q_t": 0.4058208465576172, "grad_norm": 11.31812572479248, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.9481632709503174, "logits/rejected": -1.7711522579193115, "logps/chosen": -170.9207763671875, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -217.71847534179688, "loss": 1.1336, "margin_dpo/margin_mean": 32.21687698364258, "margin_dpo/margin_std": 57.70189666748047, "step": 294 }, { "KL/chosen_KL_mean": -94.74482727050781, "KL/mean": -118.99324035644531, "KL/rejected_KL_mean": -143.2416534423828, "KL/std": 51.06188201904297, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.013623600825667381, "fcm_dpo/delta": -0.06622060388326645, "fcm_dpo/margin": 48.49683380126953, "fcm_dpo/q_t": 0.3573981821537018, "grad_norm": 10.688283920288086, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.6788742542266846, "logits/rejected": -1.5319139957427979, "logps/chosen": -172.51031494140625, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -237.48892211914062, "loss": 0.9509, "margin_dpo/margin_mean": 48.49683380126953, "margin_dpo/margin_std": 54.335487365722656, "step": 295 }, { "KL/chosen_KL_mean": -101.0079574584961, "KL/mean": -126.95489501953125, "KL/rejected_KL_mean": -152.90182495117188, "KL/std": 52.000877380371094, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.013394474983215332, "fcm_dpo/delta": -0.10308260470628738, "fcm_dpo/margin": 51.893890380859375, "fcm_dpo/q_t": 0.3481748402118683, "grad_norm": 11.960222244262695, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.359076976776123, "logits/rejected": -1.8792610168457031, "logps/chosen": -198.8643798828125, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -253.7181396484375, "loss": 0.9193, "margin_dpo/margin_mean": 51.89388656616211, "margin_dpo/margin_std": 53.45349884033203, "step": 296 }, { "KL/chosen_KL_mean": -104.16986083984375, "KL/mean": -125.88276672363281, "KL/rejected_KL_mean": -147.59567260742188, "KL/std": 50.44825744628906, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.01344512589275837, "fcm_dpo/delta": 0.01701100543141365, "fcm_dpo/margin": 43.425811767578125, "fcm_dpo/q_t": 0.37514692544937134, "grad_norm": 13.051698684692383, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.087263822555542, "logits/rejected": -0.7646397352218628, "logps/chosen": -192.73568725585938, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -249.1522216796875, "loss": 1.0022, "margin_dpo/margin_mean": 43.42580795288086, "margin_dpo/margin_std": 54.6956672668457, "step": 297 }, { "KL/chosen_KL_mean": -91.91172790527344, "KL/mean": -118.70831298828125, "KL/rejected_KL_mean": -145.50491333007812, "KL/std": 56.685333251953125, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.013244064524769783, "fcm_dpo/delta": -0.1180228590965271, "fcm_dpo/margin": 53.593177795410156, "fcm_dpo/q_t": 0.35055965185165405, "grad_norm": 14.841745376586914, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.1898951530456543, "logits/rejected": -2.130298376083374, "logps/chosen": -186.79217529296875, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -267.81591796875, "loss": 0.9227, "margin_dpo/margin_mean": 53.593177795410156, "margin_dpo/margin_std": 58.87498092651367, "step": 298 }, { "KL/chosen_KL_mean": -94.58711242675781, "KL/mean": -123.00601196289062, "KL/rejected_KL_mean": -151.42491149902344, "KL/std": 54.20139694213867, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.012805357575416565, "fcm_dpo/delta": -0.13858236372470856, "fcm_dpo/margin": 56.83780288696289, "fcm_dpo/q_t": 0.3425564765930176, "grad_norm": 10.636122703552246, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.1610912084579468, "logits/rejected": -0.9321527481079102, "logps/chosen": -174.99546813964844, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -240.9620819091797, "loss": 0.9003, "margin_dpo/margin_mean": 56.83780288696289, "margin_dpo/margin_std": 57.265296936035156, "step": 299 }, { "KL/chosen_KL_mean": -99.97329711914062, "KL/mean": -123.10993194580078, "KL/rejected_KL_mean": -146.24655151367188, "KL/std": 58.26737594604492, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.012725701555609703, "fcm_dpo/delta": 0.011522475630044937, "fcm_dpo/margin": 46.27326583862305, "fcm_dpo/q_t": 0.3730268180370331, "grad_norm": 11.511223793029785, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.569697380065918, "logits/rejected": -1.3541392087936401, "logps/chosen": -188.1322021484375, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -247.18576049804688, "loss": 1.0143, "margin_dpo/margin_mean": 46.27326202392578, "margin_dpo/margin_std": 60.53251647949219, "step": 300 }, { "KL/chosen_KL_mean": -91.66667175292969, "KL/mean": -121.84414672851562, "KL/rejected_KL_mean": -152.02163696289062, "KL/std": 57.32373809814453, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.012371879070997238, "fcm_dpo/delta": -0.16170020401477814, "fcm_dpo/margin": 60.354957580566406, "fcm_dpo/q_t": 0.34101545810699463, "grad_norm": 11.01829719543457, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.724073886871338, "logits/rejected": -1.2854777574539185, "logps/chosen": -177.17156982421875, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -250.9384765625, "loss": 0.9017, "margin_dpo/margin_mean": 60.35496139526367, "margin_dpo/margin_std": 62.5772705078125, "step": 301 }, { "KL/chosen_KL_mean": -91.52565002441406, "KL/mean": -116.25776672363281, "KL/rejected_KL_mean": -140.9898681640625, "KL/std": 59.50281524658203, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.012310752645134926, "fcm_dpo/delta": -0.00991368293762207, "fcm_dpo/margin": 49.464229583740234, "fcm_dpo/q_t": 0.3670756220817566, "grad_norm": 11.745650291442871, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.41493558883667, "logits/rejected": -1.234662413597107, "logps/chosen": -170.97605895996094, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -244.04898071289062, "loss": 0.9848, "margin_dpo/margin_mean": 49.46422576904297, "margin_dpo/margin_std": 58.744781494140625, "step": 302 }, { "KL/chosen_KL_mean": -82.72885131835938, "KL/mean": -110.55314636230469, "KL/rejected_KL_mean": -138.37744140625, "KL/std": 55.470027923583984, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.012259088456630707, "fcm_dpo/delta": -0.08832436054944992, "fcm_dpo/margin": 55.64856719970703, "fcm_dpo/q_t": 0.3503095209598541, "grad_norm": 10.342815399169922, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.9366796016693115, "logits/rejected": -1.4997478723526, "logps/chosen": -178.3271484375, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -230.4420166015625, "loss": 0.927, "margin_dpo/margin_mean": 55.64856719970703, "margin_dpo/margin_std": 56.82603454589844, "step": 303 }, { "KL/chosen_KL_mean": -82.94639587402344, "KL/mean": -112.4494400024414, "KL/rejected_KL_mean": -141.95248413085938, "KL/std": 59.29281997680664, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.011957205832004547, "fcm_dpo/delta": -0.11342738568782806, "fcm_dpo/margin": 59.00609588623047, "fcm_dpo/q_t": 0.3449639678001404, "grad_norm": 11.214903831481934, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.1100356578826904, "logits/rejected": -1.6916407346725464, "logps/chosen": -182.3995361328125, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -240.03475952148438, "loss": 0.9138, "margin_dpo/margin_mean": 59.00609588623047, "margin_dpo/margin_std": 59.92046356201172, "step": 304 }, { "KL/chosen_KL_mean": -86.91989135742188, "KL/mean": -115.43601989746094, "KL/rejected_KL_mean": -143.9521484375, "KL/std": 61.051185607910156, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.011639275588095188, "fcm_dpo/delta": -0.07061363756656647, "fcm_dpo/margin": 57.03224563598633, "fcm_dpo/q_t": 0.3542807698249817, "grad_norm": 10.13965892791748, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.2190661430358887, "logits/rejected": -1.7420494556427002, "logps/chosen": -190.18328857421875, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -241.78111267089844, "loss": 0.9428, "margin_dpo/margin_mean": 57.03224182128906, "margin_dpo/margin_std": 59.40462112426758, "step": 305 }, { "KL/chosen_KL_mean": -77.42254638671875, "KL/mean": -111.94004821777344, "KL/rejected_KL_mean": -146.45751953125, "KL/std": 57.669700622558594, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.01131722703576088, "fcm_dpo/delta": -0.197784423828125, "fcm_dpo/margin": 69.03497314453125, "fcm_dpo/q_t": 0.32400059700012207, "grad_norm": 9.995720863342285, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.128828763961792, "logits/rejected": -1.5727880001068115, "logps/chosen": -177.72637939453125, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -238.6884002685547, "loss": 0.8287, "margin_dpo/margin_mean": 69.03497314453125, "margin_dpo/margin_std": 51.950172424316406, "step": 306 }, { "KL/chosen_KL_mean": -84.56570434570312, "KL/mean": -109.37860107421875, "KL/rejected_KL_mean": -134.1914825439453, "KL/std": 57.9952507019043, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.011277512647211552, "fcm_dpo/delta": 0.04262436553835869, "fcm_dpo/margin": 49.625770568847656, "fcm_dpo/q_t": 0.3790434002876282, "grad_norm": 10.89678955078125, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.0780038833618164, "logits/rejected": -1.7523728609085083, "logps/chosen": -181.21546936035156, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -227.5400848388672, "loss": 1.0281, "margin_dpo/margin_mean": 49.62577438354492, "margin_dpo/margin_std": 66.37586975097656, "step": 307 }, { "KL/chosen_KL_mean": -87.23435974121094, "KL/mean": -111.78361511230469, "KL/rejected_KL_mean": -136.33285522460938, "KL/std": 59.368839263916016, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.011327080428600311, "fcm_dpo/delta": 0.04543329402804375, "fcm_dpo/margin": 49.098480224609375, "fcm_dpo/q_t": 0.381845623254776, "grad_norm": 11.583580017089844, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.5685229301452637, "logits/rejected": -1.2962605953216553, "logps/chosen": -172.9590301513672, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -222.75604248046875, "loss": 1.0383, "margin_dpo/margin_mean": 49.098480224609375, "margin_dpo/margin_std": 66.77845001220703, "step": 308 }, { "KL/chosen_KL_mean": -95.85281372070312, "KL/mean": -120.09855651855469, "KL/rejected_KL_mean": -144.34429931640625, "KL/std": 62.78336715698242, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.011535720899701118, "fcm_dpo/delta": 0.04292062669992447, "fcm_dpo/margin": 48.491493225097656, "fcm_dpo/q_t": 0.37761956453323364, "grad_norm": 14.172321319580078, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.7072186470031738, "logits/rejected": -1.2777293920516968, "logps/chosen": -179.0042724609375, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -230.93031311035156, "loss": 1.0173, "margin_dpo/margin_mean": 48.491493225097656, "margin_dpo/margin_std": 60.93738555908203, "step": 309 }, { "KL/chosen_KL_mean": -85.24385070800781, "KL/mean": -110.69718933105469, "KL/rejected_KL_mean": -136.15054321289062, "KL/std": 58.50639343261719, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.0115651311352849, "fcm_dpo/delta": 0.011962108314037323, "fcm_dpo/margin": 50.90667724609375, "fcm_dpo/q_t": 0.37062013149261475, "grad_norm": 10.97796630859375, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.266669750213623, "logits/rejected": -1.859579086303711, "logps/chosen": -174.98184204101562, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -223.2213592529297, "loss": 0.9912, "margin_dpo/margin_mean": 50.90667724609375, "margin_dpo/margin_std": 60.337440490722656, "step": 310 }, { "KL/chosen_KL_mean": -103.58985900878906, "KL/mean": -135.82203674316406, "KL/rejected_KL_mean": -168.05421447753906, "KL/std": 59.579872131347656, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.011330795474350452, "fcm_dpo/delta": -0.1407528668642044, "fcm_dpo/margin": 64.46435546875, "fcm_dpo/q_t": 0.33987316489219666, "grad_norm": 10.741815567016602, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.6584956645965576, "logits/rejected": -0.9308737516403198, "logps/chosen": -192.11630249023438, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -250.32029724121094, "loss": 0.8775, "margin_dpo/margin_mean": 64.46435546875, "margin_dpo/margin_std": 58.94683074951172, "step": 311 }, { "KL/chosen_KL_mean": -98.23626708984375, "KL/mean": -125.95197296142578, "KL/rejected_KL_mean": -153.66766357421875, "KL/std": 65.12993621826172, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.011263590306043625, "fcm_dpo/delta": -0.02635987475514412, "fcm_dpo/margin": 55.43140411376953, "fcm_dpo/q_t": 0.36341193318367004, "grad_norm": 10.439437866210938, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.580211877822876, "logits/rejected": -1.053236722946167, "logps/chosen": -178.636962890625, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -236.11480712890625, "loss": 0.9701, "margin_dpo/margin_mean": 55.43140411376953, "margin_dpo/margin_std": 63.09955596923828, "step": 312 }, { "KL/chosen_KL_mean": -91.67317199707031, "KL/mean": -118.2012939453125, "KL/rejected_KL_mean": -144.72943115234375, "KL/std": 65.23245239257812, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.011225591413676739, "fcm_dpo/delta": 0.004571555182337761, "fcm_dpo/margin": 53.056278228759766, "fcm_dpo/q_t": 0.37129873037338257, "grad_norm": 11.164016723632812, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.3059277534484863, "logits/rejected": -1.8885085582733154, "logps/chosen": -175.8150634765625, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -238.15628051757812, "loss": 1.0116, "margin_dpo/margin_mean": 53.05628204345703, "margin_dpo/margin_std": 69.4072036743164, "step": 313 }, { "KL/chosen_KL_mean": -92.01814270019531, "KL/mean": -118.34121704101562, "KL/rejected_KL_mean": -144.66429138183594, "KL/std": 64.55931091308594, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.01117841899394989, "fcm_dpo/delta": 0.011946924030780792, "fcm_dpo/margin": 52.64616012573242, "fcm_dpo/q_t": 0.3722578287124634, "grad_norm": 11.244053840637207, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.9729061126708984, "logits/rejected": -1.7542288303375244, "logps/chosen": -180.82786560058594, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -248.82321166992188, "loss": 0.9924, "margin_dpo/margin_mean": 52.64616012573242, "margin_dpo/margin_std": 62.83879089355469, "step": 314 }, { "KL/chosen_KL_mean": -90.331298828125, "KL/mean": -117.4103012084961, "KL/rejected_KL_mean": -144.4893035888672, "KL/std": 67.87617492675781, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.011208480224013329, "fcm_dpo/delta": -0.007499841041862965, "fcm_dpo/margin": 54.15800476074219, "fcm_dpo/q_t": 0.37134426832199097, "grad_norm": 11.795819282531738, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.559370517730713, "logits/rejected": -1.3793959617614746, "logps/chosen": -171.125, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -236.6264190673828, "loss": 1.0254, "margin_dpo/margin_mean": 54.15800476074219, "margin_dpo/margin_std": 75.20608520507812, "step": 315 }, { "KL/chosen_KL_mean": -94.75785827636719, "KL/mean": -123.07272338867188, "KL/rejected_KL_mean": -151.38758850097656, "KL/std": 66.95204162597656, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.01116903405636549, "fcm_dpo/delta": -0.034682951867580414, "fcm_dpo/margin": 56.629722595214844, "fcm_dpo/q_t": 0.36398208141326904, "grad_norm": 10.26680850982666, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.8405966758728027, "logits/rejected": -1.8399052619934082, "logps/chosen": -178.94210815429688, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -257.0783996582031, "loss": 0.9696, "margin_dpo/margin_mean": 56.629722595214844, "margin_dpo/margin_std": 67.00436401367188, "step": 316 }, { "KL/chosen_KL_mean": -96.75038146972656, "KL/mean": -132.447998046875, "KL/rejected_KL_mean": -168.14559936523438, "KL/std": 75.048095703125, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.010870996862649918, "fcm_dpo/delta": -0.19052648544311523, "fcm_dpo/margin": 71.39523315429688, "fcm_dpo/q_t": 0.3377516269683838, "grad_norm": 12.530817985534668, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.7920074462890625, "logits/rejected": -1.3009469509124756, "logps/chosen": -184.62387084960938, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -276.4107971191406, "loss": 0.9067, "margin_dpo/margin_mean": 71.39523315429688, "margin_dpo/margin_std": 77.88740539550781, "step": 317 }, { "KL/chosen_KL_mean": -93.58009338378906, "KL/mean": -125.88723754882812, "KL/rejected_KL_mean": -158.19439697265625, "KL/std": 62.276302337646484, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.010560518130660057, "fcm_dpo/delta": -0.08860616385936737, "fcm_dpo/margin": 64.61428833007812, "fcm_dpo/q_t": 0.35032060742378235, "grad_norm": 10.757477760314941, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.255362033843994, "logits/rejected": -1.5727300643920898, "logps/chosen": -191.78561401367188, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -246.3306884765625, "loss": 0.9228, "margin_dpo/margin_mean": 64.61429595947266, "margin_dpo/margin_std": 66.17192077636719, "step": 318 }, { "KL/chosen_KL_mean": -81.07221984863281, "KL/mean": -115.24949645996094, "KL/rejected_KL_mean": -149.4267578125, "KL/std": 64.6249771118164, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.010322792455554008, "fcm_dpo/delta": -0.11415446549654007, "fcm_dpo/margin": 68.35456085205078, "fcm_dpo/q_t": 0.3459065556526184, "grad_norm": 10.759950637817383, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.810572624206543, "logits/rejected": -1.729384422302246, "logps/chosen": -163.57559204101562, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -268.07745361328125, "loss": 0.9154, "margin_dpo/margin_mean": 68.35456848144531, "margin_dpo/margin_std": 70.365478515625, "step": 319 }, { "KL/chosen_KL_mean": -84.85716247558594, "KL/mean": -113.48957824707031, "KL/rejected_KL_mean": -142.12197875976562, "KL/std": 66.8855209350586, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.010286257602274418, "fcm_dpo/delta": 0.01155979372560978, "fcm_dpo/margin": 57.264808654785156, "fcm_dpo/q_t": 0.370198130607605, "grad_norm": 10.993927001953125, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.0017709732055664, "logits/rejected": -1.8126258850097656, "logps/chosen": -171.994384765625, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -245.34609985351562, "loss": 0.9768, "margin_dpo/margin_mean": 57.26481628417969, "margin_dpo/margin_std": 64.12809753417969, "step": 320 }, { "KL/chosen_KL_mean": -93.89463806152344, "KL/mean": -126.53536987304688, "KL/rejected_KL_mean": -159.17611694335938, "KL/std": 68.01840209960938, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.010162170976400375, "fcm_dpo/delta": -0.0687328651547432, "fcm_dpo/margin": 65.28146362304688, "fcm_dpo/q_t": 0.3525048792362213, "grad_norm": 10.835871696472168, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.9209041595458984, "logits/rejected": -1.5313405990600586, "logps/chosen": -186.99929809570312, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -256.64422607421875, "loss": 0.9323, "margin_dpo/margin_mean": 65.28146362304688, "margin_dpo/margin_std": 66.3851547241211, "step": 321 }, { "KL/chosen_KL_mean": -89.57185363769531, "KL/mean": -124.99256896972656, "KL/rejected_KL_mean": -160.41329956054688, "KL/std": 69.91878509521484, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.009925016202032566, "fcm_dpo/delta": -0.11320465058088303, "fcm_dpo/margin": 70.84144592285156, "fcm_dpo/q_t": 0.34773823618888855, "grad_norm": 13.703095436096191, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.627014398574829, "logits/rejected": -1.2679390907287598, "logps/chosen": -181.7647705078125, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -261.3065185546875, "loss": 0.9306, "margin_dpo/margin_mean": 70.84144592285156, "margin_dpo/margin_std": 75.77277374267578, "step": 322 }, { "KL/chosen_KL_mean": -102.78058624267578, "KL/mean": -134.3924560546875, "KL/rejected_KL_mean": -166.00433349609375, "KL/std": 69.58602905273438, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.009876185096800327, "fcm_dpo/delta": -0.02634931355714798, "fcm_dpo/margin": 63.22373962402344, "fcm_dpo/q_t": 0.3641979992389679, "grad_norm": 10.684287071228027, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.9313206672668457, "logits/rejected": -1.548772931098938, "logps/chosen": -183.0526885986328, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -246.30767822265625, "loss": 0.9582, "margin_dpo/margin_mean": 63.22373962402344, "margin_dpo/margin_std": 68.80855560302734, "step": 323 }, { "KL/chosen_KL_mean": -103.35508728027344, "KL/mean": -136.44290161132812, "KL/rejected_KL_mean": -169.53073120117188, "KL/std": 69.02296447753906, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.009825003333389759, "fcm_dpo/delta": -0.053635403513908386, "fcm_dpo/margin": 66.17564392089844, "fcm_dpo/q_t": 0.35363608598709106, "grad_norm": 10.010993957519531, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7591526508331299, "logits/rejected": -1.1103179454803467, "logps/chosen": -199.08444213867188, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -263.699951171875, "loss": 0.9224, "margin_dpo/margin_mean": 66.17564392089844, "margin_dpo/margin_std": 62.10957717895508, "step": 324 }, { "KL/chosen_KL_mean": -96.654052734375, "KL/mean": -130.71719360351562, "KL/rejected_KL_mean": -164.78033447265625, "KL/std": 66.45527648925781, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.009704859927296638, "fcm_dpo/delta": -0.06544168293476105, "fcm_dpo/margin": 68.12628936767578, "fcm_dpo/q_t": 0.3522419035434723, "grad_norm": 8.875496864318848, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.7304449081420898, "logits/rejected": -1.005537748336792, "logps/chosen": -195.36093139648438, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -241.18844604492188, "loss": 0.9239, "margin_dpo/margin_mean": 68.12629699707031, "margin_dpo/margin_std": 66.07576751708984, "step": 325 }, { "KL/chosen_KL_mean": -109.30517578125, "KL/mean": -140.28627014160156, "KL/rejected_KL_mean": -171.26736450195312, "KL/std": 66.90985107421875, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.009660892188549042, "fcm_dpo/delta": 0.0014756475575268269, "fcm_dpo/margin": 61.96220397949219, "fcm_dpo/q_t": 0.36694276332855225, "grad_norm": 10.208579063415527, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.275505542755127, "logits/rejected": -1.6036701202392578, "logps/chosen": -213.96649169921875, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -258.0927734375, "loss": 0.9748, "margin_dpo/margin_mean": 61.96220397949219, "margin_dpo/margin_std": 68.84599304199219, "step": 326 }, { "KL/chosen_KL_mean": -106.45365905761719, "KL/mean": -136.89553833007812, "KL/rejected_KL_mean": -167.33740234375, "KL/std": 61.65798568725586, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.009657494723796844, "fcm_dpo/delta": 0.012649130076169968, "fcm_dpo/margin": 60.88374710083008, "fcm_dpo/q_t": 0.3709026575088501, "grad_norm": 10.07061767578125, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.719040870666504, "logits/rejected": -1.119354009628296, "logps/chosen": -213.89389038085938, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -284.390625, "loss": 0.9777, "margin_dpo/margin_mean": 60.88374710083008, "margin_dpo/margin_std": 67.939208984375, "step": 327 }, { "KL/chosen_KL_mean": -105.53807067871094, "KL/mean": -144.69129943847656, "KL/rejected_KL_mean": -183.84451293945312, "KL/std": 73.66616821289062, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.009473506361246109, "fcm_dpo/delta": -0.15319207310676575, "fcm_dpo/margin": 78.30645751953125, "fcm_dpo/q_t": 0.3365141749382019, "grad_norm": 11.995161056518555, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6453728675842285, "logits/rejected": -1.353123426437378, "logps/chosen": -192.14877319335938, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -281.24639892578125, "loss": 0.8751, "margin_dpo/margin_mean": 78.30645751953125, "margin_dpo/margin_std": 71.34342193603516, "step": 328 }, { "KL/chosen_KL_mean": -111.27842712402344, "KL/mean": -142.10855102539062, "KL/rejected_KL_mean": -172.93869018554688, "KL/std": 67.88961029052734, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.009384993463754654, "fcm_dpo/delta": 0.022283829748630524, "fcm_dpo/margin": 61.66025161743164, "fcm_dpo/q_t": 0.3721325397491455, "grad_norm": 11.558391571044922, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.7495768070220947, "logits/rejected": -1.266786813735962, "logps/chosen": -211.89151000976562, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -277.89239501953125, "loss": 0.9839, "margin_dpo/margin_mean": 61.660247802734375, "margin_dpo/margin_std": 68.58905029296875, "step": 329 }, { "KL/chosen_KL_mean": -97.8609619140625, "KL/mean": -130.97909545898438, "KL/rejected_KL_mean": -164.0972137451172, "KL/std": 68.40953063964844, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.00938398763537407, "fcm_dpo/delta": -0.023171797394752502, "fcm_dpo/margin": 66.23626708984375, "fcm_dpo/q_t": 0.3573574721813202, "grad_norm": 9.570538520812988, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.7546844482421875, "logits/rejected": -1.6751201152801514, "logps/chosen": -174.81939697265625, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -256.91217041015625, "loss": 0.9285, "margin_dpo/margin_mean": 66.23626708984375, "margin_dpo/margin_std": 57.777565002441406, "step": 330 }, { "KL/chosen_KL_mean": -104.65771484375, "KL/mean": -141.10850524902344, "KL/rejected_KL_mean": -177.55929565429688, "KL/std": 67.31080627441406, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.009328317828476429, "fcm_dpo/delta": -0.08604207634925842, "fcm_dpo/margin": 72.90159606933594, "fcm_dpo/q_t": 0.3473587930202484, "grad_norm": 12.882772445678711, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8411427736282349, "logits/rejected": -1.5361219644546509, "logps/chosen": -187.71713256835938, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -271.6904296875, "loss": 0.8965, "margin_dpo/margin_mean": 72.90159606933594, "margin_dpo/margin_std": 62.85343551635742, "step": 331 }, { "KL/chosen_KL_mean": -109.18342590332031, "KL/mean": -135.89031982421875, "KL/rejected_KL_mean": -162.5972442626953, "KL/std": 74.63380432128906, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.009386228397488594, "fcm_dpo/delta": 0.10376466065645218, "fcm_dpo/margin": 53.41381072998047, "fcm_dpo/q_t": 0.3885642886161804, "grad_norm": 9.482118606567383, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.26485538482666, "logits/rejected": -1.8512423038482666, "logps/chosen": -191.11431884765625, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -245.44314575195312, "loss": 1.0397, "margin_dpo/margin_mean": 53.41381072998047, "margin_dpo/margin_std": 68.36637878417969, "step": 332 }, { "KL/chosen_KL_mean": -127.68972778320312, "KL/mean": -163.0169677734375, "KL/rejected_KL_mean": -198.34420776367188, "KL/std": 77.17674255371094, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.00927070900797844, "fcm_dpo/delta": -0.0606718473136425, "fcm_dpo/margin": 70.65448760986328, "fcm_dpo/q_t": 0.35932445526123047, "grad_norm": 10.047981262207031, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.603600263595581, "logits/rejected": -1.2732014656066895, "logps/chosen": -216.22230529785156, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -300.81024169921875, "loss": 0.971, "margin_dpo/margin_mean": 70.65448760986328, "margin_dpo/margin_std": 83.77717590332031, "step": 333 }, { "KL/chosen_KL_mean": -115.7264633178711, "KL/mean": -147.91635131835938, "KL/rejected_KL_mean": -180.1062774658203, "KL/std": 72.66596221923828, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.009263558313250542, "fcm_dpo/delta": 0.003100711852312088, "fcm_dpo/margin": 64.37980651855469, "fcm_dpo/q_t": 0.36814361810684204, "grad_norm": 10.953825950622559, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.413359522819519, "logits/rejected": -1.1223427057266235, "logps/chosen": -213.4842529296875, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -280.11212158203125, "loss": 1.0025, "margin_dpo/margin_mean": 64.37979888916016, "margin_dpo/margin_std": 79.87903594970703, "step": 334 }, { "KL/chosen_KL_mean": -114.42747497558594, "KL/mean": -143.55764770507812, "KL/rejected_KL_mean": -172.68780517578125, "KL/std": 71.71163940429688, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.009396743029356003, "fcm_dpo/delta": 0.05559629946947098, "fcm_dpo/margin": 58.26033020019531, "fcm_dpo/q_t": 0.38051509857177734, "grad_norm": 11.57308292388916, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.308279514312744, "logits/rejected": -1.8946013450622559, "logps/chosen": -210.8035125732422, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -267.61181640625, "loss": 1.0268, "margin_dpo/margin_mean": 58.26033020019531, "margin_dpo/margin_std": 75.99794006347656, "step": 335 }, { "KL/chosen_KL_mean": -115.80043029785156, "KL/mean": -152.26544189453125, "KL/rejected_KL_mean": -188.7304229736328, "KL/std": 75.40699005126953, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.009331358596682549, "fcm_dpo/delta": -0.08633555471897125, "fcm_dpo/margin": 72.92999267578125, "fcm_dpo/q_t": 0.3511877655982971, "grad_norm": 11.624049186706543, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.534778356552124, "logits/rejected": -1.373348355293274, "logps/chosen": -204.3299560546875, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -298.28619384765625, "loss": 0.9262, "margin_dpo/margin_mean": 72.92997741699219, "margin_dpo/margin_std": 77.302001953125, "step": 336 }, { "KL/chosen_KL_mean": -116.12496948242188, "KL/mean": -149.44821166992188, "KL/rejected_KL_mean": -182.77145385742188, "KL/std": 75.23159790039062, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.009267127141356468, "fcm_dpo/delta": -0.018923547118902206, "fcm_dpo/margin": 66.6465072631836, "fcm_dpo/q_t": 0.36777669191360474, "grad_norm": 12.077470779418945, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.219202995300293, "logits/rejected": -1.9283981323242188, "logps/chosen": -213.43734741210938, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -292.9028015136719, "loss": 1.0006, "margin_dpo/margin_mean": 66.6465072631836, "margin_dpo/margin_std": 85.69737243652344, "step": 337 }, { "KL/chosen_KL_mean": -118.52214050292969, "KL/mean": -150.74745178222656, "KL/rejected_KL_mean": -182.97274780273438, "KL/std": 76.45425415039062, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.00921938382089138, "fcm_dpo/delta": 0.00610383041203022, "fcm_dpo/margin": 64.45059967041016, "fcm_dpo/q_t": 0.36708366870880127, "grad_norm": 10.919146537780762, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7630703449249268, "logits/rejected": -1.6006054878234863, "logps/chosen": -202.15225219726562, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -276.66510009765625, "loss": 0.977, "margin_dpo/margin_mean": 64.45060729980469, "margin_dpo/margin_std": 71.33059692382812, "step": 338 }, { "KL/chosen_KL_mean": -123.42848205566406, "KL/mean": -157.4447021484375, "KL/rejected_KL_mean": -191.4609375, "KL/std": 75.60530090332031, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.009264282882213593, "fcm_dpo/delta": -0.03321116417646408, "fcm_dpo/margin": 68.03246307373047, "fcm_dpo/q_t": 0.3624188303947449, "grad_norm": 10.165472984313965, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.9429874420166016, "logits/rejected": -2.521411418914795, "logps/chosen": -223.28030395507812, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -300.5352783203125, "loss": 0.9831, "margin_dpo/margin_mean": 68.03246307373047, "margin_dpo/margin_std": 81.10905456542969, "step": 339 }, { "KL/chosen_KL_mean": -132.01622009277344, "KL/mean": -162.491943359375, "KL/rejected_KL_mean": -192.96766662597656, "KL/std": 69.73624420166016, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.00924382172524929, "fcm_dpo/delta": 0.038678016513586044, "fcm_dpo/margin": 60.951446533203125, "fcm_dpo/q_t": 0.37506335973739624, "grad_norm": 12.0796537399292, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.3655431270599365, "logits/rejected": -0.8860758543014526, "logps/chosen": -216.9901885986328, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -271.76800537109375, "loss": 1.0054, "margin_dpo/margin_mean": 60.951446533203125, "margin_dpo/margin_std": 75.20494842529297, "step": 340 }, { "KL/chosen_KL_mean": -139.78866577148438, "KL/mean": -168.49266052246094, "KL/rejected_KL_mean": -197.19664001464844, "KL/std": 72.96240234375, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.00931989960372448, "fcm_dpo/delta": 0.06866871565580368, "fcm_dpo/margin": 57.407958984375, "fcm_dpo/q_t": 0.3848504424095154, "grad_norm": 12.309844970703125, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.3784551620483398, "logits/rejected": -1.1110708713531494, "logps/chosen": -225.0526580810547, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -303.2825927734375, "loss": 1.0533, "margin_dpo/margin_mean": 57.407958984375, "margin_dpo/margin_std": 81.58512878417969, "step": 341 }, { "KL/chosen_KL_mean": -111.96800231933594, "KL/mean": -153.7183837890625, "KL/rejected_KL_mean": -195.4687957763672, "KL/std": 74.83447265625, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.009056007489562035, "fcm_dpo/delta": -0.1724531501531601, "fcm_dpo/margin": 83.50079345703125, "fcm_dpo/q_t": 0.3322939872741699, "grad_norm": 10.454486846923828, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.2058645486831665, "logits/rejected": -0.43693655729293823, "logps/chosen": -213.00660705566406, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -294.3673095703125, "loss": 0.8903, "margin_dpo/margin_mean": 83.50079345703125, "margin_dpo/margin_std": 78.72843933105469, "step": 342 }, { "KL/chosen_KL_mean": -119.92997741699219, "KL/mean": -154.64572143554688, "KL/rejected_KL_mean": -189.3614501953125, "KL/std": 76.83589172363281, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.009089207276701927, "fcm_dpo/delta": -0.03427841514348984, "fcm_dpo/margin": 69.43148803710938, "fcm_dpo/q_t": 0.362102746963501, "grad_norm": 12.223538398742676, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.164773941040039, "logits/rejected": -2.0410075187683105, "logps/chosen": -203.67733764648438, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -305.9405517578125, "loss": 0.9668, "margin_dpo/margin_mean": 69.43148803710938, "margin_dpo/margin_std": 76.89498901367188, "step": 343 }, { "KL/chosen_KL_mean": -124.14497375488281, "KL/mean": -159.22996520996094, "KL/rejected_KL_mean": -194.31497192382812, "KL/std": 82.00492095947266, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.009002451784908772, "fcm_dpo/delta": -0.034273263067007065, "fcm_dpo/margin": 70.16998291015625, "fcm_dpo/q_t": 0.3642351031303406, "grad_norm": 10.809247016906738, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.5767159461975098, "logits/rejected": -1.3183751106262207, "logps/chosen": -202.39895629882812, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -289.74365234375, "loss": 0.9816, "margin_dpo/margin_mean": 70.16998291015625, "margin_dpo/margin_std": 85.29952239990234, "step": 344 }, { "KL/chosen_KL_mean": -123.66266632080078, "KL/mean": -162.3070831298828, "KL/rejected_KL_mean": -200.9514923095703, "KL/std": 83.27194213867188, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.008799157105386257, "fcm_dpo/delta": -0.08631725609302521, "fcm_dpo/margin": 77.2888412475586, "fcm_dpo/q_t": 0.35234373807907104, "grad_norm": 11.827264785766602, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.480682611465454, "logits/rejected": -1.178969383239746, "logps/chosen": -221.74215698242188, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -304.98626708984375, "loss": 0.9316, "margin_dpo/margin_mean": 77.28884887695312, "margin_dpo/margin_std": 82.2240219116211, "step": 345 }, { "KL/chosen_KL_mean": -139.16546630859375, "KL/mean": -169.94180297851562, "KL/rejected_KL_mean": -200.7181396484375, "KL/std": 77.13172912597656, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.008861856535077095, "fcm_dpo/delta": 0.05766042694449425, "fcm_dpo/margin": 61.552696228027344, "fcm_dpo/q_t": 0.3825362026691437, "grad_norm": 10.254656791687012, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.8156230449676514, "logits/rejected": -1.500436544418335, "logps/chosen": -219.84893798828125, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -286.6107482910156, "loss": 1.0426, "margin_dpo/margin_mean": 61.552696228027344, "margin_dpo/margin_std": 86.30281066894531, "step": 346 }, { "KL/chosen_KL_mean": -130.35186767578125, "KL/mean": -159.87762451171875, "KL/rejected_KL_mean": -189.40335083007812, "KL/std": 75.50732421875, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.009010223671793938, "fcm_dpo/delta": 0.07129624485969543, "fcm_dpo/margin": 59.051456451416016, "fcm_dpo/q_t": 0.3852520287036896, "grad_norm": 11.267574310302734, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.8264482021331787, "logits/rejected": -1.4947563409805298, "logps/chosen": -224.03125, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -286.94586181640625, "loss": 1.0458, "margin_dpo/margin_mean": 59.051456451416016, "margin_dpo/margin_std": 82.06726837158203, "step": 347 }, { "KL/chosen_KL_mean": -113.24097442626953, "KL/mean": -146.83370971679688, "KL/rejected_KL_mean": -180.42645263671875, "KL/std": 72.97020721435547, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.008917481638491154, "fcm_dpo/delta": -0.0013939440250396729, "fcm_dpo/margin": 67.18547058105469, "fcm_dpo/q_t": 0.36750227212905884, "grad_norm": 9.631325721740723, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.9974944591522217, "logits/rejected": -1.7184162139892578, "logps/chosen": -203.49322509765625, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -276.19671630859375, "loss": 0.9872, "margin_dpo/margin_mean": 67.18547058105469, "margin_dpo/margin_std": 75.54493713378906, "step": 348 }, { "KL/chosen_KL_mean": -123.40333557128906, "KL/mean": -155.13833618164062, "KL/rejected_KL_mean": -186.8733673095703, "KL/std": 78.06153869628906, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.00909486599266529, "fcm_dpo/delta": 0.023321356624364853, "fcm_dpo/margin": 63.470027923583984, "fcm_dpo/q_t": 0.37466883659362793, "grad_norm": 11.78731632232666, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.8535704612731934, "logits/rejected": -1.6230392456054688, "logps/chosen": -215.46142578125, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -290.838623046875, "loss": 1.0068, "margin_dpo/margin_mean": 63.470027923583984, "margin_dpo/margin_std": 78.41249084472656, "step": 349 }, { "KL/chosen_KL_mean": -132.6189422607422, "KL/mean": -158.2574462890625, "KL/rejected_KL_mean": -183.89599609375, "KL/std": 71.72050476074219, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.009222065098583698, "fcm_dpo/delta": 0.1335791051387787, "fcm_dpo/margin": 51.27705383300781, "fcm_dpo/q_t": 0.39669013023376465, "grad_norm": 12.869258880615234, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.789060354232788, "logits/rejected": -1.560743808746338, "logps/chosen": -219.67312622070312, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -279.617919921875, "loss": 1.0738, "margin_dpo/margin_mean": 51.27705383300781, "margin_dpo/margin_std": 73.58702087402344, "step": 350 }, { "KL/chosen_KL_mean": -129.23472595214844, "KL/mean": -167.4254913330078, "KL/rejected_KL_mean": -205.61627197265625, "KL/std": 81.40641021728516, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.009186920709908009, "fcm_dpo/delta": -0.1092863529920578, "fcm_dpo/margin": 76.38153076171875, "fcm_dpo/q_t": 0.34763386845588684, "grad_norm": 10.585238456726074, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.517390489578247, "logits/rejected": -1.1918097734451294, "logps/chosen": -218.4037628173828, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -313.914306640625, "loss": 0.937, "margin_dpo/margin_mean": 76.38153076171875, "margin_dpo/margin_std": 84.33473205566406, "step": 351 }, { "KL/chosen_KL_mean": -118.77500915527344, "KL/mean": -154.91842651367188, "KL/rejected_KL_mean": -191.06185913085938, "KL/std": 80.13026428222656, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.009022538550198078, "fcm_dpo/delta": -0.055852822959423065, "fcm_dpo/margin": 72.28684997558594, "fcm_dpo/q_t": 0.36111313104629517, "grad_norm": 11.32371711730957, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.9275966882705688, "logits/rejected": -1.3902859687805176, "logps/chosen": -215.58648681640625, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -275.4666748046875, "loss": 0.9599, "margin_dpo/margin_mean": 72.28684997558594, "margin_dpo/margin_std": 83.00843811035156, "step": 352 }, { "KL/chosen_KL_mean": -136.0355682373047, "KL/mean": -158.725830078125, "KL/rejected_KL_mean": -181.41607666015625, "KL/std": 71.66810607910156, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.009231328964233398, "fcm_dpo/delta": 0.1893390417098999, "fcm_dpo/margin": 45.38052749633789, "fcm_dpo/q_t": 0.4074091911315918, "grad_norm": 12.784234046936035, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.051333427429199, "logits/rejected": -1.6680806875228882, "logps/chosen": -221.39805603027344, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -268.78338623046875, "loss": 1.13, "margin_dpo/margin_mean": 45.38052749633789, "margin_dpo/margin_std": 76.8905029296875, "step": 353 }, { "KL/chosen_KL_mean": -120.67779541015625, "KL/mean": -155.8875732421875, "KL/rejected_KL_mean": -191.09732055664062, "KL/std": 73.43081665039062, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.009293332695960999, "fcm_dpo/delta": -0.058271683752536774, "fcm_dpo/margin": 70.41951751708984, "fcm_dpo/q_t": 0.3544082045555115, "grad_norm": 10.79163646697998, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.932690143585205, "logits/rejected": -1.468198299407959, "logps/chosen": -216.47897338867188, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -285.96343994140625, "loss": 0.9172, "margin_dpo/margin_mean": 70.41951751708984, "margin_dpo/margin_std": 65.68981170654297, "step": 354 }, { "KL/chosen_KL_mean": -119.20698547363281, "KL/mean": -152.94647216796875, "KL/rejected_KL_mean": -186.68597412109375, "KL/std": 74.60952758789062, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.009151626378297806, "fcm_dpo/delta": -0.019613508135080338, "fcm_dpo/margin": 67.47898864746094, "fcm_dpo/q_t": 0.3642219305038452, "grad_norm": 11.60234260559082, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4135332107543945, "logits/rejected": -1.9346702098846436, "logps/chosen": -205.3925323486328, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -275.14752197265625, "loss": 0.9754, "margin_dpo/margin_mean": 67.47898864746094, "margin_dpo/margin_std": 75.11697387695312, "step": 355 }, { "KL/chosen_KL_mean": -140.1505889892578, "KL/mean": -165.45858764648438, "KL/rejected_KL_mean": -190.7666015625, "KL/std": 71.86222076416016, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.009413287043571472, "fcm_dpo/delta": 0.12892337143421173, "fcm_dpo/margin": 50.61601257324219, "fcm_dpo/q_t": 0.39598432183265686, "grad_norm": 13.0845308303833, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.9235329627990723, "logits/rejected": -1.4695203304290771, "logps/chosen": -230.96868896484375, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -272.8939208984375, "loss": 1.098, "margin_dpo/margin_mean": 50.61601257324219, "margin_dpo/margin_std": 81.242919921875, "step": 356 }, { "KL/chosen_KL_mean": -114.822509765625, "KL/mean": -157.2147979736328, "KL/rejected_KL_mean": -199.60708618164062, "KL/std": 81.51332092285156, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.009077783674001694, "fcm_dpo/delta": -0.18850216269493103, "fcm_dpo/margin": 84.78456115722656, "fcm_dpo/q_t": 0.3296627998352051, "grad_norm": 9.775087356567383, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.830019235610962, "logits/rejected": -1.4567103385925293, "logps/chosen": -202.3272705078125, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -291.8564453125, "loss": 0.8595, "margin_dpo/margin_mean": 84.78456115722656, "margin_dpo/margin_std": 71.80831909179688, "step": 357 }, { "KL/chosen_KL_mean": -129.39413452148438, "KL/mean": -159.16049194335938, "KL/rejected_KL_mean": -188.92684936523438, "KL/std": 73.75126647949219, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.009171983227133751, "fcm_dpo/delta": 0.05705197900533676, "fcm_dpo/margin": 59.53269958496094, "fcm_dpo/q_t": 0.38410475850105286, "grad_norm": 11.235389709472656, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.7385156154632568, "logits/rejected": -1.2678202390670776, "logps/chosen": -224.44960021972656, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -285.4029541015625, "loss": 1.0275, "margin_dpo/margin_mean": 59.53269577026367, "margin_dpo/margin_std": 79.32071685791016, "step": 358 }, { "KL/chosen_KL_mean": -142.5458984375, "KL/mean": -170.09722900390625, "KL/rejected_KL_mean": -197.64859008789062, "KL/std": 70.74909210205078, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.009253107942640781, "fcm_dpo/delta": 0.09446504712104797, "fcm_dpo/margin": 55.10267639160156, "fcm_dpo/q_t": 0.39293375611305237, "grad_norm": 11.805782318115234, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.2739100456237793, "logits/rejected": -0.6888490915298462, "logps/chosen": -229.91921997070312, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -286.42047119140625, "loss": 1.0742, "margin_dpo/margin_mean": 55.10267639160156, "margin_dpo/margin_std": 82.36737823486328, "step": 359 }, { "KL/chosen_KL_mean": -134.35415649414062, "KL/mean": -170.19949340820312, "KL/rejected_KL_mean": -206.04486083984375, "KL/std": 70.60169219970703, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.009269386529922485, "fcm_dpo/delta": -0.06916459649801254, "fcm_dpo/margin": 71.69071960449219, "fcm_dpo/q_t": 0.35194262862205505, "grad_norm": 13.87574577331543, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.8141227960586548, "logits/rejected": -1.4783596992492676, "logps/chosen": -209.57760620117188, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -282.13885498046875, "loss": 0.9305, "margin_dpo/margin_mean": 71.69071960449219, "margin_dpo/margin_std": 73.16065979003906, "step": 360 }, { "KL/chosen_KL_mean": -135.81048583984375, "KL/mean": -167.51513671875, "KL/rejected_KL_mean": -199.21975708007812, "KL/std": 80.02481079101562, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.009272661060094833, "fcm_dpo/delta": 0.012711755931377411, "fcm_dpo/margin": 63.40927505493164, "fcm_dpo/q_t": 0.37535184621810913, "grad_norm": 12.662524223327637, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.5842676162719727, "logits/rejected": -1.3275785446166992, "logps/chosen": -217.76382446289062, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -295.8989562988281, "loss": 1.0163, "margin_dpo/margin_mean": 63.40927505493164, "margin_dpo/margin_std": 84.36708068847656, "step": 361 }, { "KL/chosen_KL_mean": -140.61431884765625, "KL/mean": -173.93350219726562, "KL/rejected_KL_mean": -207.25267028808594, "KL/std": 71.98042297363281, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.009221550077199936, "fcm_dpo/delta": -0.01560905296355486, "fcm_dpo/margin": 66.63832092285156, "fcm_dpo/q_t": 0.36368370056152344, "grad_norm": 11.935662269592285, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.8818135261535645, "logits/rejected": -1.6709842681884766, "logps/chosen": -229.95404052734375, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -317.754638671875, "loss": 0.9731, "margin_dpo/margin_mean": 66.63832092285156, "margin_dpo/margin_std": 74.52210998535156, "step": 362 }, { "KL/chosen_KL_mean": -133.06466674804688, "KL/mean": -175.57992553710938, "KL/rejected_KL_mean": -218.09518432617188, "KL/std": 79.06326293945312, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.00903572328388691, "fcm_dpo/delta": -0.18192854523658752, "fcm_dpo/margin": 85.030517578125, "fcm_dpo/q_t": 0.33634427189826965, "grad_norm": 12.804475784301758, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.7099268436431885, "logits/rejected": -1.2659603357315063, "logps/chosen": -213.83966064453125, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -297.5278625488281, "loss": 0.8889, "margin_dpo/margin_mean": 85.030517578125, "margin_dpo/margin_std": 86.19454956054688, "step": 363 }, { "KL/chosen_KL_mean": -132.8457794189453, "KL/mean": -167.23611450195312, "KL/rejected_KL_mean": -201.62646484375, "KL/std": 72.91869354248047, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.008956972509622574, "fcm_dpo/delta": -0.018432609736919403, "fcm_dpo/margin": 68.78070068359375, "fcm_dpo/q_t": 0.36184197664260864, "grad_norm": 13.484635353088379, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0610973834991455, "logits/rejected": -1.9081377983093262, "logps/chosen": -209.37538146972656, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -299.188232421875, "loss": 0.9509, "margin_dpo/margin_mean": 68.78070068359375, "margin_dpo/margin_std": 67.19124603271484, "step": 364 }, { "KL/chosen_KL_mean": -129.88360595703125, "KL/mean": -167.52932739257812, "KL/rejected_KL_mean": -205.17506408691406, "KL/std": 79.31575012207031, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.008818486705422401, "fcm_dpo/delta": -0.06855962425470352, "fcm_dpo/margin": 75.29142761230469, "fcm_dpo/q_t": 0.3580206036567688, "grad_norm": 10.765987396240234, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.8091192245483398, "logits/rejected": -1.6704254150390625, "logps/chosen": -210.42849731445312, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -304.49822998046875, "loss": 0.9594, "margin_dpo/margin_mean": 75.29142761230469, "margin_dpo/margin_std": 87.83641052246094, "step": 365 }, { "KL/chosen_KL_mean": -129.66009521484375, "KL/mean": -162.356689453125, "KL/rejected_KL_mean": -195.05328369140625, "KL/std": 82.74629211425781, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.008786695078015327, "fcm_dpo/delta": 0.026960838586091995, "fcm_dpo/margin": 65.39318084716797, "fcm_dpo/q_t": 0.3776422142982483, "grad_norm": 10.304669380187988, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.8986899852752686, "logits/rejected": -1.917381763458252, "logps/chosen": -201.89816284179688, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -286.72509765625, "loss": 0.9931, "margin_dpo/margin_mean": 65.39317321777344, "margin_dpo/margin_std": 79.0001220703125, "step": 366 }, { "KL/chosen_KL_mean": -121.46359252929688, "KL/mean": -158.93527221679688, "KL/rejected_KL_mean": -196.40695190429688, "KL/std": 73.30078125, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.008727902546525002, "fcm_dpo/delta": -0.0578642264008522, "fcm_dpo/margin": 74.94334411621094, "fcm_dpo/q_t": 0.35597020387649536, "grad_norm": 10.285561561584473, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.7906179428100586, "logits/rejected": -1.4597322940826416, "logps/chosen": -206.88560485839844, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -279.92474365234375, "loss": 0.9295, "margin_dpo/margin_mean": 74.94334411621094, "margin_dpo/margin_std": 75.85501098632812, "step": 367 }, { "KL/chosen_KL_mean": -115.25946044921875, "KL/mean": -155.7138671875, "KL/rejected_KL_mean": -196.1682891845703, "KL/std": 80.23482513427734, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.008617177605628967, "fcm_dpo/delta": -0.10462093353271484, "fcm_dpo/margin": 80.90882873535156, "fcm_dpo/q_t": 0.34596121311187744, "grad_norm": 10.83174991607666, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.6348400115966797, "logits/rejected": -1.5922210216522217, "logps/chosen": -189.1005859375, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -300.86358642578125, "loss": 0.9212, "margin_dpo/margin_mean": 80.90882873535156, "margin_dpo/margin_std": 81.08068084716797, "step": 368 }, { "KL/chosen_KL_mean": -125.76205444335938, "KL/mean": -164.16033935546875, "KL/rejected_KL_mean": -202.55865478515625, "KL/std": 72.06120300292969, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.008467354811728, "fcm_dpo/delta": -0.05373477190732956, "fcm_dpo/margin": 76.79661560058594, "fcm_dpo/q_t": 0.3544258177280426, "grad_norm": 10.98386287689209, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.6508598327636719, "logits/rejected": -1.282099723815918, "logps/chosen": -224.2239990234375, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -301.71539306640625, "loss": 0.9159, "margin_dpo/margin_mean": 76.79661560058594, "margin_dpo/margin_std": 70.02391052246094, "step": 369 }, { "KL/chosen_KL_mean": -139.02850341796875, "KL/mean": -165.64266967773438, "KL/rejected_KL_mean": -192.25682067871094, "KL/std": 72.07622528076172, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.008627048693597317, "fcm_dpo/delta": 0.147303506731987, "fcm_dpo/margin": 53.22829055786133, "fcm_dpo/q_t": 0.3982418179512024, "grad_norm": 11.150107383728027, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -2.1788220405578613, "logits/rejected": -1.9844003915786743, "logps/chosen": -235.8042449951172, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -295.00091552734375, "loss": 1.0923, "margin_dpo/margin_mean": 53.228294372558594, "margin_dpo/margin_std": 81.081298828125, "step": 370 }, { "KL/chosen_KL_mean": -132.23257446289062, "KL/mean": -160.33517456054688, "KL/rejected_KL_mean": -188.43777465820312, "KL/std": 73.10238647460938, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.008849586360156536, "fcm_dpo/delta": 0.10735228657722473, "fcm_dpo/margin": 56.20519256591797, "fcm_dpo/q_t": 0.39050063490867615, "grad_norm": 12.583840370178223, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.903461217880249, "logits/rejected": -1.5428340435028076, "logps/chosen": -235.2227783203125, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -293.78839111328125, "loss": 1.0597, "margin_dpo/margin_mean": 56.20519256591797, "margin_dpo/margin_std": 78.01887512207031, "step": 371 }, { "KL/chosen_KL_mean": -112.23699188232422, "KL/mean": -156.3649139404297, "KL/rejected_KL_mean": -200.49282836914062, "KL/std": 84.24118041992188, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.008645731955766678, "fcm_dpo/delta": -0.17638391256332397, "fcm_dpo/margin": 88.25584411621094, "fcm_dpo/q_t": 0.3339630961418152, "grad_norm": 9.920493125915527, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.7640397548675537, "logits/rejected": -1.1277694702148438, "logps/chosen": -203.2632293701172, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -286.1030578613281, "loss": 0.8748, "margin_dpo/margin_mean": 88.25584411621094, "margin_dpo/margin_std": 83.14984130859375, "step": 372 }, { "KL/chosen_KL_mean": -128.0678253173828, "KL/mean": -161.85733032226562, "KL/rejected_KL_mean": -195.6468505859375, "KL/std": 78.72205352783203, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.00857684388756752, "fcm_dpo/delta": 0.02163422852754593, "fcm_dpo/margin": 67.57903289794922, "fcm_dpo/q_t": 0.373263955116272, "grad_norm": 13.231733322143555, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.1729726791381836, "logits/rejected": -1.6664485931396484, "logps/chosen": -228.14089965820312, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -286.76318359375, "loss": 1.0002, "margin_dpo/margin_mean": 67.57903289794922, "margin_dpo/margin_std": 82.235595703125, "step": 373 }, { "KL/chosen_KL_mean": -112.91627502441406, "KL/mean": -148.91268920898438, "KL/rejected_KL_mean": -184.9091033935547, "KL/std": 75.39215850830078, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.008544884622097015, "fcm_dpo/delta": -0.016500283032655716, "fcm_dpo/margin": 71.99280548095703, "fcm_dpo/q_t": 0.3630443811416626, "grad_norm": 10.26775074005127, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.243265151977539, "logits/rejected": -1.8553439378738403, "logps/chosen": -210.92861938476562, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -285.6613464355469, "loss": 0.9451, "margin_dpo/margin_mean": 71.99280548095703, "margin_dpo/margin_std": 71.0427017211914, "step": 374 }, { "KL/chosen_KL_mean": -117.73454284667969, "KL/mean": -157.49545288085938, "KL/rejected_KL_mean": -197.25640869140625, "KL/std": 73.4385986328125, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.008396909572184086, "fcm_dpo/delta": -0.07500813901424408, "fcm_dpo/margin": 79.5218505859375, "fcm_dpo/q_t": 0.35205817222595215, "grad_norm": 10.850824356079102, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.6340422630310059, "logits/rejected": -1.406312346458435, "logps/chosen": -202.13082885742188, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -292.9752502441406, "loss": 0.9257, "margin_dpo/margin_mean": 79.5218505859375, "margin_dpo/margin_std": 77.28062438964844, "step": 375 }, { "KL/chosen_KL_mean": -109.48548889160156, "KL/mean": -153.21331787109375, "KL/rejected_KL_mean": -196.94113159179688, "KL/std": 81.96785736083984, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.008266251534223557, "fcm_dpo/delta": -0.1327178031206131, "fcm_dpo/margin": 87.45562744140625, "fcm_dpo/q_t": 0.3416569232940674, "grad_norm": 11.629322052001953, "learning_rate": 2.5e-07, "logits/chosen": -1.5032219886779785, "logits/rejected": -1.1849677562713623, "logps/chosen": -205.2231903076172, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -308.9144287109375, "loss": 0.8976, "margin_dpo/margin_mean": 87.45562744140625, "margin_dpo/margin_std": 85.66285705566406, "step": 376 }, { "KL/chosen_KL_mean": -120.33424377441406, "KL/mean": -165.67254638671875, "KL/rejected_KL_mean": -211.01083374023438, "KL/std": 86.06922912597656, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.008007656782865524, "fcm_dpo/delta": -0.13748544454574585, "fcm_dpo/margin": 90.67659759521484, "fcm_dpo/q_t": 0.34062933921813965, "grad_norm": 10.181844711303711, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.4152727127075195, "logits/rejected": -1.2905082702636719, "logps/chosen": -199.26119995117188, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -316.9097595214844, "loss": 0.8933, "margin_dpo/margin_mean": 90.67658996582031, "margin_dpo/margin_std": 86.54685974121094, "step": 377 }, { "KL/chosen_KL_mean": -117.09504699707031, "KL/mean": -154.17153930664062, "KL/rejected_KL_mean": -191.248046875, "KL/std": 81.4710693359375, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.00805251207202673, "fcm_dpo/delta": 0.0021640360355377197, "fcm_dpo/margin": 74.15302276611328, "fcm_dpo/q_t": 0.36522796750068665, "grad_norm": 10.893919944763184, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.982363224029541, "logits/rejected": -1.70050847530365, "logps/chosen": -192.46743774414062, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -284.9521484375, "loss": 0.9758, "margin_dpo/margin_mean": 74.15302276611328, "margin_dpo/margin_std": 80.62299346923828, "step": 378 }, { "KL/chosen_KL_mean": -132.86514282226562, "KL/mean": -166.86306762695312, "KL/rejected_KL_mean": -200.8609619140625, "KL/std": 84.16495513916016, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.008062219247221947, "fcm_dpo/delta": 0.054829034954309464, "fcm_dpo/margin": 67.99581909179688, "fcm_dpo/q_t": 0.38279053568840027, "grad_norm": 10.907620429992676, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.9056613445281982, "logits/rejected": -1.8076817989349365, "logps/chosen": -221.7208251953125, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -322.24212646484375, "loss": 1.0432, "margin_dpo/margin_mean": 67.99581909179688, "margin_dpo/margin_std": 95.19928741455078, "step": 379 }, { "KL/chosen_KL_mean": -146.6173858642578, "KL/mean": -186.11880493164062, "KL/rejected_KL_mean": -225.62022399902344, "KL/std": 84.72185516357422, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.00804077833890915, "fcm_dpo/delta": -0.037651993334293365, "fcm_dpo/margin": 79.00283813476562, "fcm_dpo/q_t": 0.364879310131073, "grad_norm": 14.319841384887695, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.269819974899292, "logits/rejected": -1.237410068511963, "logps/chosen": -226.45059204101562, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -336.364990234375, "loss": 0.9732, "margin_dpo/margin_mean": 79.0028305053711, "margin_dpo/margin_std": 95.81834411621094, "step": 380 }, { "KL/chosen_KL_mean": -127.27312469482422, "KL/mean": -162.6256103515625, "KL/rejected_KL_mean": -197.97811889648438, "KL/std": 82.57929992675781, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.008068221621215343, "fcm_dpo/delta": 0.031310826539993286, "fcm_dpo/margin": 70.70500183105469, "fcm_dpo/q_t": 0.37292641401290894, "grad_norm": 11.665767669677734, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.8381025791168213, "logits/rejected": -1.293177604675293, "logps/chosen": -213.83995056152344, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -282.3948669433594, "loss": 0.9939, "margin_dpo/margin_mean": 70.70500183105469, "margin_dpo/margin_std": 81.96543884277344, "step": 381 }, { "KL/chosen_KL_mean": -128.27655029296875, "KL/mean": -167.087158203125, "KL/rejected_KL_mean": -205.8977508544922, "KL/std": 80.1351089477539, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.007985509932041168, "fcm_dpo/delta": -0.022745832800865173, "fcm_dpo/margin": 77.6212158203125, "fcm_dpo/q_t": 0.3634357452392578, "grad_norm": 10.567047119140625, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.8173182010650635, "logits/rejected": -1.5434061288833618, "logps/chosen": -232.73834228515625, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -320.5, "loss": 0.9579, "margin_dpo/margin_mean": 77.6212158203125, "margin_dpo/margin_std": 80.81192779541016, "step": 382 }, { "KL/chosen_KL_mean": -136.37869262695312, "KL/mean": -170.35223388671875, "KL/rejected_KL_mean": -204.3258056640625, "KL/std": 82.09567260742188, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.008084474131464958, "fcm_dpo/delta": 0.05362362414598465, "fcm_dpo/margin": 67.9471206665039, "fcm_dpo/q_t": 0.3789066672325134, "grad_norm": 11.41159439086914, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.7134125232696533, "logits/rejected": -1.596728801727295, "logps/chosen": -218.4947509765625, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -303.10296630859375, "loss": 1.0074, "margin_dpo/margin_mean": 67.9471206665039, "margin_dpo/margin_std": 79.64369201660156, "step": 383 }, { "KL/chosen_KL_mean": -148.03211975097656, "KL/mean": -181.560546875, "KL/rejected_KL_mean": -215.08895874023438, "KL/std": 85.67454528808594, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.008267415687441826, "fcm_dpo/delta": 0.04572564363479614, "fcm_dpo/margin": 67.05685424804688, "fcm_dpo/q_t": 0.3783491253852844, "grad_norm": 13.50598430633545, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.6813862323760986, "logits/rejected": -1.4510648250579834, "logps/chosen": -228.61737060546875, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -306.536865234375, "loss": 1.0519, "margin_dpo/margin_mean": 67.05685424804688, "margin_dpo/margin_std": 92.30213928222656, "step": 384 }, { "KL/chosen_KL_mean": -123.12554168701172, "KL/mean": -165.77548217773438, "KL/rejected_KL_mean": -208.42544555664062, "KL/std": 78.48614501953125, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.008027653209865093, "fcm_dpo/delta": -0.09300199896097183, "fcm_dpo/margin": 85.29989624023438, "fcm_dpo/q_t": 0.3526189625263214, "grad_norm": 11.798502922058105, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.10163950920105, "logits/rejected": -1.4805482625961304, "logps/chosen": -215.88571166992188, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -292.88543701171875, "loss": 0.936, "margin_dpo/margin_mean": 85.29989624023438, "margin_dpo/margin_std": 91.36244201660156, "step": 385 }, { "KL/chosen_KL_mean": -129.71484375, "KL/mean": -170.58624267578125, "KL/rejected_KL_mean": -211.45765686035156, "KL/std": 79.56188201904297, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.007987398654222488, "fcm_dpo/delta": -0.056592706590890884, "fcm_dpo/margin": 81.7428207397461, "fcm_dpo/q_t": 0.3530290722846985, "grad_norm": 9.782319068908691, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.436248779296875, "logits/rejected": -1.1442089080810547, "logps/chosen": -215.07379150390625, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -308.44830322265625, "loss": 0.9442, "margin_dpo/margin_mean": 81.7428207397461, "margin_dpo/margin_std": 85.953125, "step": 386 }, { "KL/chosen_KL_mean": -149.98822021484375, "KL/mean": -187.67529296875, "KL/rejected_KL_mean": -225.36236572265625, "KL/std": 76.69561767578125, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.007982458919286728, "fcm_dpo/delta": -0.001947270706295967, "fcm_dpo/margin": 75.37413024902344, "fcm_dpo/q_t": 0.36588990688323975, "grad_norm": 11.972749710083008, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.6814143657684326, "logits/rejected": -2.028057098388672, "logps/chosen": -252.67636108398438, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -324.79095458984375, "loss": 0.9627, "margin_dpo/margin_mean": 75.37413024902344, "margin_dpo/margin_std": 79.10820007324219, "step": 387 }, { "KL/chosen_KL_mean": -150.44720458984375, "KL/mean": -191.6618194580078, "KL/rejected_KL_mean": -232.87640380859375, "KL/std": 85.29866027832031, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.007888156920671463, "fcm_dpo/delta": -0.05377114564180374, "fcm_dpo/margin": 82.42919921875, "fcm_dpo/q_t": 0.3561539649963379, "grad_norm": 10.74714469909668, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.7974867820739746, "logits/rejected": -1.6686962842941284, "logps/chosen": -234.03244018554688, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -342.976806640625, "loss": 0.9397, "margin_dpo/margin_mean": 82.42919921875, "margin_dpo/margin_std": 85.2193603515625, "step": 388 }, { "KL/chosen_KL_mean": -154.55848693847656, "KL/mean": -191.21446228027344, "KL/rejected_KL_mean": -227.87042236328125, "KL/std": 88.60816192626953, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.007897498086094856, "fcm_dpo/delta": 0.02231520414352417, "fcm_dpo/margin": 73.31193542480469, "fcm_dpo/q_t": 0.37510430812835693, "grad_norm": 12.397315979003906, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.3680338859558105, "logits/rejected": -1.9208991527557373, "logps/chosen": -244.38075256347656, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -312.72418212890625, "loss": 1.0165, "margin_dpo/margin_mean": 73.31193542480469, "margin_dpo/margin_std": 96.41900634765625, "step": 389 }, { "KL/chosen_KL_mean": -156.45855712890625, "KL/mean": -190.62863159179688, "KL/rejected_KL_mean": -224.7987060546875, "KL/std": 85.41568756103516, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.00792744942009449, "fcm_dpo/delta": 0.06105216220021248, "fcm_dpo/margin": 68.34011840820312, "fcm_dpo/q_t": 0.3862881362438202, "grad_norm": 10.897446632385254, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.3670145273208618, "logits/rejected": -1.1465001106262207, "logps/chosen": -233.99099731445312, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -317.38262939453125, "loss": 1.0483, "margin_dpo/margin_mean": 68.34011840820312, "margin_dpo/margin_std": 97.50023651123047, "step": 390 }, { "KL/chosen_KL_mean": -142.4449005126953, "KL/mean": -180.11976623535156, "KL/rejected_KL_mean": -217.79464721679688, "KL/std": 82.21206665039062, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.007973108440637589, "fcm_dpo/delta": -0.0011414699256420135, "fcm_dpo/margin": 75.34974670410156, "fcm_dpo/q_t": 0.36897146701812744, "grad_norm": 10.858970642089844, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.0311970710754395, "logits/rejected": -1.5861847400665283, "logps/chosen": -230.67782592773438, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -310.80218505859375, "loss": 0.9857, "margin_dpo/margin_mean": 75.3497543334961, "margin_dpo/margin_std": 88.5110855102539, "step": 391 }, { "KL/chosen_KL_mean": -144.1068115234375, "KL/mean": -188.37294006347656, "KL/rejected_KL_mean": -232.6390838623047, "KL/std": 91.47470092773438, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.00790768675506115, "fcm_dpo/delta": -0.10751060396432877, "fcm_dpo/margin": 88.53224182128906, "fcm_dpo/q_t": 0.34837958216667175, "grad_norm": 12.842073440551758, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.6225284337997437, "logits/rejected": -1.4982194900512695, "logps/chosen": -231.85122680664062, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -341.20977783203125, "loss": 0.9363, "margin_dpo/margin_mean": 88.53224182128906, "margin_dpo/margin_std": 97.96992492675781, "step": 392 }, { "KL/chosen_KL_mean": -128.94515991210938, "KL/mean": -169.02035522460938, "KL/rejected_KL_mean": -209.0955810546875, "KL/std": 86.11857604980469, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.007797417230904102, "fcm_dpo/delta": -0.02665782906115055, "fcm_dpo/margin": 80.15042877197266, "fcm_dpo/q_t": 0.36201876401901245, "grad_norm": 10.353591918945312, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.6365962028503418, "logits/rejected": -1.3134424686431885, "logps/chosen": -205.50897216796875, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -288.82177734375, "loss": 0.956, "margin_dpo/margin_mean": 80.15042877197266, "margin_dpo/margin_std": 85.80219268798828, "step": 393 }, { "KL/chosen_KL_mean": -134.28366088867188, "KL/mean": -177.14682006835938, "KL/rejected_KL_mean": -220.010009765625, "KL/std": 81.65068054199219, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.007710480131208897, "fcm_dpo/delta": -0.06537509709596634, "fcm_dpo/margin": 85.7263412475586, "fcm_dpo/q_t": 0.3514748513698578, "grad_norm": 11.703354835510254, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6796019077301025, "logits/rejected": -1.449489951133728, "logps/chosen": -218.98233032226562, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -317.4677734375, "loss": 0.9147, "margin_dpo/margin_mean": 85.7263412475586, "margin_dpo/margin_std": 79.06229400634766, "step": 394 }, { "KL/chosen_KL_mean": -140.86691284179688, "KL/mean": -174.64691162109375, "KL/rejected_KL_mean": -208.42689514160156, "KL/std": 86.83915710449219, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.007743177004158497, "fcm_dpo/delta": 0.08096866309642792, "fcm_dpo/margin": 67.55998229980469, "fcm_dpo/q_t": 0.3921288251876831, "grad_norm": 12.043014526367188, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5334765911102295, "logits/rejected": -2.1345574855804443, "logps/chosen": -229.7545623779297, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -299.472412109375, "loss": 1.1135, "margin_dpo/margin_mean": 67.55998992919922, "margin_dpo/margin_std": 116.99273681640625, "step": 395 }, { "KL/chosen_KL_mean": -149.70863342285156, "KL/mean": -178.5358123779297, "KL/rejected_KL_mean": -207.3629913330078, "KL/std": 76.67694091796875, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.007986875250935555, "fcm_dpo/delta": 0.146285742521286, "fcm_dpo/margin": 57.654361724853516, "fcm_dpo/q_t": 0.3992249667644501, "grad_norm": 15.644421577453613, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.88157320022583, "logits/rejected": -1.3899590969085693, "logps/chosen": -252.17996215820312, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -300.5977783203125, "loss": 1.0911, "margin_dpo/margin_mean": 57.65436553955078, "margin_dpo/margin_std": 87.96942901611328, "step": 396 }, { "KL/chosen_KL_mean": -133.9219207763672, "KL/mean": -187.04779052734375, "KL/rejected_KL_mean": -240.17367553710938, "KL/std": 89.94300842285156, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.007738643325865269, "fcm_dpo/delta": -0.24319276213645935, "fcm_dpo/margin": 106.25173950195312, "fcm_dpo/q_t": 0.325309157371521, "grad_norm": 11.710845947265625, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.2405033111572266, "logits/rejected": -1.638427495956421, "logps/chosen": -234.96633911132812, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -349.02484130859375, "loss": 0.8731, "margin_dpo/margin_mean": 106.25173950195312, "margin_dpo/margin_std": 107.70158386230469, "step": 397 }, { "KL/chosen_KL_mean": -141.13134765625, "KL/mean": -179.91726684570312, "KL/rejected_KL_mean": -218.7032012939453, "KL/std": 86.9852523803711, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.007660144940018654, "fcm_dpo/delta": 0.0061048222705721855, "fcm_dpo/margin": 77.57183837890625, "fcm_dpo/q_t": 0.3725617229938507, "grad_norm": 11.762916564941406, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.3012213706970215, "logits/rejected": -2.0223350524902344, "logps/chosen": -236.18057250976562, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -332.7088317871094, "loss": 1.0149, "margin_dpo/margin_mean": 77.57183837890625, "margin_dpo/margin_std": 103.41021728515625, "step": 398 }, { "KL/chosen_KL_mean": -125.05729675292969, "KL/mean": -174.54946899414062, "KL/rejected_KL_mean": -224.0416259765625, "KL/std": 79.31352233886719, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.007529866881668568, "fcm_dpo/delta": -0.15674908459186554, "fcm_dpo/margin": 98.98433685302734, "fcm_dpo/q_t": 0.33425372838974, "grad_norm": 10.709030151367188, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.2058544158935547, "logits/rejected": -1.4292716979980469, "logps/chosen": -221.69583129882812, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -307.57806396484375, "loss": 0.8616, "margin_dpo/margin_mean": 98.98433685302734, "margin_dpo/margin_std": 83.30043029785156, "step": 399 }, { "KL/chosen_KL_mean": -129.06396484375, "KL/mean": -179.43026733398438, "KL/rejected_KL_mean": -229.79660034179688, "KL/std": 89.50796508789062, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.007276850752532482, "fcm_dpo/delta": -0.14365607500076294, "fcm_dpo/margin": 100.73262023925781, "fcm_dpo/q_t": 0.34050971269607544, "grad_norm": 14.151275634765625, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.0651931762695312, "logits/rejected": -1.571418046951294, "logps/chosen": -234.3671112060547, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -338.4521484375, "loss": 0.9048, "margin_dpo/margin_mean": 100.73262786865234, "margin_dpo/margin_std": 101.66943359375, "step": 400 }, { "KL/chosen_KL_mean": -143.39239501953125, "KL/mean": -175.595703125, "KL/rejected_KL_mean": -207.7989959716797, "KL/std": 87.65953826904297, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.007347226142883301, "fcm_dpo/delta": 0.13330283761024475, "fcm_dpo/margin": 64.4066162109375, "fcm_dpo/q_t": 0.40195322036743164, "grad_norm": 14.30765151977539, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.4367563724517822, "logits/rejected": -1.1888165473937988, "logps/chosen": -227.68154907226562, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -298.74212646484375, "loss": 1.1035, "margin_dpo/margin_mean": 64.40662384033203, "margin_dpo/margin_std": 107.70085144042969, "step": 401 }, { "KL/chosen_KL_mean": -129.20201110839844, "KL/mean": -187.98690795898438, "KL/rejected_KL_mean": -246.77183532714844, "KL/std": 94.11659240722656, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.007153146434575319, "fcm_dpo/delta": -0.2627454996109009, "fcm_dpo/margin": 117.56982421875, "fcm_dpo/q_t": 0.3123449683189392, "grad_norm": 11.226045608520508, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.7727017402648926, "logits/rejected": -1.2947998046875, "logps/chosen": -212.79513549804688, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -334.58209228515625, "loss": 0.8038, "margin_dpo/margin_mean": 117.56982421875, "margin_dpo/margin_std": 89.09761047363281, "step": 402 }, { "KL/chosen_KL_mean": -131.890380859375, "KL/mean": -175.35348510742188, "KL/rejected_KL_mean": -218.81655883789062, "KL/std": 89.7642593383789, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.007009865716099739, "fcm_dpo/delta": -0.010197965428233147, "fcm_dpo/margin": 86.92620849609375, "fcm_dpo/q_t": 0.3651764392852783, "grad_norm": 10.370346069335938, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.390258312225342, "logits/rejected": -2.0078721046447754, "logps/chosen": -205.64346313476562, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -302.7366943359375, "loss": 0.968, "margin_dpo/margin_mean": 86.92620849609375, "margin_dpo/margin_std": 95.06065368652344, "step": 403 }, { "KL/chosen_KL_mean": -158.56228637695312, "KL/mean": -195.45394897460938, "KL/rejected_KL_mean": -232.3456268310547, "KL/std": 88.53697204589844, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.007096182554960251, "fcm_dpo/delta": 0.080715611577034, "fcm_dpo/margin": 73.7833251953125, "fcm_dpo/q_t": 0.3867571949958801, "grad_norm": 15.73997974395752, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.8850417137145996, "logits/rejected": -1.541538953781128, "logps/chosen": -238.23846435546875, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -316.62615966796875, "loss": 1.0588, "margin_dpo/margin_mean": 73.7833251953125, "margin_dpo/margin_std": 107.41415405273438, "step": 404 }, { "KL/chosen_KL_mean": -159.6348419189453, "KL/mean": -197.94297790527344, "KL/rejected_KL_mean": -236.2510986328125, "KL/std": 81.5489501953125, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.00724197831004858, "fcm_dpo/delta": 0.04653295502066612, "fcm_dpo/margin": 76.61625671386719, "fcm_dpo/q_t": 0.37737810611724854, "grad_norm": 9.817625045776367, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.7342294454574585, "logits/rejected": -1.2783401012420654, "logps/chosen": -256.0968017578125, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -329.121826171875, "loss": 0.9937, "margin_dpo/margin_mean": 76.61625671386719, "margin_dpo/margin_std": 84.926513671875, "step": 405 }, { "KL/chosen_KL_mean": -150.18614196777344, "KL/mean": -185.1495361328125, "KL/rejected_KL_mean": -220.1129150390625, "KL/std": 89.36094665527344, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.007317298091948032, "fcm_dpo/delta": 0.09298479557037354, "fcm_dpo/margin": 69.92677307128906, "fcm_dpo/q_t": 0.38927412033081055, "grad_norm": 17.150859832763672, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.875551462173462, "logits/rejected": -1.7018225193023682, "logps/chosen": -228.32009887695312, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -318.3965148925781, "loss": 1.0701, "margin_dpo/margin_mean": 69.92677307128906, "margin_dpo/margin_std": 103.93281555175781, "step": 406 }, { "KL/chosen_KL_mean": -158.64053344726562, "KL/mean": -189.25546264648438, "KL/rejected_KL_mean": -219.87039184570312, "KL/std": 86.40353393554688, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.007509817369282246, "fcm_dpo/delta": 0.14689449965953827, "fcm_dpo/margin": 61.22986602783203, "fcm_dpo/q_t": 0.3991800844669342, "grad_norm": 13.676344871520996, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.235398769378662, "logits/rejected": -1.721910834312439, "logps/chosen": -242.6533660888672, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -302.65142822265625, "loss": 1.0879, "margin_dpo/margin_mean": 61.22986602783203, "margin_dpo/margin_std": 93.36032104492188, "step": 407 }, { "KL/chosen_KL_mean": -151.03475952148438, "KL/mean": -182.92340087890625, "KL/rejected_KL_mean": -214.81201171875, "KL/std": 81.0812759399414, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.007681374438107014, "fcm_dpo/delta": 0.1159222424030304, "fcm_dpo/margin": 63.777259826660156, "fcm_dpo/q_t": 0.39591002464294434, "grad_norm": 13.25179386138916, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.3091163635253906, "logits/rejected": -1.7026073932647705, "logps/chosen": -255.49652099609375, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -311.1842041015625, "loss": 1.0851, "margin_dpo/margin_mean": 63.777259826660156, "margin_dpo/margin_std": 99.56072998046875, "step": 408 }, { "KL/chosen_KL_mean": -160.72698974609375, "KL/mean": -193.19241333007812, "KL/rejected_KL_mean": -225.65785217285156, "KL/std": 86.85586547851562, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.007846983149647713, "fcm_dpo/delta": 0.09540466964244843, "fcm_dpo/margin": 64.93084716796875, "fcm_dpo/q_t": 0.38598188757896423, "grad_norm": 10.916430473327637, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5900911092758179, "logits/rejected": -1.1565730571746826, "logps/chosen": -258.39532470703125, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -315.70367431640625, "loss": 1.0732, "margin_dpo/margin_mean": 64.93084716796875, "margin_dpo/margin_std": 95.75387573242188, "step": 409 }, { "KL/chosen_KL_mean": -158.22824096679688, "KL/mean": -195.29690551757812, "KL/rejected_KL_mean": -232.3656005859375, "KL/std": 89.28762817382812, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.007911693304777145, "fcm_dpo/delta": 0.014273736625909805, "fcm_dpo/margin": 74.13735961914062, "fcm_dpo/q_t": 0.3696020245552063, "grad_norm": 11.172747611999512, "learning_rate": 2.065879555832674e-07, "logits/chosen": -2.022136688232422, "logits/rejected": -1.8946707248687744, "logps/chosen": -234.69747924804688, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -321.0062255859375, "loss": 0.9991, "margin_dpo/margin_mean": 74.13735961914062, "margin_dpo/margin_std": 90.88578796386719, "step": 410 }, { "KL/chosen_KL_mean": -166.25909423828125, "KL/mean": -202.82098388671875, "KL/rejected_KL_mean": -239.38290405273438, "KL/std": 82.69842529296875, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.007910618558526039, "fcm_dpo/delta": 0.022662775591015816, "fcm_dpo/margin": 73.12382507324219, "fcm_dpo/q_t": 0.3748021423816681, "grad_norm": 11.837505340576172, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.4981131553649902, "logits/rejected": -2.108024835586548, "logps/chosen": -253.42539978027344, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -326.47894287109375, "loss": 0.9963, "margin_dpo/margin_mean": 73.12382507324219, "margin_dpo/margin_std": 87.31697845458984, "step": 411 }, { "KL/chosen_KL_mean": -157.07594299316406, "KL/mean": -192.29541015625, "KL/rejected_KL_mean": -227.51486206054688, "KL/std": 83.54340362548828, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.007963714189827442, "fcm_dpo/delta": 0.041153714060783386, "fcm_dpo/margin": 70.43892669677734, "fcm_dpo/q_t": 0.3832489848136902, "grad_norm": 12.685282707214355, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.418454170227051, "logits/rejected": -2.1966254711151123, "logps/chosen": -236.02328491210938, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -333.62042236328125, "loss": 1.0641, "margin_dpo/margin_mean": 70.43892669677734, "margin_dpo/margin_std": 107.53343200683594, "step": 412 }, { "KL/chosen_KL_mean": -168.52987670898438, "KL/mean": -208.325439453125, "KL/rejected_KL_mean": -248.12098693847656, "KL/std": 88.9452133178711, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.007940024137496948, "fcm_dpo/delta": -0.03464067727327347, "fcm_dpo/margin": 79.59112548828125, "fcm_dpo/q_t": 0.3635329008102417, "grad_norm": 11.82545280456543, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.4438759088516235, "logits/rejected": -1.1940523386001587, "logps/chosen": -264.224609375, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -356.08184814453125, "loss": 0.9956, "margin_dpo/margin_mean": 79.59112548828125, "margin_dpo/margin_std": 102.98716735839844, "step": 413 }, { "KL/chosen_KL_mean": -155.40594482421875, "KL/mean": -191.91619873046875, "KL/rejected_KL_mean": -228.42645263671875, "KL/std": 86.4072494506836, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.007985914126038551, "fcm_dpo/delta": 0.01789022423326969, "fcm_dpo/margin": 73.02052307128906, "fcm_dpo/q_t": 0.3774893581867218, "grad_norm": 16.79780387878418, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.4796006679534912, "logits/rejected": -1.1185128688812256, "logps/chosen": -243.6826171875, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -321.2965087890625, "loss": 1.0518, "margin_dpo/margin_mean": 73.02052307128906, "margin_dpo/margin_std": 109.6328125, "step": 414 }, { "KL/chosen_KL_mean": -168.7891845703125, "KL/mean": -200.15231323242188, "KL/rejected_KL_mean": -231.5154266357422, "KL/std": 77.12353515625, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.008116893470287323, "fcm_dpo/delta": 0.09550224244594574, "fcm_dpo/margin": 62.72624969482422, "fcm_dpo/q_t": 0.39102494716644287, "grad_norm": 13.821722030639648, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.440542459487915, "logits/rejected": -1.0545800924301147, "logps/chosen": -277.705078125, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -338.9867858886719, "loss": 1.0756, "margin_dpo/margin_mean": 62.72624969482422, "margin_dpo/margin_std": 95.27058410644531, "step": 415 }, { "KL/chosen_KL_mean": -165.43106079101562, "KL/mean": -208.90426635742188, "KL/rejected_KL_mean": -252.37750244140625, "KL/std": 92.8784408569336, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.008097649551928043, "fcm_dpo/delta": -0.11324802786111832, "fcm_dpo/margin": 86.94647216796875, "fcm_dpo/q_t": 0.35047444701194763, "grad_norm": 11.12760066986084, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.76872718334198, "logits/rejected": -1.259762167930603, "logps/chosen": -258.8299255371094, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -350.05047607421875, "loss": 0.9505, "margin_dpo/margin_mean": 86.94646453857422, "margin_dpo/margin_std": 98.51605224609375, "step": 416 }, { "KL/chosen_KL_mean": -154.92340087890625, "KL/mean": -195.50640869140625, "KL/rejected_KL_mean": -236.08941650390625, "KL/std": 85.96389770507812, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.007923737168312073, "fcm_dpo/delta": -0.04610409587621689, "fcm_dpo/margin": 81.16603088378906, "fcm_dpo/q_t": 0.3590424954891205, "grad_norm": 11.668367385864258, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.2514448165893555, "logits/rejected": -1.3456957340240479, "logps/chosen": -238.458740234375, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -310.53125, "loss": 0.9581, "margin_dpo/margin_mean": 81.16603088378906, "margin_dpo/margin_std": 90.19656372070312, "step": 417 }, { "KL/chosen_KL_mean": -145.53785705566406, "KL/mean": -196.38815307617188, "KL/rejected_KL_mean": -247.23843383789062, "KL/std": 91.83435821533203, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.007756036706268787, "fcm_dpo/delta": -0.20595082640647888, "fcm_dpo/margin": 101.70059967041016, "fcm_dpo/q_t": 0.3294130265712738, "grad_norm": 13.808112144470215, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.0749683380126953, "logits/rejected": -1.4072449207305908, "logps/chosen": -253.75938415527344, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -359.10308837890625, "loss": 0.8601, "margin_dpo/margin_mean": 101.70059967041016, "margin_dpo/margin_std": 87.94700622558594, "step": 418 }, { "KL/chosen_KL_mean": -153.46119689941406, "KL/mean": -197.2794189453125, "KL/rejected_KL_mean": -241.09765625, "KL/std": 82.12565612792969, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.007541423663496971, "fcm_dpo/delta": -0.06557581573724747, "fcm_dpo/margin": 87.63645935058594, "fcm_dpo/q_t": 0.3551240861415863, "grad_norm": 11.561209678649902, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.444035291671753, "logits/rejected": -0.9937609434127808, "logps/chosen": -250.6444854736328, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -339.282958984375, "loss": 0.9388, "margin_dpo/margin_mean": 87.63645935058594, "margin_dpo/margin_std": 91.34939575195312, "step": 419 }, { "KL/chosen_KL_mean": -158.4595947265625, "KL/mean": -197.5110626220703, "KL/rejected_KL_mean": -236.5625457763672, "KL/std": 87.0154800415039, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.007524487562477589, "fcm_dpo/delta": 0.012418575584888458, "fcm_dpo/margin": 78.10295104980469, "fcm_dpo/q_t": 0.37278300523757935, "grad_norm": 15.610475540161133, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.119901180267334, "logits/rejected": -1.0082411766052246, "logps/chosen": -272.76806640625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -312.24609375, "loss": 1.0057, "margin_dpo/margin_mean": 78.10295104980469, "margin_dpo/margin_std": 97.97225952148438, "step": 420 }, { "KL/chosen_KL_mean": -153.41152954101562, "KL/mean": -196.29664611816406, "KL/rejected_KL_mean": -239.1817626953125, "KL/std": 87.52278900146484, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.007428227923810482, "fcm_dpo/delta": -0.039835311472415924, "fcm_dpo/margin": 85.77023315429688, "fcm_dpo/q_t": 0.3604215979576111, "grad_norm": 10.614893913269043, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.7837085723876953, "logits/rejected": -1.4106136560440063, "logps/chosen": -239.2913818359375, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -335.51824951171875, "loss": 0.961, "margin_dpo/margin_mean": 85.77023315429688, "margin_dpo/margin_std": 95.43048095703125, "step": 421 }, { "KL/chosen_KL_mean": -150.73809814453125, "KL/mean": -191.69874572753906, "KL/rejected_KL_mean": -232.65939331054688, "KL/std": 81.60902404785156, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.007433257065713406, "fcm_dpo/delta": -0.009581504389643669, "fcm_dpo/margin": 81.92129516601562, "fcm_dpo/q_t": 0.3669818341732025, "grad_norm": 12.77625846862793, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.3112244606018066, "logits/rejected": -1.9360928535461426, "logps/chosen": -251.21868896484375, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -327.067626953125, "loss": 0.9706, "margin_dpo/margin_mean": 81.92129516601562, "margin_dpo/margin_std": 92.4329833984375, "step": 422 }, { "KL/chosen_KL_mean": -142.02447509765625, "KL/mean": -182.4169921875, "KL/rejected_KL_mean": -222.8095245361328, "KL/std": 96.56837463378906, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.007448237389326096, "fcm_dpo/delta": -0.0022768862545490265, "fcm_dpo/margin": 80.78504943847656, "fcm_dpo/q_t": 0.3741607069969177, "grad_norm": 12.343729972839355, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.7313847541809082, "logits/rejected": -1.6104457378387451, "logps/chosen": -220.4744110107422, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -314.8560485839844, "loss": 1.0198, "margin_dpo/margin_mean": 80.78504943847656, "margin_dpo/margin_std": 109.95134735107422, "step": 423 }, { "KL/chosen_KL_mean": -157.48098754882812, "KL/mean": -198.7098388671875, "KL/rejected_KL_mean": -239.93869018554688, "KL/std": 86.86381530761719, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.007399152964353561, "fcm_dpo/delta": -0.010775750502943993, "fcm_dpo/margin": 82.45769500732422, "fcm_dpo/q_t": 0.3647323548793793, "grad_norm": 10.247640609741211, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7292550802230835, "logits/rejected": -1.1286228895187378, "logps/chosen": -245.1233367919922, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -312.3043518066406, "loss": 0.9752, "margin_dpo/margin_mean": 82.45769500732422, "margin_dpo/margin_std": 93.86041259765625, "step": 424 }, { "KL/chosen_KL_mean": -162.71875, "KL/mean": -190.65719604492188, "KL/rejected_KL_mean": -218.59568786621094, "KL/std": 88.18047332763672, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.007583575323224068, "fcm_dpo/delta": 0.18438118696212769, "fcm_dpo/margin": 55.876930236816406, "fcm_dpo/q_t": 0.4135496914386749, "grad_norm": 13.235092163085938, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.675628423690796, "logits/rejected": -1.600928544998169, "logps/chosen": -241.238525390625, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -321.3443298339844, "loss": 1.162, "margin_dpo/margin_mean": 55.876930236816406, "margin_dpo/margin_std": 110.10330200195312, "step": 425 }, { "KL/chosen_KL_mean": -139.6571044921875, "KL/mean": -180.36883544921875, "KL/rejected_KL_mean": -221.08058166503906, "KL/std": 82.65455627441406, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.007664421573281288, "fcm_dpo/delta": -0.02573956362903118, "fcm_dpo/margin": 81.42347717285156, "fcm_dpo/q_t": 0.3630499839782715, "grad_norm": 14.882542610168457, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.7016167640686035, "logits/rejected": -0.9775686860084534, "logps/chosen": -248.16293334960938, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -309.380615234375, "loss": 0.9711, "margin_dpo/margin_mean": 81.42347717285156, "margin_dpo/margin_std": 92.98140716552734, "step": 426 }, { "KL/chosen_KL_mean": -114.76704406738281, "KL/mean": -153.04989624023438, "KL/rejected_KL_mean": -191.332763671875, "KL/std": 79.32843780517578, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.007663578726351261, "fcm_dpo/delta": 0.01396836619824171, "fcm_dpo/margin": 76.56571960449219, "fcm_dpo/q_t": 0.3693893253803253, "grad_norm": 10.34302043914795, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.3676466941833496, "logits/rejected": -1.7332119941711426, "logps/chosen": -213.88751220703125, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -277.057373046875, "loss": 0.9768, "margin_dpo/margin_mean": 76.56571960449219, "margin_dpo/margin_std": 84.10894775390625, "step": 427 }, { "KL/chosen_KL_mean": -161.67515563964844, "KL/mean": -193.45359802246094, "KL/rejected_KL_mean": -225.2320556640625, "KL/std": 90.53356170654297, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.007735169492661953, "fcm_dpo/delta": 0.11382745951414108, "fcm_dpo/margin": 63.5568962097168, "fcm_dpo/q_t": 0.3944037854671478, "grad_norm": 11.407875061035156, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.7113982439041138, "logits/rejected": -1.4495681524276733, "logps/chosen": -267.6444091796875, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -334.334228515625, "loss": 1.0925, "margin_dpo/margin_mean": 63.55689239501953, "margin_dpo/margin_std": 99.98429107666016, "step": 428 }, { "KL/chosen_KL_mean": -146.55755615234375, "KL/mean": -188.26776123046875, "KL/rejected_KL_mean": -229.97799682617188, "KL/std": 92.2166748046875, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0078426543623209, "fcm_dpo/delta": -0.05955342948436737, "fcm_dpo/margin": 83.42044067382812, "fcm_dpo/q_t": 0.3626925051212311, "grad_norm": 12.069082260131836, "learning_rate": 1.828194884925749e-07, "logits/chosen": -2.145521640777588, "logits/rejected": -1.4594614505767822, "logps/chosen": -260.1024169921875, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -328.2200012207031, "loss": 0.9847, "margin_dpo/margin_mean": 83.42044067382812, "margin_dpo/margin_std": 102.86378479003906, "step": 429 }, { "KL/chosen_KL_mean": -144.6695098876953, "KL/mean": -175.7419891357422, "KL/rejected_KL_mean": -206.81448364257812, "KL/std": 85.18883514404297, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.007902027107775211, "fcm_dpo/delta": 0.1137969121336937, "fcm_dpo/margin": 62.144962310791016, "fcm_dpo/q_t": 0.39485034346580505, "grad_norm": 11.791912078857422, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4742017984390259, "logits/rejected": -1.4342076778411865, "logps/chosen": -235.98887634277344, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -316.92413330078125, "loss": 1.0873, "margin_dpo/margin_mean": 62.14495849609375, "margin_dpo/margin_std": 96.9000244140625, "step": 430 }, { "KL/chosen_KL_mean": -143.3631134033203, "KL/mean": -180.73953247070312, "KL/rejected_KL_mean": -218.11595153808594, "KL/std": 86.56428527832031, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.00791468657553196, "fcm_dpo/delta": 0.008839213289320469, "fcm_dpo/margin": 74.7528305053711, "fcm_dpo/q_t": 0.3723512887954712, "grad_norm": 12.585562705993652, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8056966066360474, "logits/rejected": -1.4107317924499512, "logps/chosen": -236.54434204101562, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -316.24822998046875, "loss": 1.0119, "margin_dpo/margin_mean": 74.75283813476562, "margin_dpo/margin_std": 97.22952270507812, "step": 431 }, { "KL/chosen_KL_mean": -125.29377746582031, "KL/mean": -171.48114013671875, "KL/rejected_KL_mean": -217.6685028076172, "KL/std": 91.92903137207031, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.007812817580997944, "fcm_dpo/delta": -0.13099418580532074, "fcm_dpo/margin": 92.37474060058594, "fcm_dpo/q_t": 0.34232357144355774, "grad_norm": 10.862214088439941, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.5360217094421387, "logits/rejected": -0.9828450679779053, "logps/chosen": -229.72854614257812, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -322.758056640625, "loss": 0.9016, "margin_dpo/margin_mean": 92.37474060058594, "margin_dpo/margin_std": 89.4026107788086, "step": 432 }, { "KL/chosen_KL_mean": -149.347900390625, "KL/mean": -185.40853881835938, "KL/rejected_KL_mean": -221.46917724609375, "KL/std": 85.85087585449219, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.007823634892702103, "fcm_dpo/delta": 0.036593928933143616, "fcm_dpo/margin": 72.12126922607422, "fcm_dpo/q_t": 0.3788387179374695, "grad_norm": 11.844096183776855, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.8435044288635254, "logits/rejected": -1.5245897769927979, "logps/chosen": -239.19113159179688, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -323.20263671875, "loss": 1.0333, "margin_dpo/margin_mean": 72.12126159667969, "margin_dpo/margin_std": 97.50343322753906, "step": 433 }, { "KL/chosen_KL_mean": -133.3025665283203, "KL/mean": -173.20021057128906, "KL/rejected_KL_mean": -213.09783935546875, "KL/std": 94.53541564941406, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.007767610251903534, "fcm_dpo/delta": -0.021131768822669983, "fcm_dpo/margin": 79.79530334472656, "fcm_dpo/q_t": 0.3717169165611267, "grad_norm": 11.61451530456543, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.294811725616455, "logits/rejected": -1.6203594207763672, "logps/chosen": -230.99508666992188, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -297.18914794921875, "loss": 1.0071, "margin_dpo/margin_mean": 79.79530334472656, "margin_dpo/margin_std": 108.62364196777344, "step": 434 }, { "KL/chosen_KL_mean": -128.1294403076172, "KL/mean": -170.17886352539062, "KL/rejected_KL_mean": -212.22830200195312, "KL/std": 88.63929748535156, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.007703529670834541, "fcm_dpo/delta": -0.05114363878965378, "fcm_dpo/margin": 84.09884643554688, "fcm_dpo/q_t": 0.36057594418525696, "grad_norm": 12.488868713378906, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.4954402446746826, "logits/rejected": -2.051886796951294, "logps/chosen": -214.30136108398438, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -305.97998046875, "loss": 0.9593, "margin_dpo/margin_mean": 84.09884643554688, "margin_dpo/margin_std": 96.5893325805664, "step": 435 }, { "KL/chosen_KL_mean": -106.29578399658203, "KL/mean": -151.04168701171875, "KL/rejected_KL_mean": -195.78758239746094, "KL/std": 77.59100341796875, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.007572680711746216, "fcm_dpo/delta": -0.08339563012123108, "fcm_dpo/margin": 89.49180603027344, "fcm_dpo/q_t": 0.34820860624313354, "grad_norm": 11.454240798950195, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.8888766765594482, "logits/rejected": -1.0374984741210938, "logps/chosen": -212.0828857421875, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -284.41229248046875, "loss": 0.9038, "margin_dpo/margin_mean": 89.49180603027344, "margin_dpo/margin_std": 81.09136962890625, "step": 436 }, { "KL/chosen_KL_mean": -126.3140869140625, "KL/mean": -160.70887756347656, "KL/rejected_KL_mean": -195.10366821289062, "KL/std": 81.00616455078125, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.007641713134944439, "fcm_dpo/delta": 0.07833351939916611, "fcm_dpo/margin": 68.78958129882812, "fcm_dpo/q_t": 0.38094091415405273, "grad_norm": 11.858981132507324, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.8064277172088623, "logits/rejected": -1.13604736328125, "logps/chosen": -222.37612915039062, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -284.1158752441406, "loss": 1.0169, "margin_dpo/margin_mean": 68.78958129882812, "margin_dpo/margin_std": 81.00286102294922, "step": 437 }, { "KL/chosen_KL_mean": -125.9722900390625, "KL/mean": -161.98843383789062, "KL/rejected_KL_mean": -198.00454711914062, "KL/std": 82.3248291015625, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.007679305039346218, "fcm_dpo/delta": 0.04927004873752594, "fcm_dpo/margin": 72.0322494506836, "fcm_dpo/q_t": 0.3776671290397644, "grad_norm": 12.564921379089355, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1967129707336426, "logits/rejected": -1.3897695541381836, "logps/chosen": -227.82766723632812, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -287.45220947265625, "loss": 1.0131, "margin_dpo/margin_mean": 72.0322494506836, "margin_dpo/margin_std": 87.12477111816406, "step": 438 }, { "KL/chosen_KL_mean": -137.8834686279297, "KL/mean": -174.15020751953125, "KL/rejected_KL_mean": -210.41696166992188, "KL/std": 88.90861511230469, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.007719128858298063, "fcm_dpo/delta": 0.041249364614486694, "fcm_dpo/margin": 72.53349304199219, "fcm_dpo/q_t": 0.3781670928001404, "grad_norm": 13.368515968322754, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.4943352937698364, "logits/rejected": -1.243110179901123, "logps/chosen": -219.63909912109375, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -301.0033264160156, "loss": 1.015, "margin_dpo/margin_mean": 72.53348541259766, "margin_dpo/margin_std": 88.86139678955078, "step": 439 }, { "KL/chosen_KL_mean": -123.79306030273438, "KL/mean": -167.35960388183594, "KL/rejected_KL_mean": -210.9261474609375, "KL/std": 89.94277954101562, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.007662579417228699, "fcm_dpo/delta": -0.074125736951828, "fcm_dpo/margin": 87.1330795288086, "fcm_dpo/q_t": 0.3566509485244751, "grad_norm": 10.177237510681152, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.3716518878936768, "logits/rejected": -1.8743007183074951, "logps/chosen": -229.43414306640625, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -314.3271484375, "loss": 0.9511, "margin_dpo/margin_mean": 87.13307189941406, "margin_dpo/margin_std": 98.03084564208984, "step": 440 }, { "KL/chosen_KL_mean": -152.3690185546875, "KL/mean": -193.87954711914062, "KL/rejected_KL_mean": -235.39004516601562, "KL/std": 85.96257781982422, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.007626797072589397, "fcm_dpo/delta": -0.03568700700998306, "fcm_dpo/margin": 83.02101135253906, "fcm_dpo/q_t": 0.36243635416030884, "grad_norm": 10.942414283752441, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.7188664674758911, "logits/rejected": -1.3507673740386963, "logps/chosen": -243.8985595703125, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -339.0094299316406, "loss": 0.9435, "margin_dpo/margin_mean": 83.02101135253906, "margin_dpo/margin_std": 87.77552795410156, "step": 441 }, { "KL/chosen_KL_mean": -151.49996948242188, "KL/mean": -200.1038818359375, "KL/rejected_KL_mean": -248.70779418945312, "KL/std": 91.57186889648438, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.007478709332644939, "fcm_dpo/delta": -0.13698464632034302, "fcm_dpo/margin": 97.20780944824219, "fcm_dpo/q_t": 0.3391960859298706, "grad_norm": 11.492963790893555, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.730852723121643, "logits/rejected": -1.2672916650772095, "logps/chosen": -236.27752685546875, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -332.5319519042969, "loss": 0.8827, "margin_dpo/margin_mean": 97.20781707763672, "margin_dpo/margin_std": 88.82469940185547, "step": 442 }, { "KL/chosen_KL_mean": -146.25778198242188, "KL/mean": -178.2651824951172, "KL/rejected_KL_mean": -210.27259826660156, "KL/std": 88.54417419433594, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.007515427656471729, "fcm_dpo/delta": 0.124794140458107, "fcm_dpo/margin": 64.01481628417969, "fcm_dpo/q_t": 0.39310336112976074, "grad_norm": 11.644486427307129, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.975780725479126, "logits/rejected": -1.4380849599838257, "logps/chosen": -248.90704345703125, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -303.3106689453125, "loss": 1.0595, "margin_dpo/margin_mean": 64.01481628417969, "margin_dpo/margin_std": 84.06665802001953, "step": 443 }, { "KL/chosen_KL_mean": -153.3967742919922, "KL/mean": -187.35519409179688, "KL/rejected_KL_mean": -221.31362915039062, "KL/std": 84.9640121459961, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.007713917642831802, "fcm_dpo/delta": 0.08020992577075958, "fcm_dpo/margin": 67.91683959960938, "fcm_dpo/q_t": 0.38746434450149536, "grad_norm": 12.805763244628906, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5735952854156494, "logits/rejected": -1.4024622440338135, "logps/chosen": -241.31649780273438, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -324.6370849609375, "loss": 1.0585, "margin_dpo/margin_mean": 67.91683959960938, "margin_dpo/margin_std": 98.29344177246094, "step": 444 }, { "KL/chosen_KL_mean": -134.63693237304688, "KL/mean": -174.95574951171875, "KL/rejected_KL_mean": -215.27456665039062, "KL/std": 82.31544494628906, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.007680041715502739, "fcm_dpo/delta": -0.021080173552036285, "fcm_dpo/margin": 80.63761901855469, "fcm_dpo/q_t": 0.36301201581954956, "grad_norm": 11.405637741088867, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.8492155075073242, "logits/rejected": -1.4306448698043823, "logps/chosen": -236.03781127929688, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -314.3124694824219, "loss": 0.9583, "margin_dpo/margin_mean": 80.63761901855469, "margin_dpo/margin_std": 86.10155487060547, "step": 445 }, { "KL/chosen_KL_mean": -129.3259735107422, "KL/mean": -166.48141479492188, "KL/rejected_KL_mean": -203.6368408203125, "KL/std": 89.19482421875, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.007745532318949699, "fcm_dpo/delta": 0.025847410783171654, "fcm_dpo/margin": 74.31085205078125, "fcm_dpo/q_t": 0.3730408847332001, "grad_norm": 13.862944602966309, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9598513841629028, "logits/rejected": -1.8122164011001587, "logps/chosen": -216.74832153320312, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -310.33758544921875, "loss": 0.9868, "margin_dpo/margin_mean": 74.31085968017578, "margin_dpo/margin_std": 84.18939208984375, "step": 446 }, { "KL/chosen_KL_mean": -129.60382080078125, "KL/mean": -176.58319091796875, "KL/rejected_KL_mean": -223.56259155273438, "KL/std": 90.30181884765625, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.007595124188810587, "fcm_dpo/delta": -0.12259222567081451, "fcm_dpo/margin": 93.958740234375, "fcm_dpo/q_t": 0.345831036567688, "grad_norm": 11.759650230407715, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.016045093536377, "logits/rejected": -1.9688349962234497, "logps/chosen": -213.21908569335938, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -328.4749755859375, "loss": 0.9233, "margin_dpo/margin_mean": 93.95874786376953, "margin_dpo/margin_std": 100.654052734375, "step": 447 }, { "KL/chosen_KL_mean": -139.2518768310547, "KL/mean": -184.98501586914062, "KL/rejected_KL_mean": -230.71817016601562, "KL/std": 96.35697937011719, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.007501625455915928, "fcm_dpo/delta": -0.09270685166120529, "fcm_dpo/margin": 91.46629333496094, "fcm_dpo/q_t": 0.3555144965648651, "grad_norm": 12.258155822753906, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.790082573890686, "logits/rejected": -1.5212831497192383, "logps/chosen": -231.82766723632812, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -338.407958984375, "loss": 0.9722, "margin_dpo/margin_mean": 91.46629333496094, "margin_dpo/margin_std": 112.03981018066406, "step": 448 }, { "KL/chosen_KL_mean": -138.13804626464844, "KL/mean": -185.43478393554688, "KL/rejected_KL_mean": -232.73153686523438, "KL/std": 89.34208679199219, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.007319760043174028, "fcm_dpo/delta": -0.09918862581253052, "fcm_dpo/margin": 94.59349060058594, "fcm_dpo/q_t": 0.3487839698791504, "grad_norm": 10.345510482788086, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4379135370254517, "logits/rejected": -0.9959603548049927, "logps/chosen": -240.5369873046875, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -327.8804016113281, "loss": 0.9299, "margin_dpo/margin_mean": 94.59349060058594, "margin_dpo/margin_std": 100.87744903564453, "step": 449 }, { "KL/chosen_KL_mean": -122.0594711303711, "KL/mean": -166.82351684570312, "KL/rejected_KL_mean": -211.58758544921875, "KL/std": 90.43101501464844, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.007214718498289585, "fcm_dpo/delta": -0.04914788156747818, "fcm_dpo/margin": 89.528076171875, "fcm_dpo/q_t": 0.35765063762664795, "grad_norm": 10.69074535369873, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3885655403137207, "logits/rejected": -2.191129207611084, "logps/chosen": -209.05233764648438, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -320.11962890625, "loss": 0.9577, "margin_dpo/margin_mean": 89.528076171875, "margin_dpo/margin_std": 99.930908203125, "step": 450 }, { "KL/chosen_KL_mean": -138.0839385986328, "KL/mean": -175.00811767578125, "KL/rejected_KL_mean": -211.93231201171875, "KL/std": 91.09397888183594, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.007291129790246487, "fcm_dpo/delta": 0.06475840508937836, "fcm_dpo/margin": 73.84837341308594, "fcm_dpo/q_t": 0.3860728144645691, "grad_norm": 19.69459342956543, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.465888023376465, "logits/rejected": -1.9691853523254395, "logps/chosen": -224.8952178955078, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -291.78790283203125, "loss": 1.0649, "margin_dpo/margin_mean": 73.84837341308594, "margin_dpo/margin_std": 111.01652526855469, "step": 451 }, { "KL/chosen_KL_mean": -120.84186553955078, "KL/mean": -161.16848754882812, "KL/rejected_KL_mean": -201.4951171875, "KL/std": 87.38921356201172, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.007241596933454275, "fcm_dpo/delta": 0.015081025660037994, "fcm_dpo/margin": 80.65325927734375, "fcm_dpo/q_t": 0.36896568536758423, "grad_norm": 10.307807922363281, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.755597472190857, "logits/rejected": -1.6914863586425781, "logps/chosen": -200.22149658203125, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -305.21051025390625, "loss": 0.9661, "margin_dpo/margin_mean": 80.65325927734375, "margin_dpo/margin_std": 77.11204528808594, "step": 452 }, { "KL/chosen_KL_mean": -145.3657989501953, "KL/mean": -193.8863525390625, "KL/rejected_KL_mean": -242.40692138671875, "KL/std": 88.58659362792969, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.007213181350380182, "fcm_dpo/delta": -0.10739025473594666, "fcm_dpo/margin": 97.04109191894531, "fcm_dpo/q_t": 0.3463546931743622, "grad_norm": 14.060781478881836, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.9360098838806152, "logits/rejected": -1.4647655487060547, "logps/chosen": -233.0609893798828, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -332.46514892578125, "loss": 0.8978, "margin_dpo/margin_mean": 97.04109191894531, "margin_dpo/margin_std": 90.18641662597656, "step": 453 }, { "KL/chosen_KL_mean": -155.0008087158203, "KL/mean": -194.22735595703125, "KL/rejected_KL_mean": -233.45388793945312, "KL/std": 90.16275024414062, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.007237846963107586, "fcm_dpo/delta": 0.03331391140818596, "fcm_dpo/margin": 78.45308685302734, "fcm_dpo/q_t": 0.3791477680206299, "grad_norm": 12.366720199584961, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.9179167747497559, "logits/rejected": -1.464126467704773, "logps/chosen": -244.56704711914062, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -326.37493896484375, "loss": 1.057, "margin_dpo/margin_mean": 78.45307922363281, "margin_dpo/margin_std": 116.14363098144531, "step": 454 }, { "KL/chosen_KL_mean": -142.74266052246094, "KL/mean": -180.58753967285156, "KL/rejected_KL_mean": -218.43240356445312, "KL/std": 88.77230834960938, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.00729729887098074, "fcm_dpo/delta": 0.04981229454278946, "fcm_dpo/margin": 75.68976593017578, "fcm_dpo/q_t": 0.37943828105926514, "grad_norm": 10.894377708435059, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8686200380325317, "logits/rejected": -1.8191096782684326, "logps/chosen": -220.37265014648438, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -337.4103698730469, "loss": 1.0285, "margin_dpo/margin_mean": 75.68975830078125, "margin_dpo/margin_std": 99.00331115722656, "step": 455 }, { "KL/chosen_KL_mean": -145.16265869140625, "KL/mean": -182.11422729492188, "KL/rejected_KL_mean": -219.06581115722656, "KL/std": 91.41963195800781, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.007368436083197594, "fcm_dpo/delta": 0.05828423798084259, "fcm_dpo/margin": 73.90313720703125, "fcm_dpo/q_t": 0.3799523115158081, "grad_norm": 13.644865989685059, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.1006388664245605, "logits/rejected": -1.970228672027588, "logps/chosen": -234.779541015625, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -328.6255187988281, "loss": 1.0156, "margin_dpo/margin_mean": 73.90313720703125, "margin_dpo/margin_std": 90.61260986328125, "step": 456 }, { "KL/chosen_KL_mean": -121.26397705078125, "KL/mean": -170.35433959960938, "KL/rejected_KL_mean": -219.44467163085938, "KL/std": 89.22004699707031, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.007170416414737701, "fcm_dpo/delta": -0.1151975691318512, "fcm_dpo/margin": 98.18070983886719, "fcm_dpo/q_t": 0.3407749533653259, "grad_norm": 11.8727388381958, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.0681633949279785, "logits/rejected": -1.7544293403625488, "logps/chosen": -208.58566284179688, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -310.2113037109375, "loss": 0.8787, "margin_dpo/margin_mean": 98.18070983886719, "margin_dpo/margin_std": 78.03376770019531, "step": 457 }, { "KL/chosen_KL_mean": -138.24534606933594, "KL/mean": -175.2338409423828, "KL/rejected_KL_mean": -212.2223358154297, "KL/std": 90.48780059814453, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.007289388217031956, "fcm_dpo/delta": 0.06388363242149353, "fcm_dpo/margin": 73.97697448730469, "fcm_dpo/q_t": 0.3858475089073181, "grad_norm": 12.586189270019531, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5384387969970703, "logits/rejected": -1.2887548208236694, "logps/chosen": -231.7657928466797, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -315.59130859375, "loss": 1.0455, "margin_dpo/margin_mean": 73.97698211669922, "margin_dpo/margin_std": 105.01176452636719, "step": 458 }, { "KL/chosen_KL_mean": -132.4233856201172, "KL/mean": -177.91165161132812, "KL/rejected_KL_mean": -223.39993286132812, "KL/std": 94.88827514648438, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.007186271250247955, "fcm_dpo/delta": -0.05866532772779465, "fcm_dpo/margin": 90.97654724121094, "fcm_dpo/q_t": 0.35942912101745605, "grad_norm": 9.064743995666504, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6968202590942383, "logits/rejected": -1.4410802125930786, "logps/chosen": -208.11160278320312, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -315.5704345703125, "loss": 0.9553, "margin_dpo/margin_mean": 90.97654724121094, "margin_dpo/margin_std": 102.06742858886719, "step": 459 }, { "KL/chosen_KL_mean": -136.92373657226562, "KL/mean": -178.42486572265625, "KL/rejected_KL_mean": -219.926025390625, "KL/std": 89.75792694091797, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0071813929826021194, "fcm_dpo/delta": 0.003812413662672043, "fcm_dpo/margin": 83.0022964477539, "fcm_dpo/q_t": 0.3707672953605652, "grad_norm": 13.253705024719238, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0925867557525635, "logits/rejected": -2.105555534362793, "logps/chosen": -218.0416259765625, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -330.2384033203125, "loss": 0.9962, "margin_dpo/margin_mean": 83.00228881835938, "margin_dpo/margin_std": 101.75143432617188, "step": 460 }, { "KL/chosen_KL_mean": -167.90345764160156, "KL/mean": -200.69967651367188, "KL/rejected_KL_mean": -233.49588012695312, "KL/std": 80.10353088378906, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.007322758436203003, "fcm_dpo/delta": 0.1259041726589203, "fcm_dpo/margin": 65.59241485595703, "fcm_dpo/q_t": 0.39550164341926575, "grad_norm": 11.27901840209961, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.525556206703186, "logits/rejected": -1.328979730606079, "logps/chosen": -249.4869842529297, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -327.37298583984375, "loss": 1.0683, "margin_dpo/margin_mean": 65.59241485595703, "margin_dpo/margin_std": 91.58177185058594, "step": 461 }, { "KL/chosen_KL_mean": -126.33505249023438, "KL/mean": -170.46343994140625, "KL/rejected_KL_mean": -214.59182739257812, "KL/std": 86.14350891113281, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.00735663203522563, "fcm_dpo/delta": -0.05268000438809395, "fcm_dpo/margin": 88.25678253173828, "fcm_dpo/q_t": 0.3572591543197632, "grad_norm": 10.841658592224121, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.040835380554199, "logits/rejected": -1.5203356742858887, "logps/chosen": -220.53359985351562, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -300.22344970703125, "loss": 0.9402, "margin_dpo/margin_mean": 88.25677490234375, "margin_dpo/margin_std": 91.75244140625, "step": 462 }, { "KL/chosen_KL_mean": -123.44120788574219, "KL/mean": -174.50894165039062, "KL/rejected_KL_mean": -225.57669067382812, "KL/std": 91.79393005371094, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.007229425944387913, "fcm_dpo/delta": -0.14988577365875244, "fcm_dpo/margin": 102.13551330566406, "fcm_dpo/q_t": 0.33738774061203003, "grad_norm": 14.31781005859375, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.6217007637023926, "logits/rejected": -1.1783033609390259, "logps/chosen": -209.36593627929688, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -322.4785461425781, "loss": 0.9042, "margin_dpo/margin_mean": 102.13551330566406, "margin_dpo/margin_std": 98.32884216308594, "step": 463 }, { "KL/chosen_KL_mean": -151.6400146484375, "KL/mean": -193.388916015625, "KL/rejected_KL_mean": -235.13784790039062, "KL/std": 91.62759399414062, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.007098211906850338, "fcm_dpo/delta": 0.007712244987487793, "fcm_dpo/margin": 83.49783325195312, "fcm_dpo/q_t": 0.37005606293678284, "grad_norm": 14.45534896850586, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.8857734203338623, "logits/rejected": -1.9608622789382935, "logps/chosen": -231.32920837402344, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -342.43017578125, "loss": 0.994, "margin_dpo/margin_mean": 83.49783325195312, "margin_dpo/margin_std": 101.7740249633789, "step": 464 }, { "KL/chosen_KL_mean": -143.3654022216797, "KL/mean": -192.7783203125, "KL/rejected_KL_mean": -242.19125366210938, "KL/std": 100.45069885253906, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.006991433911025524, "fcm_dpo/delta": -0.09815485030412674, "fcm_dpo/margin": 98.82586669921875, "fcm_dpo/q_t": 0.34755587577819824, "grad_norm": 12.071995735168457, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6059271097183228, "logits/rejected": -1.3003222942352295, "logps/chosen": -235.22561645507812, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -360.9012756347656, "loss": 0.9066, "margin_dpo/margin_mean": 98.82586669921875, "margin_dpo/margin_std": 94.66999053955078, "step": 465 }, { "KL/chosen_KL_mean": -127.98948669433594, "KL/mean": -178.72219848632812, "KL/rejected_KL_mean": -229.45492553710938, "KL/std": 96.27102661132812, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.006854129955172539, "fcm_dpo/delta": -0.10296843945980072, "fcm_dpo/margin": 101.46543884277344, "fcm_dpo/q_t": 0.3466642498970032, "grad_norm": 8.528982162475586, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.5530818700790405, "logits/rejected": -1.2282238006591797, "logps/chosen": -212.69088745117188, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -325.5057678222656, "loss": 0.9083, "margin_dpo/margin_mean": 101.4654312133789, "margin_dpo/margin_std": 98.88075256347656, "step": 466 }, { "KL/chosen_KL_mean": -159.43203735351562, "KL/mean": -202.68478393554688, "KL/rejected_KL_mean": -245.93753051757812, "KL/std": 99.59646606445312, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.006804898381233215, "fcm_dpo/delta": 0.011367086321115494, "fcm_dpo/margin": 86.5054931640625, "fcm_dpo/q_t": 0.3719228506088257, "grad_norm": 13.879347801208496, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.2178666591644287, "logits/rejected": -1.6320037841796875, "logps/chosen": -268.7303466796875, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -354.78118896484375, "loss": 1.0145, "margin_dpo/margin_mean": 86.5054931640625, "margin_dpo/margin_std": 111.64726257324219, "step": 467 }, { "KL/chosen_KL_mean": -126.99529266357422, "KL/mean": -165.2655029296875, "KL/rejected_KL_mean": -203.53570556640625, "KL/std": 96.21795654296875, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.00691565964370966, "fcm_dpo/delta": 0.07468912750482559, "fcm_dpo/margin": 76.54039764404297, "fcm_dpo/q_t": 0.3880506157875061, "grad_norm": 11.378750801086426, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.747143268585205, "logits/rejected": -1.5470695495605469, "logps/chosen": -225.3169403076172, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -310.2161865234375, "loss": 1.0494, "margin_dpo/margin_mean": 76.5404052734375, "margin_dpo/margin_std": 108.8714828491211, "step": 468 }, { "KL/chosen_KL_mean": -155.73753356933594, "KL/mean": -203.06048583984375, "KL/rejected_KL_mean": -250.38345336914062, "KL/std": 95.64952850341797, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.006854588631540537, "fcm_dpo/delta": -0.05331520363688469, "fcm_dpo/margin": 94.64590454101562, "fcm_dpo/q_t": 0.35796457529067993, "grad_norm": 14.049504280090332, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.1694085597991943, "logits/rejected": -1.7432684898376465, "logps/chosen": -252.50173950195312, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -359.97845458984375, "loss": 0.9545, "margin_dpo/margin_mean": 94.64591217041016, "margin_dpo/margin_std": 103.98637390136719, "step": 469 }, { "KL/chosen_KL_mean": -144.50656127929688, "KL/mean": -187.97662353515625, "KL/rejected_KL_mean": -231.44671630859375, "KL/std": 91.09492492675781, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.006883541122078896, "fcm_dpo/delta": 0.0016384599730372429, "fcm_dpo/margin": 86.9401626586914, "fcm_dpo/q_t": 0.3668821156024933, "grad_norm": 10.772821426391602, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.588070273399353, "logits/rejected": -1.160801649093628, "logps/chosen": -228.00320434570312, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -319.9324951171875, "loss": 0.9626, "margin_dpo/margin_mean": 86.9401626586914, "margin_dpo/margin_std": 91.24411010742188, "step": 470 }, { "KL/chosen_KL_mean": -165.17459106445312, "KL/mean": -207.63034057617188, "KL/rejected_KL_mean": -250.08609008789062, "KL/std": 89.90997314453125, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.006891671568155289, "fcm_dpo/delta": 0.01573711819946766, "fcm_dpo/margin": 84.9114990234375, "fcm_dpo/q_t": 0.3686991333961487, "grad_norm": 10.700573921203613, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6725733280181885, "logits/rejected": -1.2738275527954102, "logps/chosen": -253.64889526367188, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -340.56781005859375, "loss": 0.97, "margin_dpo/margin_mean": 84.9114990234375, "margin_dpo/margin_std": 89.22756958007812, "step": 471 }, { "KL/chosen_KL_mean": -150.9773712158203, "KL/mean": -202.50778198242188, "KL/rejected_KL_mean": -254.03819274902344, "KL/std": 89.41836547851562, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.006761905737221241, "fcm_dpo/delta": -0.1052173525094986, "fcm_dpo/margin": 103.06082916259766, "fcm_dpo/q_t": 0.34531599283218384, "grad_norm": 9.26842212677002, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.1756153106689453, "logits/rejected": -1.482186198234558, "logps/chosen": -276.2110595703125, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -375.0916748046875, "loss": 0.8894, "margin_dpo/margin_mean": 103.06082916259766, "margin_dpo/margin_std": 90.4927978515625, "step": 472 }, { "KL/chosen_KL_mean": -172.31854248046875, "KL/mean": -204.76565551757812, "KL/rejected_KL_mean": -237.21275329589844, "KL/std": 91.29168701171875, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.006935113575309515, "fcm_dpo/delta": 0.1565394401550293, "fcm_dpo/margin": 64.89421081542969, "fcm_dpo/q_t": 0.4042420983314514, "grad_norm": 12.41799545288086, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -2.024327278137207, "logits/rejected": -1.5501500368118286, "logps/chosen": -267.929931640625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -325.3638916015625, "loss": 1.1042, "margin_dpo/margin_mean": 64.89421081542969, "margin_dpo/margin_std": 105.02009582519531, "step": 473 }, { "KL/chosen_KL_mean": -165.1923828125, "KL/mean": -196.608642578125, "KL/rejected_KL_mean": -228.02487182617188, "KL/std": 85.1009292602539, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.007198813371360302, "fcm_dpo/delta": 0.15160080790519714, "fcm_dpo/margin": 62.832481384277344, "fcm_dpo/q_t": 0.4017202854156494, "grad_norm": 15.320625305175781, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.5810729265213013, "logits/rejected": -1.390292763710022, "logps/chosen": -246.67214965820312, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -324.4905090332031, "loss": 1.1172, "margin_dpo/margin_mean": 62.83248519897461, "margin_dpo/margin_std": 101.14838409423828, "step": 474 }, { "KL/chosen_KL_mean": -122.60350036621094, "KL/mean": -171.00616455078125, "KL/rejected_KL_mean": -219.4088134765625, "KL/std": 92.81784057617188, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.00709480931982398, "fcm_dpo/delta": -0.09316974878311157, "fcm_dpo/margin": 96.80530548095703, "fcm_dpo/q_t": 0.35160699486732483, "grad_norm": 10.550143241882324, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.217155933380127, "logits/rejected": -2.1621439456939697, "logps/chosen": -196.7994842529297, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -314.1012268066406, "loss": 0.9245, "margin_dpo/margin_mean": 96.80530548095703, "margin_dpo/margin_std": 101.42073059082031, "step": 475 }, { "KL/chosen_KL_mean": -145.65090942382812, "KL/mean": -188.01510620117188, "KL/rejected_KL_mean": -230.37933349609375, "KL/std": 94.66455841064453, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.007026867941021919, "fcm_dpo/delta": 0.0044501833617687225, "fcm_dpo/margin": 84.72840881347656, "fcm_dpo/q_t": 0.372454434633255, "grad_norm": 14.49284553527832, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.8008003234863281, "logits/rejected": -1.4075559377670288, "logps/chosen": -217.62200927734375, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -310.6415710449219, "loss": 0.9918, "margin_dpo/margin_mean": 84.72840118408203, "margin_dpo/margin_std": 102.58384704589844, "step": 476 }, { "KL/chosen_KL_mean": -145.33737182617188, "KL/mean": -189.6104736328125, "KL/rejected_KL_mean": -233.88357543945312, "KL/std": 85.21902465820312, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.007047213613986969, "fcm_dpo/delta": -0.025613542646169662, "fcm_dpo/margin": 88.54618835449219, "fcm_dpo/q_t": 0.3627077043056488, "grad_norm": 11.680216789245605, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.1934521198272705, "logits/rejected": -1.2940565347671509, "logps/chosen": -250.34292602539062, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -315.76202392578125, "loss": 0.9798, "margin_dpo/margin_mean": 88.54618835449219, "margin_dpo/margin_std": 105.13601684570312, "step": 477 }, { "KL/chosen_KL_mean": -137.80645751953125, "KL/mean": -183.3409423828125, "KL/rejected_KL_mean": -228.87545776367188, "KL/std": 91.83963012695312, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.006991152185946703, "fcm_dpo/delta": -0.03915346413850784, "fcm_dpo/margin": 91.06900024414062, "fcm_dpo/q_t": 0.359982430934906, "grad_norm": 10.428479194641113, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.7466964721679688, "logits/rejected": -1.576913833618164, "logps/chosen": -214.5946502685547, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -319.31536865234375, "loss": 0.9575, "margin_dpo/margin_mean": 91.06900024414062, "margin_dpo/margin_std": 100.99594116210938, "step": 478 }, { "KL/chosen_KL_mean": -145.67306518554688, "KL/mean": -188.06900024414062, "KL/rejected_KL_mean": -230.4649200439453, "KL/std": 90.71542358398438, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.007011481560766697, "fcm_dpo/delta": 0.0053132884204387665, "fcm_dpo/margin": 84.79185485839844, "fcm_dpo/q_t": 0.36689460277557373, "grad_norm": 13.254948616027832, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.7028346061706543, "logits/rejected": -1.5661453008651733, "logps/chosen": -227.03536987304688, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -331.5560607910156, "loss": 0.9758, "margin_dpo/margin_mean": 84.79185485839844, "margin_dpo/margin_std": 92.27081298828125, "step": 479 }, { "KL/chosen_KL_mean": -166.86093139648438, "KL/mean": -202.53759765625, "KL/rejected_KL_mean": -238.21426391601562, "KL/std": 87.69963073730469, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.007081994786858559, "fcm_dpo/delta": 0.09973622858524323, "fcm_dpo/margin": 71.35332489013672, "fcm_dpo/q_t": 0.38974419236183167, "grad_norm": 13.62548542022705, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.6021537780761719, "logits/rejected": -0.9071184396743774, "logps/chosen": -263.63482666015625, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -324.6190185546875, "loss": 1.0557, "margin_dpo/margin_mean": 71.35332489013672, "margin_dpo/margin_std": 99.06027221679688, "step": 480 }, { "KL/chosen_KL_mean": -163.15403747558594, "KL/mean": -205.4266357421875, "KL/rejected_KL_mean": -247.69923400878906, "KL/std": 91.01664733886719, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.00711432471871376, "fcm_dpo/delta": -0.0015836535021662712, "fcm_dpo/margin": 84.54520416259766, "fcm_dpo/q_t": 0.36751770973205566, "grad_norm": 14.930974006652832, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.840137004852295, "logits/rejected": -1.3913183212280273, "logps/chosen": -254.82420349121094, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -346.3941345214844, "loss": 0.989, "margin_dpo/margin_mean": 84.54520416259766, "margin_dpo/margin_std": 101.47366333007812, "step": 481 }, { "KL/chosen_KL_mean": -155.1199188232422, "KL/mean": -197.50784301757812, "KL/rejected_KL_mean": -239.89576721191406, "KL/std": 88.74147033691406, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.007095282431691885, "fcm_dpo/delta": -0.0017808638513088226, "fcm_dpo/margin": 84.7758560180664, "fcm_dpo/q_t": 0.36606836318969727, "grad_norm": 14.282244682312012, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.5931556224822998, "logits/rejected": -1.2323875427246094, "logps/chosen": -253.64002990722656, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -334.7252197265625, "loss": 0.9776, "margin_dpo/margin_mean": 84.77584838867188, "margin_dpo/margin_std": 95.80502319335938, "step": 482 }, { "KL/chosen_KL_mean": -162.1891326904297, "KL/mean": -206.25668334960938, "KL/rejected_KL_mean": -250.32424926757812, "KL/std": 95.36003112792969, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.007137489505112171, "fcm_dpo/delta": -0.0320398211479187, "fcm_dpo/margin": 88.13511657714844, "fcm_dpo/q_t": 0.3602200448513031, "grad_norm": 9.462681770324707, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.0883445739746094, "logits/rejected": -1.4533038139343262, "logps/chosen": -269.3077392578125, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -351.4392395019531, "loss": 0.9552, "margin_dpo/margin_mean": 88.13511657714844, "margin_dpo/margin_std": 92.6663818359375, "step": 483 }, { "KL/chosen_KL_mean": -151.9749298095703, "KL/mean": -199.53945922851562, "KL/rejected_KL_mean": -247.10398864746094, "KL/std": 98.01980590820312, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.007024774327874184, "fcm_dpo/delta": -0.07323425263166428, "fcm_dpo/margin": 95.12903594970703, "fcm_dpo/q_t": 0.35387182235717773, "grad_norm": 10.318730354309082, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.407949447631836, "logits/rejected": -1.9258768558502197, "logps/chosen": -238.95484924316406, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -337.82763671875, "loss": 0.9481, "margin_dpo/margin_mean": 95.12904357910156, "margin_dpo/margin_std": 105.02281951904297, "step": 484 }, { "KL/chosen_KL_mean": -139.44183349609375, "KL/mean": -193.00660705566406, "KL/rejected_KL_mean": -246.57138061523438, "KL/std": 99.2025146484375, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.006825345568358898, "fcm_dpo/delta": -0.14149078726768494, "fcm_dpo/margin": 107.12954711914062, "fcm_dpo/q_t": 0.33951207995414734, "grad_norm": 10.631072998046875, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.5572237968444824, "logits/rejected": -1.1177480220794678, "logps/chosen": -233.50057983398438, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -355.1343688964844, "loss": 0.8819, "margin_dpo/margin_mean": 107.12954711914062, "margin_dpo/margin_std": 98.83341217041016, "step": 485 }, { "KL/chosen_KL_mean": -161.92755126953125, "KL/mean": -209.1256866455078, "KL/rejected_KL_mean": -256.32379150390625, "KL/std": 96.28413391113281, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.006753925234079361, "fcm_dpo/delta": -0.040186379104852676, "fcm_dpo/margin": 94.396240234375, "fcm_dpo/q_t": 0.3598681688308716, "grad_norm": 12.52357006072998, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.0489330291748047, "logits/rejected": -1.3353488445281982, "logps/chosen": -255.67343139648438, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -354.39447021484375, "loss": 0.9603, "margin_dpo/margin_mean": 94.396240234375, "margin_dpo/margin_std": 104.20707702636719, "step": 486 }, { "KL/chosen_KL_mean": -155.47312927246094, "KL/mean": -198.93951416015625, "KL/rejected_KL_mean": -242.4058837890625, "KL/std": 93.13529968261719, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.006725200451910496, "fcm_dpo/delta": 0.01629689894616604, "fcm_dpo/margin": 86.9327392578125, "fcm_dpo/q_t": 0.37093690037727356, "grad_norm": 11.055121421813965, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.066627025604248, "logits/rejected": -1.7517499923706055, "logps/chosen": -243.5067138671875, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -340.87799072265625, "loss": 0.9788, "margin_dpo/margin_mean": 86.9327392578125, "margin_dpo/margin_std": 96.72283935546875, "step": 487 }, { "KL/chosen_KL_mean": -145.4847412109375, "KL/mean": -191.16683959960938, "KL/rejected_KL_mean": -236.84893798828125, "KL/std": 97.32330322265625, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.006729425862431526, "fcm_dpo/delta": -0.01579986698925495, "fcm_dpo/margin": 91.36419677734375, "fcm_dpo/q_t": 0.36102399230003357, "grad_norm": 12.586343765258789, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.506693720817566, "logits/rejected": -1.243421196937561, "logps/chosen": -230.27439880371094, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -327.1224060058594, "loss": 0.9397, "margin_dpo/margin_mean": 91.36419677734375, "margin_dpo/margin_std": 84.7784652709961, "step": 488 }, { "KL/chosen_KL_mean": -159.97665405273438, "KL/mean": -203.03207397460938, "KL/rejected_KL_mean": -246.08749389648438, "KL/std": 88.75929260253906, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.006741112098097801, "fcm_dpo/delta": 0.02071393094956875, "fcm_dpo/margin": 86.1108627319336, "fcm_dpo/q_t": 0.3723028898239136, "grad_norm": 11.298696517944336, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.5678930282592773, "logits/rejected": -1.0701305866241455, "logps/chosen": -250.44595336914062, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -332.485107421875, "loss": 0.9855, "margin_dpo/margin_mean": 86.1108627319336, "margin_dpo/margin_std": 98.28038024902344, "step": 489 }, { "KL/chosen_KL_mean": -162.8048553466797, "KL/mean": -201.76683044433594, "KL/rejected_KL_mean": -240.7288055419922, "KL/std": 94.84097290039062, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.006876502651721239, "fcm_dpo/delta": 0.0653955340385437, "fcm_dpo/margin": 77.9239501953125, "fcm_dpo/q_t": 0.387326717376709, "grad_norm": 12.050117492675781, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -1.9649832248687744, "logits/rejected": -2.036487102508545, "logps/chosen": -248.12498474121094, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -356.72265625, "loss": 1.0875, "margin_dpo/margin_mean": 77.9239501953125, "margin_dpo/margin_std": 123.45890808105469, "step": 490 }, { "KL/chosen_KL_mean": -164.49462890625, "KL/mean": -201.833251953125, "KL/rejected_KL_mean": -239.17184448242188, "KL/std": 98.12358093261719, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.006923770997673273, "fcm_dpo/delta": 0.0874912291765213, "fcm_dpo/margin": 74.67724609375, "fcm_dpo/q_t": 0.3857119381427765, "grad_norm": 11.188833236694336, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.919226050376892, "logits/rejected": -1.4274805784225464, "logps/chosen": -250.63815307617188, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -319.851318359375, "loss": 1.0419, "margin_dpo/margin_mean": 74.67724609375, "margin_dpo/margin_std": 99.10111999511719, "step": 491 }, { "KL/chosen_KL_mean": -152.11325073242188, "KL/mean": -181.71475219726562, "KL/rejected_KL_mean": -211.3162841796875, "KL/std": 95.902587890625, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.007147323340177536, "fcm_dpo/delta": 0.184751957654953, "fcm_dpo/margin": 59.20301818847656, "fcm_dpo/q_t": 0.40982648730278015, "grad_norm": 16.34548568725586, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.018230438232422, "logits/rejected": -1.6268681287765503, "logps/chosen": -233.2208251953125, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -292.0682678222656, "loss": 1.1703, "margin_dpo/margin_mean": 59.20301818847656, "margin_dpo/margin_std": 119.07235717773438, "step": 492 }, { "KL/chosen_KL_mean": -150.44866943359375, "KL/mean": -189.8380126953125, "KL/rejected_KL_mean": -229.22732543945312, "KL/std": 87.80055236816406, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0072105275467038155, "fcm_dpo/delta": 0.033341605216264725, "fcm_dpo/margin": 78.77863311767578, "fcm_dpo/q_t": 0.37309813499450684, "grad_norm": 15.570918083190918, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.584127187728882, "logits/rejected": -1.8709397315979004, "logps/chosen": -262.656005859375, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -322.83453369140625, "loss": 0.9945, "margin_dpo/margin_mean": 78.77863311767578, "margin_dpo/margin_std": 89.12460327148438, "step": 493 }, { "KL/chosen_KL_mean": -147.27923583984375, "KL/mean": -199.09445190429688, "KL/rejected_KL_mean": -250.90966796875, "KL/std": 96.9630126953125, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.007058255840092897, "fcm_dpo/delta": -0.1441689133644104, "fcm_dpo/margin": 103.63043212890625, "fcm_dpo/q_t": 0.3394149839878082, "grad_norm": 12.425756454467773, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.359938621520996, "logits/rejected": -1.275738000869751, "logps/chosen": -220.39413452148438, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -343.07269287109375, "loss": 0.8962, "margin_dpo/margin_mean": 103.63043212890625, "margin_dpo/margin_std": 100.42266845703125, "step": 494 }, { "KL/chosen_KL_mean": -148.38043212890625, "KL/mean": -192.36709594726562, "KL/rejected_KL_mean": -236.35374450683594, "KL/std": 92.07487487792969, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.007029523607343435, "fcm_dpo/delta": -0.01969076506793499, "fcm_dpo/margin": 87.97331237792969, "fcm_dpo/q_t": 0.36528149247169495, "grad_norm": 12.305154800415039, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.1386590003967285, "logits/rejected": -1.6702306270599365, "logps/chosen": -247.9007568359375, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -334.2846374511719, "loss": 0.9862, "margin_dpo/margin_mean": 87.97332000732422, "margin_dpo/margin_std": 107.82037353515625, "step": 495 }, { "KL/chosen_KL_mean": -164.38775634765625, "KL/mean": -201.72543334960938, "KL/rejected_KL_mean": -239.0631103515625, "KL/std": 92.50668334960938, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.007135157939046621, "fcm_dpo/delta": 0.0703272819519043, "fcm_dpo/margin": 74.67535400390625, "fcm_dpo/q_t": 0.3810383081436157, "grad_norm": 11.635498046875, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.0952539443969727, "logits/rejected": -1.4422355890274048, "logps/chosen": -272.2445068359375, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -331.83367919921875, "loss": 1.023, "margin_dpo/margin_mean": 74.67535400390625, "margin_dpo/margin_std": 91.55549621582031, "step": 496 }, { "KL/chosen_KL_mean": -179.580078125, "KL/mean": -205.242431640625, "KL/rejected_KL_mean": -230.90478515625, "KL/std": 94.54544830322266, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0073593915440142155, "fcm_dpo/delta": 0.23153796792030334, "fcm_dpo/margin": 51.3246955871582, "fcm_dpo/q_t": 0.41949695348739624, "grad_norm": 11.79470443725586, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.7509794235229492, "logits/rejected": -1.3017889261245728, "logps/chosen": -271.6632995605469, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -312.6998291015625, "loss": 1.1697, "margin_dpo/margin_mean": 51.32469177246094, "margin_dpo/margin_std": 99.1965103149414, "step": 497 }, { "KL/chosen_KL_mean": -147.97528076171875, "KL/mean": -190.26853942871094, "KL/rejected_KL_mean": -232.5618133544922, "KL/std": 89.61714172363281, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0074294013902544975, "fcm_dpo/delta": -0.03037802129983902, "fcm_dpo/margin": 84.58653259277344, "fcm_dpo/q_t": 0.3614158630371094, "grad_norm": 13.488200187683105, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.7572612762451172, "logits/rejected": -1.2598570585250854, "logps/chosen": -246.16964721679688, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -323.249267578125, "loss": 0.9601, "margin_dpo/margin_mean": 84.58654022216797, "margin_dpo/margin_std": 93.3172607421875, "step": 498 }, { "KL/chosen_KL_mean": -159.33102416992188, "KL/mean": -201.79490661621094, "KL/rejected_KL_mean": -244.2587890625, "KL/std": 99.46517181396484, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0074063753709197044, "fcm_dpo/delta": -0.03098292089998722, "fcm_dpo/margin": 84.92778015136719, "fcm_dpo/q_t": 0.36475786566734314, "grad_norm": 15.313008308410645, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.943324327468872, "logits/rejected": -1.889524221420288, "logps/chosen": -240.3709259033203, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -351.18048095703125, "loss": 0.9897, "margin_dpo/margin_mean": 84.92778015136719, "margin_dpo/margin_std": 106.61343383789062, "step": 499 }, { "KL/chosen_KL_mean": -172.66177368164062, "KL/mean": -210.90797424316406, "KL/rejected_KL_mean": -249.1541748046875, "KL/std": 96.51409149169922, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.007405046373605728, "fcm_dpo/delta": 0.035586465150117874, "fcm_dpo/margin": 76.49237823486328, "fcm_dpo/q_t": 0.38237982988357544, "grad_norm": 14.309152603149414, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.519291639328003, "logits/rejected": -1.341374397277832, "logps/chosen": -261.90985107421875, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -349.56439208984375, "loss": 1.0404, "margin_dpo/margin_mean": 76.49237823486328, "margin_dpo/margin_std": 109.17184448242188, "step": 500 }, { "KL/chosen_KL_mean": -151.75601196289062, "KL/mean": -192.32716369628906, "KL/rejected_KL_mean": -232.8983154296875, "KL/std": 105.01946258544922, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.007501913234591484, "fcm_dpo/delta": -0.011712707579135895, "fcm_dpo/margin": 81.14228820800781, "fcm_dpo/q_t": 0.3733324706554413, "grad_norm": 13.749055862426758, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.4936261177062988, "logits/rejected": -1.0492372512817383, "logps/chosen": -242.3529510498047, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -318.195556640625, "loss": 1.0455, "margin_dpo/margin_mean": 81.14228820800781, "margin_dpo/margin_std": 116.85734558105469, "step": 501 }, { "KL/chosen_KL_mean": -154.0494384765625, "KL/mean": -190.75802612304688, "KL/rejected_KL_mean": -227.46661376953125, "KL/std": 87.3137435913086, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.007522047497332096, "fcm_dpo/delta": 0.04913443699479103, "fcm_dpo/margin": 73.41716766357422, "fcm_dpo/q_t": 0.3808327913284302, "grad_norm": 12.678153038024902, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.8433648347854614, "logits/rejected": -1.0966087579727173, "logps/chosen": -257.5201721191406, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -316.5107727050781, "loss": 1.0627, "margin_dpo/margin_mean": 73.41717529296875, "margin_dpo/margin_std": 107.966796875, "step": 502 }, { "KL/chosen_KL_mean": -149.55116271972656, "KL/mean": -192.50390625, "KL/rejected_KL_mean": -235.4566650390625, "KL/std": 93.12542724609375, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.007491983473300934, "fcm_dpo/delta": -0.04744477570056915, "fcm_dpo/margin": 85.9054946899414, "fcm_dpo/q_t": 0.36096590757369995, "grad_norm": 11.915144920349121, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.777030348777771, "logits/rejected": -1.274646520614624, "logps/chosen": -243.28115844726562, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -323.06561279296875, "loss": 0.9749, "margin_dpo/margin_mean": 85.90550231933594, "margin_dpo/margin_std": 100.52666473388672, "step": 503 }, { "KL/chosen_KL_mean": -150.59280395507812, "KL/mean": -203.32711791992188, "KL/rejected_KL_mean": -256.06146240234375, "KL/std": 95.94749450683594, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0072372350841760635, "fcm_dpo/delta": -0.17656481266021729, "fcm_dpo/margin": 105.46864318847656, "fcm_dpo/q_t": 0.3318415582180023, "grad_norm": 15.024852752685547, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.2676479816436768, "logits/rejected": -1.9383832216262817, "logps/chosen": -242.88467407226562, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -365.09912109375, "loss": 0.8713, "margin_dpo/margin_mean": 105.4686508178711, "margin_dpo/margin_std": 97.4165267944336, "step": 504 }, { "KL/chosen_KL_mean": -168.31396484375, "KL/mean": -208.23486328125, "KL/rejected_KL_mean": -248.15573120117188, "KL/std": 96.56719207763672, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.007191069424152374, "fcm_dpo/delta": 0.027413375675678253, "fcm_dpo/margin": 79.841796875, "fcm_dpo/q_t": 0.37392929196357727, "grad_norm": 10.838815689086914, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.4139366149902344, "logits/rejected": -1.8580589294433594, "logps/chosen": -274.59088134765625, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -342.36187744140625, "loss": 0.9962, "margin_dpo/margin_mean": 79.841796875, "margin_dpo/margin_std": 93.74464416503906, "step": 505 }, { "KL/chosen_KL_mean": -164.06072998046875, "KL/mean": -206.4884033203125, "KL/rejected_KL_mean": -248.9160919189453, "KL/std": 88.7039566040039, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.007159810047596693, "fcm_dpo/delta": -0.00842544436454773, "fcm_dpo/margin": 84.85535430908203, "fcm_dpo/q_t": 0.3628551959991455, "grad_norm": 11.072511672973633, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9906580448150635, "logits/rejected": -1.6289644241333008, "logps/chosen": -245.5015869140625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -334.58050537109375, "loss": 0.967, "margin_dpo/margin_mean": 84.85535430908203, "margin_dpo/margin_std": 90.92335510253906, "step": 506 }, { "KL/chosen_KL_mean": -170.71316528320312, "KL/mean": -215.78811645507812, "KL/rejected_KL_mean": -260.8630065917969, "KL/std": 94.89112854003906, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.007146242540329695, "fcm_dpo/delta": -0.04725296422839165, "fcm_dpo/margin": 90.14984893798828, "fcm_dpo/q_t": 0.361957311630249, "grad_norm": 11.26477336883545, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.199552536010742, "logits/rejected": -1.7580262422561646, "logps/chosen": -270.14013671875, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -363.115966796875, "loss": 0.9836, "margin_dpo/margin_mean": 90.14984130859375, "margin_dpo/margin_std": 111.27189636230469, "step": 507 }, { "KL/chosen_KL_mean": -167.46017456054688, "KL/mean": -204.1432647705078, "KL/rejected_KL_mean": -240.8263702392578, "KL/std": 101.66572570800781, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0071931928396224976, "fcm_dpo/delta": 0.07634302973747253, "fcm_dpo/margin": 73.36620330810547, "fcm_dpo/q_t": 0.3894597589969635, "grad_norm": 15.375066757202148, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.9920873641967773, "logits/rejected": -1.5051259994506836, "logps/chosen": -265.3758239746094, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -344.62420654296875, "loss": 1.0829, "margin_dpo/margin_mean": 73.36619567871094, "margin_dpo/margin_std": 114.41323852539062, "step": 508 }, { "KL/chosen_KL_mean": -156.606689453125, "KL/mean": -192.43637084960938, "KL/rejected_KL_mean": -228.26602172851562, "KL/std": 78.88925170898438, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.007329905871301889, "fcm_dpo/delta": 0.07860207557678223, "fcm_dpo/margin": 71.65933990478516, "fcm_dpo/q_t": 0.3829195201396942, "grad_norm": 17.32098388671875, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.075132369995117, "logits/rejected": -1.416548490524292, "logps/chosen": -257.3365173339844, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -315.2093811035156, "loss": 1.058, "margin_dpo/margin_mean": 71.65933990478516, "margin_dpo/margin_std": 102.01838684082031, "step": 509 }, { "KL/chosen_KL_mean": -184.4112548828125, "KL/mean": -226.18917846679688, "KL/rejected_KL_mean": -267.96710205078125, "KL/std": 97.15513610839844, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.007343881763517857, "fcm_dpo/delta": -0.014591187238693237, "fcm_dpo/margin": 83.55584716796875, "fcm_dpo/q_t": 0.36610209941864014, "grad_norm": 14.39696216583252, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.3620381355285645, "logits/rejected": -1.1644854545593262, "logps/chosen": -273.58062744140625, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -375.886474609375, "loss": 0.9864, "margin_dpo/margin_mean": 83.55584716796875, "margin_dpo/margin_std": 98.40934753417969, "step": 510 }, { "KL/chosen_KL_mean": -167.38563537597656, "KL/mean": -212.77101135253906, "KL/rejected_KL_mean": -258.1564025878906, "KL/std": 94.86567687988281, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.007273542694747448, "fcm_dpo/delta": -0.06447415798902512, "fcm_dpo/margin": 90.77078247070312, "fcm_dpo/q_t": 0.3582109212875366, "grad_norm": 12.435418128967285, "learning_rate": 9.028829858700973e-08, "logits/chosen": -1.9940211772918701, "logits/rejected": -1.415341854095459, "logps/chosen": -257.13543701171875, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -355.8876953125, "loss": 0.9518, "margin_dpo/margin_mean": 90.77078247070312, "margin_dpo/margin_std": 102.87515258789062, "step": 511 }, { "KL/chosen_KL_mean": -159.04629516601562, "KL/mean": -202.300537109375, "KL/rejected_KL_mean": -245.55479431152344, "KL/std": 90.62704467773438, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.007244492881000042, "fcm_dpo/delta": -0.029211711138486862, "fcm_dpo/margin": 86.50847625732422, "fcm_dpo/q_t": 0.3642181158065796, "grad_norm": 16.3244571685791, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.7922022342681885, "logits/rejected": -1.3424029350280762, "logps/chosen": -244.7814483642578, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -333.1796875, "loss": 0.9948, "margin_dpo/margin_mean": 86.50848388671875, "margin_dpo/margin_std": 108.11572265625, "step": 512 }, { "KL/chosen_KL_mean": -169.8643035888672, "KL/mean": -207.1676025390625, "KL/rejected_KL_mean": -244.4709014892578, "KL/std": 88.17761993408203, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0072096060030162334, "fcm_dpo/delta": 0.06522329151630402, "fcm_dpo/margin": 74.6065673828125, "fcm_dpo/q_t": 0.3793642520904541, "grad_norm": 13.025800704956055, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.402454376220703, "logits/rejected": -1.7203654050827026, "logps/chosen": -266.0548095703125, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -327.3886413574219, "loss": 1.0326, "margin_dpo/margin_mean": 74.6065673828125, "margin_dpo/margin_std": 95.31332397460938, "step": 513 }, { "KL/chosen_KL_mean": -163.21192932128906, "KL/mean": -203.258056640625, "KL/rejected_KL_mean": -243.30418395996094, "KL/std": 86.67200469970703, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.007290694862604141, "fcm_dpo/delta": 0.01706370897591114, "fcm_dpo/margin": 80.09225463867188, "fcm_dpo/q_t": 0.37475594878196716, "grad_norm": 14.206077575683594, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.691982626914978, "logits/rejected": -1.1168920993804932, "logps/chosen": -261.03558349609375, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -328.67401123046875, "loss": 1.0039, "margin_dpo/margin_mean": 80.09225463867188, "margin_dpo/margin_std": 100.39649200439453, "step": 514 }, { "KL/chosen_KL_mean": -159.92181396484375, "KL/mean": -196.81234741210938, "KL/rejected_KL_mean": -233.70286560058594, "KL/std": 89.96261596679688, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.007366856560111046, "fcm_dpo/delta": 0.059732090681791306, "fcm_dpo/margin": 73.78108215332031, "fcm_dpo/q_t": 0.38095155358314514, "grad_norm": 16.975486755371094, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.9173033237457275, "logits/rejected": -1.6894665956497192, "logps/chosen": -241.85536193847656, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -327.2794189453125, "loss": 1.0335, "margin_dpo/margin_mean": 73.78107452392578, "margin_dpo/margin_std": 97.51678466796875, "step": 515 }, { "KL/chosen_KL_mean": -161.52035522460938, "KL/mean": -199.38150024414062, "KL/rejected_KL_mean": -237.24266052246094, "KL/std": 103.18201446533203, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.007447918877005577, "fcm_dpo/delta": 0.03804297372698784, "fcm_dpo/margin": 75.72230529785156, "fcm_dpo/q_t": 0.38302987813949585, "grad_norm": 13.479026794433594, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.2004756927490234, "logits/rejected": -2.0080819129943848, "logps/chosen": -258.6120300292969, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -355.79931640625, "loss": 1.0792, "margin_dpo/margin_mean": 75.72230529785156, "margin_dpo/margin_std": 121.2779541015625, "step": 516 }, { "KL/chosen_KL_mean": -172.22064208984375, "KL/mean": -218.17388916015625, "KL/rejected_KL_mean": -264.1271667480469, "KL/std": 94.94319152832031, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.00729462131857872, "fcm_dpo/delta": -0.07804064452648163, "fcm_dpo/margin": 91.90653991699219, "fcm_dpo/q_t": 0.3568037450313568, "grad_norm": 11.62269115447998, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.997402310371399, "logits/rejected": -1.5780152082443237, "logps/chosen": -265.4071960449219, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -359.719970703125, "loss": 0.9677, "margin_dpo/margin_mean": 91.90653991699219, "margin_dpo/margin_std": 107.52702331542969, "step": 517 }, { "KL/chosen_KL_mean": -181.310302734375, "KL/mean": -215.19924926757812, "KL/rejected_KL_mean": -249.08819580078125, "KL/std": 88.77427673339844, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.007440718822181225, "fcm_dpo/delta": 0.1008116751909256, "fcm_dpo/margin": 67.77787780761719, "fcm_dpo/q_t": 0.3929661810398102, "grad_norm": 15.219815254211426, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.994368553161621, "logits/rejected": -1.6646612882614136, "logps/chosen": -268.3316650390625, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -340.332763671875, "loss": 1.0888, "margin_dpo/margin_mean": 67.77787780761719, "margin_dpo/margin_std": 108.64458465576172, "step": 518 }, { "KL/chosen_KL_mean": -167.17276000976562, "KL/mean": -205.11465454101562, "KL/rejected_KL_mean": -243.05661010742188, "KL/std": 84.35797119140625, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0074867261573672295, "fcm_dpo/delta": 0.033532094210386276, "fcm_dpo/margin": 75.88383483886719, "fcm_dpo/q_t": 0.37545472383499146, "grad_norm": 12.742801666259766, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.9141077995300293, "logits/rejected": -1.5343555212020874, "logps/chosen": -267.86029052734375, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -352.0870361328125, "loss": 0.9896, "margin_dpo/margin_mean": 75.88383483886719, "margin_dpo/margin_std": 85.7808837890625, "step": 519 }, { "KL/chosen_KL_mean": -152.71682739257812, "KL/mean": -197.3311309814453, "KL/rejected_KL_mean": -241.9454345703125, "KL/std": 94.4261245727539, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.007437916472554207, "fcm_dpo/delta": -0.06851769983768463, "fcm_dpo/margin": 89.2286376953125, "fcm_dpo/q_t": 0.352560818195343, "grad_norm": 11.588751792907715, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.4851148128509521, "logits/rejected": -1.0368211269378662, "logps/chosen": -251.49444580078125, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -348.8911437988281, "loss": 0.9281, "margin_dpo/margin_mean": 89.2286376953125, "margin_dpo/margin_std": 89.18794250488281, "step": 520 }, { "KL/chosen_KL_mean": -158.48617553710938, "KL/mean": -203.23171997070312, "KL/rejected_KL_mean": -247.97726440429688, "KL/std": 88.81007385253906, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.00739981047809124, "fcm_dpo/delta": -0.06692080199718475, "fcm_dpo/margin": 89.4911117553711, "fcm_dpo/q_t": 0.3538839817047119, "grad_norm": 12.586703300476074, "learning_rate": 8.062991975753378e-08, "logits/chosen": -2.184523105621338, "logits/rejected": -1.8619673252105713, "logps/chosen": -244.83724975585938, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -338.37042236328125, "loss": 0.9286, "margin_dpo/margin_mean": 89.49110412597656, "margin_dpo/margin_std": 88.91680145263672, "step": 521 }, { "KL/chosen_KL_mean": -161.5187225341797, "KL/mean": -203.96006774902344, "KL/rejected_KL_mean": -246.4014129638672, "KL/std": 96.26828002929688, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.007325354032218456, "fcm_dpo/delta": -0.02331504598259926, "fcm_dpo/margin": 84.88267517089844, "fcm_dpo/q_t": 0.3662179112434387, "grad_norm": 11.958304405212402, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.6722010374069214, "logits/rejected": -1.2992022037506104, "logps/chosen": -237.5267791748047, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -320.8426513671875, "loss": 0.9814, "margin_dpo/margin_mean": 84.88268280029297, "margin_dpo/margin_std": 101.8878173828125, "step": 522 }, { "KL/chosen_KL_mean": -181.6602325439453, "KL/mean": -211.32655334472656, "KL/rejected_KL_mean": -240.99285888671875, "KL/std": 94.10973358154297, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0074586449190974236, "fcm_dpo/delta": 0.16505371034145355, "fcm_dpo/margin": 59.332637786865234, "fcm_dpo/q_t": 0.40665629506111145, "grad_norm": 14.599053382873535, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.0645651817321777, "logits/rejected": -1.3628551959991455, "logps/chosen": -278.156494140625, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -332.0081481933594, "loss": 1.1406, "margin_dpo/margin_mean": 59.33263397216797, "margin_dpo/margin_std": 108.85240173339844, "step": 523 }, { "KL/chosen_KL_mean": -166.5956268310547, "KL/mean": -207.5314178466797, "KL/rejected_KL_mean": -248.46722412109375, "KL/std": 94.31497192382812, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.007518031634390354, "fcm_dpo/delta": -0.016558989882469177, "fcm_dpo/margin": 81.87158203125, "fcm_dpo/q_t": 0.36865517497062683, "grad_norm": 15.54129409790039, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.2280433177948, "logits/rejected": -1.5488274097442627, "logps/chosen": -270.0011291503906, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -336.37738037109375, "loss": 0.9841, "margin_dpo/margin_mean": 81.87158203125, "margin_dpo/margin_std": 99.62823486328125, "step": 524 }, { "KL/chosen_KL_mean": -156.50653076171875, "KL/mean": -202.77642822265625, "KL/rejected_KL_mean": -249.04632568359375, "KL/std": 94.91960144042969, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.00731679005548358, "fcm_dpo/delta": -0.08805843442678452, "fcm_dpo/margin": 92.539794921875, "fcm_dpo/q_t": 0.35933879017829895, "grad_norm": 11.998452186584473, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.9037154912948608, "logits/rejected": -1.819291353225708, "logps/chosen": -240.7130126953125, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -355.30792236328125, "loss": 0.9645, "margin_dpo/margin_mean": 92.539794921875, "margin_dpo/margin_std": 107.42744445800781, "step": 525 }, { "KL/chosen_KL_mean": -162.74497985839844, "KL/mean": -206.12979125976562, "KL/rejected_KL_mean": -249.51458740234375, "KL/std": 91.01008605957031, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.007348902523517609, "fcm_dpo/delta": -0.04020724445581436, "fcm_dpo/margin": 86.76959228515625, "fcm_dpo/q_t": 0.3628358244895935, "grad_norm": 10.35740852355957, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.6899495124816895, "logits/rejected": -1.0300250053405762, "logps/chosen": -265.5990905761719, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -343.1660461425781, "loss": 0.9812, "margin_dpo/margin_mean": 86.76959991455078, "margin_dpo/margin_std": 105.84957885742188, "step": 526 }, { "KL/chosen_KL_mean": -161.47030639648438, "KL/mean": -205.30160522460938, "KL/rejected_KL_mean": -249.1329345703125, "KL/std": 94.66036224365234, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.007281571626663208, "fcm_dpo/delta": -0.04094438999891281, "fcm_dpo/margin": 87.66262817382812, "fcm_dpo/q_t": 0.35734421014785767, "grad_norm": 12.663978576660156, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.358595371246338, "logits/rejected": -2.5829100608825684, "logps/chosen": -258.2357177734375, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -334.52813720703125, "loss": 0.9293, "margin_dpo/margin_mean": 87.66262817382812, "margin_dpo/margin_std": 83.78001403808594, "step": 527 }, { "KL/chosen_KL_mean": -150.44598388671875, "KL/mean": -194.7101593017578, "KL/rejected_KL_mean": -238.97433471679688, "KL/std": 95.09722900390625, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.007235691882669926, "fcm_dpo/delta": -0.04333607852458954, "fcm_dpo/margin": 88.5283432006836, "fcm_dpo/q_t": 0.3608816862106323, "grad_norm": 12.477225303649902, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.266801357269287, "logits/rejected": -1.772531509399414, "logps/chosen": -244.61090087890625, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -339.3466491699219, "loss": 0.9682, "margin_dpo/margin_mean": 88.5283432006836, "margin_dpo/margin_std": 103.05467224121094, "step": 528 }, { "KL/chosen_KL_mean": -175.1003875732422, "KL/mean": -214.10821533203125, "KL/rejected_KL_mean": -253.11605834960938, "KL/std": 94.58468627929688, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.007253101095557213, "fcm_dpo/delta": 0.03611200675368309, "fcm_dpo/margin": 78.01565551757812, "fcm_dpo/q_t": 0.3809089958667755, "grad_norm": 14.23095417022705, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8932833671569824, "logits/rejected": -1.7140226364135742, "logps/chosen": -255.877197265625, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -350.3072204589844, "loss": 1.0421, "margin_dpo/margin_mean": 78.01566314697266, "margin_dpo/margin_std": 112.27008056640625, "step": 529 }, { "KL/chosen_KL_mean": -160.29080200195312, "KL/mean": -200.02166748046875, "KL/rejected_KL_mean": -239.7525634765625, "KL/std": 92.73977661132812, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.007250873371958733, "fcm_dpo/delta": 0.024967875331640244, "fcm_dpo/margin": 79.46176147460938, "fcm_dpo/q_t": 0.37311893701553345, "grad_norm": 10.898365020751953, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.6589449644088745, "logits/rejected": -1.3356523513793945, "logps/chosen": -240.0665283203125, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -331.80645751953125, "loss": 0.9777, "margin_dpo/margin_mean": 79.4617691040039, "margin_dpo/margin_std": 85.9366226196289, "step": 530 }, { "KL/chosen_KL_mean": -155.42813110351562, "KL/mean": -204.94253540039062, "KL/rejected_KL_mean": -254.4569549560547, "KL/std": 93.4742431640625, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.007165520451962948, "fcm_dpo/delta": -0.11799405515193939, "fcm_dpo/margin": 99.02880859375, "fcm_dpo/q_t": 0.34166431427001953, "grad_norm": 11.582938194274902, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.8681902885437012, "logits/rejected": -1.2654600143432617, "logps/chosen": -234.74734497070312, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -332.5652770996094, "loss": 0.8983, "margin_dpo/margin_mean": 99.02880859375, "margin_dpo/margin_std": 93.77027893066406, "step": 531 }, { "KL/chosen_KL_mean": -165.63726806640625, "KL/mean": -219.7882843017578, "KL/rejected_KL_mean": -273.9393005371094, "KL/std": 91.99006652832031, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0069693634286522865, "fcm_dpo/delta": -0.1671288013458252, "fcm_dpo/margin": 108.30204772949219, "fcm_dpo/q_t": 0.3347044587135315, "grad_norm": 11.61130428314209, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.7596850395202637, "logits/rejected": -1.1451879739761353, "logps/chosen": -267.274169921875, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -367.2846984863281, "loss": 0.8778, "margin_dpo/margin_mean": 108.30204772949219, "margin_dpo/margin_std": 100.13478088378906, "step": 532 }, { "KL/chosen_KL_mean": -161.74237060546875, "KL/mean": -193.59860229492188, "KL/rejected_KL_mean": -225.45480346679688, "KL/std": 96.08778381347656, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.007060919888317585, "fcm_dpo/delta": 0.1569821983575821, "fcm_dpo/margin": 63.71247100830078, "fcm_dpo/q_t": 0.4043995141983032, "grad_norm": 12.174015998840332, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.411881685256958, "logits/rejected": -1.8178207874298096, "logps/chosen": -252.489013671875, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -308.34332275390625, "loss": 1.1031, "margin_dpo/margin_mean": 63.71247100830078, "margin_dpo/margin_std": 102.6718521118164, "step": 533 }, { "KL/chosen_KL_mean": -145.4346923828125, "KL/mean": -191.54725646972656, "KL/rejected_KL_mean": -237.6598358154297, "KL/std": 91.78729248046875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.007060560863465071, "fcm_dpo/delta": -0.054688625037670135, "fcm_dpo/margin": 92.22514343261719, "fcm_dpo/q_t": 0.35382717847824097, "grad_norm": 10.377558708190918, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0666205883026123, "logits/rejected": -1.6837546825408936, "logps/chosen": -232.06552124023438, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -322.53179931640625, "loss": 0.9209, "margin_dpo/margin_mean": 92.22514343261719, "margin_dpo/margin_std": 86.01315307617188, "step": 534 }, { "KL/chosen_KL_mean": -146.13845825195312, "KL/mean": -194.52484130859375, "KL/rejected_KL_mean": -242.91123962402344, "KL/std": 92.94698333740234, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.006991185247898102, "fcm_dpo/delta": -0.08243025094270706, "fcm_dpo/margin": 96.77279663085938, "fcm_dpo/q_t": 0.35292863845825195, "grad_norm": 10.114188194274902, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.6884734630584717, "logits/rejected": -1.4974993467330933, "logps/chosen": -236.4838409423828, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -343.689208984375, "loss": 0.9392, "margin_dpo/margin_mean": 96.77279663085938, "margin_dpo/margin_std": 103.85552978515625, "step": 535 }, { "KL/chosen_KL_mean": -148.230224609375, "KL/mean": -194.54148864746094, "KL/rejected_KL_mean": -240.85275268554688, "KL/std": 93.497314453125, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.006861628964543343, "fcm_dpo/delta": -0.038155511021614075, "fcm_dpo/margin": 92.62251281738281, "fcm_dpo/q_t": 0.358638197183609, "grad_norm": 10.421487808227539, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.967271327972412, "logits/rejected": -1.7180171012878418, "logps/chosen": -227.18978881835938, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -333.9405517578125, "loss": 0.9487, "margin_dpo/margin_mean": 92.62251281738281, "margin_dpo/margin_std": 97.25619506835938, "step": 536 }, { "KL/chosen_KL_mean": -127.88273620605469, "KL/mean": -179.58120727539062, "KL/rejected_KL_mean": -231.2797088623047, "KL/std": 98.67975616455078, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0068073393777012825, "fcm_dpo/delta": -0.1122007742524147, "fcm_dpo/margin": 103.39696502685547, "fcm_dpo/q_t": 0.34417790174484253, "grad_norm": 9.629549026489258, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.5832747220993042, "logits/rejected": -0.8845180869102478, "logps/chosen": -214.59213256835938, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -310.1873779296875, "loss": 0.9007, "margin_dpo/margin_mean": 103.39697265625, "margin_dpo/margin_std": 94.66325378417969, "step": 537 }, { "KL/chosen_KL_mean": -171.3836669921875, "KL/mean": -213.23033142089844, "KL/rejected_KL_mean": -255.0769805908203, "KL/std": 85.55362701416016, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0067585245706140995, "fcm_dpo/delta": 0.036311738193035126, "fcm_dpo/margin": 83.69332885742188, "fcm_dpo/q_t": 0.3727618455886841, "grad_norm": 11.511000633239746, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.9283943176269531, "logits/rejected": -1.4007277488708496, "logps/chosen": -269.8258056640625, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -346.171630859375, "loss": 0.978, "margin_dpo/margin_mean": 83.69332885742188, "margin_dpo/margin_std": 87.42132568359375, "step": 538 }, { "KL/chosen_KL_mean": -152.94686889648438, "KL/mean": -195.83172607421875, "KL/rejected_KL_mean": -238.71656799316406, "KL/std": 95.05938720703125, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.006788298022001982, "fcm_dpo/delta": 0.01881079003214836, "fcm_dpo/margin": 85.76971435546875, "fcm_dpo/q_t": 0.37270689010620117, "grad_norm": 10.446345329284668, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.2400712966918945, "logits/rejected": -1.7471708059310913, "logps/chosen": -243.87896728515625, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -336.16204833984375, "loss": 1.006, "margin_dpo/margin_mean": 85.76971435546875, "margin_dpo/margin_std": 107.61327362060547, "step": 539 }, { "KL/chosen_KL_mean": -158.2256317138672, "KL/mean": -209.83056640625, "KL/rejected_KL_mean": -261.43548583984375, "KL/std": 100.06329345703125, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.006718725897371769, "fcm_dpo/delta": -0.10039995610713959, "fcm_dpo/margin": 103.2098617553711, "fcm_dpo/q_t": 0.35086995363235474, "grad_norm": 12.099160194396973, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.0485808849334717, "logits/rejected": -1.8069610595703125, "logps/chosen": -266.09552001953125, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -382.8289794921875, "loss": 0.9363, "margin_dpo/margin_mean": 103.2098617553711, "margin_dpo/margin_std": 114.02565002441406, "step": 540 }, { "KL/chosen_KL_mean": -178.15621948242188, "KL/mean": -211.8179931640625, "KL/rejected_KL_mean": -245.479736328125, "KL/std": 86.69386291503906, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.006757402326911688, "fcm_dpo/delta": 0.15205374360084534, "fcm_dpo/margin": 67.32351684570312, "fcm_dpo/q_t": 0.39957159757614136, "grad_norm": 11.66394329071045, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.477358818054199, "logits/rejected": -2.0188894271850586, "logps/chosen": -284.72369384765625, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -354.54608154296875, "loss": 1.0741, "margin_dpo/margin_mean": 67.32352447509766, "margin_dpo/margin_std": 90.02629089355469, "step": 541 }, { "KL/chosen_KL_mean": -150.12908935546875, "KL/mean": -201.1271514892578, "KL/rejected_KL_mean": -252.12522888183594, "KL/std": 96.16487121582031, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.006764009594917297, "fcm_dpo/delta": -0.09650899469852448, "fcm_dpo/margin": 101.99613952636719, "fcm_dpo/q_t": 0.34668439626693726, "grad_norm": 11.439042091369629, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.1498732566833496, "logits/rejected": -2.0217621326446533, "logps/chosen": -237.4983673095703, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -354.945556640625, "loss": 0.9151, "margin_dpo/margin_mean": 101.99615478515625, "margin_dpo/margin_std": 100.45806884765625, "step": 542 }, { "KL/chosen_KL_mean": -174.71502685546875, "KL/mean": -201.3974609375, "KL/rejected_KL_mean": -228.07989501953125, "KL/std": 99.91960906982422, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.006936088670045137, "fcm_dpo/delta": 0.23937858641147614, "fcm_dpo/margin": 53.36488342285156, "fcm_dpo/q_t": 0.42293015122413635, "grad_norm": 14.080509185791016, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.7714061737060547, "logits/rejected": -1.8032736778259277, "logps/chosen": -261.86761474609375, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -339.61163330078125, "loss": 1.2006, "margin_dpo/margin_mean": 53.36487579345703, "margin_dpo/margin_std": 116.28689575195312, "step": 543 }, { "KL/chosen_KL_mean": -180.16729736328125, "KL/mean": -213.11669921875, "KL/rejected_KL_mean": -246.06610107421875, "KL/std": 94.28871154785156, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.007191224955022335, "fcm_dpo/delta": 0.13246020674705505, "fcm_dpo/margin": 65.89881134033203, "fcm_dpo/q_t": 0.39769721031188965, "grad_norm": 13.778532981872559, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.8070670366287231, "logits/rejected": -1.7060999870300293, "logps/chosen": -266.314208984375, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -352.46002197265625, "loss": 1.0692, "margin_dpo/margin_mean": 65.89881134033203, "margin_dpo/margin_std": 94.7223129272461, "step": 544 }, { "KL/chosen_KL_mean": -162.654541015625, "KL/mean": -203.83657836914062, "KL/rejected_KL_mean": -245.01864624023438, "KL/std": 91.90994262695312, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.007262915372848511, "fcm_dpo/delta": 0.0018051397055387497, "fcm_dpo/margin": 82.3641128540039, "fcm_dpo/q_t": 0.37072643637657166, "grad_norm": 12.418686866760254, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.5399014949798584, "logits/rejected": -1.1145203113555908, "logps/chosen": -259.07879638671875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -345.752685546875, "loss": 1.0253, "margin_dpo/margin_mean": 82.3641128540039, "margin_dpo/margin_std": 112.54133605957031, "step": 545 }, { "KL/chosen_KL_mean": -165.35411071777344, "KL/mean": -204.07907104492188, "KL/rejected_KL_mean": -242.80406188964844, "KL/std": 95.4853744506836, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.007268503308296204, "fcm_dpo/delta": 0.03920260816812515, "fcm_dpo/margin": 77.44993591308594, "fcm_dpo/q_t": 0.3755866289138794, "grad_norm": 12.30596923828125, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.473224401473999, "logits/rejected": -1.1077535152435303, "logps/chosen": -251.375244140625, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -330.17669677734375, "loss": 1.0248, "margin_dpo/margin_mean": 77.44993591308594, "margin_dpo/margin_std": 101.83229064941406, "step": 546 }, { "KL/chosen_KL_mean": -159.3666534423828, "KL/mean": -204.10128784179688, "KL/rejected_KL_mean": -248.83592224121094, "KL/std": 93.58941650390625, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0072801122441887856, "fcm_dpo/delta": -0.055135831236839294, "fcm_dpo/margin": 89.46925354003906, "fcm_dpo/q_t": 0.3535280227661133, "grad_norm": 11.077335357666016, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.8800299167633057, "logits/rejected": -1.423275351524353, "logps/chosen": -260.0176696777344, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -346.7476806640625, "loss": 0.9202, "margin_dpo/margin_mean": 89.46925354003906, "margin_dpo/margin_std": 82.25728607177734, "step": 547 }, { "KL/chosen_KL_mean": -158.9522705078125, "KL/mean": -193.7176055908203, "KL/rejected_KL_mean": -228.48294067382812, "KL/std": 95.19538116455078, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.007336183916777372, "fcm_dpo/delta": 0.09458817541599274, "fcm_dpo/margin": 69.53064727783203, "fcm_dpo/q_t": 0.3907926082611084, "grad_norm": 13.771510124206543, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.2350082397460938, "logits/rejected": -1.8791561126708984, "logps/chosen": -254.98239135742188, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -326.89617919921875, "loss": 1.1058, "margin_dpo/margin_mean": 69.53065490722656, "margin_dpo/margin_std": 116.71221923828125, "step": 548 }, { "KL/chosen_KL_mean": -147.95013427734375, "KL/mean": -187.0211181640625, "KL/rejected_KL_mean": -226.09210205078125, "KL/std": 88.18083953857422, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.00742876622825861, "fcm_dpo/delta": 0.019837401807308197, "fcm_dpo/margin": 78.14199829101562, "fcm_dpo/q_t": 0.3744286894798279, "grad_norm": 13.959664344787598, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.9458718299865723, "logits/rejected": -1.5294342041015625, "logps/chosen": -231.40109252929688, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -307.47235107421875, "loss": 1.0354, "margin_dpo/margin_mean": 78.14199829101562, "margin_dpo/margin_std": 107.734375, "step": 549 }, { "KL/chosen_KL_mean": -166.4427490234375, "KL/mean": -203.54141235351562, "KL/rejected_KL_mean": -240.6400604248047, "KL/std": 86.77885437011719, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.007470706012099981, "fcm_dpo/delta": 0.047952547669410706, "fcm_dpo/margin": 74.19732666015625, "fcm_dpo/q_t": 0.378238320350647, "grad_norm": 11.96721076965332, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.6765552759170532, "logits/rejected": -1.456495761871338, "logps/chosen": -256.0370788574219, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -341.95556640625, "loss": 1.0101, "margin_dpo/margin_mean": 74.19732666015625, "margin_dpo/margin_std": 88.376708984375, "step": 550 }, { "KL/chosen_KL_mean": -146.36953735351562, "KL/mean": -191.58425903320312, "KL/rejected_KL_mean": -236.79898071289062, "KL/std": 96.68698120117188, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.007328840903937817, "fcm_dpo/delta": -0.06903297454118729, "fcm_dpo/margin": 90.42945098876953, "fcm_dpo/q_t": 0.35807156562805176, "grad_norm": 14.75256633758545, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.668421745300293, "logits/rejected": -1.5208580493927002, "logps/chosen": -241.45169067382812, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -354.51824951171875, "loss": 0.9406, "margin_dpo/margin_mean": 90.429443359375, "margin_dpo/margin_std": 97.238525390625, "step": 551 }, { "KL/chosen_KL_mean": -138.39337158203125, "KL/mean": -201.09271240234375, "KL/rejected_KL_mean": -263.79205322265625, "KL/std": 105.53106689453125, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.006960398517549038, "fcm_dpo/delta": -0.30471882224082947, "fcm_dpo/margin": 125.39868927001953, "fcm_dpo/q_t": 0.31042754650115967, "grad_norm": 10.09952163696289, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.1602134704589844, "logits/rejected": -1.7994890213012695, "logps/chosen": -239.8025665283203, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -374.6304931640625, "loss": 0.818, "margin_dpo/margin_mean": 125.398681640625, "margin_dpo/margin_std": 107.41957092285156, "step": 552 }, { "KL/chosen_KL_mean": -153.48486328125, "KL/mean": -191.7346649169922, "KL/rejected_KL_mean": -229.98448181152344, "KL/std": 98.21002197265625, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.00701237004250288, "fcm_dpo/delta": 0.06685532629489899, "fcm_dpo/margin": 76.49961853027344, "fcm_dpo/q_t": 0.38146859407424927, "grad_norm": 14.589842796325684, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.6235713958740234, "logits/rejected": -1.3018136024475098, "logps/chosen": -251.44873046875, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -333.5616760253906, "loss": 1.0392, "margin_dpo/margin_mean": 76.49961853027344, "margin_dpo/margin_std": 102.50001525878906, "step": 553 }, { "KL/chosen_KL_mean": -160.215087890625, "KL/mean": -202.601318359375, "KL/rejected_KL_mean": -244.987548828125, "KL/std": 97.26549530029297, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.007061859592795372, "fcm_dpo/delta": 0.0006809048354625702, "fcm_dpo/margin": 84.77247619628906, "fcm_dpo/q_t": 0.3730701208114624, "grad_norm": 11.793261528015137, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8047308921813965, "logits/rejected": -1.0229729413986206, "logps/chosen": -263.9018859863281, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -337.97967529296875, "loss": 1.0453, "margin_dpo/margin_mean": 84.7724838256836, "margin_dpo/margin_std": 123.09761047363281, "step": 554 }, { "KL/chosen_KL_mean": -169.90585327148438, "KL/mean": -209.47129821777344, "KL/rejected_KL_mean": -249.0367431640625, "KL/std": 101.43125915527344, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.007061750628054142, "fcm_dpo/delta": 0.043651994317770004, "fcm_dpo/margin": 79.13093566894531, "fcm_dpo/q_t": 0.38212668895721436, "grad_norm": 12.643020629882812, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.8334777355194092, "logits/rejected": -1.738342523574829, "logps/chosen": -256.1094665527344, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -363.5839538574219, "loss": 1.0484, "margin_dpo/margin_mean": 79.13093566894531, "margin_dpo/margin_std": 114.6536865234375, "step": 555 }, { "KL/chosen_KL_mean": -149.85992431640625, "KL/mean": -193.0934600830078, "KL/rejected_KL_mean": -236.32699584960938, "KL/std": 91.51927185058594, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0070508792996406555, "fcm_dpo/delta": -0.010478297248482704, "fcm_dpo/margin": 86.46705627441406, "fcm_dpo/q_t": 0.3665311932563782, "grad_norm": 11.99555778503418, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.6652047634124756, "logits/rejected": -1.3543239831924438, "logps/chosen": -238.263427734375, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -342.96612548828125, "loss": 0.9863, "margin_dpo/margin_mean": 86.46705627441406, "margin_dpo/margin_std": 103.92250061035156, "step": 556 }, { "KL/chosen_KL_mean": -160.8556365966797, "KL/mean": -209.45912170410156, "KL/rejected_KL_mean": -258.0625915527344, "KL/std": 103.90011596679688, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.006937894970178604, "fcm_dpo/delta": -0.08112950623035431, "fcm_dpo/margin": 97.20693969726562, "fcm_dpo/q_t": 0.3558619022369385, "grad_norm": 10.271981239318848, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -2.020752191543579, "logits/rejected": -1.8483147621154785, "logps/chosen": -259.69085693359375, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -373.71148681640625, "loss": 0.9516, "margin_dpo/margin_mean": 97.20693969726562, "margin_dpo/margin_std": 110.42082214355469, "step": 557 }, { "KL/chosen_KL_mean": -161.1572265625, "KL/mean": -211.66665649414062, "KL/rejected_KL_mean": -262.17608642578125, "KL/std": 90.06619262695312, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.006835636682808399, "fcm_dpo/delta": -0.09761544317007065, "fcm_dpo/margin": 101.01885986328125, "fcm_dpo/q_t": 0.3463051915168762, "grad_norm": 11.9409818649292, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.366330623626709, "logits/rejected": -1.7524259090423584, "logps/chosen": -267.1690673828125, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -370.5674133300781, "loss": 0.9007, "margin_dpo/margin_mean": 101.01885986328125, "margin_dpo/margin_std": 92.4937744140625, "step": 558 }, { "KL/chosen_KL_mean": -170.56629943847656, "KL/mean": -213.20443725585938, "KL/rejected_KL_mean": -255.8426055908203, "KL/std": 91.45774841308594, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.006840849295258522, "fcm_dpo/delta": 0.01764558255672455, "fcm_dpo/margin": 85.27630615234375, "fcm_dpo/q_t": 0.370486319065094, "grad_norm": 13.069741249084473, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.4860687255859375, "logits/rejected": -2.014636993408203, "logps/chosen": -270.99102783203125, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -356.25531005859375, "loss": 1.0009, "margin_dpo/margin_mean": 85.27630615234375, "margin_dpo/margin_std": 103.90190124511719, "step": 559 }, { "KL/chosen_KL_mean": -166.81585693359375, "KL/mean": -207.29583740234375, "KL/rejected_KL_mean": -247.77584838867188, "KL/std": 90.43592071533203, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.006881291978061199, "fcm_dpo/delta": 0.045446332544088364, "fcm_dpo/margin": 80.9599609375, "fcm_dpo/q_t": 0.37735241651535034, "grad_norm": 14.210784912109375, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.4517383575439453, "logits/rejected": -1.0044516324996948, "logps/chosen": -254.45138549804688, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -335.15570068359375, "loss": 1.0197, "margin_dpo/margin_mean": 80.9599609375, "margin_dpo/margin_std": 103.73191833496094, "step": 560 }, { "KL/chosen_KL_mean": -173.20079040527344, "KL/mean": -211.69427490234375, "KL/rejected_KL_mean": -250.187744140625, "KL/std": 91.07765197753906, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.006995262578129768, "fcm_dpo/delta": 0.06448754668235779, "fcm_dpo/margin": 76.98695373535156, "fcm_dpo/q_t": 0.38048505783081055, "grad_norm": 10.257858276367188, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.8223521709442139, "logits/rejected": -1.7895530462265015, "logps/chosen": -248.08218383789062, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -356.2357177734375, "loss": 1.0029, "margin_dpo/margin_mean": 76.98695373535156, "margin_dpo/margin_std": 87.4388427734375, "step": 561 }, { "KL/chosen_KL_mean": -158.42449951171875, "KL/mean": -195.20791625976562, "KL/rejected_KL_mean": -231.99134826660156, "KL/std": 90.1038818359375, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.007062805816531181, "fcm_dpo/delta": 0.08489162474870682, "fcm_dpo/margin": 73.56687927246094, "fcm_dpo/q_t": 0.38869065046310425, "grad_norm": 14.900578498840332, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.331298351287842, "logits/rejected": -1.89418625831604, "logps/chosen": -259.08770751953125, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -334.23223876953125, "loss": 1.0738, "margin_dpo/margin_mean": 73.56687927246094, "margin_dpo/margin_std": 113.28826904296875, "step": 562 }, { "KL/chosen_KL_mean": -169.32388305664062, "KL/mean": -202.22296142578125, "KL/rejected_KL_mean": -235.12203979492188, "KL/std": 91.1452865600586, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.007279085461050272, "fcm_dpo/delta": 0.12626494467258453, "fcm_dpo/margin": 65.79817199707031, "fcm_dpo/q_t": 0.3950991928577423, "grad_norm": 12.44426155090332, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.8495442867279053, "logits/rejected": -1.515937328338623, "logps/chosen": -265.5830993652344, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -336.32708740234375, "loss": 1.0759, "margin_dpo/margin_mean": 65.79816436767578, "margin_dpo/margin_std": 96.37681579589844, "step": 563 }, { "KL/chosen_KL_mean": -164.00857543945312, "KL/mean": -200.47817993164062, "KL/rejected_KL_mean": -236.94778442382812, "KL/std": 92.02906799316406, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0073974947445094585, "fcm_dpo/delta": 0.06319880485534668, "fcm_dpo/margin": 72.93922424316406, "fcm_dpo/q_t": 0.3795182704925537, "grad_norm": 13.147255897521973, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.5232372283935547, "logits/rejected": -1.9759347438812256, "logps/chosen": -276.968994140625, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -336.475830078125, "loss": 1.0071, "margin_dpo/margin_mean": 72.93922424316406, "margin_dpo/margin_std": 83.47322082519531, "step": 564 }, { "KL/chosen_KL_mean": -157.000732421875, "KL/mean": -202.46435546875, "KL/rejected_KL_mean": -247.92799377441406, "KL/std": 98.46769714355469, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.007297782227396965, "fcm_dpo/delta": -0.06834682822227478, "fcm_dpo/margin": 90.92727661132812, "fcm_dpo/q_t": 0.3567652106285095, "grad_norm": 11.99504280090332, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.7485618591308594, "logits/rejected": -1.2431488037109375, "logps/chosen": -252.36630249023438, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -340.1453857421875, "loss": 0.9566, "margin_dpo/margin_mean": 90.92728424072266, "margin_dpo/margin_std": 103.99984741210938, "step": 565 }, { "KL/chosen_KL_mean": -164.79942321777344, "KL/mean": -208.4412841796875, "KL/rejected_KL_mean": -252.08319091796875, "KL/std": 101.68280029296875, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.007268537301570177, "fcm_dpo/delta": -0.0368087962269783, "fcm_dpo/margin": 87.28376007080078, "fcm_dpo/q_t": 0.3603999614715576, "grad_norm": 10.494162559509277, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.8696147203445435, "logits/rejected": -1.431383490562439, "logps/chosen": -263.77264404296875, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -362.90411376953125, "loss": 0.9693, "margin_dpo/margin_mean": 87.28376007080078, "margin_dpo/margin_std": 100.51423645019531, "step": 566 }, { "KL/chosen_KL_mean": -166.3544921875, "KL/mean": -204.57791137695312, "KL/rejected_KL_mean": -242.80133056640625, "KL/std": 91.59230041503906, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.007246246561408043, "fcm_dpo/delta": 0.04835917800664902, "fcm_dpo/margin": 76.44681549072266, "fcm_dpo/q_t": 0.3763677477836609, "grad_norm": 13.168147087097168, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.2939541339874268, "logits/rejected": -1.627565622329712, "logps/chosen": -259.05810546875, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -334.025634765625, "loss": 1.0195, "margin_dpo/margin_mean": 76.44681549072266, "margin_dpo/margin_std": 95.33842468261719, "step": 567 }, { "KL/chosen_KL_mean": -157.3588104248047, "KL/mean": -204.95120239257812, "KL/rejected_KL_mean": -252.5435791015625, "KL/std": 96.53971862792969, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0071782516315579414, "fcm_dpo/delta": -0.09011013805866241, "fcm_dpo/margin": 95.18476867675781, "fcm_dpo/q_t": 0.35065847635269165, "grad_norm": 10.581900596618652, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3834645748138428, "logits/rejected": -1.2438427209854126, "logps/chosen": -230.49107360839844, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -347.4962158203125, "loss": 0.9185, "margin_dpo/margin_mean": 95.18476867675781, "margin_dpo/margin_std": 95.39273071289062, "step": 568 }, { "KL/chosen_KL_mean": -167.24899291992188, "KL/mean": -206.6490478515625, "KL/rejected_KL_mean": -246.04908752441406, "KL/std": 95.64614868164062, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.007231033407151699, "fcm_dpo/delta": 0.031725455075502396, "fcm_dpo/margin": 78.80008697509766, "fcm_dpo/q_t": 0.3755494952201843, "grad_norm": 17.419673919677734, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.8433568477630615, "logits/rejected": -1.4613895416259766, "logps/chosen": -256.466064453125, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -332.46014404296875, "loss": 1.0498, "margin_dpo/margin_mean": 78.80007934570312, "margin_dpo/margin_std": 113.16255187988281, "step": 569 }, { "KL/chosen_KL_mean": -148.6803436279297, "KL/mean": -199.75543212890625, "KL/rejected_KL_mean": -250.8305206298828, "KL/std": 92.63615417480469, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.007071505766361952, "fcm_dpo/delta": -0.13209094107151031, "fcm_dpo/margin": 102.15017700195312, "fcm_dpo/q_t": 0.33921074867248535, "grad_norm": 11.006324768066406, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.4460861682891846, "logits/rejected": -1.7758216857910156, "logps/chosen": -243.443115234375, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -339.67742919921875, "loss": 0.8887, "margin_dpo/margin_mean": 102.15017700195312, "margin_dpo/margin_std": 94.72120666503906, "step": 570 }, { "KL/chosen_KL_mean": -152.91600036621094, "KL/mean": -199.33856201171875, "KL/rejected_KL_mean": -245.7611083984375, "KL/std": 92.43782043457031, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.006998921744525433, "fcm_dpo/delta": -0.05331359803676605, "fcm_dpo/margin": 92.84512329101562, "fcm_dpo/q_t": 0.3595590591430664, "grad_norm": 15.042098045349121, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7447779178619385, "logits/rejected": -1.5860135555267334, "logps/chosen": -233.73727416992188, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -342.7763977050781, "loss": 0.9713, "margin_dpo/margin_mean": 92.84512329101562, "margin_dpo/margin_std": 109.89712524414062, "step": 571 }, { "KL/chosen_KL_mean": -157.2875213623047, "KL/mean": -203.38389587402344, "KL/rejected_KL_mean": -249.48028564453125, "KL/std": 96.81280517578125, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.006902849301695824, "fcm_dpo/delta": -0.03904286399483681, "fcm_dpo/margin": 92.19275665283203, "fcm_dpo/q_t": 0.3637348413467407, "grad_norm": 11.643832206726074, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.7506237030029297, "logits/rejected": -1.5804955959320068, "logps/chosen": -234.66744995117188, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -340.0462646484375, "loss": 0.965, "margin_dpo/margin_mean": 92.19275665283203, "margin_dpo/margin_std": 106.50337219238281, "step": 572 }, { "KL/chosen_KL_mean": -153.27682495117188, "KL/mean": -196.26321411132812, "KL/rejected_KL_mean": -239.2495880126953, "KL/std": 98.52392578125, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0069714998826384544, "fcm_dpo/delta": -0.0013410970568656921, "fcm_dpo/margin": 85.9727554321289, "fcm_dpo/q_t": 0.367115318775177, "grad_norm": 10.912205696105957, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.2844080924987793, "logits/rejected": -2.0628132820129395, "logps/chosen": -236.98291015625, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -329.9470520019531, "loss": 0.9946, "margin_dpo/margin_mean": 85.97274780273438, "margin_dpo/margin_std": 100.9037857055664, "step": 573 }, { "KL/chosen_KL_mean": -160.82730102539062, "KL/mean": -206.00900268554688, "KL/rejected_KL_mean": -251.1907196044922, "KL/std": 95.63417053222656, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.00690201111137867, "fcm_dpo/delta": -0.025466814637184143, "fcm_dpo/margin": 90.36341857910156, "fcm_dpo/q_t": 0.3641034960746765, "grad_norm": 13.697463035583496, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.200556755065918, "logits/rejected": -1.8752466440200806, "logps/chosen": -245.956787109375, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -350.2447509765625, "loss": 0.9821, "margin_dpo/margin_mean": 90.36341857910156, "margin_dpo/margin_std": 108.13163757324219, "step": 574 }, { "KL/chosen_KL_mean": -166.03488159179688, "KL/mean": -210.36410522460938, "KL/rejected_KL_mean": -254.69332885742188, "KL/std": 98.46319580078125, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0068479301407933235, "fcm_dpo/delta": -0.007652823813259602, "fcm_dpo/margin": 88.658447265625, "fcm_dpo/q_t": 0.3685096502304077, "grad_norm": 15.038473129272461, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.431783676147461, "logits/rejected": -0.9720022082328796, "logps/chosen": -271.80572509765625, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -355.64276123046875, "loss": 1.0017, "margin_dpo/margin_mean": 88.658447265625, "margin_dpo/margin_std": 113.37554931640625, "step": 575 }, { "KL/chosen_KL_mean": -166.90965270996094, "KL/mean": -209.5780029296875, "KL/rejected_KL_mean": -252.24635314941406, "KL/std": 92.27449035644531, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.006887336727231741, "fcm_dpo/delta": 0.012847473844885826, "fcm_dpo/margin": 85.33670043945312, "fcm_dpo/q_t": 0.37189146876335144, "grad_norm": 12.313560485839844, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.093674659729004, "logits/rejected": -2.0223402976989746, "logps/chosen": -248.1141815185547, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -357.65576171875, "loss": 1.0139, "margin_dpo/margin_mean": 85.33670043945312, "margin_dpo/margin_std": 110.64700317382812, "step": 576 }, { "KL/chosen_KL_mean": -147.89366149902344, "KL/mean": -200.31820678710938, "KL/rejected_KL_mean": -252.74276733398438, "KL/std": 99.57530212402344, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.006724217906594276, "fcm_dpo/delta": -0.11411580443382263, "fcm_dpo/margin": 104.84907531738281, "fcm_dpo/q_t": 0.3461261987686157, "grad_norm": 9.61844253540039, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.1019248962402344, "logits/rejected": -1.4144123792648315, "logps/chosen": -249.3578643798828, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -342.0691223144531, "loss": 0.9273, "margin_dpo/margin_mean": 104.84907531738281, "margin_dpo/margin_std": 111.84803771972656, "step": 577 }, { "KL/chosen_KL_mean": -175.5157012939453, "KL/mean": -213.64007568359375, "KL/rejected_KL_mean": -251.76446533203125, "KL/std": 94.15430450439453, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.006808901205658913, "fcm_dpo/delta": 0.0851910412311554, "fcm_dpo/margin": 76.24874877929688, "fcm_dpo/q_t": 0.3856717050075531, "grad_norm": 16.132356643676758, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.6696621179580688, "logits/rejected": -1.4401416778564453, "logps/chosen": -252.80812072753906, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -340.011474609375, "loss": 1.0502, "margin_dpo/margin_mean": 76.24876403808594, "margin_dpo/margin_std": 105.67674255371094, "step": 578 }, { "KL/chosen_KL_mean": -158.7632598876953, "KL/mean": -197.84092712402344, "KL/rejected_KL_mean": -236.91860961914062, "KL/std": 94.7284927368164, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.006867557764053345, "fcm_dpo/delta": 0.06666059792041779, "fcm_dpo/margin": 78.15533447265625, "fcm_dpo/q_t": 0.38308876752853394, "grad_norm": 11.416373252868652, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7951991558074951, "logits/rejected": -1.480039119720459, "logps/chosen": -246.1063232421875, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -339.03961181640625, "loss": 1.0276, "margin_dpo/margin_mean": 78.15533447265625, "margin_dpo/margin_std": 97.09781646728516, "step": 579 }, { "KL/chosen_KL_mean": -152.34707641601562, "KL/mean": -198.67047119140625, "KL/rejected_KL_mean": -244.99391174316406, "KL/std": 92.14573669433594, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.006846962496638298, "fcm_dpo/delta": -0.03761524707078934, "fcm_dpo/margin": 92.64680480957031, "fcm_dpo/q_t": 0.3634788393974304, "grad_norm": 11.89334487915039, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.4906351566314697, "logits/rejected": -1.1779499053955078, "logps/chosen": -241.27586364746094, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -333.6905212402344, "loss": 0.9715, "margin_dpo/margin_mean": 92.64680480957031, "margin_dpo/margin_std": 107.83760070800781, "step": 580 }, { "KL/chosen_KL_mean": -161.93408203125, "KL/mean": -203.45254516601562, "KL/rejected_KL_mean": -244.9709930419922, "KL/std": 94.04299926757812, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.006915360223501921, "fcm_dpo/delta": 0.02720755897462368, "fcm_dpo/margin": 83.03692626953125, "fcm_dpo/q_t": 0.37539559602737427, "grad_norm": 13.468917846679688, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.0158138275146484, "logits/rejected": -1.6706271171569824, "logps/chosen": -254.1573028564453, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -344.36041259765625, "loss": 1.0411, "margin_dpo/margin_mean": 83.03692626953125, "margin_dpo/margin_std": 116.85882568359375, "step": 581 }, { "KL/chosen_KL_mean": -141.75462341308594, "KL/mean": -180.81407165527344, "KL/rejected_KL_mean": -219.87353515625, "KL/std": 96.92404174804688, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.007044246420264244, "fcm_dpo/delta": 0.04923056811094284, "fcm_dpo/margin": 78.1189193725586, "fcm_dpo/q_t": 0.38219770789146423, "grad_norm": 11.272939682006836, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.083263874053955, "logits/rejected": -2.0692882537841797, "logps/chosen": -229.09024047851562, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -335.0279541015625, "loss": 1.0367, "margin_dpo/margin_mean": 78.11892700195312, "margin_dpo/margin_std": 103.89439392089844, "step": 582 }, { "KL/chosen_KL_mean": -173.0093994140625, "KL/mean": -214.61105346679688, "KL/rejected_KL_mean": -256.21270751953125, "KL/std": 90.92886352539062, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.006969665177166462, "fcm_dpo/delta": 0.02099817991256714, "fcm_dpo/margin": 83.20330047607422, "fcm_dpo/q_t": 0.3759821653366089, "grad_norm": 13.42957592010498, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.3295899629592896, "logits/rejected": -1.0903382301330566, "logps/chosen": -256.7264709472656, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -350.79736328125, "loss": 1.0085, "margin_dpo/margin_mean": 83.20329284667969, "margin_dpo/margin_std": 105.92809295654297, "step": 583 }, { "KL/chosen_KL_mean": -151.97482299804688, "KL/mean": -196.396484375, "KL/rejected_KL_mean": -240.818115234375, "KL/std": 95.97360229492188, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.007047527004033327, "fcm_dpo/delta": -0.02955196052789688, "fcm_dpo/margin": 88.84330749511719, "fcm_dpo/q_t": 0.36357468366622925, "grad_norm": 10.742506980895996, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.8395295143127441, "logits/rejected": -1.3646323680877686, "logps/chosen": -246.23439025878906, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -345.3008117675781, "loss": 0.9919, "margin_dpo/margin_mean": 88.84329986572266, "margin_dpo/margin_std": 107.34136962890625, "step": 584 }, { "KL/chosen_KL_mean": -161.69956970214844, "KL/mean": -212.964599609375, "KL/rejected_KL_mean": -264.22967529296875, "KL/std": 109.10899353027344, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.006887997500598431, "fcm_dpo/delta": -0.1142788827419281, "fcm_dpo/margin": 102.53008270263672, "fcm_dpo/q_t": 0.3510194420814514, "grad_norm": 11.300503730773926, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.9613196849822998, "logits/rejected": -1.682650089263916, "logps/chosen": -247.33633422851562, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -354.51513671875, "loss": 0.9554, "margin_dpo/margin_mean": 102.53009033203125, "margin_dpo/margin_std": 122.8785171508789, "step": 585 }, { "KL/chosen_KL_mean": -179.94041442871094, "KL/mean": -214.73471069335938, "KL/rejected_KL_mean": -249.5290069580078, "KL/std": 96.40656280517578, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.006941286846995354, "fcm_dpo/delta": 0.12280081957578659, "fcm_dpo/margin": 69.58859252929688, "fcm_dpo/q_t": 0.39285314083099365, "grad_norm": 13.459778785705566, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.7809075117111206, "logits/rejected": -1.5119850635528564, "logps/chosen": -263.6215515136719, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -347.88818359375, "loss": 1.0499, "margin_dpo/margin_mean": 69.58859252929688, "margin_dpo/margin_std": 89.92185974121094, "step": 586 }, { "KL/chosen_KL_mean": -144.9281005859375, "KL/mean": -182.7052001953125, "KL/rejected_KL_mean": -220.48226928710938, "KL/std": 92.46125793457031, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.007075323723256588, "fcm_dpo/delta": 0.06865807622671127, "fcm_dpo/margin": 75.55415344238281, "fcm_dpo/q_t": 0.38294386863708496, "grad_norm": 11.850054740905762, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.3777475357055664, "logits/rejected": -2.087808132171631, "logps/chosen": -222.505615234375, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -309.8026123046875, "loss": 1.0261, "margin_dpo/margin_mean": 75.55415344238281, "margin_dpo/margin_std": 96.13864135742188, "step": 587 }, { "KL/chosen_KL_mean": -158.02529907226562, "KL/mean": -193.29965209960938, "KL/rejected_KL_mean": -228.57400512695312, "KL/std": 95.06059265136719, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.007158635649830103, "fcm_dpo/delta": 0.10012312233448029, "fcm_dpo/margin": 70.54869842529297, "fcm_dpo/q_t": 0.39292752742767334, "grad_norm": 14.43738842010498, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.6819374561309814, "logits/rejected": -1.6309006214141846, "logps/chosen": -241.82955932617188, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -341.69342041015625, "loss": 1.0886, "margin_dpo/margin_mean": 70.54869842529297, "margin_dpo/margin_std": 111.80137634277344, "step": 588 }, { "KL/chosen_KL_mean": -169.903564453125, "KL/mean": -216.04766845703125, "KL/rejected_KL_mean": -262.1917724609375, "KL/std": 93.09346008300781, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.007146029267460108, "fcm_dpo/delta": -0.06375712156295776, "fcm_dpo/margin": 92.28823852539062, "fcm_dpo/q_t": 0.3529452085494995, "grad_norm": 11.580463409423828, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.6725620031356812, "logits/rejected": -1.2709966897964478, "logps/chosen": -266.36859130859375, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -378.2289733886719, "loss": 0.923, "margin_dpo/margin_mean": 92.28823852539062, "margin_dpo/margin_std": 89.24899291992188, "step": 589 }, { "KL/chosen_KL_mean": -153.86683654785156, "KL/mean": -199.42849731445312, "KL/rejected_KL_mean": -244.99017333984375, "KL/std": 87.46002197265625, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.007117594126611948, "fcm_dpo/delta": -0.052433304488658905, "fcm_dpo/margin": 91.12332153320312, "fcm_dpo/q_t": 0.3571138381958008, "grad_norm": 13.589166641235352, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.9763000011444092, "logits/rejected": -1.8135721683502197, "logps/chosen": -242.29470825195312, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -355.1889343261719, "loss": 0.933, "margin_dpo/margin_mean": 91.12332153320312, "margin_dpo/margin_std": 90.67710876464844, "step": 590 }, { "KL/chosen_KL_mean": -149.21063232421875, "KL/mean": -198.02288818359375, "KL/rejected_KL_mean": -246.83514404296875, "KL/std": 96.46595764160156, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.006944627501070499, "fcm_dpo/delta": -0.08409854769706726, "fcm_dpo/margin": 97.62451171875, "fcm_dpo/q_t": 0.3504767417907715, "grad_norm": 9.119867324829102, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9293599128723145, "logits/rejected": -1.3039813041687012, "logps/chosen": -237.06031799316406, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -336.4143371582031, "loss": 0.9144, "margin_dpo/margin_mean": 97.62451934814453, "margin_dpo/margin_std": 94.39678955078125, "step": 591 }, { "KL/chosen_KL_mean": -174.9619140625, "KL/mean": -213.5303955078125, "KL/rejected_KL_mean": -252.09886169433594, "KL/std": 89.46687316894531, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0070076314732432365, "fcm_dpo/delta": 0.06277268379926682, "fcm_dpo/margin": 77.13695526123047, "fcm_dpo/q_t": 0.38230037689208984, "grad_norm": 13.260658264160156, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.850945234298706, "logits/rejected": -1.523915410041809, "logps/chosen": -249.93765258789062, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -333.6514892578125, "loss": 1.0472, "margin_dpo/margin_mean": 77.13695526123047, "margin_dpo/margin_std": 108.44454956054688, "step": 592 }, { "KL/chosen_KL_mean": -162.62896728515625, "KL/mean": -195.5267791748047, "KL/rejected_KL_mean": -228.42459106445312, "KL/std": 95.44747924804688, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.007130770478397608, "fcm_dpo/delta": 0.1374119222164154, "fcm_dpo/margin": 65.7956314086914, "fcm_dpo/q_t": 0.3996197581291199, "grad_norm": 17.701610565185547, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.6751659512519836, "logits/rejected": -0.3620113134384155, "logps/chosen": -234.8679962158203, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -298.5211181640625, "loss": 1.098, "margin_dpo/margin_mean": 65.7956314086914, "margin_dpo/margin_std": 103.87567901611328, "step": 593 }, { "KL/chosen_KL_mean": -146.5657958984375, "KL/mean": -195.8118133544922, "KL/rejected_KL_mean": -245.05783081054688, "KL/std": 93.22430419921875, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.007133793085813522, "fcm_dpo/delta": -0.11029151827096939, "fcm_dpo/margin": 98.49205017089844, "fcm_dpo/q_t": 0.34480544924736023, "grad_norm": 10.516873359680176, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9148788452148438, "logits/rejected": -1.3886020183563232, "logps/chosen": -237.00350952148438, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -337.3254089355469, "loss": 0.8939, "margin_dpo/margin_mean": 98.49205017089844, "margin_dpo/margin_std": 89.31375122070312, "step": 594 }, { "KL/chosen_KL_mean": -155.29342651367188, "KL/mean": -196.1044921875, "KL/rejected_KL_mean": -236.91555786132812, "KL/std": 91.73303985595703, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.007110237143933773, "fcm_dpo/delta": 0.020485006272792816, "fcm_dpo/margin": 81.62213897705078, "fcm_dpo/q_t": 0.37600505352020264, "grad_norm": 12.029863357543945, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.7504065036773682, "logits/rejected": -1.3301124572753906, "logps/chosen": -236.8082275390625, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -323.43603515625, "loss": 1.0151, "margin_dpo/margin_mean": 81.62213897705078, "margin_dpo/margin_std": 105.995361328125, "step": 595 }, { "KL/chosen_KL_mean": -159.89453125, "KL/mean": -196.36422729492188, "KL/rejected_KL_mean": -232.83392333984375, "KL/std": 89.90532684326172, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.007183522917330265, "fcm_dpo/delta": 0.080133356153965, "fcm_dpo/margin": 72.93937683105469, "fcm_dpo/q_t": 0.3879748582839966, "grad_norm": 15.7341890335083, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.7777928113937378, "logits/rejected": -1.2113715410232544, "logps/chosen": -248.1025848388672, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -318.92767333984375, "loss": 1.0723, "margin_dpo/margin_mean": 72.93938446044922, "margin_dpo/margin_std": 110.53237915039062, "step": 596 }, { "KL/chosen_KL_mean": -155.95252990722656, "KL/mean": -204.29464721679688, "KL/rejected_KL_mean": -252.63674926757812, "KL/std": 98.88459777832031, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.007086900528520346, "fcm_dpo/delta": -0.09191150963306427, "fcm_dpo/margin": 96.68421936035156, "fcm_dpo/q_t": 0.352841317653656, "grad_norm": 10.76797103881836, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.549586296081543, "logits/rejected": -1.4302891492843628, "logps/chosen": -239.54251098632812, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -359.5811767578125, "loss": 0.9414, "margin_dpo/margin_mean": 96.68421173095703, "margin_dpo/margin_std": 108.06651306152344, "step": 597 }, { "KL/chosen_KL_mean": -150.19944763183594, "KL/mean": -189.96823120117188, "KL/rejected_KL_mean": -229.7370147705078, "KL/std": 91.25550842285156, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.007075078785419464, "fcm_dpo/delta": 0.039041392505168915, "fcm_dpo/margin": 79.53754425048828, "fcm_dpo/q_t": 0.37561431527137756, "grad_norm": 12.29248046875, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.923269271850586, "logits/rejected": -1.6066948175430298, "logps/chosen": -240.382080078125, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -324.53558349609375, "loss": 1.0246, "margin_dpo/margin_mean": 79.53754425048828, "margin_dpo/margin_std": 102.55648803710938, "step": 598 }, { "KL/chosen_KL_mean": -154.5861358642578, "KL/mean": -190.308837890625, "KL/rejected_KL_mean": -226.0315704345703, "KL/std": 95.17540740966797, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.007177642080932856, "fcm_dpo/delta": 0.09154414385557175, "fcm_dpo/margin": 71.44544982910156, "fcm_dpo/q_t": 0.3887494206428528, "grad_norm": 12.270193099975586, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.731421709060669, "logits/rejected": -1.6055083274841309, "logps/chosen": -232.42840576171875, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -315.22918701171875, "loss": 1.0674, "margin_dpo/margin_mean": 71.44544982910156, "margin_dpo/margin_std": 102.84925842285156, "step": 599 }, { "KL/chosen_KL_mean": -165.75469970703125, "KL/mean": -207.47216796875, "KL/rejected_KL_mean": -249.18966674804688, "KL/std": 92.00709533691406, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.007261472754180431, "fcm_dpo/delta": -0.006251073908060789, "fcm_dpo/margin": 83.4349365234375, "fcm_dpo/q_t": 0.3688989579677582, "grad_norm": 10.376070022583008, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.4058241844177246, "logits/rejected": -2.060559034347534, "logps/chosen": -257.9425048828125, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -342.1439514160156, "loss": 0.9868, "margin_dpo/margin_mean": 83.4349365234375, "margin_dpo/margin_std": 99.55221557617188, "step": 600 }, { "KL/chosen_KL_mean": -170.6857452392578, "KL/mean": -213.8529052734375, "KL/rejected_KL_mean": -257.02008056640625, "KL/std": 92.41392517089844, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0071959784254431725, "fcm_dpo/delta": -0.02316192165017128, "fcm_dpo/margin": 86.3343276977539, "fcm_dpo/q_t": 0.3630520701408386, "grad_norm": 11.42811107635498, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.8483115434646606, "logits/rejected": -1.5277241468429565, "logps/chosen": -259.9018859863281, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -357.19061279296875, "loss": 0.9536, "margin_dpo/margin_mean": 86.33432006835938, "margin_dpo/margin_std": 90.5497817993164, "step": 601 }, { "KL/chosen_KL_mean": -134.19573974609375, "KL/mean": -189.27093505859375, "KL/rejected_KL_mean": -244.3461151123047, "KL/std": 100.06878662109375, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.007020370103418827, "fcm_dpo/delta": -0.18784166872501373, "fcm_dpo/margin": 110.1503677368164, "fcm_dpo/q_t": 0.333892285823822, "grad_norm": 10.603527069091797, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.328240156173706, "logits/rejected": -2.084373712539673, "logps/chosen": -214.2533416748047, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -337.54364013671875, "loss": 0.8872, "margin_dpo/margin_mean": 110.15037536621094, "margin_dpo/margin_std": 112.4236831665039, "step": 602 }, { "KL/chosen_KL_mean": -144.7084503173828, "KL/mean": -191.54823303222656, "KL/rejected_KL_mean": -238.38803100585938, "KL/std": 91.71591186523438, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.006912807002663612, "fcm_dpo/delta": -0.05085933953523636, "fcm_dpo/margin": 93.6795654296875, "fcm_dpo/q_t": 0.35853111743927, "grad_norm": 10.603703498840332, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1642088890075684, "logits/rejected": -1.5833070278167725, "logps/chosen": -247.01803588867188, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -346.4568786621094, "loss": 0.9513, "margin_dpo/margin_mean": 93.6795654296875, "margin_dpo/margin_std": 104.23471069335938, "step": 603 }, { "KL/chosen_KL_mean": -151.46255493164062, "KL/mean": -198.5010986328125, "KL/rejected_KL_mean": -245.53964233398438, "KL/std": 95.33517456054688, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.006810622755438089, "fcm_dpo/delta": -0.04400404542684555, "fcm_dpo/margin": 94.07707977294922, "fcm_dpo/q_t": 0.35498708486557007, "grad_norm": 10.123159408569336, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.6935317516326904, "logits/rejected": -1.5362553596496582, "logps/chosen": -229.63665771484375, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -345.0357666015625, "loss": 0.9381, "margin_dpo/margin_mean": 94.07707977294922, "margin_dpo/margin_std": 93.10210418701172, "step": 604 }, { "KL/chosen_KL_mean": -164.04217529296875, "KL/mean": -203.26278686523438, "KL/rejected_KL_mean": -242.48341369628906, "KL/std": 86.12641906738281, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.006879772990942001, "fcm_dpo/delta": 0.06382867693901062, "fcm_dpo/margin": 78.44123077392578, "fcm_dpo/q_t": 0.3811225891113281, "grad_norm": 10.797769546508789, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.89601731300354, "logits/rejected": -1.3985096216201782, "logps/chosen": -258.81549072265625, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -349.788330078125, "loss": 1.0155, "margin_dpo/margin_mean": 78.44123840332031, "margin_dpo/margin_std": 95.44406127929688, "step": 605 }, { "KL/chosen_KL_mean": -158.40057373046875, "KL/mean": -200.04696655273438, "KL/rejected_KL_mean": -241.69338989257812, "KL/std": 94.16796875, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.006981974001973867, "fcm_dpo/delta": 0.018229767680168152, "fcm_dpo/margin": 83.29283142089844, "fcm_dpo/q_t": 0.3704386353492737, "grad_norm": 11.523370742797852, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.9156712293624878, "logits/rejected": -1.5112608671188354, "logps/chosen": -246.15396118164062, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -345.93560791015625, "loss": 0.9984, "margin_dpo/margin_mean": 83.2928237915039, "margin_dpo/margin_std": 98.36637878417969, "step": 606 }, { "KL/chosen_KL_mean": -155.29617309570312, "KL/mean": -201.50587463378906, "KL/rejected_KL_mean": -247.715576171875, "KL/std": 100.3311767578125, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0068945810198783875, "fcm_dpo/delta": -0.03972142934799194, "fcm_dpo/margin": 92.41940307617188, "fcm_dpo/q_t": 0.36313295364379883, "grad_norm": 14.075140953063965, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.6992088556289673, "logits/rejected": -1.1591174602508545, "logps/chosen": -243.62521362304688, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -334.4836730957031, "loss": 1.0062, "margin_dpo/margin_mean": 92.41940307617188, "margin_dpo/margin_std": 122.5616455078125, "step": 607 }, { "KL/chosen_KL_mean": -140.97390747070312, "KL/mean": -189.37582397460938, "KL/rejected_KL_mean": -237.77772521972656, "KL/std": 98.44059753417969, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.006758525036275387, "fcm_dpo/delta": -0.06021926924586296, "fcm_dpo/margin": 96.80380249023438, "fcm_dpo/q_t": 0.35397228598594666, "grad_norm": 13.05864429473877, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.8913509845733643, "logits/rejected": -1.5980541706085205, "logps/chosen": -226.7432861328125, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -338.01055908203125, "loss": 0.9573, "margin_dpo/margin_mean": 96.80380249023438, "margin_dpo/margin_std": 104.85302734375, "step": 608 }, { "KL/chosen_KL_mean": -153.7366943359375, "KL/mean": -201.2283935546875, "KL/rejected_KL_mean": -248.72010803222656, "KL/std": 98.95556640625, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.006722399964928627, "fcm_dpo/delta": -0.04184867441654205, "fcm_dpo/margin": 94.98341369628906, "fcm_dpo/q_t": 0.3607417345046997, "grad_norm": 13.247615814208984, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.1930959224700928, "logits/rejected": -1.7636018991470337, "logps/chosen": -246.70327758789062, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -344.6382751464844, "loss": 0.9611, "margin_dpo/margin_mean": 94.98341369628906, "margin_dpo/margin_std": 103.45018005371094, "step": 609 }, { "KL/chosen_KL_mean": -145.52410888671875, "KL/mean": -194.569580078125, "KL/rejected_KL_mean": -243.6150665283203, "KL/std": 95.2448959350586, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.006666187662631273, "fcm_dpo/delta": -0.057946156710386276, "fcm_dpo/margin": 98.09097290039062, "fcm_dpo/q_t": 0.35316145420074463, "grad_norm": 10.085182189941406, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.811213731765747, "logits/rejected": -1.2720458507537842, "logps/chosen": -240.22439575195312, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -333.3024597167969, "loss": 0.936, "margin_dpo/margin_mean": 98.09097290039062, "margin_dpo/margin_std": 99.8183822631836, "step": 610 }, { "KL/chosen_KL_mean": -151.08074951171875, "KL/mean": -199.22885131835938, "KL/rejected_KL_mean": -247.37696838378906, "KL/std": 98.63188171386719, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.006660032086074352, "fcm_dpo/delta": -0.04466771334409714, "fcm_dpo/margin": 96.29621887207031, "fcm_dpo/q_t": 0.3601592481136322, "grad_norm": 12.406939506530762, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.235910415649414, "logits/rejected": -1.8958334922790527, "logps/chosen": -237.8640899658203, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -337.22003173828125, "loss": 0.9462, "margin_dpo/margin_mean": 96.29621887207031, "margin_dpo/margin_std": 100.7657470703125, "step": 611 }, { "KL/chosen_KL_mean": -164.71823120117188, "KL/mean": -214.001953125, "KL/rejected_KL_mean": -263.2856750488281, "KL/std": 93.3245849609375, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.006541106849908829, "fcm_dpo/delta": -0.048063430935144424, "fcm_dpo/margin": 98.56745910644531, "fcm_dpo/q_t": 0.35592710971832275, "grad_norm": 10.668329238891602, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.0953712463378906, "logits/rejected": -1.2490897178649902, "logps/chosen": -265.7383728027344, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -357.0686950683594, "loss": 0.927, "margin_dpo/margin_mean": 98.56745910644531, "margin_dpo/margin_std": 94.75941467285156, "step": 612 }, { "KL/chosen_KL_mean": -136.13201904296875, "KL/mean": -189.58966064453125, "KL/rejected_KL_mean": -243.04733276367188, "KL/std": 95.88484954833984, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.006434359587728977, "fcm_dpo/delta": -0.09476737678050995, "fcm_dpo/margin": 106.91532135009766, "fcm_dpo/q_t": 0.34565889835357666, "grad_norm": 10.253174781799316, "learning_rate": 1.551886292185553e-08, "logits/chosen": -2.0383970737457275, "logits/rejected": -1.8415833711624146, "logps/chosen": -225.12063598632812, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -353.0428466796875, "loss": 0.8898, "margin_dpo/margin_mean": 106.91531372070312, "margin_dpo/margin_std": 92.33283996582031, "step": 613 }, { "KL/chosen_KL_mean": -167.2001953125, "KL/mean": -210.2760009765625, "KL/rejected_KL_mean": -253.35182189941406, "KL/std": 89.31161499023438, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.00644969567656517, "fcm_dpo/delta": 0.046966638416051865, "fcm_dpo/margin": 86.15164947509766, "fcm_dpo/q_t": 0.3764270544052124, "grad_norm": 12.235685348510742, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.6406195163726807, "logits/rejected": -1.626504898071289, "logps/chosen": -247.40023803710938, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -363.2142333984375, "loss": 1.0164, "margin_dpo/margin_mean": 86.15164947509766, "margin_dpo/margin_std": 106.30526733398438, "step": 614 }, { "KL/chosen_KL_mean": -155.805908203125, "KL/mean": -193.97439575195312, "KL/rejected_KL_mean": -232.14288330078125, "KL/std": 91.5688705444336, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.006631205789744854, "fcm_dpo/delta": 0.09628231823444366, "fcm_dpo/margin": 76.33699035644531, "fcm_dpo/q_t": 0.3851287364959717, "grad_norm": 11.916601181030273, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.7134699821472168, "logits/rejected": -1.3067545890808105, "logps/chosen": -256.24114990234375, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -333.32293701171875, "loss": 1.0237, "margin_dpo/margin_mean": 76.33699035644531, "margin_dpo/margin_std": 85.07723999023438, "step": 615 }, { "KL/chosen_KL_mean": -158.3206329345703, "KL/mean": -209.11627197265625, "KL/rejected_KL_mean": -259.91192626953125, "KL/std": 91.82144927978516, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.006565681658685207, "fcm_dpo/delta": -0.07192869484424591, "fcm_dpo/margin": 101.5912857055664, "fcm_dpo/q_t": 0.3529027998447418, "grad_norm": 10.566946029663086, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.7258317470550537, "logits/rejected": -1.2233316898345947, "logps/chosen": -250.81356811523438, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -341.97259521484375, "loss": 0.9345, "margin_dpo/margin_mean": 101.5912857055664, "margin_dpo/margin_std": 105.95802307128906, "step": 616 }, { "KL/chosen_KL_mean": -164.21475219726562, "KL/mean": -204.23574829101562, "KL/rejected_KL_mean": -244.2567596435547, "KL/std": 88.38334655761719, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.006581010762602091, "fcm_dpo/delta": 0.07735887169837952, "fcm_dpo/margin": 80.04202270507812, "fcm_dpo/q_t": 0.3821824789047241, "grad_norm": 11.055672645568848, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6958956718444824, "logits/rejected": -1.0639562606811523, "logps/chosen": -255.31173706054688, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -314.66680908203125, "loss": 1.0264, "margin_dpo/margin_mean": 80.04202270507812, "margin_dpo/margin_std": 99.28726196289062, "step": 617 }, { "KL/chosen_KL_mean": -165.8621826171875, "KL/mean": -209.3111572265625, "KL/rejected_KL_mean": -252.76014709472656, "KL/std": 96.65109252929688, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.006619461812078953, "fcm_dpo/delta": 0.02621281147003174, "fcm_dpo/margin": 86.89796447753906, "fcm_dpo/q_t": 0.3712009787559509, "grad_norm": 11.887856483459473, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.1773295402526855, "logits/rejected": -1.830935001373291, "logps/chosen": -267.8827819824219, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -352.56134033203125, "loss": 1.0047, "margin_dpo/margin_mean": 86.89795684814453, "margin_dpo/margin_std": 105.34744262695312, "step": 618 }, { "KL/chosen_KL_mean": -158.95233154296875, "KL/mean": -198.61920166015625, "KL/rejected_KL_mean": -238.28607177734375, "KL/std": 95.9604721069336, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.006716095842421055, "fcm_dpo/delta": 0.0709773451089859, "fcm_dpo/margin": 79.333740234375, "fcm_dpo/q_t": 0.3864196538925171, "grad_norm": 11.145162582397461, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.4948410987854004, "logits/rejected": -1.1198971271514893, "logps/chosen": -248.69369506835938, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -338.18743896484375, "loss": 1.0507, "margin_dpo/margin_mean": 79.333740234375, "margin_dpo/margin_std": 113.49344635009766, "step": 619 }, { "KL/chosen_KL_mean": -153.293701171875, "KL/mean": -197.50906372070312, "KL/rejected_KL_mean": -241.72442626953125, "KL/std": 100.93150329589844, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.006662972271442413, "fcm_dpo/delta": 0.008081376552581787, "fcm_dpo/margin": 88.43071746826172, "fcm_dpo/q_t": 0.3721146583557129, "grad_norm": 11.053604125976562, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.227707862854004, "logits/rejected": -1.7064398527145386, "logps/chosen": -238.41799926757812, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -327.136962890625, "loss": 1.0161, "margin_dpo/margin_mean": 88.43071746826172, "margin_dpo/margin_std": 108.6497802734375, "step": 620 }, { "KL/chosen_KL_mean": -156.15496826171875, "KL/mean": -198.55673217773438, "KL/rejected_KL_mean": -240.95852661132812, "KL/std": 97.85447692871094, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.006850396748632193, "fcm_dpo/delta": 0.017163343727588654, "fcm_dpo/margin": 84.80354309082031, "fcm_dpo/q_t": 0.3750428557395935, "grad_norm": 11.195357322692871, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.99844491481781, "logits/rejected": -1.7369494438171387, "logps/chosen": -245.40338134765625, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -336.42315673828125, "loss": 1.0357, "margin_dpo/margin_mean": 84.80354309082031, "margin_dpo/margin_std": 113.91090393066406, "step": 621 }, { "KL/chosen_KL_mean": -154.9768524169922, "KL/mean": -199.65017700195312, "KL/rejected_KL_mean": -244.32351684570312, "KL/std": 87.04754638671875, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.006754318717867136, "fcm_dpo/delta": -0.0038462337106466293, "fcm_dpo/margin": 89.34666442871094, "fcm_dpo/q_t": 0.3658301830291748, "grad_norm": 12.479233741760254, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.6989606618881226, "logits/rejected": -1.050009846687317, "logps/chosen": -254.77096557617188, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -335.1517333984375, "loss": 0.9637, "margin_dpo/margin_mean": 89.34666442871094, "margin_dpo/margin_std": 94.03802490234375, "step": 622 }, { "KL/chosen_KL_mean": -160.8984375, "KL/mean": -193.15658569335938, "KL/rejected_KL_mean": -225.4147491455078, "KL/std": 94.82798767089844, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.006910811644047499, "fcm_dpo/delta": 0.16164974868297577, "fcm_dpo/margin": 64.51629638671875, "fcm_dpo/q_t": 0.4056931436061859, "grad_norm": 13.483115196228027, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.3651726245880127, "logits/rejected": -1.9365458488464355, "logps/chosen": -251.35400390625, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -316.73748779296875, "loss": 1.1332, "margin_dpo/margin_mean": 64.51629638671875, "margin_dpo/margin_std": 114.68384552001953, "step": 623 }, { "KL/chosen_KL_mean": -163.44888305664062, "KL/mean": -212.97695922851562, "KL/rejected_KL_mean": -262.5050048828125, "KL/std": 93.59333801269531, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.006898221559822559, "fcm_dpo/delta": -0.08945785462856293, "fcm_dpo/margin": 99.05614471435547, "fcm_dpo/q_t": 0.35129615664482117, "grad_norm": 12.25908088684082, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.6282312870025635, "logits/rejected": -1.711545705795288, "logps/chosen": -273.3240966796875, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -367.2782287597656, "loss": 0.9329, "margin_dpo/margin_mean": 99.05613708496094, "margin_dpo/margin_std": 106.00656127929688, "step": 624 }, { "KL/chosen_KL_mean": -172.53701782226562, "KL/mean": -209.47097778320312, "KL/rejected_KL_mean": -246.40493774414062, "KL/std": 93.85888671875, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.006977788172662258, "fcm_dpo/delta": 0.08880499005317688, "fcm_dpo/margin": 73.86790466308594, "fcm_dpo/q_t": 0.38692906498908997, "grad_norm": 10.714426040649414, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.6439225673675537, "logits/rejected": -1.2619787454605103, "logps/chosen": -259.7052001953125, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -338.26641845703125, "loss": 1.0385, "margin_dpo/margin_mean": 73.86790466308594, "margin_dpo/margin_std": 96.79435729980469, "step": 625 }, { "KL/chosen_KL_mean": -163.43185424804688, "KL/mean": -199.673583984375, "KL/rejected_KL_mean": -235.91529846191406, "KL/std": 102.98783874511719, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.007108000572770834, "fcm_dpo/delta": 0.08888818323612213, "fcm_dpo/margin": 72.48345184326172, "fcm_dpo/q_t": 0.3912317752838135, "grad_norm": 13.16305160522461, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.5785739421844482, "logits/rejected": -1.1544768810272217, "logps/chosen": -267.65606689453125, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -340.0927734375, "loss": 1.0791, "margin_dpo/margin_mean": 72.48344421386719, "margin_dpo/margin_std": 113.23786926269531, "step": 626 }, { "KL/chosen_KL_mean": -146.79246520996094, "KL/mean": -191.98263549804688, "KL/rejected_KL_mean": -237.1728515625, "KL/std": 106.24773406982422, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.007069021929055452, "fcm_dpo/delta": -0.04159139096736908, "fcm_dpo/margin": 90.38038635253906, "fcm_dpo/q_t": 0.3646802008152008, "grad_norm": 9.287775993347168, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.7292776107788086, "logits/rejected": -1.8644907474517822, "logps/chosen": -221.72708129882812, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -349.7457275390625, "loss": 0.9837, "margin_dpo/margin_mean": 90.38040161132812, "margin_dpo/margin_std": 113.69436645507812, "step": 627 }, { "KL/chosen_KL_mean": -152.13571166992188, "KL/mean": -200.39727783203125, "KL/rejected_KL_mean": -248.65884399414062, "KL/std": 95.0730209350586, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0069746533408761024, "fcm_dpo/delta": -0.07852533459663391, "fcm_dpo/margin": 96.52310180664062, "fcm_dpo/q_t": 0.3527430295944214, "grad_norm": 12.535661697387695, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.076289653778076, "logits/rejected": -1.8133248090744019, "logps/chosen": -245.82679748535156, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -356.0028076171875, "loss": 0.9298, "margin_dpo/margin_mean": 96.52310180664062, "margin_dpo/margin_std": 99.45779418945312, "step": 628 }, { "KL/chosen_KL_mean": -163.8128662109375, "KL/mean": -207.00637817382812, "KL/rejected_KL_mean": -250.1999053955078, "KL/std": 95.40560913085938, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0069231558591127396, "fcm_dpo/delta": 0.0017434991896152496, "fcm_dpo/margin": 86.38703155517578, "fcm_dpo/q_t": 0.3670782744884491, "grad_norm": 11.485631942749023, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.9891417026519775, "logits/rejected": -1.5540037155151367, "logps/chosen": -267.0432434082031, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -347.36834716796875, "loss": 0.9877, "margin_dpo/margin_mean": 86.38703155517578, "margin_dpo/margin_std": 100.86512756347656, "step": 629 }, { "KL/chosen_KL_mean": -150.00772094726562, "KL/mean": -179.866943359375, "KL/rejected_KL_mean": -209.72616577148438, "KL/std": 89.26080322265625, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.007179318927228451, "fcm_dpo/delta": 0.17715156078338623, "fcm_dpo/margin": 59.71842956542969, "fcm_dpo/q_t": 0.405933141708374, "grad_norm": 15.68066120147705, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.8523951768875122, "logits/rejected": -1.3003599643707275, "logps/chosen": -243.9052734375, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -299.1005554199219, "loss": 1.1219, "margin_dpo/margin_mean": 59.71842956542969, "margin_dpo/margin_std": 98.19818878173828, "step": 630 }, { "KL/chosen_KL_mean": -147.9554443359375, "KL/mean": -189.91558837890625, "KL/rejected_KL_mean": -231.875732421875, "KL/std": 87.26608276367188, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.00718712667003274, "fcm_dpo/delta": -0.003399255685508251, "fcm_dpo/margin": 83.92032623291016, "fcm_dpo/q_t": 0.3663654029369354, "grad_norm": 11.044988632202148, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.320514678955078, "logits/rejected": -1.877066969871521, "logps/chosen": -235.72625732421875, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -320.55816650390625, "loss": 0.9779, "margin_dpo/margin_mean": 83.92032623291016, "margin_dpo/margin_std": 95.57085418701172, "step": 631 }, { "KL/chosen_KL_mean": -144.65615844726562, "KL/mean": -191.09259033203125, "KL/rejected_KL_mean": -237.529052734375, "KL/std": 94.00450134277344, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.00707068620249629, "fcm_dpo/delta": -0.06263597309589386, "fcm_dpo/margin": 92.87287902832031, "fcm_dpo/q_t": 0.35927510261535645, "grad_norm": 12.786091804504395, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.128201961517334, "logits/rejected": -1.7098519802093506, "logps/chosen": -233.28268432617188, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -328.97998046875, "loss": 0.9771, "margin_dpo/margin_mean": 92.87287902832031, "margin_dpo/margin_std": 112.5600814819336, "step": 632 }, { "KL/chosen_KL_mean": -180.24676513671875, "KL/mean": -208.8311767578125, "KL/rejected_KL_mean": -237.4156036376953, "KL/std": 95.34663391113281, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.007364756893366575, "fcm_dpo/delta": 0.18393391370773315, "fcm_dpo/margin": 57.16883087158203, "fcm_dpo/q_t": 0.4145483374595642, "grad_norm": 14.513399124145508, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.165386199951172, "logits/rejected": -1.9722667932510376, "logps/chosen": -261.6211853027344, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -336.04132080078125, "loss": 1.1985, "margin_dpo/margin_mean": 57.16883087158203, "margin_dpo/margin_std": 124.85458374023438, "step": 633 }, { "KL/chosen_KL_mean": -166.4353790283203, "KL/mean": -203.2814483642578, "KL/rejected_KL_mean": -240.12753295898438, "KL/std": 85.76390075683594, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.007516602054238319, "fcm_dpo/delta": 0.04500632733106613, "fcm_dpo/margin": 73.69215393066406, "fcm_dpo/q_t": 0.3773460388183594, "grad_norm": 13.822626113891602, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.5553464889526367, "logits/rejected": -1.3772296905517578, "logps/chosen": -262.5582275390625, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -352.975341796875, "loss": 1.0117, "margin_dpo/margin_mean": 73.69215393066406, "margin_dpo/margin_std": 84.48291015625, "step": 634 }, { "KL/chosen_KL_mean": -172.04931640625, "KL/mean": -200.4110870361328, "KL/rejected_KL_mean": -228.77285766601562, "KL/std": 90.09078979492188, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.007676552515476942, "fcm_dpo/delta": 0.17031870782375336, "fcm_dpo/margin": 56.7235221862793, "fcm_dpo/q_t": 0.4077800214290619, "grad_norm": 16.368589401245117, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.6753485202789307, "logits/rejected": -1.1345549821853638, "logps/chosen": -270.7334289550781, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -318.67205810546875, "loss": 1.1455, "margin_dpo/margin_mean": 56.7235221862793, "margin_dpo/margin_std": 105.44437408447266, "step": 635 }, { "KL/chosen_KL_mean": -157.0723876953125, "KL/mean": -202.601318359375, "KL/rejected_KL_mean": -248.1302490234375, "KL/std": 97.06875610351562, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.007597176358103752, "fcm_dpo/delta": -0.09850575774908066, "fcm_dpo/margin": 91.057861328125, "fcm_dpo/q_t": 0.3509254455566406, "grad_norm": 12.046630859375, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.602974772453308, "logits/rejected": -1.1308594942092896, "logps/chosen": -247.4903564453125, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -335.837158203125, "loss": 0.9296, "margin_dpo/margin_mean": 91.057861328125, "margin_dpo/margin_std": 97.74097442626953, "step": 636 }, { "KL/chosen_KL_mean": -158.05020141601562, "KL/mean": -206.7764129638672, "KL/rejected_KL_mean": -255.50262451171875, "KL/std": 105.88407897949219, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.007414502557367086, "fcm_dpo/delta": -0.13194185495376587, "fcm_dpo/margin": 97.45243835449219, "fcm_dpo/q_t": 0.3436763882637024, "grad_norm": 10.844650268554688, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.0370514392852783, "logits/rejected": -1.9342517852783203, "logps/chosen": -240.49990844726562, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -359.5312194824219, "loss": 0.917, "margin_dpo/margin_mean": 97.45243835449219, "margin_dpo/margin_std": 102.07131958007812, "step": 637 }, { "KL/chosen_KL_mean": -146.01309204101562, "KL/mean": -194.66883850097656, "KL/rejected_KL_mean": -243.32456970214844, "KL/std": 94.4871597290039, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0072778514586389065, "fcm_dpo/delta": -0.11670075356960297, "fcm_dpo/margin": 97.31146240234375, "fcm_dpo/q_t": 0.34191834926605225, "grad_norm": 10.339118003845215, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.6270800828933716, "logits/rejected": -0.947350025177002, "logps/chosen": -237.93807983398438, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -329.6116027832031, "loss": 0.8835, "margin_dpo/margin_mean": 97.31146240234375, "margin_dpo/margin_std": 81.40766143798828, "step": 638 }, { "KL/chosen_KL_mean": -192.86886596679688, "KL/mean": -217.66488647460938, "KL/rejected_KL_mean": -242.4609375, "KL/std": 89.75401306152344, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.007475035265088081, "fcm_dpo/delta": 0.23652800917625427, "fcm_dpo/margin": 49.59208679199219, "fcm_dpo/q_t": 0.41979682445526123, "grad_norm": 13.991231918334961, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.4492952823638916, "logits/rejected": -0.6927109956741333, "logps/chosen": -297.39642333984375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -323.9412841796875, "loss": 1.205, "margin_dpo/margin_mean": 49.59208679199219, "margin_dpo/margin_std": 110.02244567871094, "step": 639 }, { "KL/chosen_KL_mean": -162.25596618652344, "KL/mean": -206.34469604492188, "KL/rejected_KL_mean": -250.43344116210938, "KL/std": 95.22294616699219, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.007545283064246178, "fcm_dpo/delta": -0.07240527868270874, "fcm_dpo/margin": 88.177490234375, "fcm_dpo/q_t": 0.35440731048583984, "grad_norm": 13.203383445739746, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.6212174892425537, "logits/rejected": -1.0797920227050781, "logps/chosen": -260.1412353515625, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -363.13848876953125, "loss": 0.9525, "margin_dpo/margin_mean": 88.177490234375, "margin_dpo/margin_std": 92.44313049316406, "step": 640 }, { "KL/chosen_KL_mean": -149.6476287841797, "KL/mean": -195.84298706054688, "KL/rejected_KL_mean": -242.038330078125, "KL/std": 109.9680404663086, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.007337721064686775, "fcm_dpo/delta": -0.08353941142559052, "fcm_dpo/margin": 92.39068603515625, "fcm_dpo/q_t": 0.35952630639076233, "grad_norm": 10.76613712310791, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.4462890625, "logits/rejected": -1.9225687980651855, "logps/chosen": -246.0933074951172, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -337.17401123046875, "loss": 0.9733, "margin_dpo/margin_mean": 92.39068603515625, "margin_dpo/margin_std": 118.24760437011719, "step": 641 }, { "KL/chosen_KL_mean": -168.4007568359375, "KL/mean": -210.71316528320312, "KL/rejected_KL_mean": -253.02560424804688, "KL/std": 93.6077880859375, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.007304553873836994, "fcm_dpo/delta": -0.01956186443567276, "fcm_dpo/margin": 84.62486267089844, "fcm_dpo/q_t": 0.37027326226234436, "grad_norm": 12.884522438049316, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.527350902557373, "logits/rejected": -1.1907775402069092, "logps/chosen": -269.1606140136719, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -367.7332458496094, "loss": 1.0071, "margin_dpo/margin_mean": 84.62486267089844, "margin_dpo/margin_std": 114.21339416503906, "step": 642 }, { "KL/chosen_KL_mean": -157.51873779296875, "KL/mean": -198.27658081054688, "KL/rejected_KL_mean": -239.03440856933594, "KL/std": 86.57600402832031, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.007206802722066641, "fcm_dpo/delta": 0.011197030544281006, "fcm_dpo/margin": 81.51566314697266, "fcm_dpo/q_t": 0.3707771897315979, "grad_norm": 13.636649131774902, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.2618513107299805, "logits/rejected": -1.8777098655700684, "logps/chosen": -242.26239013671875, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -333.35284423828125, "loss": 1.0048, "margin_dpo/margin_mean": 81.51565551757812, "margin_dpo/margin_std": 97.57747650146484, "step": 643 }, { "KL/chosen_KL_mean": -161.81661987304688, "KL/mean": -195.3797607421875, "KL/rejected_KL_mean": -228.94290161132812, "KL/std": 87.48783874511719, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0074090855196118355, "fcm_dpo/delta": 0.10794378817081451, "fcm_dpo/margin": 67.12629699707031, "fcm_dpo/q_t": 0.3900315761566162, "grad_norm": 14.409489631652832, "learning_rate": 4.741290495811873e-09, "logits/chosen": -2.059009075164795, "logits/rejected": -1.791301965713501, "logps/chosen": -247.1393585205078, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -323.551513671875, "loss": 1.0519, "margin_dpo/margin_mean": 67.12629699707031, "margin_dpo/margin_std": 90.55181884765625, "step": 644 }, { "KL/chosen_KL_mean": -173.952880859375, "KL/mean": -196.98455810546875, "KL/rejected_KL_mean": -220.01626586914062, "KL/std": 97.39523315429688, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.007755122147500515, "fcm_dpo/delta": 0.25067955255508423, "fcm_dpo/margin": 46.06340789794922, "fcm_dpo/q_t": 0.4255422353744507, "grad_norm": 14.249744415283203, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.5394432544708252, "logits/rejected": -1.3632748126983643, "logps/chosen": -256.5431213378906, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -323.02001953125, "loss": 1.2224, "margin_dpo/margin_mean": 46.06340789794922, "margin_dpo/margin_std": 107.89938354492188, "step": 645 }, { "KL/chosen_KL_mean": -151.3134307861328, "KL/mean": -195.1842041015625, "KL/rejected_KL_mean": -239.05502319335938, "KL/std": 84.86918640136719, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.007684169337153435, "fcm_dpo/delta": -0.08099155128002167, "fcm_dpo/margin": 87.7415771484375, "fcm_dpo/q_t": 0.3519039750099182, "grad_norm": 11.975767135620117, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.9051244258880615, "logits/rejected": -1.3055133819580078, "logps/chosen": -242.50546264648438, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -322.8333435058594, "loss": 0.9284, "margin_dpo/margin_mean": 87.7415771484375, "margin_dpo/margin_std": 88.87870025634766, "step": 646 }, { "KL/chosen_KL_mean": -166.53721618652344, "KL/mean": -203.9755859375, "KL/rejected_KL_mean": -241.4139404296875, "KL/std": 101.45904541015625, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.007719170302152634, "fcm_dpo/delta": 0.023361235857009888, "fcm_dpo/margin": 74.87675476074219, "fcm_dpo/q_t": 0.380698025226593, "grad_norm": 12.581117630004883, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.3682063817977905, "logits/rejected": -1.2257659435272217, "logps/chosen": -255.3816680908203, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -340.91229248046875, "loss": 1.0557, "margin_dpo/margin_mean": 74.87674713134766, "margin_dpo/margin_std": 113.31559753417969, "step": 647 }, { "KL/chosen_KL_mean": -154.18048095703125, "KL/mean": -207.08187866210938, "KL/rejected_KL_mean": -259.9832763671875, "KL/std": 103.12141418457031, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.007524555549025536, "fcm_dpo/delta": -0.21260285377502441, "fcm_dpo/margin": 105.80282592773438, "fcm_dpo/q_t": 0.3298163414001465, "grad_norm": 12.21097469329834, "learning_rate": 3.798061746947995e-09, "logits/chosen": -2.143336296081543, "logits/rejected": -2.0671629905700684, "logps/chosen": -242.02857971191406, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -364.6533508300781, "loss": 0.8833, "margin_dpo/margin_mean": 105.80282592773438, "margin_dpo/margin_std": 108.880615234375, "step": 648 }, { "KL/chosen_KL_mean": -140.1556854248047, "KL/mean": -183.87490844726562, "KL/rejected_KL_mean": -227.59414672851562, "KL/std": 90.37361145019531, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0073182238265872, "fcm_dpo/delta": -0.043864499777555466, "fcm_dpo/margin": 87.43848419189453, "fcm_dpo/q_t": 0.3582548201084137, "grad_norm": 13.286715507507324, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.3732261657714844, "logits/rejected": -1.969871997833252, "logps/chosen": -229.8481903076172, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -316.30072021484375, "loss": 0.939, "margin_dpo/margin_mean": 87.43848419189453, "margin_dpo/margin_std": 86.00944519042969, "step": 649 }, { "KL/chosen_KL_mean": -165.79159545898438, "KL/mean": -207.6180419921875, "KL/rejected_KL_mean": -249.44448852539062, "KL/std": 96.58186340332031, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.007345967926084995, "fcm_dpo/delta": -0.015489364974200726, "fcm_dpo/margin": 83.65290832519531, "fcm_dpo/q_t": 0.3671303987503052, "grad_norm": 12.898774147033691, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.7597496509552002, "logits/rejected": -1.577094316482544, "logps/chosen": -255.72219848632812, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -352.05731201171875, "loss": 1.0065, "margin_dpo/margin_mean": 83.65290832519531, "margin_dpo/margin_std": 110.4447250366211, "step": 650 }, { "KL/chosen_KL_mean": -140.00192260742188, "KL/mean": -190.60765075683594, "KL/rejected_KL_mean": -241.21339416503906, "KL/std": 94.40056610107422, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.007173136807978153, "fcm_dpo/delta": -0.13607516884803772, "fcm_dpo/margin": 101.21147155761719, "fcm_dpo/q_t": 0.33891594409942627, "grad_norm": 11.239595413208008, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.1632893085479736, "logits/rejected": -1.9913432598114014, "logps/chosen": -219.18923950195312, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -347.146728515625, "loss": 0.8863, "margin_dpo/margin_mean": 101.21147155761719, "margin_dpo/margin_std": 93.14625549316406, "step": 651 }, { "KL/chosen_KL_mean": -145.6478729248047, "KL/mean": -188.7807159423828, "KL/rejected_KL_mean": -231.91357421875, "KL/std": 86.44781494140625, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.00713393185287714, "fcm_dpo/delta": -0.016536947339773178, "fcm_dpo/margin": 86.26571655273438, "fcm_dpo/q_t": 0.3621070981025696, "grad_norm": 10.493731498718262, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.5675411224365234, "logits/rejected": -2.2689785957336426, "logps/chosen": -247.43809509277344, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -348.2381591796875, "loss": 0.9382, "margin_dpo/margin_mean": 86.26571655273438, "margin_dpo/margin_std": 81.84165954589844, "step": 652 }, { "KL/chosen_KL_mean": -165.29269409179688, "KL/mean": -206.13844299316406, "KL/rejected_KL_mean": -246.9841766357422, "KL/std": 94.78298950195312, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.007145079784095287, "fcm_dpo/delta": 0.017129333689808846, "fcm_dpo/margin": 81.69148254394531, "fcm_dpo/q_t": 0.3686487078666687, "grad_norm": 13.028366088867188, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.175969362258911, "logits/rejected": -1.9279592037200928, "logps/chosen": -258.64630126953125, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -356.107421875, "loss": 0.9887, "margin_dpo/margin_mean": 81.69148254394531, "margin_dpo/margin_std": 93.285400390625, "step": 653 }, { "KL/chosen_KL_mean": -166.05368041992188, "KL/mean": -201.95323181152344, "KL/rejected_KL_mean": -237.852783203125, "KL/std": 92.52761840820312, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0072245229966938496, "fcm_dpo/delta": 0.08572724461555481, "fcm_dpo/margin": 71.79907989501953, "fcm_dpo/q_t": 0.39214521646499634, "grad_norm": 15.528448104858398, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.3204503059387207, "logits/rejected": -2.1371216773986816, "logps/chosen": -255.16921997070312, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -342.772705078125, "loss": 1.0874, "margin_dpo/margin_mean": 71.799072265625, "margin_dpo/margin_std": 116.86124420166016, "step": 654 }, { "KL/chosen_KL_mean": -153.19053649902344, "KL/mean": -196.3241424560547, "KL/rejected_KL_mean": -239.45774841308594, "KL/std": 99.70976257324219, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.007245707791298628, "fcm_dpo/delta": -0.02680610865354538, "fcm_dpo/margin": 86.2672119140625, "fcm_dpo/q_t": 0.362676739692688, "grad_norm": 11.32379150390625, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.1040616035461426, "logits/rejected": -2.1013011932373047, "logps/chosen": -234.2266387939453, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -342.26007080078125, "loss": 0.9848, "margin_dpo/margin_mean": 86.2672119140625, "margin_dpo/margin_std": 103.9677963256836, "step": 655 }, { "KL/chosen_KL_mean": -153.10015869140625, "KL/mean": -206.53195190429688, "KL/rejected_KL_mean": -259.9637451171875, "KL/std": 99.60165405273438, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.007024797610938549, "fcm_dpo/delta": -0.16354700922966003, "fcm_dpo/margin": 106.86357879638672, "fcm_dpo/q_t": 0.3370662331581116, "grad_norm": 12.331055641174316, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.5201234817504883, "logits/rejected": -1.5891121625900269, "logps/chosen": -238.42550659179688, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -378.3023986816406, "loss": 0.8764, "margin_dpo/margin_mean": 106.86358642578125, "margin_dpo/margin_std": 100.95097351074219, "step": 656 }, { "KL/chosen_KL_mean": -163.20968627929688, "KL/mean": -204.79476928710938, "KL/rejected_KL_mean": -246.37985229492188, "KL/std": 93.09001922607422, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0069674099795520306, "fcm_dpo/delta": 0.021239254623651505, "fcm_dpo/margin": 83.17017364501953, "fcm_dpo/q_t": 0.3766845762729645, "grad_norm": 14.734891891479492, "learning_rate": 2.055847060721566e-09, "logits/chosen": -2.1014785766601562, "logits/rejected": -1.9322123527526855, "logps/chosen": -243.40740966796875, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -348.9611511230469, "loss": 1.0243, "margin_dpo/margin_mean": 83.170166015625, "margin_dpo/margin_std": 111.3800048828125, "step": 657 }, { "KL/chosen_KL_mean": -167.74118041992188, "KL/mean": -204.26473999023438, "KL/rejected_KL_mean": -240.78826904296875, "KL/std": 87.66316223144531, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0071324799209833145, "fcm_dpo/delta": 0.0825737938284874, "fcm_dpo/margin": 73.04708862304688, "fcm_dpo/q_t": 0.38528192043304443, "grad_norm": 11.66353702545166, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.9648735523223877, "logits/rejected": -1.4902524948120117, "logps/chosen": -262.3757019042969, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -342.4198913574219, "loss": 1.0357, "margin_dpo/margin_mean": 73.04708862304688, "margin_dpo/margin_std": 94.07040405273438, "step": 658 }, { "KL/chosen_KL_mean": -147.51528930664062, "KL/mean": -193.81939697265625, "KL/rejected_KL_mean": -240.12347412109375, "KL/std": 95.66726684570312, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.00703673530369997, "fcm_dpo/delta": -0.05606871470808983, "fcm_dpo/margin": 92.60818481445312, "fcm_dpo/q_t": 0.3625422716140747, "grad_norm": 11.699962615966797, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.0455338954925537, "logits/rejected": -1.0685107707977295, "logps/chosen": -260.0711669921875, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -334.04510498046875, "loss": 1.0093, "margin_dpo/margin_mean": 92.60818481445312, "margin_dpo/margin_std": 123.62088012695312, "step": 659 }, { "KL/chosen_KL_mean": -153.9538116455078, "KL/mean": -201.45053100585938, "KL/rejected_KL_mean": -248.94723510742188, "KL/std": 93.70677185058594, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.006978246383368969, "fcm_dpo/delta": -0.06739085912704468, "fcm_dpo/margin": 94.993408203125, "fcm_dpo/q_t": 0.3552727699279785, "grad_norm": 14.54831314086914, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.941504716873169, "logits/rejected": -1.3270399570465088, "logps/chosen": -247.33123779296875, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -330.342041015625, "loss": 0.9429, "margin_dpo/margin_mean": 94.993408203125, "margin_dpo/margin_std": 103.41921997070312, "step": 660 }, { "KL/chosen_KL_mean": -157.39431762695312, "KL/mean": -208.59371948242188, "KL/rejected_KL_mean": -259.7930908203125, "KL/std": 100.78553771972656, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.006859012879431248, "fcm_dpo/delta": -0.10994887351989746, "fcm_dpo/margin": 102.39876556396484, "fcm_dpo/q_t": 0.3489990234375, "grad_norm": 11.098326683044434, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.954606056213379, "logits/rejected": -1.445054531097412, "logps/chosen": -245.24948120117188, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -352.1964111328125, "loss": 0.9318, "margin_dpo/margin_mean": 102.39877319335938, "margin_dpo/margin_std": 112.15310668945312, "step": 661 }, { "KL/chosen_KL_mean": -153.62173461914062, "KL/mean": -199.94505310058594, "KL/rejected_KL_mean": -246.2683868408203, "KL/std": 89.34446716308594, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.006790856830775738, "fcm_dpo/delta": -0.031149927526712418, "fcm_dpo/margin": 92.64666748046875, "fcm_dpo/q_t": 0.3599088191986084, "grad_norm": 10.121819496154785, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.6876024007797241, "logits/rejected": -1.1185039281845093, "logps/chosen": -256.4015197753906, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -341.49371337890625, "loss": 0.9413, "margin_dpo/margin_mean": 92.64667510986328, "margin_dpo/margin_std": 91.3426284790039, "step": 662 }, { "KL/chosen_KL_mean": -152.91787719726562, "KL/mean": -196.0003662109375, "KL/rejected_KL_mean": -239.0828857421875, "KL/std": 93.09788513183594, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.006799261551350355, "fcm_dpo/delta": 0.014787685126066208, "fcm_dpo/margin": 86.16500854492188, "fcm_dpo/q_t": 0.37257158756256104, "grad_norm": 11.509709358215332, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.3177809715270996, "logits/rejected": -2.0966362953186035, "logps/chosen": -249.26446533203125, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -359.6093444824219, "loss": 1.0082, "margin_dpo/margin_mean": 86.16500854492188, "margin_dpo/margin_std": 108.53549194335938, "step": 663 }, { "KL/chosen_KL_mean": -168.0064697265625, "KL/mean": -207.24758911132812, "KL/rejected_KL_mean": -246.48867797851562, "KL/std": 93.65726470947266, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.006866086274385452, "fcm_dpo/delta": 0.064328633248806, "fcm_dpo/margin": 78.48221588134766, "fcm_dpo/q_t": 0.3796159327030182, "grad_norm": 11.65186882019043, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.6768138408660889, "logits/rejected": -1.4965018033981323, "logps/chosen": -259.848876953125, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -358.32537841796875, "loss": 1.0122, "margin_dpo/margin_mean": 78.48222351074219, "margin_dpo/margin_std": 91.91385650634766, "step": 664 }, { "KL/chosen_KL_mean": -144.41152954101562, "KL/mean": -187.48663330078125, "KL/rejected_KL_mean": -230.56173706054688, "KL/std": 92.62655639648438, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.006894375197589397, "fcm_dpo/delta": 0.006300309672951698, "fcm_dpo/margin": 86.15023803710938, "fcm_dpo/q_t": 0.3659161329269409, "grad_norm": 9.967409133911133, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.5664970874786377, "logits/rejected": -2.1480026245117188, "logps/chosen": -232.59771728515625, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -322.4737548828125, "loss": 0.9696, "margin_dpo/margin_mean": 86.15022277832031, "margin_dpo/margin_std": 92.16411590576172, "step": 665 }, { "KL/chosen_KL_mean": -161.1727294921875, "KL/mean": -203.71856689453125, "KL/rejected_KL_mean": -246.264404296875, "KL/std": 90.6947250366211, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.006868130527436733, "fcm_dpo/delta": 0.016250133514404297, "fcm_dpo/margin": 85.0916748046875, "fcm_dpo/q_t": 0.3737221658229828, "grad_norm": 14.828961372375488, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.4998377561569214, "logits/rejected": -1.114161729812622, "logps/chosen": -262.1473388671875, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -347.51434326171875, "loss": 1.0102, "margin_dpo/margin_mean": 85.0916748046875, "margin_dpo/margin_std": 107.07771301269531, "step": 666 }, { "KL/chosen_KL_mean": -172.7552947998047, "KL/mean": -219.11123657226562, "KL/rejected_KL_mean": -265.4671630859375, "KL/std": 94.48756408691406, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0068450733087956905, "fcm_dpo/delta": -0.03724222630262375, "fcm_dpo/margin": 92.71189880371094, "fcm_dpo/q_t": 0.3578525185585022, "grad_norm": 11.334739685058594, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.7010917663574219, "logits/rejected": -1.0621293783187866, "logps/chosen": -270.32647705078125, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -359.05194091796875, "loss": 0.9347, "margin_dpo/margin_mean": 92.7118911743164, "margin_dpo/margin_std": 89.50836181640625, "step": 667 }, { "KL/chosen_KL_mean": -153.27774047851562, "KL/mean": -205.93966674804688, "KL/rejected_KL_mean": -258.60162353515625, "KL/std": 101.70606994628906, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.006745823193341494, "fcm_dpo/delta": -0.11879193782806396, "fcm_dpo/margin": 105.3238525390625, "fcm_dpo/q_t": 0.34798485040664673, "grad_norm": 13.517265319824219, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.452906608581543, "logits/rejected": -1.941100001335144, "logps/chosen": -255.85279846191406, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -367.4193115234375, "loss": 0.9112, "margin_dpo/margin_mean": 105.3238525390625, "margin_dpo/margin_std": 108.94068145751953, "step": 668 }, { "KL/chosen_KL_mean": -139.63104248046875, "KL/mean": -190.40127563476562, "KL/rejected_KL_mean": -241.17153930664062, "KL/std": 97.60295104980469, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.006616000086069107, "fcm_dpo/delta": -0.07694902271032333, "fcm_dpo/margin": 101.54048156738281, "fcm_dpo/q_t": 0.35119104385375977, "grad_norm": 11.759913444519043, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.7834802865982056, "logits/rejected": -1.3853893280029297, "logps/chosen": -229.12582397460938, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -323.6910400390625, "loss": 0.9205, "margin_dpo/margin_mean": 101.54048156738281, "margin_dpo/margin_std": 99.7247085571289, "step": 669 }, { "KL/chosen_KL_mean": -139.7811737060547, "KL/mean": -195.49923706054688, "KL/rejected_KL_mean": -251.21726989746094, "KL/std": 99.77938079833984, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.006465866696089506, "fcm_dpo/delta": -0.12983879446983337, "fcm_dpo/margin": 111.43609619140625, "fcm_dpo/q_t": 0.3381672203540802, "grad_norm": 9.688863754272461, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.8412344455718994, "logits/rejected": -1.3089523315429688, "logps/chosen": -235.23577880859375, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -352.75018310546875, "loss": 0.8885, "margin_dpo/margin_mean": 111.43609619140625, "margin_dpo/margin_std": 102.03959655761719, "step": 670 }, { "KL/chosen_KL_mean": -172.14599609375, "KL/mean": -209.2783203125, "KL/rejected_KL_mean": -246.41065979003906, "KL/std": 91.93510437011719, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.00657990388572216, "fcm_dpo/delta": 0.11502684652805328, "fcm_dpo/margin": 74.26466369628906, "fcm_dpo/q_t": 0.3923582136631012, "grad_norm": 11.69428825378418, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.7768421173095703, "logits/rejected": -1.556856632232666, "logps/chosen": -254.26913452148438, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -336.63037109375, "loss": 1.0602, "margin_dpo/margin_mean": 74.26466369628906, "margin_dpo/margin_std": 100.87190246582031, "step": 671 }, { "KL/chosen_KL_mean": -170.41151428222656, "KL/mean": -203.3292694091797, "KL/rejected_KL_mean": -236.24700927734375, "KL/std": 90.0072021484375, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0067269327118992805, "fcm_dpo/delta": 0.16452553868293762, "fcm_dpo/margin": 65.83549499511719, "fcm_dpo/q_t": 0.4019850492477417, "grad_norm": 13.540038108825684, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.153766632080078, "logits/rejected": -1.8220579624176025, "logps/chosen": -260.4734191894531, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -336.70025634765625, "loss": 1.0869, "margin_dpo/margin_mean": 65.83549499511719, "margin_dpo/margin_std": 93.11683654785156, "step": 672 }, { "KL/chosen_KL_mean": -151.94189453125, "KL/mean": -189.88955688476562, "KL/rejected_KL_mean": -227.83721923828125, "KL/std": 91.64775085449219, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.006891964469105005, "fcm_dpo/delta": 0.08073317259550095, "fcm_dpo/margin": 75.89530944824219, "fcm_dpo/q_t": 0.3837796747684479, "grad_norm": 13.549480438232422, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.9158183336257935, "logits/rejected": -1.543032169342041, "logps/chosen": -231.20504760742188, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -319.18646240234375, "loss": 1.0214, "margin_dpo/margin_mean": 75.89530181884766, "margin_dpo/margin_std": 91.94528198242188, "step": 673 }, { "KL/chosen_KL_mean": -143.9180450439453, "KL/mean": -195.2746124267578, "KL/rejected_KL_mean": -246.63119506835938, "KL/std": 93.14541625976562, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.00677870586514473, "fcm_dpo/delta": -0.10380436480045319, "fcm_dpo/margin": 102.71315002441406, "fcm_dpo/q_t": 0.34793785214424133, "grad_norm": 10.500953674316406, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.5134446620941162, "logits/rejected": -1.1061439514160156, "logps/chosen": -219.37635803222656, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -322.8348388671875, "loss": 0.904, "margin_dpo/margin_mean": 102.7131576538086, "margin_dpo/margin_std": 99.90264129638672, "step": 674 }, { "KL/chosen_KL_mean": -159.94601440429688, "KL/mean": -205.0975341796875, "KL/rejected_KL_mean": -250.24908447265625, "KL/std": 99.26661682128906, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.006694559007883072, "fcm_dpo/delta": -0.006357654929161072, "fcm_dpo/margin": 90.30307006835938, "fcm_dpo/q_t": 0.3679022490978241, "grad_norm": 13.010387420654297, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.4409756660461426, "logits/rejected": -1.0996551513671875, "logps/chosen": -239.8555450439453, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -331.46734619140625, "loss": 1.0148, "margin_dpo/margin_mean": 90.30307006835938, "margin_dpo/margin_std": 118.03688049316406, "step": 675 }, { "KL/chosen_KL_mean": -170.8087158203125, "KL/mean": -216.36708068847656, "KL/rejected_KL_mean": -261.9254150390625, "KL/std": 96.16000366210938, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.006710603833198547, "fcm_dpo/delta": -0.012545458972454071, "fcm_dpo/margin": 91.11669921875, "fcm_dpo/q_t": 0.3650050163269043, "grad_norm": 11.123711585998535, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.9316630363464355, "logits/rejected": -1.344076156616211, "logps/chosen": -268.9798583984375, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -356.9500732421875, "loss": 0.9817, "margin_dpo/margin_mean": 91.11669921875, "margin_dpo/margin_std": 105.396484375, "step": 676 }, { "KL/chosen_KL_mean": -153.04042053222656, "KL/mean": -200.0087127685547, "KL/rejected_KL_mean": -246.97698974609375, "KL/std": 94.11901092529297, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.006684846710413694, "fcm_dpo/delta": -0.03003844805061817, "fcm_dpo/margin": 93.93658447265625, "fcm_dpo/q_t": 0.35938170552253723, "grad_norm": 10.223237037658691, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.734102487564087, "logits/rejected": -1.0533475875854492, "logps/chosen": -244.41970825195312, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -329.8547668457031, "loss": 0.9359, "margin_dpo/margin_mean": 93.93658447265625, "margin_dpo/margin_std": 89.13810729980469, "step": 677 }, { "KL/chosen_KL_mean": -144.5040283203125, "KL/mean": -200.08770751953125, "KL/rejected_KL_mean": -255.67135620117188, "KL/std": 100.24153137207031, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.006594943813979626, "fcm_dpo/delta": -0.1436343640089035, "fcm_dpo/margin": 111.16732788085938, "fcm_dpo/q_t": 0.3370254933834076, "grad_norm": 10.624103546142578, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.707273006439209, "logits/rejected": -1.3784533739089966, "logps/chosen": -220.14990234375, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -342.6374816894531, "loss": 0.8789, "margin_dpo/margin_mean": 111.16732788085938, "margin_dpo/margin_std": 98.06687927246094, "step": 678 }, { "KL/chosen_KL_mean": -165.6080322265625, "KL/mean": -210.481689453125, "KL/rejected_KL_mean": -255.35531616210938, "KL/std": 81.93534088134766, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.006514072418212891, "fcm_dpo/delta": 0.01633685640990734, "fcm_dpo/margin": 89.74730682373047, "fcm_dpo/q_t": 0.3685380518436432, "grad_norm": 10.37939453125, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.2724196910858154, "logits/rejected": -1.7573773860931396, "logps/chosen": -246.3814697265625, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -338.23382568359375, "loss": 0.9535, "margin_dpo/margin_mean": 89.74730682373047, "margin_dpo/margin_std": 85.75448608398438, "step": 679 }, { "KL/chosen_KL_mean": -168.92379760742188, "KL/mean": -215.94813537597656, "KL/rejected_KL_mean": -262.97247314453125, "KL/std": 99.12411499023438, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.006572544574737549, "fcm_dpo/delta": -0.021431609988212585, "fcm_dpo/margin": 94.04866027832031, "fcm_dpo/q_t": 0.36517924070358276, "grad_norm": 9.254494667053223, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9595396518707275, "logits/rejected": -1.6371653079986572, "logps/chosen": -276.60675048828125, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -379.06732177734375, "loss": 0.9741, "margin_dpo/margin_mean": 94.04866027832031, "margin_dpo/margin_std": 103.90563201904297, "step": 680 }, { "KL/chosen_KL_mean": -151.71780395507812, "KL/mean": -193.87814331054688, "KL/rejected_KL_mean": -236.03846740722656, "KL/std": 94.12771606445312, "epoch": 1.0, "fcm_dpo/beta": 0.0065569630824029446, "fcm_dpo/delta": 0.0496843159198761, "fcm_dpo/margin": 84.3206787109375, "fcm_dpo/q_t": 0.37772709131240845, "grad_norm": 10.958782196044922, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.3419761657714844, "logits/rejected": -2.0358190536499023, "logps/chosen": -244.72886657714844, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -330.86065673828125, "loss": 1.0063, "margin_dpo/margin_mean": 84.3206787109375, "margin_dpo/margin_std": 100.86186218261719, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0154666916389297, "train_runtime": 2022.2966, "train_samples_per_second": 21.559, "train_steps_per_second": 0.337 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }