{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.818904876708984, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.756305694580078, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.023387908935546875, "KL/mean": 0.010738633573055267, "KL/rejected_KL_mean": -0.001911163330078125, "KL/std": 0.2956864833831787, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025298938155174255, "fcm_dpo/q_t": 0.49936774373054504, "grad_norm": 26.003320693969727, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4270673096179962, "logits/rejected": 0.6503190994262695, "logps/chosen": -93.78759765625, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.23141479492188, "loss": 1.3843, "margin_dpo/margin_mean": 0.02529902756214142, "margin_dpo/margin_std": 0.4536363482475281, "step": 3 }, { "KL/chosen_KL_mean": -0.0164947509765625, "KL/mean": 0.009604737162590027, "KL/rejected_KL_mean": 0.035701751708984375, "KL/std": 0.3031823933124542, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.052198439836502075, "fcm_dpo/q_t": 0.5013041496276855, "grad_norm": 32.9527587890625, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6463137269020081, "logits/rejected": 0.7780138254165649, "logps/chosen": -87.3089599609375, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.78855895996094, "loss": 1.3919, "margin_dpo/margin_mean": -0.052198320627212524, "margin_dpo/margin_std": 0.3748226463794708, "step": 4 }, { "KL/chosen_KL_mean": 0.019054412841796875, "KL/mean": -0.03161180764436722, "KL/rejected_KL_mean": -0.08228683471679688, "KL/std": 0.2768011689186096, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10134117305278778, "fcm_dpo/q_t": 0.4974674880504608, "grad_norm": 34.702606201171875, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4460350275039673, "logits/rejected": 0.6576099395751953, "logps/chosen": -89.31770324707031, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.83012390136719, "loss": 1.3766, "margin_dpo/margin_mean": 0.10134080052375793, "margin_dpo/margin_std": 0.37679600715637207, "step": 5 }, { "KL/chosen_KL_mean": 0.004322052001953125, "KL/mean": 0.004646636545658112, "KL/rejected_KL_mean": 0.004978179931640625, "KL/std": 0.2969215512275696, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0006594210863113403, "fcm_dpo/q_t": 0.500015139579773, "grad_norm": 36.070594787597656, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.7293883562088013, "logits/rejected": 1.0672513246536255, "logps/chosen": -97.31715393066406, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.87847137451172, "loss": 1.3868, "margin_dpo/margin_mean": -0.0006603598594665527, "margin_dpo/margin_std": 0.42816463112831116, "step": 6 }, { "KL/chosen_KL_mean": -0.021518707275390625, "KL/mean": -0.03818559646606445, "KL/rejected_KL_mean": -0.054859161376953125, "KL/std": 0.30022090673446655, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03333783149719238, "fcm_dpo/q_t": 0.4991675615310669, "grad_norm": 37.84027099609375, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.49979984760284424, "logits/rejected": 0.5474007725715637, "logps/chosen": -86.67005157470703, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.67103576660156, "loss": 1.3835, "margin_dpo/margin_mean": 0.03333774209022522, "margin_dpo/margin_std": 0.46710824966430664, "step": 7 }, { "KL/chosen_KL_mean": -0.030429840087890625, "KL/mean": 0.0012157410383224487, "KL/rejected_KL_mean": 0.03285980224609375, "KL/std": 0.3102671504020691, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06328606605529785, "fcm_dpo/q_t": 0.5015826225280762, "grad_norm": 32.95698928833008, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.34600841999053955, "logits/rejected": 0.6492912769317627, "logps/chosen": -89.95411682128906, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.19517517089844, "loss": 1.3931, "margin_dpo/margin_mean": -0.06328605115413666, "margin_dpo/margin_std": 0.4384044408798218, "step": 8 }, { "KL/chosen_KL_mean": 0.06787490844726562, "KL/mean": 0.019995570182800293, "KL/rejected_KL_mean": -0.027881622314453125, "KL/std": 0.2999148666858673, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09575149416923523, "fcm_dpo/q_t": 0.4976067543029785, "grad_norm": 36.0135498046875, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7142183780670166, "logits/rejected": 0.8972364664077759, "logps/chosen": -103.78926086425781, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.34719848632812, "loss": 1.3771, "margin_dpo/margin_mean": 0.09575074911117554, "margin_dpo/margin_std": 0.36230137944221497, "step": 9 }, { "KL/chosen_KL_mean": 0.010662078857421875, "KL/mean": 0.06309156119823456, "KL/rejected_KL_mean": 0.11552047729492188, "KL/std": 0.28032273054122925, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.10486075282096863, "fcm_dpo/q_t": 0.5026201605796814, "grad_norm": 33.765830993652344, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.6176000833511353, "logits/rejected": 0.834958553314209, "logps/chosen": -76.1937026977539, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.03658294677734, "loss": 1.3972, "margin_dpo/margin_mean": -0.1048608124256134, "margin_dpo/margin_std": 0.4106290638446808, "step": 10 }, { "KL/chosen_KL_mean": -0.024150848388671875, "KL/mean": -0.04581160843372345, "KL/rejected_KL_mean": -0.067474365234375, "KL/std": 0.3222964406013489, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.043324023485183716, "fcm_dpo/q_t": 0.49891823530197144, "grad_norm": 37.01686477661133, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.18801042437553406, "logits/rejected": 0.3988957405090332, "logps/chosen": -82.3906478881836, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.33209228515625, "loss": 1.3825, "margin_dpo/margin_mean": 0.043324440717697144, "margin_dpo/margin_std": 0.4597359597682953, "step": 11 }, { "KL/chosen_KL_mean": 0.0594482421875, "KL/mean": 0.00034242868423461914, "KL/rejected_KL_mean": -0.0587615966796875, "KL/std": 0.3795194625854492, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11820477247238159, "fcm_dpo/q_t": 0.4970482885837555, "grad_norm": 39.987281799316406, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.24496988952159882, "logits/rejected": 0.3676449954509735, "logps/chosen": -99.04605102539062, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.33016967773438, "loss": 1.3753, "margin_dpo/margin_mean": 0.11820492148399353, "margin_dpo/margin_std": 0.533420205116272, "step": 12 }, { "KL/chosen_KL_mean": 0.02411651611328125, "KL/mean": 0.007204167544841766, "KL/rejected_KL_mean": -0.009708404541015625, "KL/std": 0.36061999201774597, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0338258296251297, "fcm_dpo/q_t": 0.49915415048599243, "grad_norm": 41.97040557861328, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.2832522988319397, "logits/rejected": 0.5227876305580139, "logps/chosen": -90.53561401367188, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.7008056640625, "loss": 1.3836, "margin_dpo/margin_mean": 0.03382605314254761, "margin_dpo/margin_std": 0.5202242136001587, "step": 13 }, { "KL/chosen_KL_mean": -0.08250808715820312, "KL/mean": -0.01590893417596817, "KL/rejected_KL_mean": 0.050685882568359375, "KL/std": 0.30720195174217224, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.1332002431154251, "fcm_dpo/q_t": 0.5033278465270996, "grad_norm": 35.9677734375, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.902587890625, "logits/rejected": 0.9918005466461182, "logps/chosen": -99.90968322753906, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.89131164550781, "loss": 1.4001, "margin_dpo/margin_mean": -0.13320046663284302, "margin_dpo/margin_std": 0.4106825590133667, "step": 14 }, { "KL/chosen_KL_mean": 0.027408599853515625, "KL/mean": 0.026289641857147217, "KL/rejected_KL_mean": 0.025173187255859375, "KL/std": 0.3058145046234131, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.002240121364593506, "fcm_dpo/q_t": 0.49994394183158875, "grad_norm": 31.969240188598633, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.32524988055229187, "logits/rejected": 0.4884376525878906, "logps/chosen": -78.88256072998047, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.03717041015625, "loss": 1.3865, "margin_dpo/margin_mean": 0.0022400319576263428, "margin_dpo/margin_std": 0.40432286262512207, "step": 15 }, { "KL/chosen_KL_mean": 0.0057373046875, "KL/mean": 0.009102389216423035, "KL/rejected_KL_mean": 0.012462615966796875, "KL/std": 0.3465569019317627, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006727665662765503, "fcm_dpo/q_t": 0.5001651048660278, "grad_norm": 34.717857360839844, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.5384898781776428, "logits/rejected": 0.7929645776748657, "logps/chosen": -97.41754150390625, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.58699798583984, "loss": 1.3876, "margin_dpo/margin_mean": -0.00672796368598938, "margin_dpo/margin_std": 0.5135875940322876, "step": 16 }, { "KL/chosen_KL_mean": 0.029277801513671875, "KL/mean": 0.013889744877815247, "KL/rejected_KL_mean": -0.001499176025390625, "KL/std": 0.35586756467819214, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.030777662992477417, "fcm_dpo/q_t": 0.4992300271987915, "grad_norm": 35.604637145996094, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.4608882963657379, "logits/rejected": 0.688048243522644, "logps/chosen": -104.33502960205078, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.46922302246094, "loss": 1.3839, "margin_dpo/margin_mean": 0.03077736496925354, "margin_dpo/margin_std": 0.4947991967201233, "step": 17 }, { "KL/chosen_KL_mean": -0.06040191650390625, "KL/mean": -0.04041974991559982, "KL/rejected_KL_mean": -0.020435333251953125, "KL/std": 0.3180393874645233, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.039970383048057556, "fcm_dpo/q_t": 0.5009981393814087, "grad_norm": 41.891845703125, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.5752621293067932, "logits/rejected": 0.940882682800293, "logps/chosen": -87.15235900878906, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.87115478515625, "loss": 1.3908, "margin_dpo/margin_mean": -0.039969831705093384, "margin_dpo/margin_std": 0.4589792490005493, "step": 18 }, { "KL/chosen_KL_mean": -0.0795745849609375, "KL/mean": -0.08217489719390869, "KL/rejected_KL_mean": -0.08477783203125, "KL/std": 0.29568102955818176, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005194127559661865, "fcm_dpo/q_t": 0.499871164560318, "grad_norm": 31.30753517150879, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6364111304283142, "logits/rejected": 0.9154881238937378, "logps/chosen": -105.95311737060547, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -97.0150146484375, "loss": 1.3862, "margin_dpo/margin_mean": 0.0051940977573394775, "margin_dpo/margin_std": 0.4034244120121002, "step": 19 }, { "KL/chosen_KL_mean": 0.032978057861328125, "KL/mean": -0.006814613938331604, "KL/rejected_KL_mean": -0.04660797119140625, "KL/std": 0.28955304622650146, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07958313822746277, "fcm_dpo/q_t": 0.49801105260849, "grad_norm": 32.589141845703125, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.27756258845329285, "logits/rejected": 0.5554755926132202, "logps/chosen": -90.72514343261719, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.95892333984375, "loss": 1.3787, "margin_dpo/margin_mean": 0.07958298921585083, "margin_dpo/margin_std": 0.3736102879047394, "step": 20 }, { "KL/chosen_KL_mean": 0.03226470947265625, "KL/mean": -0.008472487330436707, "KL/rejected_KL_mean": -0.049213409423828125, "KL/std": 0.24999001622200012, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08147940039634705, "fcm_dpo/q_t": 0.49796390533447266, "grad_norm": 31.579269409179688, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5736658573150635, "logits/rejected": 0.7546557188034058, "logps/chosen": -80.30119323730469, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.98294830322266, "loss": 1.3785, "margin_dpo/margin_mean": 0.08147972822189331, "margin_dpo/margin_std": 0.34619683027267456, "step": 21 }, { "KL/chosen_KL_mean": -0.02725982666015625, "KL/mean": -0.03122386336326599, "KL/rejected_KL_mean": -0.035182952880859375, "KL/std": 0.383489727973938, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007913768291473389, "fcm_dpo/q_t": 0.4998001158237457, "grad_norm": 42.92274475097656, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.5900495052337646, "logits/rejected": 0.7075623273849487, "logps/chosen": -95.42256164550781, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.50869750976562, "loss": 1.3862, "margin_dpo/margin_mean": 0.007913529872894287, "margin_dpo/margin_std": 0.5263677835464478, "step": 22 }, { "KL/chosen_KL_mean": 0.00165557861328125, "KL/mean": -0.003539755940437317, "KL/rejected_KL_mean": -0.008731842041015625, "KL/std": 0.3166351914405823, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.010387152433395386, "fcm_dpo/q_t": 0.4997401833534241, "grad_norm": 31.37350082397461, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.8437352776527405, "logits/rejected": 1.0287058353424072, "logps/chosen": -90.63585662841797, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.60298919677734, "loss": 1.3857, "margin_dpo/margin_mean": 0.010387152433395386, "margin_dpo/margin_std": 0.4008484184741974, "step": 23 }, { "KL/chosen_KL_mean": -0.016431808471679688, "KL/mean": -0.023091726005077362, "KL/rejected_KL_mean": -0.029750823974609375, "KL/std": 0.28409337997436523, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013323381543159485, "fcm_dpo/q_t": 0.4996671676635742, "grad_norm": 44.422889709472656, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.6848082542419434, "logits/rejected": 0.7146521806716919, "logps/chosen": -69.93372344970703, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.6396484375, "loss": 1.3854, "margin_dpo/margin_mean": 0.013324141502380371, "margin_dpo/margin_std": 0.40111660957336426, "step": 24 }, { "KL/chosen_KL_mean": -0.000293731689453125, "KL/mean": -0.04660933464765549, "KL/rejected_KL_mean": -0.09293365478515625, "KL/std": 0.3227173089981079, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09263212978839874, "fcm_dpo/q_t": 0.4976845383644104, "grad_norm": 36.841094970703125, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.4053041338920593, "logits/rejected": 0.5679289698600769, "logps/chosen": -80.82577514648438, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.05004119873047, "loss": 1.3775, "margin_dpo/margin_mean": 0.092631995677948, "margin_dpo/margin_std": 0.43129491806030273, "step": 25 }, { "KL/chosen_KL_mean": -0.043743133544921875, "KL/mean": -0.04767926037311554, "KL/rejected_KL_mean": -0.05161285400390625, "KL/std": 0.34445613622665405, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007874280214309692, "fcm_dpo/q_t": 0.4998001456260681, "grad_norm": 41.15323257446289, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.44155511260032654, "logits/rejected": 0.6535449028015137, "logps/chosen": -88.94491577148438, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.87980651855469, "loss": 1.3861, "margin_dpo/margin_mean": 0.007874369621276855, "margin_dpo/margin_std": 0.4711059331893921, "step": 26 }, { "KL/chosen_KL_mean": 0.008525848388671875, "KL/mean": -0.04025181382894516, "KL/rejected_KL_mean": -0.0890350341796875, "KL/std": 0.325300931930542, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09756244719028473, "fcm_dpo/q_t": 0.4975633919239044, "grad_norm": 44.07526397705078, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.07119900733232498, "logits/rejected": 0.11365430057048798, "logps/chosen": -77.58748626708984, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.02754211425781, "loss": 1.3771, "margin_dpo/margin_mean": 0.0975627601146698, "margin_dpo/margin_std": 0.45653122663497925, "step": 27 }, { "KL/chosen_KL_mean": 0.040985107421875, "KL/mean": -0.003113970160484314, "KL/rejected_KL_mean": -0.047206878662109375, "KL/std": 0.30647993087768555, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0881892740726471, "fcm_dpo/q_t": 0.49779510498046875, "grad_norm": 36.52345275878906, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.6403088569641113, "logits/rejected": 0.8656172156333923, "logps/chosen": -102.18758392333984, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.00669860839844, "loss": 1.378, "margin_dpo/margin_mean": 0.0881899893283844, "margin_dpo/margin_std": 0.4476761817932129, "step": 28 }, { "KL/chosen_KL_mean": 0.0053253173828125, "KL/mean": -0.06289912015199661, "KL/rejected_KL_mean": -0.1311187744140625, "KL/std": 0.29980552196502686, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13645385205745697, "fcm_dpo/q_t": 0.4965905547142029, "grad_norm": 42.413578033447266, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.34219616651535034, "logits/rejected": 0.6024200916290283, "logps/chosen": -88.6417236328125, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.09123229980469, "loss": 1.3731, "margin_dpo/margin_mean": 0.13645510375499725, "margin_dpo/margin_std": 0.4073440730571747, "step": 29 }, { "KL/chosen_KL_mean": -0.000919342041015625, "KL/mean": -0.06342763453722, "KL/rejected_KL_mean": -0.12594223022460938, "KL/std": 0.3242136538028717, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12502776086330414, "fcm_dpo/q_t": 0.4968774914741516, "grad_norm": 38.961124420166016, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.4537328779697418, "logits/rejected": 0.5650418996810913, "logps/chosen": -88.3893051147461, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.44483947753906, "loss": 1.3743, "margin_dpo/margin_mean": 0.12502801418304443, "margin_dpo/margin_std": 0.44598862528800964, "step": 30 }, { "KL/chosen_KL_mean": -0.01622772216796875, "KL/mean": -0.06588798761367798, "KL/rejected_KL_mean": -0.11554718017578125, "KL/std": 0.30449333786964417, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09931725263595581, "fcm_dpo/q_t": 0.4975195527076721, "grad_norm": 31.207820892333984, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.7096586227416992, "logits/rejected": 0.9546472430229187, "logps/chosen": -101.14189147949219, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.97396850585938, "loss": 1.3768, "margin_dpo/margin_mean": 0.09931749105453491, "margin_dpo/margin_std": 0.41561049222946167, "step": 31 }, { "KL/chosen_KL_mean": -0.0418853759765625, "KL/mean": -0.1025150939822197, "KL/rejected_KL_mean": -0.16314315795898438, "KL/std": 0.3464288115501404, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1212615817785263, "fcm_dpo/q_t": 0.4969692826271057, "grad_norm": 35.18325424194336, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.5164635181427002, "logits/rejected": 0.663323700428009, "logps/chosen": -96.6589126586914, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.38751220703125, "loss": 1.3749, "margin_dpo/margin_mean": 0.12126168608665466, "margin_dpo/margin_std": 0.5217796564102173, "step": 32 }, { "KL/chosen_KL_mean": 0.054073333740234375, "KL/mean": 0.009096838533878326, "KL/rejected_KL_mean": -0.03588104248046875, "KL/std": 0.33035674691200256, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08994944393634796, "fcm_dpo/q_t": 0.4977532625198364, "grad_norm": 33.768409729003906, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5179184079170227, "logits/rejected": 0.6965822577476501, "logps/chosen": -81.4669418334961, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.84183502197266, "loss": 1.3778, "margin_dpo/margin_mean": 0.08994972705841064, "margin_dpo/margin_std": 0.45166170597076416, "step": 33 }, { "KL/chosen_KL_mean": 0.06499481201171875, "KL/mean": -0.050529010593891144, "KL/rejected_KL_mean": -0.16604995727539062, "KL/std": 0.3994750380516052, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23105041682720184, "fcm_dpo/q_t": 0.4942375123500824, "grad_norm": 40.79511642456055, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.4207673668861389, "logits/rejected": 0.5935499668121338, "logps/chosen": -77.13703918457031, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.88481140136719, "loss": 1.3641, "margin_dpo/margin_mean": 0.23105067014694214, "margin_dpo/margin_std": 0.5517414808273315, "step": 34 }, { "KL/chosen_KL_mean": -0.03212738037109375, "KL/mean": -0.1007416844367981, "KL/rejected_KL_mean": -0.1693572998046875, "KL/std": 0.3389318883419037, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1372414231300354, "fcm_dpo/q_t": 0.4965740144252777, "grad_norm": 42.6923713684082, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.3400149345397949, "logits/rejected": 0.4948399066925049, "logps/chosen": -77.60247802734375, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.35791015625, "loss": 1.3733, "margin_dpo/margin_mean": 0.13724187016487122, "margin_dpo/margin_std": 0.5151667594909668, "step": 35 }, { "KL/chosen_KL_mean": -0.0783843994140625, "KL/mean": -0.14783477783203125, "KL/rejected_KL_mean": -0.21728515625, "KL/std": 0.4493231773376465, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13890428841114044, "fcm_dpo/q_t": 0.49653494358062744, "grad_norm": 32.5205192565918, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.36493980884552, "logits/rejected": 0.5516271591186523, "logps/chosen": -83.36662292480469, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.97470092773438, "loss": 1.3735, "margin_dpo/margin_mean": 0.1389044225215912, "margin_dpo/margin_std": 0.633370041847229, "step": 36 }, { "KL/chosen_KL_mean": -0.04113006591796875, "KL/mean": -0.17032013833522797, "KL/rejected_KL_mean": -0.2995147705078125, "KL/std": 0.4677668809890747, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25837984681129456, "fcm_dpo/q_t": 0.4935556650161743, "grad_norm": 35.754600524902344, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.43330591917037964, "logits/rejected": 0.5744740962982178, "logps/chosen": -94.81222534179688, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01678466796875, "loss": 1.3619, "margin_dpo/margin_mean": 0.2583801746368408, "margin_dpo/margin_std": 0.7177779078483582, "step": 37 }, { "KL/chosen_KL_mean": 0.08501815795898438, "KL/mean": -0.040863797068595886, "KL/rejected_KL_mean": -0.16674041748046875, "KL/std": 0.42607080936431885, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2517622411251068, "fcm_dpo/q_t": 0.49371832609176636, "grad_norm": 36.91084671020508, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.263674259185791, "logits/rejected": 0.3999466300010681, "logps/chosen": -75.89348602294922, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.27075958251953, "loss": 1.3623, "margin_dpo/margin_mean": 0.25176334381103516, "margin_dpo/margin_std": 0.6318477392196655, "step": 38 }, { "KL/chosen_KL_mean": -0.0670166015625, "KL/mean": -0.19623425602912903, "KL/rejected_KL_mean": -0.3254547119140625, "KL/std": 0.38931113481521606, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25843316316604614, "fcm_dpo/q_t": 0.49354708194732666, "grad_norm": 36.221702575683594, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.5231233835220337, "logits/rejected": 0.7136045694351196, "logps/chosen": -81.25279235839844, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.52137756347656, "loss": 1.3614, "margin_dpo/margin_mean": 0.25843340158462524, "margin_dpo/margin_std": 0.5414197444915771, "step": 39 }, { "KL/chosen_KL_mean": 0.009578704833984375, "KL/mean": -0.11047288775444031, "KL/rejected_KL_mean": -0.23051834106445312, "KL/std": 0.37187638878822327, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24009698629379272, "fcm_dpo/q_t": 0.4940013289451599, "grad_norm": 29.124425888061523, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.6448565125465393, "logits/rejected": 0.8434960842132568, "logps/chosen": -83.322998046875, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.48643493652344, "loss": 1.3629, "margin_dpo/margin_mean": 0.24009615182876587, "margin_dpo/margin_std": 0.44556406140327454, "step": 40 }, { "KL/chosen_KL_mean": -0.028778076171875, "KL/mean": -0.18186727166175842, "KL/rejected_KL_mean": -0.3349571228027344, "KL/std": 0.4871253967285156, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.30618101358413696, "fcm_dpo/q_t": 0.49235761165618896, "grad_norm": 33.40583419799805, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6443256735801697, "logits/rejected": 0.8323005437850952, "logps/chosen": -93.17744445800781, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.41416931152344, "loss": 1.3571, "margin_dpo/margin_mean": 0.30618149042129517, "margin_dpo/margin_std": 0.6774824857711792, "step": 41 }, { "KL/chosen_KL_mean": -0.027935028076171875, "KL/mean": -0.19727501273155212, "KL/rejected_KL_mean": -0.3666191101074219, "KL/std": 0.5067838430404663, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.33867692947387695, "fcm_dpo/q_t": 0.4915480613708496, "grad_norm": 40.983646392822266, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.46488580107688904, "logits/rejected": 0.6078572273254395, "logps/chosen": -90.72955322265625, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.23126220703125, "loss": 1.3538, "margin_dpo/margin_mean": 0.3386765122413635, "margin_dpo/margin_std": 0.6516327261924744, "step": 42 }, { "KL/chosen_KL_mean": -0.013671875, "KL/mean": -0.23529259860515594, "KL/rejected_KL_mean": -0.4569129943847656, "KL/std": 0.5269670486450195, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4432455599308014, "fcm_dpo/q_t": 0.48896199464797974, "grad_norm": 37.6937141418457, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.5572470426559448, "logits/rejected": 0.747031569480896, "logps/chosen": -89.65769958496094, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.40876770019531, "loss": 1.3441, "margin_dpo/margin_mean": 0.4432455897331238, "margin_dpo/margin_std": 0.7670128345489502, "step": 43 }, { "KL/chosen_KL_mean": -0.034244537353515625, "KL/mean": -0.26017698645591736, "KL/rejected_KL_mean": -0.486114501953125, "KL/std": 0.5511221289634705, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.45187342166900635, "fcm_dpo/q_t": 0.48871976137161255, "grad_norm": 34.282066345214844, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.5257748365402222, "logits/rejected": 0.6549081206321716, "logps/chosen": -81.91256713867188, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.90774536132812, "loss": 1.3425, "margin_dpo/margin_mean": 0.45187410712242126, "margin_dpo/margin_std": 0.5888241529464722, "step": 44 }, { "KL/chosen_KL_mean": -0.10094070434570312, "KL/mean": -0.18020549416542053, "KL/rejected_KL_mean": -0.2594718933105469, "KL/std": 0.3995981216430664, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1585323065519333, "fcm_dpo/q_t": 0.4960416853427887, "grad_norm": 24.59613037109375, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.48959749937057495, "logits/rejected": 0.5982831120491028, "logps/chosen": -77.44554138183594, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.50721740722656, "loss": 1.3713, "margin_dpo/margin_mean": 0.15853281319141388, "margin_dpo/margin_std": 0.5481212735176086, "step": 45 }, { "KL/chosen_KL_mean": -0.038299560546875, "KL/mean": -0.24129709601402283, "KL/rejected_KL_mean": -0.4442863464355469, "KL/std": 0.5440940260887146, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40598946809768677, "fcm_dpo/q_t": 0.48986393213272095, "grad_norm": 33.16312026977539, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.29832375049591064, "logits/rejected": 0.5310325026512146, "logps/chosen": -90.37911987304688, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.12080383300781, "loss": 1.3471, "margin_dpo/margin_mean": 0.40598899126052856, "margin_dpo/margin_std": 0.6098673343658447, "step": 46 }, { "KL/chosen_KL_mean": 0.042675018310546875, "KL/mean": -0.1954789161682129, "KL/rejected_KL_mean": -0.43363189697265625, "KL/std": 0.5276778340339661, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4763045310974121, "fcm_dpo/q_t": 0.4881129860877991, "grad_norm": 33.9340705871582, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.3664342761039734, "logits/rejected": 0.5427783727645874, "logps/chosen": -104.36862182617188, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.68852996826172, "loss": 1.3404, "margin_dpo/margin_mean": 0.47630399465560913, "margin_dpo/margin_std": 0.6682124137878418, "step": 47 }, { "KL/chosen_KL_mean": -0.04547119140625, "KL/mean": -0.30655574798583984, "KL/rejected_KL_mean": -0.5676383972167969, "KL/std": 0.6637288331985474, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.522168755531311, "fcm_dpo/q_t": 0.4869752526283264, "grad_norm": 39.10186004638672, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.29671046137809753, "logits/rejected": 0.5020701885223389, "logps/chosen": -91.14574432373047, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.6482162475586, "loss": 1.3364, "margin_dpo/margin_mean": 0.5221691727638245, "margin_dpo/margin_std": 0.8175289630889893, "step": 48 }, { "KL/chosen_KL_mean": 0.011516571044921875, "KL/mean": -0.3529120087623596, "KL/rejected_KL_mean": -0.7173423767089844, "KL/std": 0.773455798625946, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7288556098937988, "fcm_dpo/q_t": 0.481844961643219, "grad_norm": 39.57054138183594, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.1616140753030777, "logits/rejected": 0.6406388878822327, "logps/chosen": -92.9921646118164, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.46633911132812, "loss": 1.3171, "margin_dpo/margin_mean": 0.7288557291030884, "margin_dpo/margin_std": 0.9757804870605469, "step": 49 }, { "KL/chosen_KL_mean": -0.07786941528320312, "KL/mean": -0.40695878863334656, "KL/rejected_KL_mean": -0.7360343933105469, "KL/std": 0.7618446350097656, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6581700444221497, "fcm_dpo/q_t": 0.4836057424545288, "grad_norm": 33.17576217651367, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.42797011137008667, "logits/rejected": 0.5943090319633484, "logps/chosen": -94.70468139648438, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.31039428710938, "loss": 1.3238, "margin_dpo/margin_mean": 0.6581701040267944, "margin_dpo/margin_std": 0.9293594360351562, "step": 50 }, { "KL/chosen_KL_mean": -0.03026580810546875, "KL/mean": -0.38186758756637573, "KL/rejected_KL_mean": -0.7334671020507812, "KL/std": 0.8783245086669922, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.703199028968811, "fcm_dpo/q_t": 0.48251873254776, "grad_norm": 29.88286018371582, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.7391107082366943, "logits/rejected": 0.8871128559112549, "logps/chosen": -87.53754425048828, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.20582580566406, "loss": 1.3202, "margin_dpo/margin_mean": 0.7031992077827454, "margin_dpo/margin_std": 1.0769977569580078, "step": 51 }, { "KL/chosen_KL_mean": -0.02695465087890625, "KL/mean": -0.46134573221206665, "KL/rejected_KL_mean": -0.8957481384277344, "KL/std": 0.7852196097373962, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8687930107116699, "fcm_dpo/q_t": 0.47836267948150635, "grad_norm": 32.15827941894531, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.3686307370662689, "logits/rejected": 0.5769591331481934, "logps/chosen": -90.65721893310547, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.934814453125, "loss": 1.3035, "margin_dpo/margin_mean": 0.868792712688446, "margin_dpo/margin_std": 0.9224597215652466, "step": 52 }, { "KL/chosen_KL_mean": -0.007266998291015625, "KL/mean": -0.564792811870575, "KL/rejected_KL_mean": -1.1223220825195312, "KL/std": 1.1043574810028076, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.115053415298462, "fcm_dpo/q_t": 0.47237205505371094, "grad_norm": 39.23210906982422, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.0012142658233642578, "logits/rejected": 0.1092616617679596, "logps/chosen": -81.59034729003906, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.78385162353516, "loss": 1.2828, "margin_dpo/margin_mean": 1.1150538921356201, "margin_dpo/margin_std": 1.3047435283660889, "step": 53 }, { "KL/chosen_KL_mean": -0.001567840576171875, "KL/mean": -0.5511602163314819, "KL/rejected_KL_mean": -1.1007499694824219, "KL/std": 1.2253997325897217, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0991829633712769, "fcm_dpo/q_t": 0.4726998209953308, "grad_norm": 39.8622932434082, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.34692272543907166, "logits/rejected": 0.5243130922317505, "logps/chosen": -88.91173553466797, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.21830749511719, "loss": 1.2835, "margin_dpo/margin_mean": 1.0991826057434082, "margin_dpo/margin_std": 1.2754499912261963, "step": 54 }, { "KL/chosen_KL_mean": 0.136749267578125, "KL/mean": -0.4430486559867859, "KL/rejected_KL_mean": -1.0228424072265625, "KL/std": 1.1745827198028564, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1595911979675293, "fcm_dpo/q_t": 0.47130468487739563, "grad_norm": 35.34449005126953, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.0935109406709671, "logits/rejected": 0.39841151237487793, "logps/chosen": -92.31916809082031, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -93.9937744140625, "loss": 1.2798, "margin_dpo/margin_mean": 1.1595909595489502, "margin_dpo/margin_std": 1.5505378246307373, "step": 55 }, { "KL/chosen_KL_mean": -0.13799285888671875, "KL/mean": -0.6348596811294556, "KL/rejected_KL_mean": -1.1317214965820312, "KL/std": 1.156689167022705, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.993730902671814, "fcm_dpo/q_t": 0.4754391610622406, "grad_norm": 33.31754684448242, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.0948457196354866, "logits/rejected": 0.42848700284957886, "logps/chosen": -87.37464904785156, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.77726745605469, "loss": 1.296, "margin_dpo/margin_mean": 0.993730902671814, "margin_dpo/margin_std": 1.6280632019042969, "step": 56 }, { "KL/chosen_KL_mean": -0.08787918090820312, "KL/mean": -0.6388577222824097, "KL/rejected_KL_mean": -1.1898384094238281, "KL/std": 1.2657309770584106, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1019731760025024, "fcm_dpo/q_t": 0.4726974368095398, "grad_norm": 33.89845275878906, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.475721538066864, "logits/rejected": 0.7005236148834229, "logps/chosen": -98.23861694335938, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.24267578125, "loss": 1.2847, "margin_dpo/margin_mean": 1.101973295211792, "margin_dpo/margin_std": 1.4976792335510254, "step": 57 }, { "KL/chosen_KL_mean": 0.2559051513671875, "KL/mean": -0.49702948331832886, "KL/rejected_KL_mean": -1.2499618530273438, "KL/std": 1.3708233833312988, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.505864143371582, "fcm_dpo/q_t": 0.4629475772380829, "grad_norm": 38.31918716430664, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.4398585855960846, "logits/rejected": 0.7864245772361755, "logps/chosen": -99.32506561279297, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.47223663330078, "loss": 1.2502, "margin_dpo/margin_mean": 1.5058640241622925, "margin_dpo/margin_std": 1.7969675064086914, "step": 58 }, { "KL/chosen_KL_mean": -0.050567626953125, "KL/mean": -0.6296656131744385, "KL/rejected_KL_mean": -1.2087593078613281, "KL/std": 1.121105670928955, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1581916809082031, "fcm_dpo/q_t": 0.4712894558906555, "grad_norm": 34.28953552246094, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.17875322699546814, "logits/rejected": 0.37366271018981934, "logps/chosen": -89.85289001464844, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.48543548583984, "loss": 1.279, "margin_dpo/margin_mean": 1.1581910848617554, "margin_dpo/margin_std": 1.4262681007385254, "step": 59 }, { "KL/chosen_KL_mean": -0.000759124755859375, "KL/mean": -0.5722290873527527, "KL/rejected_KL_mean": -1.1437034606933594, "KL/std": 1.2497632503509521, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1429455280303955, "fcm_dpo/q_t": 0.4716324806213379, "grad_norm": 28.717697143554688, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.3798879384994507, "logits/rejected": 0.6497697234153748, "logps/chosen": -95.1564712524414, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.51614379882812, "loss": 1.2801, "margin_dpo/margin_mean": 1.1429455280303955, "margin_dpo/margin_std": 1.3910194635391235, "step": 60 }, { "KL/chosen_KL_mean": -0.235382080078125, "KL/mean": -0.813821017742157, "KL/rejected_KL_mean": -1.3922576904296875, "KL/std": 1.4219188690185547, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1568758487701416, "fcm_dpo/q_t": 0.4716115891933441, "grad_norm": 28.384286880493164, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.41926372051239014, "logits/rejected": 0.5636248588562012, "logps/chosen": -85.96768951416016, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.35801696777344, "loss": 1.283, "margin_dpo/margin_mean": 1.1568758487701416, "margin_dpo/margin_std": 1.860975980758667, "step": 61 }, { "KL/chosen_KL_mean": -0.16065597534179688, "KL/mean": -0.756861686706543, "KL/rejected_KL_mean": -1.3530731201171875, "KL/std": 1.4292011260986328, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1924232244491577, "fcm_dpo/q_t": 0.4705609381198883, "grad_norm": 28.452159881591797, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.3273305296897888, "logits/rejected": 0.5312330722808838, "logps/chosen": -81.79603576660156, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.39138793945312, "loss": 1.2807, "margin_dpo/margin_mean": 1.1924223899841309, "margin_dpo/margin_std": 1.887484073638916, "step": 62 }, { "KL/chosen_KL_mean": 0.00029754638671875, "KL/mean": -0.784099817276001, "KL/rejected_KL_mean": -1.5685005187988281, "KL/std": 1.3912111520767212, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5688031911849976, "fcm_dpo/q_t": 0.46117645502090454, "grad_norm": 34.983123779296875, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.549911379814148, "logits/rejected": 0.7721988558769226, "logps/chosen": -103.62376403808594, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.97152709960938, "loss": 1.242, "margin_dpo/margin_mean": 1.568803071975708, "margin_dpo/margin_std": 1.5832934379577637, "step": 63 }, { "KL/chosen_KL_mean": -0.16749954223632812, "KL/mean": -1.0290528535842896, "KL/rejected_KL_mean": -1.8906135559082031, "KL/std": 1.8355145454406738, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10139448940753937, "fcm_dpo/delta": 0.0692426934838295, "fcm_dpo/margin": 1.7231167554855347, "fcm_dpo/q_t": 0.45756059885025024, "grad_norm": 35.4733772277832, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.6644800901412964, "logits/rejected": 0.8888744711875916, "logps/chosen": -87.16908264160156, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.47608947753906, "loss": 1.2309, "margin_dpo/margin_mean": 1.723116397857666, "margin_dpo/margin_std": 2.0706851482391357, "step": 64 }, { "KL/chosen_KL_mean": -0.10486602783203125, "KL/mean": -0.9995708465576172, "KL/rejected_KL_mean": -1.8942832946777344, "KL/std": 1.688509225845337, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10139448940753937, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.789414882659912, "fcm_dpo/q_t": 0.45547154545783997, "grad_norm": 38.618289947509766, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.11418561637401581, "logits/rejected": 0.029267169535160065, "logps/chosen": -91.3267822265625, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.22981262207031, "loss": 1.2245, "margin_dpo/margin_mean": 1.7894150018692017, "margin_dpo/margin_std": 2.1142139434814453, "step": 65 }, { "KL/chosen_KL_mean": -0.3168983459472656, "KL/mean": -0.97967928647995, "KL/rejected_KL_mean": -1.6424636840820312, "KL/std": 1.6315829753875732, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10139448940753937, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.325562596321106, "fcm_dpo/q_t": 0.4669954478740692, "grad_norm": 25.854677200317383, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.13995397090911865, "logits/rejected": 0.34246283769607544, "logps/chosen": -84.10111999511719, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.39456176757812, "loss": 1.2665, "margin_dpo/margin_mean": 1.3255627155303955, "margin_dpo/margin_std": 1.9230554103851318, "step": 66 }, { "KL/chosen_KL_mean": -0.8661651611328125, "KL/mean": -1.4250576496124268, "KL/rejected_KL_mean": -1.9839515686035156, "KL/std": 1.9923858642578125, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10139448940753937, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1177891492843628, "fcm_dpo/q_t": 0.4726470708847046, "grad_norm": 24.6008358001709, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.3365855813026428, "logits/rejected": 0.6116951107978821, "logps/chosen": -88.53912353515625, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.0707015991211, "loss": 1.2948, "margin_dpo/margin_mean": 1.1177890300750732, "margin_dpo/margin_std": 2.703024387359619, "step": 67 }, { "KL/chosen_KL_mean": -0.24016952514648438, "KL/mean": -1.2630674839019775, "KL/rejected_KL_mean": -2.2859573364257812, "KL/std": 2.185187816619873, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10279335081577301, "fcm_dpo/delta": 0.06850970536470413, "fcm_dpo/margin": 2.0457961559295654, "fcm_dpo/q_t": 0.4497773051261902, "grad_norm": 28.97216033935547, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.03972816839814186, "logits/rejected": 0.25442707538604736, "logps/chosen": -97.31901550292969, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.4046859741211, "loss": 1.2071, "margin_dpo/margin_mean": 2.0457961559295654, "margin_dpo/margin_std": 2.7876174449920654, "step": 68 }, { "KL/chosen_KL_mean": -0.3192901611328125, "KL/mean": -1.4097886085510254, "KL/rejected_KL_mean": -2.5002899169921875, "KL/std": 2.44561767578125, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.10483173280954361, "fcm_dpo/delta": 0.12404301762580872, "fcm_dpo/margin": 2.1809980869293213, "fcm_dpo/q_t": 0.4456480145454407, "grad_norm": 31.76175308227539, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.3194381594657898, "logits/rejected": 0.4281793236732483, "logps/chosen": -86.03900146484375, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.98049926757812, "loss": 1.1943, "margin_dpo/margin_mean": 2.180999279022217, "margin_dpo/margin_std": 2.9800069332122803, "step": 69 }, { "KL/chosen_KL_mean": -0.6002616882324219, "KL/mean": -1.8815968036651611, "KL/rejected_KL_mean": -3.162933349609375, "KL/std": 2.759925603866577, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.10665775835514069, "fcm_dpo/delta": 0.07879804074764252, "fcm_dpo/margin": 2.562671661376953, "fcm_dpo/q_t": 0.4358280897140503, "grad_norm": 30.406497955322266, "learning_rate": 5e-07, "logits/chosen": 0.2655646800994873, "logits/rejected": 0.443000853061676, "logps/chosen": -95.61021423339844, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.37565612792969, "loss": 1.1635, "margin_dpo/margin_mean": 2.562671661376953, "margin_dpo/margin_std": 3.3409719467163086, "step": 70 }, { "KL/chosen_KL_mean": -0.045772552490234375, "KL/mean": -1.9278644323349, "KL/rejected_KL_mean": -3.809955596923828, "KL/std": 3.593599319458008, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.10591454803943634, "fcm_dpo/delta": -0.051049306988716125, "fcm_dpo/margin": 3.764186382293701, "fcm_dpo/q_t": 0.40720999240875244, "grad_norm": 35.11409378051758, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.09832069277763367, "logits/rejected": 0.2702038288116455, "logps/chosen": -91.02312469482422, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.40098571777344, "loss": 1.0697, "margin_dpo/margin_mean": 3.764185905456543, "margin_dpo/margin_std": 4.114628791809082, "step": 71 }, { "KL/chosen_KL_mean": -0.5525970458984375, "KL/mean": -1.9819421768188477, "KL/rejected_KL_mean": -3.4112930297851562, "KL/std": 2.8850550651550293, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.10658645629882812, "fcm_dpo/delta": 0.046751730144023895, "fcm_dpo/margin": 2.858701229095459, "fcm_dpo/q_t": 0.4274483323097229, "grad_norm": 30.89877700805664, "learning_rate": 4.999868246217933e-07, "logits/chosen": 0.17424926161766052, "logits/rejected": 0.5288213491439819, "logps/chosen": -98.44412231445312, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.60301208496094, "loss": 1.1338, "margin_dpo/margin_mean": 2.858700752258301, "margin_dpo/margin_std": 3.2687463760375977, "step": 72 }, { "KL/chosen_KL_mean": -0.7794876098632812, "KL/mean": -2.3231043815612793, "KL/rejected_KL_mean": -3.8667221069335938, "KL/std": 3.8273966312408447, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.10722959041595459, "fcm_dpo/delta": 0.019610531628131866, "fcm_dpo/margin": 3.087235927581787, "fcm_dpo/q_t": 0.4260888993740082, "grad_norm": 29.200252532958984, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.4935552179813385, "logits/rejected": -0.2012936770915985, "logps/chosen": -96.54850769042969, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.79916381835938, "loss": 1.1465, "margin_dpo/margin_mean": 3.087235927581787, "margin_dpo/margin_std": 4.968820571899414, "step": 73 }, { "KL/chosen_KL_mean": -1.0069122314453125, "KL/mean": -2.7828691005706787, "KL/rejected_KL_mean": -4.558826446533203, "KL/std": 3.8079068660736084, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.10672082751989365, "fcm_dpo/delta": -0.030310316011309624, "fcm_dpo/margin": 3.551910400390625, "fcm_dpo/q_t": 0.41235238313674927, "grad_norm": 31.894132614135742, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.27401676774024963, "logits/rejected": -0.13221508264541626, "logps/chosen": -79.8153076171875, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.20559692382812, "loss": 1.0966, "margin_dpo/margin_mean": 3.551910400390625, "margin_dpo/margin_std": 4.516332626342773, "step": 74 }, { "KL/chosen_KL_mean": -1.0713386535644531, "KL/mean": -3.339719772338867, "KL/rejected_KL_mean": -5.608104705810547, "KL/std": 4.549798011779785, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.10504598915576935, "fcm_dpo/delta": -0.13269458711147308, "fcm_dpo/margin": 4.536768913269043, "fcm_dpo/q_t": 0.3928720951080322, "grad_norm": 33.14641571044922, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.20562829077243805, "logits/rejected": -0.004346504807472229, "logps/chosen": -79.35319519042969, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.01121520996094, "loss": 1.0362, "margin_dpo/margin_mean": 4.536768913269043, "margin_dpo/margin_std": 5.380740642547607, "step": 75 }, { "KL/chosen_KL_mean": -1.4899826049804688, "KL/mean": -3.0548512935638428, "KL/rejected_KL_mean": -4.619720458984375, "KL/std": 3.861638069152832, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.10427361726760864, "fcm_dpo/delta": 0.02445552870631218, "fcm_dpo/margin": 3.129734516143799, "fcm_dpo/q_t": 0.4254528284072876, "grad_norm": 23.580114364624023, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6873964071273804, "logits/rejected": -0.3151237368583679, "logps/chosen": -89.36712646484375, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.33940124511719, "loss": 1.1408, "margin_dpo/margin_mean": 3.129734516143799, "margin_dpo/margin_std": 4.669698238372803, "step": 76 }, { "KL/chosen_KL_mean": -1.6541862487792969, "KL/mean": -3.3420519828796387, "KL/rejected_KL_mean": -5.029914855957031, "KL/std": 4.441540718078613, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.1042061597108841, "fcm_dpo/delta": -0.0019206497818231583, "fcm_dpo/margin": 3.3757307529449463, "fcm_dpo/q_t": 0.4192841351032257, "grad_norm": 27.99580192565918, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.09842056781053543, "logits/rejected": 0.28008967638015747, "logps/chosen": -101.35453796386719, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.23567199707031, "loss": 1.1309, "margin_dpo/margin_mean": 3.375730514526367, "margin_dpo/margin_std": 5.072809219360352, "step": 77 }, { "KL/chosen_KL_mean": -0.6558341979980469, "KL/mean": -2.4886598587036133, "KL/rejected_KL_mean": -4.321479797363281, "KL/std": 3.729641914367676, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.1038551926612854, "fcm_dpo/delta": -0.031932283192873, "fcm_dpo/margin": 3.665645122528076, "fcm_dpo/q_t": 0.4116262197494507, "grad_norm": 25.61400032043457, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.0882125273346901, "logits/rejected": 0.3234598636627197, "logps/chosen": -90.95254516601562, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.4592056274414, "loss": 1.091, "margin_dpo/margin_mean": 3.6656455993652344, "margin_dpo/margin_std": 4.4862260818481445, "step": 78 }, { "KL/chosen_KL_mean": -0.8090019226074219, "KL/mean": -3.247934579849243, "KL/rejected_KL_mean": -5.6868743896484375, "KL/std": 5.422076225280762, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.10156351327896118, "fcm_dpo/delta": -0.152810201048851, "fcm_dpo/margin": 4.877870559692383, "fcm_dpo/q_t": 0.3887721300125122, "grad_norm": 26.971776962280273, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.46887022256851196, "logits/rejected": -0.07375195622444153, "logps/chosen": -87.18733215332031, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.79464721679688, "loss": 1.0324, "margin_dpo/margin_mean": 4.877870559692383, "margin_dpo/margin_std": 6.032390117645264, "step": 79 }, { "KL/chosen_KL_mean": -1.5632705688476562, "KL/mean": -3.40891170501709, "KL/rejected_KL_mean": -5.254550933837891, "KL/std": 4.644263744354248, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.100312739610672, "fcm_dpo/delta": -0.021059703081846237, "fcm_dpo/margin": 3.691276788711548, "fcm_dpo/q_t": 0.415848970413208, "grad_norm": 25.73511505126953, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.6991344094276428, "logits/rejected": -0.34425169229507446, "logps/chosen": -95.53358459472656, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.82896423339844, "loss": 1.1088, "margin_dpo/margin_mean": 3.691276788711548, "margin_dpo/margin_std": 5.062727451324463, "step": 80 }, { "KL/chosen_KL_mean": -1.0953369140625, "KL/mean": -3.704251289367676, "KL/rejected_KL_mean": -6.313152313232422, "KL/std": 5.730771064758301, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.09722723066806793, "fcm_dpo/delta": -0.1671447902917862, "fcm_dpo/margin": 5.217823028564453, "fcm_dpo/q_t": 0.3895528316497803, "grad_norm": 27.52021598815918, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.4917618930339813, "logits/rejected": -0.06423325836658478, "logps/chosen": -100.92546081542969, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -139.98562622070312, "loss": 1.0355, "margin_dpo/margin_mean": 5.217823028564453, "margin_dpo/margin_std": 6.809062957763672, "step": 81 }, { "KL/chosen_KL_mean": -2.026813507080078, "KL/mean": -3.8705193996429443, "KL/rejected_KL_mean": -5.714221954345703, "KL/std": 4.605215072631836, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.09731516987085342, "fcm_dpo/delta": -0.009781494736671448, "fcm_dpo/margin": 3.687406301498413, "fcm_dpo/q_t": 0.4172978401184082, "grad_norm": 25.63909339904785, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.18969501554965973, "logits/rejected": 0.028823871165513992, "logps/chosen": -85.07279968261719, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.24017333984375, "loss": 1.1182, "margin_dpo/margin_mean": 3.687405586242676, "margin_dpo/margin_std": 5.041373252868652, "step": 82 }, { "KL/chosen_KL_mean": -2.40423583984375, "KL/mean": -4.514264106750488, "KL/rejected_KL_mean": -6.624290466308594, "KL/std": 5.223086357116699, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.09584876894950867, "fcm_dpo/delta": -0.05681682005524635, "fcm_dpo/margin": 4.220065116882324, "fcm_dpo/q_t": 0.41026076674461365, "grad_norm": 25.85201072692871, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6005710363388062, "logits/rejected": -0.17953182756900787, "logps/chosen": -94.58045196533203, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.36893463134766, "loss": 1.1116, "margin_dpo/margin_mean": 4.220065116882324, "margin_dpo/margin_std": 6.576836109161377, "step": 83 }, { "KL/chosen_KL_mean": -2.9761505126953125, "KL/mean": -4.302863597869873, "KL/rejected_KL_mean": -5.62957763671875, "KL/std": 4.731132984161377, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.09510910511016846, "fcm_dpo/delta": -0.02661805786192417, "fcm_dpo/margin": 2.6534271240234375, "fcm_dpo/q_t": 0.44257473945617676, "grad_norm": 27.1372013092041, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.36952438950538635, "logits/rejected": 0.027232438325881958, "logps/chosen": -104.50263977050781, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.0556640625, "loss": 1.2088, "margin_dpo/margin_mean": 2.653428077697754, "margin_dpo/margin_std": 5.027728080749512, "step": 84 }, { "KL/chosen_KL_mean": -2.6829147338867188, "KL/mean": -4.421186923980713, "KL/rejected_KL_mean": -6.159458160400391, "KL/std": 5.156990051269531, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.09572692960500717, "fcm_dpo/delta": 0.017420435324311256, "fcm_dpo/margin": 3.4765474796295166, "fcm_dpo/q_t": 0.4250291585922241, "grad_norm": 25.948848724365234, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.3868050277233124, "logits/rejected": -0.05528336018323898, "logps/chosen": -98.81028747558594, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.21464538574219, "loss": 1.166, "margin_dpo/margin_mean": 3.476547956466675, "margin_dpo/margin_std": 6.274026870727539, "step": 85 }, { "KL/chosen_KL_mean": -2.1462631225585938, "KL/mean": -4.347163200378418, "KL/rejected_KL_mean": -6.548065185546875, "KL/std": 5.885907173156738, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.09438899159431458, "fcm_dpo/delta": -0.06839875876903534, "fcm_dpo/margin": 4.401805877685547, "fcm_dpo/q_t": 0.4117809534072876, "grad_norm": 21.302061080932617, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.5865161418914795, "logits/rejected": -0.3425723612308502, "logps/chosen": -83.85052490234375, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.20356750488281, "loss": 1.1146, "margin_dpo/margin_mean": 4.401805877685547, "margin_dpo/margin_std": 7.349972724914551, "step": 86 }, { "KL/chosen_KL_mean": -2.4608192443847656, "KL/mean": -4.924564361572266, "KL/rejected_KL_mean": -7.388301849365234, "KL/std": 6.633878707885742, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.09233886003494263, "fcm_dpo/delta": -0.1105005070567131, "fcm_dpo/margin": 4.927480697631836, "fcm_dpo/q_t": 0.39787834882736206, "grad_norm": 24.893230438232422, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.9215160608291626, "logits/rejected": -0.5508020520210266, "logps/chosen": -95.14678955078125, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.30648040771484, "loss": 1.0974, "margin_dpo/margin_mean": 4.927481174468994, "margin_dpo/margin_std": 7.818390846252441, "step": 87 }, { "KL/chosen_KL_mean": -3.0319290161132812, "KL/mean": -4.9255170822143555, "KL/rejected_KL_mean": -6.819095611572266, "KL/std": 6.308513641357422, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.0915941447019577, "fcm_dpo/delta": 0.002568814903497696, "fcm_dpo/margin": 3.7871644496917725, "fcm_dpo/q_t": 0.42249244451522827, "grad_norm": 22.95734977722168, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.2744530439376831, "logits/rejected": -0.08346293121576309, "logps/chosen": -103.79490661621094, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.96946716308594, "loss": 1.1534, "margin_dpo/margin_mean": 3.7871646881103516, "margin_dpo/margin_std": 6.490657806396484, "step": 88 }, { "KL/chosen_KL_mean": -2.386798858642578, "KL/mean": -4.489686012268066, "KL/rejected_KL_mean": -6.5925750732421875, "KL/std": 5.350257873535156, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.09140130877494812, "fcm_dpo/delta": -0.036023057997226715, "fcm_dpo/margin": 4.205777645111084, "fcm_dpo/q_t": 0.416969358921051, "grad_norm": 22.42654037475586, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.6200143694877625, "logits/rejected": -0.2757211923599243, "logps/chosen": -92.077880859375, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.16090393066406, "loss": 1.1255, "margin_dpo/margin_mean": 4.205777645111084, "margin_dpo/margin_std": 7.0749616622924805, "step": 89 }, { "KL/chosen_KL_mean": -2.1324539184570312, "KL/mean": -4.854391098022461, "KL/rejected_KL_mean": -7.576320648193359, "KL/std": 6.304018020629883, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.08985205739736557, "fcm_dpo/delta": -0.1460399329662323, "fcm_dpo/margin": 5.4438700675964355, "fcm_dpo/q_t": 0.39336252212524414, "grad_norm": 25.00754737854004, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.5265606641769409, "logits/rejected": -0.4013633131980896, "logps/chosen": -83.5150146484375, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.35346221923828, "loss": 1.057, "margin_dpo/margin_mean": 5.443870544433594, "margin_dpo/margin_std": 7.612011909484863, "step": 90 }, { "KL/chosen_KL_mean": -2.538330078125, "KL/mean": -4.833251953125, "KL/rejected_KL_mean": -7.128170013427734, "KL/std": 6.2117438316345215, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.08816000819206238, "fcm_dpo/delta": -0.05691235512495041, "fcm_dpo/margin": 4.589841842651367, "fcm_dpo/q_t": 0.4123058021068573, "grad_norm": 23.74753761291504, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.2958667576313019, "logits/rejected": 0.13216130435466766, "logps/chosen": -99.76020812988281, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.10491943359375, "loss": 1.1128, "margin_dpo/margin_mean": 4.589841365814209, "margin_dpo/margin_std": 7.322835445404053, "step": 91 }, { "KL/chosen_KL_mean": -2.7231979370117188, "KL/mean": -4.748151779174805, "KL/rejected_KL_mean": -6.773101806640625, "KL/std": 6.136410713195801, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.08784030377864838, "fcm_dpo/delta": -0.00596965616568923, "fcm_dpo/margin": 4.049901962280273, "fcm_dpo/q_t": 0.420840859413147, "grad_norm": 21.455860137939453, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.9186328649520874, "logits/rejected": -0.5581549406051636, "logps/chosen": -87.25347900390625, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.27442169189453, "loss": 1.1377, "margin_dpo/margin_mean": 4.049901962280273, "margin_dpo/margin_std": 6.61488151550293, "step": 92 }, { "KL/chosen_KL_mean": -2.9775352478027344, "KL/mean": -4.9512939453125, "KL/rejected_KL_mean": -6.925056457519531, "KL/std": 5.842714309692383, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.08789533376693726, "fcm_dpo/delta": 0.0030937613919377327, "fcm_dpo/margin": 3.9475252628326416, "fcm_dpo/q_t": 0.4226209819316864, "grad_norm": 23.67386245727539, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.9560178518295288, "logits/rejected": -0.3847508132457733, "logps/chosen": -93.22796630859375, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -91.01927185058594, "loss": 1.1595, "margin_dpo/margin_mean": 3.9475252628326416, "margin_dpo/margin_std": 7.102597236633301, "step": 93 }, { "KL/chosen_KL_mean": -3.073925018310547, "KL/mean": -5.990779876708984, "KL/rejected_KL_mean": -8.907630920410156, "KL/std": 6.763996124267578, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.08511851727962494, "fcm_dpo/delta": -0.15656156837940216, "fcm_dpo/margin": 5.833703517913818, "fcm_dpo/q_t": 0.38814353942871094, "grad_norm": 23.80930519104004, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.6005295515060425, "logits/rejected": -0.29256099462509155, "logps/chosen": -95.52814483642578, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.15492248535156, "loss": 1.052, "margin_dpo/margin_mean": 5.83370304107666, "margin_dpo/margin_std": 7.788494110107422, "step": 94 }, { "KL/chosen_KL_mean": -2.566295623779297, "KL/mean": -4.896753787994385, "KL/rejected_KL_mean": -7.227203369140625, "KL/std": 5.7789506912231445, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.08482074737548828, "fcm_dpo/delta": -0.04734663665294647, "fcm_dpo/margin": 4.660906791687012, "fcm_dpo/q_t": 0.41282984614372253, "grad_norm": 20.1112003326416, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8500373959541321, "logits/rejected": -0.508613109588623, "logps/chosen": -81.44000244140625, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.81842041015625, "loss": 1.113, "margin_dpo/margin_mean": 4.6609063148498535, "margin_dpo/margin_std": 7.096644878387451, "step": 95 }, { "KL/chosen_KL_mean": -2.2249526977539062, "KL/mean": -5.056510925292969, "KL/rejected_KL_mean": -7.888065338134766, "KL/std": 6.842815399169922, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.08254378288984299, "fcm_dpo/delta": -0.12362420558929443, "fcm_dpo/margin": 5.66312313079834, "fcm_dpo/q_t": 0.3960241675376892, "grad_norm": 20.8452205657959, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.6390334367752075, "logits/rejected": -0.17932797968387604, "logps/chosen": -108.8171157836914, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.65609741210938, "loss": 1.0601, "margin_dpo/margin_mean": 5.663122653961182, "margin_dpo/margin_std": 7.532233238220215, "step": 96 }, { "KL/chosen_KL_mean": -3.3381195068359375, "KL/mean": -5.9296875, "KL/rejected_KL_mean": -8.521251678466797, "KL/std": 6.045007228851318, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.08183018118143082, "fcm_dpo/delta": -0.07783898711204529, "fcm_dpo/margin": 5.183134078979492, "fcm_dpo/q_t": 0.40284299850463867, "grad_norm": 23.164897918701172, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.5162617564201355, "logits/rejected": -0.11061953753232956, "logps/chosen": -92.01800537109375, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.33354187011719, "loss": 1.0748, "margin_dpo/margin_mean": 5.183133602142334, "margin_dpo/margin_std": 6.442495346069336, "step": 97 }, { "KL/chosen_KL_mean": -2.483776092529297, "KL/mean": -5.439664840698242, "KL/rejected_KL_mean": -8.395553588867188, "KL/std": 5.801393985748291, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.07908406108617783, "fcm_dpo/delta": -0.12435489892959595, "fcm_dpo/margin": 5.911775588989258, "fcm_dpo/q_t": 0.3928989768028259, "grad_norm": 21.540822982788086, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.7344412803649902, "logits/rejected": -0.4690203070640564, "logps/chosen": -94.72627258300781, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.9150390625, "loss": 1.0352, "margin_dpo/margin_mean": 5.911775588989258, "margin_dpo/margin_std": 6.517614364624023, "step": 98 }, { "KL/chosen_KL_mean": -1.87371826171875, "KL/mean": -5.167026519775391, "KL/rejected_KL_mean": -8.460338592529297, "KL/std": 6.939482688903809, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.07697322964668274, "fcm_dpo/delta": -0.16553995013237, "fcm_dpo/margin": 6.586621284484863, "fcm_dpo/q_t": 0.3904907703399658, "grad_norm": 21.246572494506836, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.5023515224456787, "logits/rejected": -0.2898815870285034, "logps/chosen": -74.05836486816406, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.34165954589844, "loss": 1.0352, "margin_dpo/margin_mean": 6.586621284484863, "margin_dpo/margin_std": 8.755260467529297, "step": 99 }, { "KL/chosen_KL_mean": -2.2669830322265625, "KL/mean": -4.591638565063477, "KL/rejected_KL_mean": -6.916297912597656, "KL/std": 5.266744136810303, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.07607864588499069, "fcm_dpo/delta": -0.004105091094970703, "fcm_dpo/margin": 4.649309158325195, "fcm_dpo/q_t": 0.4173815846443176, "grad_norm": 18.144428253173828, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.7095901370048523, "logits/rejected": -0.22270354628562927, "logps/chosen": -91.78080749511719, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -88.1334228515625, "loss": 1.1071, "margin_dpo/margin_mean": 4.649308681488037, "margin_dpo/margin_std": 5.635429382324219, "step": 100 }, { "KL/chosen_KL_mean": -3.6976585388183594, "KL/mean": -5.748890399932861, "KL/rejected_KL_mean": -7.8001251220703125, "KL/std": 6.906227111816406, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.07642011344432831, "fcm_dpo/delta": 0.03744722902774811, "fcm_dpo/margin": 4.10246467590332, "fcm_dpo/q_t": 0.42777490615844727, "grad_norm": 21.825387954711914, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.5674424171447754, "logits/rejected": -0.2781991958618164, "logps/chosen": -98.1008529663086, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.71014404296875, "loss": 1.1736, "margin_dpo/margin_mean": 4.10246467590332, "margin_dpo/margin_std": 7.3672943115234375, "step": 101 }, { "KL/chosen_KL_mean": -4.232578277587891, "KL/mean": -6.027883529663086, "KL/rejected_KL_mean": -7.823184967041016, "KL/std": 6.668657302856445, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.07657626271247864, "fcm_dpo/delta": -0.04209199547767639, "fcm_dpo/margin": 3.5906057357788086, "fcm_dpo/q_t": 0.43709588050842285, "grad_norm": 25.945026397705078, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.9546716213226318, "logits/rejected": -0.5920330286026001, "logps/chosen": -95.84200286865234, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.59210205078125, "loss": 1.213, "margin_dpo/margin_mean": 3.590606212615967, "margin_dpo/margin_std": 7.614558219909668, "step": 102 }, { "KL/chosen_KL_mean": -3.2196044921875, "KL/mean": -6.32435417175293, "KL/rejected_KL_mean": -9.429107666015625, "KL/std": 6.99813985824585, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.07475174218416214, "fcm_dpo/delta": -0.1200585663318634, "fcm_dpo/margin": 6.209504127502441, "fcm_dpo/q_t": 0.3936261236667633, "grad_norm": 20.863868713378906, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.7048640251159668, "logits/rejected": -0.3003638982772827, "logps/chosen": -100.62199401855469, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.41079711914062, "loss": 1.0567, "margin_dpo/margin_mean": 6.209505081176758, "margin_dpo/margin_std": 7.9016923904418945, "step": 103 }, { "KL/chosen_KL_mean": -2.5501327514648438, "KL/mean": -5.828807830810547, "KL/rejected_KL_mean": -9.107479095458984, "KL/std": 7.560544490814209, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.0728510171175003, "fcm_dpo/delta": -0.13439568877220154, "fcm_dpo/margin": 6.557347297668457, "fcm_dpo/q_t": 0.3954158425331116, "grad_norm": 17.711952209472656, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.0180977582931519, "logits/rejected": -0.6655277609825134, "logps/chosen": -97.406494140625, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -106.13417053222656, "loss": 1.053, "margin_dpo/margin_mean": 6.557348251342773, "margin_dpo/margin_std": 8.977165222167969, "step": 104 }, { "KL/chosen_KL_mean": -3.4648704528808594, "KL/mean": -6.718199253082275, "KL/rejected_KL_mean": -9.971527099609375, "KL/std": 6.907395362854004, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.07151034474372864, "fcm_dpo/delta": -0.1207953542470932, "fcm_dpo/margin": 6.506649017333984, "fcm_dpo/q_t": 0.39465856552124023, "grad_norm": 20.835006713867188, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.9848097562789917, "logits/rejected": -0.5035390257835388, "logps/chosen": -100.22277069091797, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.54901885986328, "loss": 1.0608, "margin_dpo/margin_mean": 6.506649017333984, "margin_dpo/margin_std": 8.597970962524414, "step": 105 }, { "KL/chosen_KL_mean": -3.339824676513672, "KL/mean": -6.5228271484375, "KL/rejected_KL_mean": -9.705821990966797, "KL/std": 7.2118425369262695, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.06991012394428253, "fcm_dpo/delta": -0.09935948997735977, "fcm_dpo/margin": 6.365994453430176, "fcm_dpo/q_t": 0.4011589288711548, "grad_norm": 18.261722564697266, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.8636033535003662, "logits/rejected": -0.4333842396736145, "logps/chosen": -95.7777328491211, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.71552276611328, "loss": 1.0734, "margin_dpo/margin_mean": 6.365994453430176, "margin_dpo/margin_std": 8.751908302307129, "step": 106 }, { "KL/chosen_KL_mean": -2.5301284790039062, "KL/mean": -6.219145774841309, "KL/rejected_KL_mean": -9.908164978027344, "KL/std": 9.108587265014648, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.06770555675029755, "fcm_dpo/delta": -0.15749549865722656, "fcm_dpo/margin": 7.378036975860596, "fcm_dpo/q_t": 0.39625290036201477, "grad_norm": 18.47820281982422, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.7350085973739624, "logits/rejected": -0.20037809014320374, "logps/chosen": -107.65513610839844, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -111.09066772460938, "loss": 1.0608, "margin_dpo/margin_mean": 7.378037452697754, "margin_dpo/margin_std": 11.334686279296875, "step": 107 }, { "KL/chosen_KL_mean": -3.674163818359375, "KL/mean": -7.204038619995117, "KL/rejected_KL_mean": -10.73391342163086, "KL/std": 8.58387565612793, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.0657070130109787, "fcm_dpo/delta": -0.12037976831197739, "fcm_dpo/margin": 7.059749126434326, "fcm_dpo/q_t": 0.3972952365875244, "grad_norm": 18.625791549682617, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.0117151737213135, "logits/rejected": -0.5060229897499084, "logps/chosen": -92.88352966308594, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -112.57176208496094, "loss": 1.0629, "margin_dpo/margin_mean": 7.059749603271484, "margin_dpo/margin_std": 9.428449630737305, "step": 108 }, { "KL/chosen_KL_mean": -2.991596221923828, "KL/mean": -7.262105464935303, "KL/rejected_KL_mean": -11.532623291015625, "KL/std": 8.540365219116211, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.0634729340672493, "fcm_dpo/delta": -0.20383548736572266, "fcm_dpo/margin": 8.541025161743164, "fcm_dpo/q_t": 0.3797753155231476, "grad_norm": 17.538047790527344, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.7754598259925842, "logits/rejected": -0.5477880835533142, "logps/chosen": -103.17868041992188, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -115.07002258300781, "loss": 1.0124, "margin_dpo/margin_mean": 8.541025161743164, "margin_dpo/margin_std": 10.587366104125977, "step": 109 }, { "KL/chosen_KL_mean": -3.875316619873047, "KL/mean": -6.411931037902832, "KL/rejected_KL_mean": -8.94854736328125, "KL/std": 7.0772528648376465, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.0625348836183548, "fcm_dpo/delta": -0.03540291264653206, "fcm_dpo/margin": 5.073228359222412, "fcm_dpo/q_t": 0.42672330141067505, "grad_norm": 18.63017463684082, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.31317082047462463, "logits/rejected": 0.10748951882123947, "logps/chosen": -110.77103424072266, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.75975036621094, "loss": 1.1554, "margin_dpo/margin_mean": 5.073227882385254, "margin_dpo/margin_std": 8.065149307250977, "step": 110 }, { "KL/chosen_KL_mean": -3.425312042236328, "KL/mean": -8.454314231872559, "KL/rejected_KL_mean": -13.483318328857422, "KL/std": 9.922119140625, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.05978306382894516, "fcm_dpo/delta": -0.26926857233047485, "fcm_dpo/margin": 10.058002471923828, "fcm_dpo/q_t": 0.3652436137199402, "grad_norm": 16.215923309326172, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.9643905758857727, "logits/rejected": -0.6720935106277466, "logps/chosen": -96.75468444824219, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -117.66080474853516, "loss": 0.9705, "margin_dpo/margin_mean": 10.058002471923828, "margin_dpo/margin_std": 11.052976608276367, "step": 111 }, { "KL/chosen_KL_mean": -3.964916229248047, "KL/mean": -6.516231536865234, "KL/rejected_KL_mean": -9.06753921508789, "KL/std": 5.842938423156738, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.059595316648483276, "fcm_dpo/delta": 0.04737403988838196, "fcm_dpo/margin": 5.102624893188477, "fcm_dpo/q_t": 0.42947596311569214, "grad_norm": 15.84349250793457, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.1275852918624878, "logits/rejected": -0.595390796661377, "logps/chosen": -86.71195983886719, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -80.62599182128906, "loss": 1.1588, "margin_dpo/margin_mean": 5.102625370025635, "margin_dpo/margin_std": 7.953078269958496, "step": 112 }, { "KL/chosen_KL_mean": -4.950626373291016, "KL/mean": -7.722280025482178, "KL/rejected_KL_mean": -10.493934631347656, "KL/std": 7.667716026306152, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.0600103922188282, "fcm_dpo/delta": 0.017926234751939774, "fcm_dpo/margin": 5.543315887451172, "fcm_dpo/q_t": 0.42304813861846924, "grad_norm": 19.43992042541504, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.938984751701355, "logits/rejected": -0.3901151418685913, "logps/chosen": -102.8501205444336, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -94.91352844238281, "loss": 1.1324, "margin_dpo/margin_mean": 5.543315887451172, "margin_dpo/margin_std": 7.504870414733887, "step": 113 }, { "KL/chosen_KL_mean": -4.623615264892578, "KL/mean": -8.515734672546387, "KL/rejected_KL_mean": -12.407852172851562, "KL/std": 8.945812225341797, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.05887102335691452, "fcm_dpo/delta": -0.11390817165374756, "fcm_dpo/margin": 7.784242153167725, "fcm_dpo/q_t": 0.39992207288742065, "grad_norm": 18.6238956451416, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.2416443824768066, "logits/rejected": -0.6701104640960693, "logps/chosen": -101.11872863769531, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -104.4232177734375, "loss": 1.086, "margin_dpo/margin_mean": 7.784242153167725, "margin_dpo/margin_std": 11.729823112487793, "step": 114 }, { "KL/chosen_KL_mean": -4.824565887451172, "KL/mean": -8.752584457397461, "KL/rejected_KL_mean": -12.68060302734375, "KL/std": 9.566091537475586, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.05777555704116821, "fcm_dpo/delta": -0.10876966267824173, "fcm_dpo/margin": 7.856038570404053, "fcm_dpo/q_t": 0.3983760476112366, "grad_norm": 16.46400260925293, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.9016156792640686, "logits/rejected": -0.48098286986351013, "logps/chosen": -108.00469207763672, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -118.39656066894531, "loss": 1.0678, "margin_dpo/margin_mean": 7.8560380935668945, "margin_dpo/margin_std": 10.47158432006836, "step": 115 }, { "KL/chosen_KL_mean": -4.276515960693359, "KL/mean": -7.578589916229248, "KL/rejected_KL_mean": -10.880657196044922, "KL/std": 7.524150848388672, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.05736490339040756, "fcm_dpo/delta": -0.030128024518489838, "fcm_dpo/margin": 6.604145050048828, "fcm_dpo/q_t": 0.4125102162361145, "grad_norm": 15.849674224853516, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.1804680824279785, "logits/rejected": -0.7526981234550476, "logps/chosen": -85.43489074707031, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -97.18651580810547, "loss": 1.0999, "margin_dpo/margin_mean": 6.604145526885986, "margin_dpo/margin_std": 8.501105308532715, "step": 116 }, { "KL/chosen_KL_mean": -5.2355194091796875, "KL/mean": -7.87876033782959, "KL/rejected_KL_mean": -10.522003173828125, "KL/std": 7.714868545532227, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.05674366652965546, "fcm_dpo/delta": -0.029128016903996468, "fcm_dpo/margin": 5.286484241485596, "fcm_dpo/q_t": 0.430641770362854, "grad_norm": 22.272796630859375, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.884135365486145, "logits/rejected": -0.46612879633903503, "logps/chosen": -102.31647491455078, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -85.66191864013672, "loss": 1.1797, "margin_dpo/margin_mean": 5.2864837646484375, "margin_dpo/margin_std": 9.565271377563477, "step": 117 }, { "KL/chosen_KL_mean": -3.8243370056152344, "KL/mean": -7.633500099182129, "KL/rejected_KL_mean": -11.442657470703125, "KL/std": 8.34169864654541, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.056010037660598755, "fcm_dpo/delta": -0.08023056387901306, "fcm_dpo/margin": 7.618321418762207, "fcm_dpo/q_t": 0.4006776511669159, "grad_norm": 15.016273498535156, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.3599293231964111, "logits/rejected": -1.0341523885726929, "logps/chosen": -90.3605728149414, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -94.3782958984375, "loss": 1.0611, "margin_dpo/margin_mean": 7.618321418762207, "margin_dpo/margin_std": 8.856171607971191, "step": 118 }, { "KL/chosen_KL_mean": -5.725929260253906, "KL/mean": -9.748357772827148, "KL/rejected_KL_mean": -13.77078628540039, "KL/std": 10.038764953613281, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.05491935834288597, "fcm_dpo/delta": -0.09638936817646027, "fcm_dpo/margin": 8.044849395751953, "fcm_dpo/q_t": 0.3997253179550171, "grad_norm": 16.917255401611328, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.4333970546722412, "logits/rejected": -1.1162283420562744, "logps/chosen": -110.33500671386719, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -122.92859649658203, "loss": 1.1009, "margin_dpo/margin_mean": 8.044849395751953, "margin_dpo/margin_std": 12.681705474853516, "step": 119 }, { "KL/chosen_KL_mean": -4.571308135986328, "KL/mean": -9.850593566894531, "KL/rejected_KL_mean": -15.1298828125, "KL/std": 11.86029052734375, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.05294711887836456, "fcm_dpo/delta": -0.22167858481407166, "fcm_dpo/margin": 10.558568954467773, "fcm_dpo/q_t": 0.3793621063232422, "grad_norm": 19.136680603027344, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.4057508707046509, "logits/rejected": -0.8586447238922119, "logps/chosen": -119.25873565673828, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -126.31575012207031, "loss": 1.039, "margin_dpo/margin_mean": 10.558568954467773, "margin_dpo/margin_std": 15.06926155090332, "step": 120 }, { "KL/chosen_KL_mean": -4.1710052490234375, "KL/mean": -9.03736686706543, "KL/rejected_KL_mean": -13.903732299804688, "KL/std": 9.341870307922363, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.05103442072868347, "fcm_dpo/delta": -0.15511475503444672, "fcm_dpo/margin": 9.73273754119873, "fcm_dpo/q_t": 0.3853793442249298, "grad_norm": 16.917587280273438, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.305532455444336, "logits/rejected": -0.9791836142539978, "logps/chosen": -96.28766632080078, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -128.8766326904297, "loss": 1.0141, "margin_dpo/margin_mean": 9.732736587524414, "margin_dpo/margin_std": 10.43644905090332, "step": 121 }, { "KL/chosen_KL_mean": -4.484130859375, "KL/mean": -10.061508178710938, "KL/rejected_KL_mean": -15.638885498046875, "KL/std": 10.169212341308594, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.049420587718486786, "fcm_dpo/delta": -0.2124975621700287, "fcm_dpo/margin": 11.154748916625977, "fcm_dpo/q_t": 0.3736308217048645, "grad_norm": 16.989023208618164, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.1924018859863281, "logits/rejected": -1.098008155822754, "logps/chosen": -89.36170196533203, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -126.72187042236328, "loss": 0.9905, "margin_dpo/margin_mean": 11.154748916625977, "margin_dpo/margin_std": 11.976485252380371, "step": 122 }, { "KL/chosen_KL_mean": -5.073841094970703, "KL/mean": -9.080595016479492, "KL/rejected_KL_mean": -13.087348937988281, "KL/std": 9.257241249084473, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.04813563823699951, "fcm_dpo/delta": -0.03804235905408859, "fcm_dpo/margin": 8.013504981994629, "fcm_dpo/q_t": 0.4109480381011963, "grad_norm": 14.522604942321777, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.2759168148040771, "logits/rejected": -0.9575897455215454, "logps/chosen": -103.5525894165039, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -106.9415283203125, "loss": 1.102, "margin_dpo/margin_mean": 8.013504981994629, "margin_dpo/margin_std": 10.739602088928223, "step": 123 }, { "KL/chosen_KL_mean": -4.854072570800781, "KL/mean": -9.780332565307617, "KL/rejected_KL_mean": -14.706588745117188, "KL/std": 11.589552879333496, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.047443144023418427, "fcm_dpo/delta": -0.1231820285320282, "fcm_dpo/margin": 9.852516174316406, "fcm_dpo/q_t": 0.40023332834243774, "grad_norm": 14.876575469970703, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.2135165929794312, "logits/rejected": -0.9485418796539307, "logps/chosen": -107.61697387695312, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -137.7122039794922, "loss": 1.0811, "margin_dpo/margin_mean": 9.852516174316406, "margin_dpo/margin_std": 15.079033851623535, "step": 124 }, { "KL/chosen_KL_mean": -6.386928558349609, "KL/mean": -9.834383964538574, "KL/rejected_KL_mean": -13.281845092773438, "KL/std": 9.454874038696289, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.04728318750858307, "fcm_dpo/delta": 0.02480250410735607, "fcm_dpo/margin": 6.89491081237793, "fcm_dpo/q_t": 0.4257546067237854, "grad_norm": 15.146885871887207, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.8383991718292236, "logits/rejected": -0.6098995804786682, "logps/chosen": -96.81320190429688, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -102.5308609008789, "loss": 1.1584, "margin_dpo/margin_mean": 6.894911289215088, "margin_dpo/margin_std": 11.633110046386719, "step": 125 }, { "KL/chosen_KL_mean": -5.268161773681641, "KL/mean": -10.460610389709473, "KL/rejected_KL_mean": -15.653064727783203, "KL/std": 10.324319839477539, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.04642191529273987, "fcm_dpo/delta": -0.13873659074306488, "fcm_dpo/margin": 10.384900093078613, "fcm_dpo/q_t": 0.3872610926628113, "grad_norm": 14.729775428771973, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.0370118618011475, "logits/rejected": -0.7533065676689148, "logps/chosen": -99.6910400390625, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -119.53462219238281, "loss": 1.0197, "margin_dpo/margin_mean": 10.384899139404297, "margin_dpo/margin_std": 10.857223510742188, "step": 126 }, { "KL/chosen_KL_mean": -5.377170562744141, "KL/mean": -9.208480834960938, "KL/rejected_KL_mean": -13.039787292480469, "KL/std": 9.427553176879883, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.046155430376529694, "fcm_dpo/delta": -0.0039015356451272964, "fcm_dpo/margin": 7.662618637084961, "fcm_dpo/q_t": 0.4199431538581848, "grad_norm": 14.784599304199219, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.7573096752166748, "logits/rejected": -0.39354532957077026, "logps/chosen": -90.12821960449219, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -96.58502197265625, "loss": 1.1327, "margin_dpo/margin_mean": 7.6626176834106445, "margin_dpo/margin_std": 11.851057052612305, "step": 127 }, { "KL/chosen_KL_mean": -5.373893737792969, "KL/mean": -9.355546951293945, "KL/rejected_KL_mean": -13.337198257446289, "KL/std": 11.683955192565918, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.046129319816827774, "fcm_dpo/delta": -0.018353182822465897, "fcm_dpo/margin": 7.963303565979004, "fcm_dpo/q_t": 0.4164101481437683, "grad_norm": 16.565847396850586, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7623624801635742, "logits/rejected": -0.2705897092819214, "logps/chosen": -95.99353790283203, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -96.440673828125, "loss": 1.1494, "margin_dpo/margin_mean": 7.9633026123046875, "margin_dpo/margin_std": 14.547991752624512, "step": 128 }, { "KL/chosen_KL_mean": -5.888336181640625, "KL/mean": -10.060571670532227, "KL/rejected_KL_mean": -14.23281478881836, "KL/std": 9.486391067504883, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.045767392963171005, "fcm_dpo/delta": -0.03322559967637062, "fcm_dpo/margin": 8.34447193145752, "fcm_dpo/q_t": 0.4153265655040741, "grad_norm": 13.491514205932617, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.7597717046737671, "logits/rejected": -0.44169121980667114, "logps/chosen": -104.80704498291016, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -117.51701354980469, "loss": 1.1236, "margin_dpo/margin_mean": 8.34447193145752, "margin_dpo/margin_std": 12.964457511901855, "step": 129 }, { "KL/chosen_KL_mean": -4.6703643798828125, "KL/mean": -10.085119247436523, "KL/rejected_KL_mean": -15.499874114990234, "KL/std": 11.046258926391602, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.04492930695414543, "fcm_dpo/delta": -0.14356231689453125, "fcm_dpo/margin": 10.82951545715332, "fcm_dpo/q_t": 0.39121997356414795, "grad_norm": 12.920951843261719, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.4051958322525024, "logits/rejected": -1.1697733402252197, "logps/chosen": -77.85404968261719, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -103.52141571044922, "loss": 1.0432, "margin_dpo/margin_mean": 10.82951545715332, "margin_dpo/margin_std": 13.267864227294922, "step": 130 }, { "KL/chosen_KL_mean": -4.097496032714844, "KL/mean": -9.045454025268555, "KL/rejected_KL_mean": -13.993415832519531, "KL/std": 11.060049057006836, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.043949924409389496, "fcm_dpo/delta": -0.08894640952348709, "fcm_dpo/margin": 9.895912170410156, "fcm_dpo/q_t": 0.40065979957580566, "grad_norm": 15.642345428466797, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.2610573768615723, "logits/rejected": -1.0502772331237793, "logps/chosen": -78.05393981933594, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -108.4492416381836, "loss": 1.0724, "margin_dpo/margin_mean": 9.895912170410156, "margin_dpo/margin_std": 12.646656036376953, "step": 131 }, { "KL/chosen_KL_mean": -5.8685455322265625, "KL/mean": -10.756608963012695, "KL/rejected_KL_mean": -15.644664764404297, "KL/std": 11.65256118774414, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.0428718626499176, "fcm_dpo/delta": -0.0726904422044754, "fcm_dpo/margin": 9.776111602783203, "fcm_dpo/q_t": 0.40363162755966187, "grad_norm": 15.035387992858887, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.4740582704544067, "logits/rejected": -1.0587754249572754, "logps/chosen": -98.30802917480469, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -109.77718353271484, "loss": 1.088, "margin_dpo/margin_mean": 9.776111602783203, "margin_dpo/margin_std": 13.517314910888672, "step": 132 }, { "KL/chosen_KL_mean": -5.125308990478516, "KL/mean": -10.218056678771973, "KL/rejected_KL_mean": -15.310802459716797, "KL/std": 11.554460525512695, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.04236079007387161, "fcm_dpo/delta": -0.0850902795791626, "fcm_dpo/margin": 10.185494422912598, "fcm_dpo/q_t": 0.4010116755962372, "grad_norm": 13.845327377319336, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.6811132431030273, "logits/rejected": -1.2901538610458374, "logps/chosen": -98.24197387695312, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -117.40571594238281, "loss": 1.0614, "margin_dpo/margin_mean": 10.185493469238281, "margin_dpo/margin_std": 12.19244384765625, "step": 133 }, { "KL/chosen_KL_mean": -5.123981475830078, "KL/mean": -10.673818588256836, "KL/rejected_KL_mean": -16.22365951538086, "KL/std": 10.854654312133789, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.04168469458818436, "fcm_dpo/delta": -0.11824632436037064, "fcm_dpo/margin": 11.0996732711792, "fcm_dpo/q_t": 0.39240792393684387, "grad_norm": 13.495899200439453, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.6795830726623535, "logits/rejected": -1.341775894165039, "logps/chosen": -88.67410278320312, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -110.105224609375, "loss": 1.0368, "margin_dpo/margin_mean": 11.0996732711792, "margin_dpo/margin_std": 11.773730278015137, "step": 134 }, { "KL/chosen_KL_mean": -4.790737152099609, "KL/mean": -10.40603256225586, "KL/rejected_KL_mean": -16.021331787109375, "KL/std": 12.111650466918945, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.04057490825653076, "fcm_dpo/delta": -0.11058482527732849, "fcm_dpo/margin": 11.230587005615234, "fcm_dpo/q_t": 0.39757055044174194, "grad_norm": 17.271394729614258, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.4120547771453857, "logits/rejected": -1.2964329719543457, "logps/chosen": -82.37336730957031, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -137.07781982421875, "loss": 1.0539, "margin_dpo/margin_mean": 11.230588912963867, "margin_dpo/margin_std": 13.703414916992188, "step": 135 }, { "KL/chosen_KL_mean": -5.659854888916016, "KL/mean": -11.027205467224121, "KL/rejected_KL_mean": -16.39455795288086, "KL/std": 12.330917358398438, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.039757680147886276, "fcm_dpo/delta": -0.08027178049087524, "fcm_dpo/margin": 10.734698295593262, "fcm_dpo/q_t": 0.4015490412712097, "grad_norm": 13.11953067779541, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.5372681617736816, "logits/rejected": -1.1036169528961182, "logps/chosen": -99.83258819580078, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -119.65884399414062, "loss": 1.0608, "margin_dpo/margin_mean": 10.734697341918945, "margin_dpo/margin_std": 12.618770599365234, "step": 136 }, { "KL/chosen_KL_mean": -4.452968597412109, "KL/mean": -13.074161529541016, "KL/rejected_KL_mean": -21.69534683227539, "KL/std": 16.00282859802246, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.037560656666755676, "fcm_dpo/delta": -0.32083800435066223, "fcm_dpo/margin": 17.242389678955078, "fcm_dpo/q_t": 0.35592639446258545, "grad_norm": 15.567719459533691, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.4399515390396118, "logits/rejected": -1.228621482849121, "logps/chosen": -82.57170867919922, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -139.6177215576172, "loss": 0.9311, "margin_dpo/margin_mean": 17.24239158630371, "margin_dpo/margin_std": 17.642850875854492, "step": 137 }, { "KL/chosen_KL_mean": -7.457889556884766, "KL/mean": -12.637964248657227, "KL/rejected_KL_mean": -17.818038940429688, "KL/std": 11.56195068359375, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.03691154718399048, "fcm_dpo/delta": -0.033806901425123215, "fcm_dpo/margin": 10.360157012939453, "fcm_dpo/q_t": 0.4112199544906616, "grad_norm": 12.170183181762695, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.0821491479873657, "logits/rejected": -0.8153581619262695, "logps/chosen": -100.28783416748047, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -116.46644592285156, "loss": 1.0938, "margin_dpo/margin_mean": 10.360157012939453, "margin_dpo/margin_std": 13.029619216918945, "step": 138 }, { "KL/chosen_KL_mean": -5.118339538574219, "KL/mean": -11.23638916015625, "KL/rejected_KL_mean": -17.35443878173828, "KL/std": 12.633675575256348, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.03629777580499649, "fcm_dpo/delta": -0.09879275411367416, "fcm_dpo/margin": 12.236104965209961, "fcm_dpo/q_t": 0.3955083191394806, "grad_norm": 12.361343383789062, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.3805100917816162, "logits/rejected": -1.052200436592102, "logps/chosen": -84.12135314941406, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -103.18177032470703, "loss": 1.059, "margin_dpo/margin_mean": 12.236104011535645, "margin_dpo/margin_std": 15.10415267944336, "step": 139 }, { "KL/chosen_KL_mean": -5.667591094970703, "KL/mean": -11.191991806030273, "KL/rejected_KL_mean": -16.71639633178711, "KL/std": 11.858210563659668, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.03594401478767395, "fcm_dpo/delta": -0.04906575381755829, "fcm_dpo/margin": 11.048803329467773, "fcm_dpo/q_t": 0.40684184432029724, "grad_norm": 11.632204055786133, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.7377097606658936, "logits/rejected": -1.3297992944717407, "logps/chosen": -94.02003479003906, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -107.29116821289062, "loss": 1.0771, "margin_dpo/margin_mean": 11.048803329467773, "margin_dpo/margin_std": 12.700379371643066, "step": 140 }, { "KL/chosen_KL_mean": -7.751041412353516, "KL/mean": -13.960451126098633, "KL/rejected_KL_mean": -20.169857025146484, "KL/std": 15.80606746673584, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.035352494567632675, "fcm_dpo/delta": -0.09309424459934235, "fcm_dpo/margin": 12.418829917907715, "fcm_dpo/q_t": 0.3980710506439209, "grad_norm": 14.47197151184082, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.5156145095825195, "logits/rejected": -1.1269365549087524, "logps/chosen": -114.97120666503906, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -124.88208770751953, "loss": 1.0703, "margin_dpo/margin_mean": 12.418828964233398, "margin_dpo/margin_std": 16.27509880065918, "step": 141 }, { "KL/chosen_KL_mean": -7.716632843017578, "KL/mean": -13.891855239868164, "KL/rejected_KL_mean": -20.06707763671875, "KL/std": 14.70584774017334, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.034686386585235596, "fcm_dpo/delta": -0.08209466934204102, "fcm_dpo/margin": 12.350446701049805, "fcm_dpo/q_t": 0.4027446508407593, "grad_norm": 11.564453125, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.4815151691436768, "logits/rejected": -1.2283344268798828, "logps/chosen": -96.4455795288086, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -119.7974624633789, "loss": 1.0658, "margin_dpo/margin_mean": 12.350444793701172, "margin_dpo/margin_std": 15.368958473205566, "step": 142 }, { "KL/chosen_KL_mean": -5.958866119384766, "KL/mean": -10.938465118408203, "KL/rejected_KL_mean": -15.91805648803711, "KL/std": 12.011764526367188, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.03450293838977814, "fcm_dpo/delta": 0.006346460431814194, "fcm_dpo/margin": 9.95919132232666, "fcm_dpo/q_t": 0.41954851150512695, "grad_norm": 11.775642395019531, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.6499781608581543, "logits/rejected": -1.193598747253418, "logps/chosen": -91.46343994140625, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -101.72371673583984, "loss": 1.1223, "margin_dpo/margin_mean": 9.959192276000977, "margin_dpo/margin_std": 13.181422233581543, "step": 143 }, { "KL/chosen_KL_mean": -7.248882293701172, "KL/mean": -12.18206787109375, "KL/rejected_KL_mean": -17.115257263183594, "KL/std": 12.83199691772461, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.034853022545576096, "fcm_dpo/delta": 0.005768917500972748, "fcm_dpo/margin": 9.866374969482422, "fcm_dpo/q_t": 0.4208351671695709, "grad_norm": 15.040413856506348, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.40469491481781, "logits/rejected": -0.836674690246582, "logps/chosen": -111.93597412109375, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -104.83433532714844, "loss": 1.1452, "margin_dpo/margin_mean": 9.866375923156738, "margin_dpo/margin_std": 15.891203880310059, "step": 144 }, { "KL/chosen_KL_mean": -6.002056121826172, "KL/mean": -11.974849700927734, "KL/rejected_KL_mean": -17.947647094726562, "KL/std": 12.241619110107422, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.03435234725475311, "fcm_dpo/delta": -0.06295502930879593, "fcm_dpo/margin": 11.945587158203125, "fcm_dpo/q_t": 0.4039357304573059, "grad_norm": 12.138075828552246, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.9973472952842712, "logits/rejected": -0.39669573307037354, "logps/chosen": -94.52194213867188, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -101.71278381347656, "loss": 1.07, "margin_dpo/margin_mean": 11.945587158203125, "margin_dpo/margin_std": 13.794803619384766, "step": 145 }, { "KL/chosen_KL_mean": -6.582672119140625, "KL/mean": -13.16024398803711, "KL/rejected_KL_mean": -19.737808227539062, "KL/std": 13.315376281738281, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.03360557556152344, "fcm_dpo/delta": -0.09719344973564148, "fcm_dpo/margin": 13.155132293701172, "fcm_dpo/q_t": 0.39635294675827026, "grad_norm": 11.917059898376465, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.3230942487716675, "logits/rejected": -0.6191867589950562, "logps/chosen": -109.48854064941406, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -108.68717193603516, "loss": 1.0501, "margin_dpo/margin_mean": 13.155134201049805, "margin_dpo/margin_std": 14.687067031860352, "step": 146 }, { "KL/chosen_KL_mean": -6.306705474853516, "KL/mean": -13.122514724731445, "KL/rejected_KL_mean": -19.93832778930664, "KL/std": 12.71150016784668, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.033081118017435074, "fcm_dpo/delta": -0.10567206144332886, "fcm_dpo/margin": 13.631625175476074, "fcm_dpo/q_t": 0.3941296339035034, "grad_norm": 11.245645523071289, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.4389277696609497, "logits/rejected": -0.8396052122116089, "logps/chosen": -95.56889343261719, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -96.43844604492188, "loss": 1.0328, "margin_dpo/margin_mean": 13.631625175476074, "margin_dpo/margin_std": 13.747417449951172, "step": 147 }, { "KL/chosen_KL_mean": -4.648410797119141, "KL/mean": -11.151263236999512, "KL/rejected_KL_mean": -17.654117584228516, "KL/std": 13.530643463134766, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.03274247795343399, "fcm_dpo/delta": -0.07974278926849365, "fcm_dpo/margin": 13.00571060180664, "fcm_dpo/q_t": 0.4036399722099304, "grad_norm": 11.717710494995117, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.683523178100586, "logits/rejected": -1.2902233600616455, "logps/chosen": -96.41129302978516, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -128.03964233398438, "loss": 1.0723, "margin_dpo/margin_mean": 13.005708694458008, "margin_dpo/margin_std": 15.472522735595703, "step": 148 }, { "KL/chosen_KL_mean": -7.9540863037109375, "KL/mean": -14.032675743103027, "KL/rejected_KL_mean": -20.11125946044922, "KL/std": 14.313365936279297, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.032043710350990295, "fcm_dpo/delta": -0.04162246733903885, "fcm_dpo/margin": 12.157170295715332, "fcm_dpo/q_t": 0.4136727750301361, "grad_norm": 11.683693885803223, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.4312667846679688, "logits/rejected": -1.0987895727157593, "logps/chosen": -101.43810272216797, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -125.75495910644531, "loss": 1.111, "margin_dpo/margin_mean": 12.157171249389648, "margin_dpo/margin_std": 18.070232391357422, "step": 149 }, { "KL/chosen_KL_mean": -6.377616882324219, "KL/mean": -12.794892311096191, "KL/rejected_KL_mean": -19.21216583251953, "KL/std": 15.180438041687012, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.03173564374446869, "fcm_dpo/delta": -0.06001156568527222, "fcm_dpo/margin": 12.83454704284668, "fcm_dpo/q_t": 0.40694186091423035, "grad_norm": 10.440954208374023, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.525697946548462, "logits/rejected": -1.2680137157440186, "logps/chosen": -82.15630340576172, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -109.23892974853516, "loss": 1.0853, "margin_dpo/margin_mean": 12.83454704284668, "margin_dpo/margin_std": 16.98514175415039, "step": 150 }, { "KL/chosen_KL_mean": -8.494384765625, "KL/mean": -14.637149810791016, "KL/rejected_KL_mean": -20.7799072265625, "KL/std": 14.73770809173584, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.03151886910200119, "fcm_dpo/delta": -0.03874630481004715, "fcm_dpo/margin": 12.285518646240234, "fcm_dpo/q_t": 0.4112705588340759, "grad_norm": 12.733338356018066, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.1176414489746094, "logits/rejected": -1.520703911781311, "logps/chosen": -111.22860717773438, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -126.81771850585938, "loss": 1.1005, "margin_dpo/margin_mean": 12.285518646240234, "margin_dpo/margin_std": 16.96334457397461, "step": 151 }, { "KL/chosen_KL_mean": -5.681407928466797, "KL/mean": -13.222940444946289, "KL/rejected_KL_mean": -20.76447296142578, "KL/std": 14.692083358764648, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.030862461775541306, "fcm_dpo/delta": -0.12136073410511017, "fcm_dpo/margin": 15.083060264587402, "fcm_dpo/q_t": 0.39185649156570435, "grad_norm": 11.363607406616211, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.6583338975906372, "logits/rejected": -1.2720584869384766, "logps/chosen": -103.27215576171875, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -131.26361083984375, "loss": 1.0398, "margin_dpo/margin_mean": 15.083061218261719, "margin_dpo/margin_std": 17.409503936767578, "step": 152 }, { "KL/chosen_KL_mean": -7.995357513427734, "KL/mean": -13.394023895263672, "KL/rejected_KL_mean": -18.792694091796875, "KL/std": 14.231164932250977, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.030807677656412125, "fcm_dpo/delta": 0.01794874295592308, "fcm_dpo/margin": 10.797345161437988, "fcm_dpo/q_t": 0.422918438911438, "grad_norm": 11.251535415649414, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.9146442413330078, "logits/rejected": -1.5979907512664795, "logps/chosen": -95.02095794677734, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -109.86123657226562, "loss": 1.1442, "margin_dpo/margin_mean": 10.797345161437988, "margin_dpo/margin_std": 16.576967239379883, "step": 153 }, { "KL/chosen_KL_mean": -6.381282806396484, "KL/mean": -15.102710723876953, "KL/rejected_KL_mean": -23.824142456054688, "KL/std": 15.617969512939453, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.03017614409327507, "fcm_dpo/delta": -0.18579283356666565, "fcm_dpo/margin": 17.44286346435547, "fcm_dpo/q_t": 0.3779833912849426, "grad_norm": 11.18238639831543, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.6261560916900635, "logits/rejected": -1.3064229488372803, "logps/chosen": -97.48695373535156, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -135.196533203125, "loss": 0.991, "margin_dpo/margin_mean": 17.44286346435547, "margin_dpo/margin_std": 16.86060905456543, "step": 154 }, { "KL/chosen_KL_mean": -8.497291564941406, "KL/mean": -14.918011665344238, "KL/rejected_KL_mean": -21.338729858398438, "KL/std": 16.065532684326172, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.0294441320002079, "fcm_dpo/delta": -0.02971264347434044, "fcm_dpo/margin": 12.84144115447998, "fcm_dpo/q_t": 0.4102928042411804, "grad_norm": 13.14407730102539, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.3509496450424194, "logits/rejected": -0.9485398530960083, "logps/chosen": -103.05826568603516, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -106.11231994628906, "loss": 1.1249, "margin_dpo/margin_mean": 12.84144115447998, "margin_dpo/margin_std": 20.130081176757812, "step": 155 }, { "KL/chosen_KL_mean": -10.232257843017578, "KL/mean": -16.02643585205078, "KL/rejected_KL_mean": -21.820613861083984, "KL/std": 14.39579963684082, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.029434023424983025, "fcm_dpo/delta": 0.008858952671289444, "fcm_dpo/margin": 11.588356018066406, "fcm_dpo/q_t": 0.42265188694000244, "grad_norm": 11.814373016357422, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.6615185737609863, "logits/rejected": -1.1536394357681274, "logps/chosen": -104.88764953613281, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -106.23995971679688, "loss": 1.1409, "margin_dpo/margin_mean": 11.588356018066406, "margin_dpo/margin_std": 17.849079132080078, "step": 156 }, { "KL/chosen_KL_mean": -9.593612670898438, "KL/mean": -17.874143600463867, "KL/rejected_KL_mean": -26.154674530029297, "KL/std": 15.583829879760742, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.029012426733970642, "fcm_dpo/delta": -0.136943057179451, "fcm_dpo/margin": 16.561058044433594, "fcm_dpo/q_t": 0.3872585892677307, "grad_norm": 12.01373291015625, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.173445463180542, "logits/rejected": -1.7241896390914917, "logps/chosen": -106.60600280761719, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -137.85601806640625, "loss": 1.0171, "margin_dpo/margin_mean": 16.561058044433594, "margin_dpo/margin_std": 16.628910064697266, "step": 157 }, { "KL/chosen_KL_mean": -8.652706146240234, "KL/mean": -17.275146484375, "KL/rejected_KL_mean": -25.8975830078125, "KL/std": 16.84441375732422, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.02821245789527893, "fcm_dpo/delta": -0.14334173500537872, "fcm_dpo/margin": 17.244876861572266, "fcm_dpo/q_t": 0.3877744674682617, "grad_norm": 10.951716423034668, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.4806451797485352, "logits/rejected": -1.2188292741775513, "logps/chosen": -103.36566925048828, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -146.83493041992188, "loss": 1.0188, "margin_dpo/margin_mean": 17.244876861572266, "margin_dpo/margin_std": 18.460861206054688, "step": 158 }, { "KL/chosen_KL_mean": -8.36319351196289, "KL/mean": -15.912189483642578, "KL/rejected_KL_mean": -23.461181640625, "KL/std": 16.206531524658203, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.027666477486491203, "fcm_dpo/delta": -0.0706782341003418, "fcm_dpo/margin": 15.098000526428223, "fcm_dpo/q_t": 0.40305566787719727, "grad_norm": 10.194029808044434, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.1804912090301514, "logits/rejected": -1.6951813697814941, "logps/chosen": -116.58143615722656, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -118.30020141601562, "loss": 1.0728, "margin_dpo/margin_mean": 15.098001480102539, "margin_dpo/margin_std": 18.622940063476562, "step": 159 }, { "KL/chosen_KL_mean": -7.051700592041016, "KL/mean": -14.658958435058594, "KL/rejected_KL_mean": -22.266223907470703, "KL/std": 14.836772918701172, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.027247663587331772, "fcm_dpo/delta": -0.06754288077354431, "fcm_dpo/margin": 15.21451473236084, "fcm_dpo/q_t": 0.4019056558609009, "grad_norm": 10.355915069580078, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.656933307647705, "logits/rejected": -1.1049742698669434, "logps/chosen": -105.2151107788086, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -124.54720306396484, "loss": 1.059, "margin_dpo/margin_mean": 15.214515686035156, "margin_dpo/margin_std": 16.062129974365234, "step": 160 }, { "KL/chosen_KL_mean": -9.146942138671875, "KL/mean": -15.815595626831055, "KL/rejected_KL_mean": -22.48424530029297, "KL/std": 13.54593276977539, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.02706938050687313, "fcm_dpo/delta": -0.011691456660628319, "fcm_dpo/margin": 13.337309837341309, "fcm_dpo/q_t": 0.4155295789241791, "grad_norm": 10.343181610107422, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.8467233180999756, "logits/rejected": -1.3479657173156738, "logps/chosen": -95.73881530761719, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -111.50067901611328, "loss": 1.1133, "margin_dpo/margin_mean": 13.337310791015625, "margin_dpo/margin_std": 17.730674743652344, "step": 161 }, { "KL/chosen_KL_mean": -8.271827697753906, "KL/mean": -15.78854751586914, "KL/rejected_KL_mean": -23.305255889892578, "KL/std": 15.79774284362793, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.02683880925178528, "fcm_dpo/delta": -0.05604541674256325, "fcm_dpo/margin": 15.03343391418457, "fcm_dpo/q_t": 0.40475332736968994, "grad_norm": 9.991804122924805, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.4179744720458984, "logits/rejected": -1.018592357635498, "logps/chosen": -105.67471313476562, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -119.17448425292969, "loss": 1.0719, "margin_dpo/margin_mean": 15.033432960510254, "margin_dpo/margin_std": 16.965835571289062, "step": 162 }, { "KL/chosen_KL_mean": -7.985736846923828, "KL/mean": -16.890939712524414, "KL/rejected_KL_mean": -25.79615020751953, "KL/std": 17.884336471557617, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.02628672868013382, "fcm_dpo/delta": -0.1243613064289093, "fcm_dpo/margin": 17.81041145324707, "fcm_dpo/q_t": 0.391821950674057, "grad_norm": 9.037556648254395, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.829420804977417, "logits/rejected": -1.4375957250595093, "logps/chosen": -90.1611328125, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -108.660888671875, "loss": 1.0293, "margin_dpo/margin_mean": 17.810409545898438, "margin_dpo/margin_std": 19.26786231994629, "step": 163 }, { "KL/chosen_KL_mean": -11.888832092285156, "KL/mean": -17.985944747924805, "KL/rejected_KL_mean": -24.083053588867188, "KL/std": 14.205061912536621, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.025939736515283585, "fcm_dpo/delta": -0.04571648687124252, "fcm_dpo/margin": 12.194215774536133, "fcm_dpo/q_t": 0.42463418841362, "grad_norm": 10.517411231994629, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.4460028409957886, "logits/rejected": -0.9690171480178833, "logps/chosen": -102.36309814453125, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -111.11114501953125, "loss": 1.1393, "margin_dpo/margin_mean": 12.194215774536133, "margin_dpo/margin_std": 16.009143829345703, "step": 164 }, { "KL/chosen_KL_mean": -8.19302749633789, "KL/mean": -16.925189971923828, "KL/rejected_KL_mean": -25.657352447509766, "KL/std": 16.608652114868164, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.025597594678401947, "fcm_dpo/delta": -0.10146971046924591, "fcm_dpo/margin": 17.46432876586914, "fcm_dpo/q_t": 0.3971170485019684, "grad_norm": 8.622611045837402, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.8205021619796753, "logits/rejected": -1.3892595767974854, "logps/chosen": -95.50625610351562, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -107.5599365234375, "loss": 1.0417, "margin_dpo/margin_mean": 17.46432876586914, "margin_dpo/margin_std": 19.07748794555664, "step": 165 }, { "KL/chosen_KL_mean": -13.014354705810547, "KL/mean": -21.751365661621094, "KL/rejected_KL_mean": -30.488372802734375, "KL/std": 18.580320358276367, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.025066830217838287, "fcm_dpo/delta": -0.0920732319355011, "fcm_dpo/margin": 17.474014282226562, "fcm_dpo/q_t": 0.401882529258728, "grad_norm": 9.913846969604492, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.617943286895752, "logits/rejected": -1.4172170162200928, "logps/chosen": -94.60041809082031, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -133.65354919433594, "loss": 1.0641, "margin_dpo/margin_mean": 17.474016189575195, "margin_dpo/margin_std": 22.546428680419922, "step": 166 }, { "KL/chosen_KL_mean": -11.613056182861328, "KL/mean": -20.89676284790039, "KL/rejected_KL_mean": -30.18047332763672, "KL/std": 17.018280029296875, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.024435367435216904, "fcm_dpo/delta": -0.10960444808006287, "fcm_dpo/margin": 18.567419052124023, "fcm_dpo/q_t": 0.39307233691215515, "grad_norm": 9.140959739685059, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.1415822505950928, "logits/rejected": -0.9725895524024963, "logps/chosen": -95.17472839355469, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -129.2335205078125, "loss": 1.0277, "margin_dpo/margin_mean": 18.567419052124023, "margin_dpo/margin_std": 17.700233459472656, "step": 167 }, { "KL/chosen_KL_mean": -11.15762710571289, "KL/mean": -19.14666175842285, "KL/rejected_KL_mean": -27.135700225830078, "KL/std": 16.737571716308594, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.0243010763078928, "fcm_dpo/delta": -0.039813559502363205, "fcm_dpo/margin": 15.978068351745605, "fcm_dpo/q_t": 0.40819162130355835, "grad_norm": 9.737680435180664, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.6294573545455933, "logits/rejected": -1.2833266258239746, "logps/chosen": -91.0888671875, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -107.23786926269531, "loss": 1.0758, "margin_dpo/margin_mean": 15.978069305419922, "margin_dpo/margin_std": 16.98413848876953, "step": 168 }, { "KL/chosen_KL_mean": -9.66867446899414, "KL/mean": -18.89518165588379, "KL/rejected_KL_mean": -28.121688842773438, "KL/std": 18.21259307861328, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.023862838745117188, "fcm_dpo/delta": -0.09458985924720764, "fcm_dpo/margin": 18.45301628112793, "fcm_dpo/q_t": 0.3972659111022949, "grad_norm": 9.477913856506348, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.643841028213501, "logits/rejected": -1.4205901622772217, "logps/chosen": -92.57238006591797, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -118.24359893798828, "loss": 1.044, "margin_dpo/margin_mean": 18.453018188476562, "margin_dpo/margin_std": 19.574180603027344, "step": 169 }, { "KL/chosen_KL_mean": -11.250534057617188, "KL/mean": -19.194137573242188, "KL/rejected_KL_mean": -27.137741088867188, "KL/std": 15.084205627441406, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.023574069142341614, "fcm_dpo/delta": -0.025959761813282967, "fcm_dpo/margin": 15.887207984924316, "fcm_dpo/q_t": 0.41016313433647156, "grad_norm": 9.88397216796875, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.2437496185302734, "logits/rejected": -1.0211937427520752, "logps/chosen": -86.23250579833984, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -109.3033447265625, "loss": 1.0783, "margin_dpo/margin_mean": 15.887207984924316, "margin_dpo/margin_std": 15.057684898376465, "step": 170 }, { "KL/chosen_KL_mean": -14.587520599365234, "KL/mean": -23.490673065185547, "KL/rejected_KL_mean": -32.39383316040039, "KL/std": 22.18875503540039, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.023419927805662155, "fcm_dpo/delta": -0.06989271193742752, "fcm_dpo/margin": 17.806324005126953, "fcm_dpo/q_t": 0.4084351062774658, "grad_norm": 11.277585983276367, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.9766175746917725, "logits/rejected": -1.4484509229660034, "logps/chosen": -133.5660400390625, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -148.16314697265625, "loss": 1.0973, "margin_dpo/margin_mean": 17.80632781982422, "margin_dpo/margin_std": 26.921417236328125, "step": 171 }, { "KL/chosen_KL_mean": -15.526805877685547, "KL/mean": -23.321094512939453, "KL/rejected_KL_mean": -31.11538314819336, "KL/std": 15.691038131713867, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.023221470415592194, "fcm_dpo/delta": -0.012561071664094925, "fcm_dpo/margin": 15.588586807250977, "fcm_dpo/q_t": 0.4124225974082947, "grad_norm": 9.398983001708984, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.2336301803588867, "logits/rejected": -1.4383389949798584, "logps/chosen": -116.68059539794922, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -122.12106323242188, "loss": 1.0788, "margin_dpo/margin_mean": 15.58858585357666, "margin_dpo/margin_std": 12.82138729095459, "step": 172 }, { "KL/chosen_KL_mean": -13.728534698486328, "KL/mean": -20.643814086914062, "KL/rejected_KL_mean": -27.55908966064453, "KL/std": 14.858856201171875, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.02338273636996746, "fcm_dpo/delta": 0.02742098644375801, "fcm_dpo/margin": 13.830552101135254, "fcm_dpo/q_t": 0.4228430390357971, "grad_norm": 8.972540855407715, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.6489722728729248, "logits/rejected": -1.1633257865905762, "logps/chosen": -112.78521728515625, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -111.40982055664062, "loss": 1.1216, "margin_dpo/margin_mean": 13.830552101135254, "margin_dpo/margin_std": 15.766674041748047, "step": 173 }, { "KL/chosen_KL_mean": -13.361562728881836, "KL/mean": -22.9661808013916, "KL/rejected_KL_mean": -32.57079315185547, "KL/std": 20.430253982543945, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.02307542972266674, "fcm_dpo/delta": -0.09749956429004669, "fcm_dpo/margin": 19.209232330322266, "fcm_dpo/q_t": 0.3962975740432739, "grad_norm": 10.469078063964844, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.7691709995269775, "logits/rejected": -1.734276533126831, "logps/chosen": -93.77046966552734, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -140.7725067138672, "loss": 1.0439, "margin_dpo/margin_mean": 19.209232330322266, "margin_dpo/margin_std": 20.71190643310547, "step": 174 }, { "KL/chosen_KL_mean": -17.040775299072266, "KL/mean": -26.235076904296875, "KL/rejected_KL_mean": -35.42937469482422, "KL/std": 18.155494689941406, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.022737935185432434, "fcm_dpo/delta": -0.0710313469171524, "fcm_dpo/margin": 18.38860321044922, "fcm_dpo/q_t": 0.40028661489486694, "grad_norm": 10.17194652557373, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.1522672176361084, "logits/rejected": -0.878129243850708, "logps/chosen": -99.75726318359375, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -132.09649658203125, "loss": 1.0503, "margin_dpo/margin_mean": 18.38860321044922, "margin_dpo/margin_std": 17.479402542114258, "step": 175 }, { "KL/chosen_KL_mean": -14.032093048095703, "KL/mean": -23.51911163330078, "KL/rejected_KL_mean": -33.00613021850586, "KL/std": 19.632362365722656, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.022342292591929436, "fcm_dpo/delta": -0.0772285982966423, "fcm_dpo/margin": 18.974029541015625, "fcm_dpo/q_t": 0.40168508887290955, "grad_norm": 10.091586112976074, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.610479712486267, "logits/rejected": -1.455884575843811, "logps/chosen": -93.66139221191406, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -135.69473266601562, "loss": 1.067, "margin_dpo/margin_mean": 18.974029541015625, "margin_dpo/margin_std": 22.825580596923828, "step": 176 }, { "KL/chosen_KL_mean": -15.76540756225586, "KL/mean": -26.87215805053711, "KL/rejected_KL_mean": -37.97890853881836, "KL/std": 22.69171714782715, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.021877482533454895, "fcm_dpo/delta": -0.14267373085021973, "fcm_dpo/margin": 22.213497161865234, "fcm_dpo/q_t": 0.38919317722320557, "grad_norm": 9.17241382598877, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.7591780424118042, "logits/rejected": -1.4251071214675903, "logps/chosen": -108.32650756835938, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -141.31729125976562, "loss": 1.0188, "margin_dpo/margin_mean": 22.213497161865234, "margin_dpo/margin_std": 23.680465698242188, "step": 177 }, { "KL/chosen_KL_mean": -14.546836853027344, "KL/mean": -27.779781341552734, "KL/rejected_KL_mean": -41.012718200683594, "KL/std": 22.072113037109375, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.020935973152518272, "fcm_dpo/delta": -0.21625497937202454, "fcm_dpo/margin": 26.46588134765625, "fcm_dpo/q_t": 0.37107616662979126, "grad_norm": 9.964146614074707, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.613564133644104, "logits/rejected": -1.403324842453003, "logps/chosen": -98.81263732910156, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -151.81002807617188, "loss": 0.9623, "margin_dpo/margin_mean": 26.465879440307617, "margin_dpo/margin_std": 23.91272735595703, "step": 178 }, { "KL/chosen_KL_mean": -18.509944915771484, "KL/mean": -26.86016082763672, "KL/rejected_KL_mean": -35.21037292480469, "KL/std": 21.234390258789062, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.020671211183071136, "fcm_dpo/delta": 0.004771493375301361, "fcm_dpo/margin": 16.70042610168457, "fcm_dpo/q_t": 0.4203554093837738, "grad_norm": 9.489779472351074, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.237297773361206, "logits/rejected": -0.7884641289710999, "logps/chosen": -111.95791625976562, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -129.46578979492188, "loss": 1.1335, "margin_dpo/margin_mean": 16.700428009033203, "margin_dpo/margin_std": 24.83700180053711, "step": 179 }, { "KL/chosen_KL_mean": -15.494674682617188, "KL/mean": -26.44276237487793, "KL/rejected_KL_mean": -37.39085388183594, "KL/std": 20.48491668701172, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.020381076261401176, "fcm_dpo/delta": -0.10113057494163513, "fcm_dpo/margin": 21.89618682861328, "fcm_dpo/q_t": 0.3950251340866089, "grad_norm": 11.414496421813965, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.3396464586257935, "logits/rejected": -1.1032745838165283, "logps/chosen": -104.33021545410156, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -141.29782104492188, "loss": 1.0308, "margin_dpo/margin_mean": 21.896188735961914, "margin_dpo/margin_std": 21.12126922607422, "step": 180 }, { "KL/chosen_KL_mean": -15.436534881591797, "KL/mean": -23.689956665039062, "KL/rejected_KL_mean": -31.943378448486328, "KL/std": 19.671337127685547, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.02038200944662094, "fcm_dpo/delta": 0.014009272679686546, "fcm_dpo/margin": 16.506847381591797, "fcm_dpo/q_t": 0.42158788442611694, "grad_norm": 8.655712127685547, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.8371130228042603, "logits/rejected": -1.3892309665679932, "logps/chosen": -94.67050170898438, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -101.1177978515625, "loss": 1.121, "margin_dpo/margin_mean": 16.506847381591797, "margin_dpo/margin_std": 21.159317016601562, "step": 181 }, { "KL/chosen_KL_mean": -19.305824279785156, "KL/mean": -30.197402954101562, "KL/rejected_KL_mean": -41.0889892578125, "KL/std": 22.73550796508789, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.020052295178174973, "fcm_dpo/delta": -0.09117116034030914, "fcm_dpo/margin": 21.78315544128418, "fcm_dpo/q_t": 0.398590087890625, "grad_norm": 9.367116928100586, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.3684989213943481, "logits/rejected": -1.158937931060791, "logps/chosen": -95.38800811767578, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -136.05758666992188, "loss": 1.0547, "margin_dpo/margin_mean": 21.783157348632812, "margin_dpo/margin_std": 25.065933227539062, "step": 182 }, { "KL/chosen_KL_mean": -20.384262084960938, "KL/mean": -30.81032943725586, "KL/rejected_KL_mean": -41.236392974853516, "KL/std": 20.640527725219727, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.019749049097299576, "fcm_dpo/delta": -0.06497211754322052, "fcm_dpo/margin": 20.852128982543945, "fcm_dpo/q_t": 0.4024851322174072, "grad_norm": 9.770333290100098, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.6421141624450684, "logits/rejected": -1.0372426509857178, "logps/chosen": -114.29216766357422, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -124.79701232910156, "loss": 1.0587, "margin_dpo/margin_mean": 20.852128982543945, "margin_dpo/margin_std": 21.184059143066406, "step": 183 }, { "KL/chosen_KL_mean": -18.156002044677734, "KL/mean": -25.2950439453125, "KL/rejected_KL_mean": -32.43408203125, "KL/std": 21.361175537109375, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.0199723057448864, "fcm_dpo/delta": 0.06665405631065369, "fcm_dpo/margin": 14.278075218200684, "fcm_dpo/q_t": 0.43249446153640747, "grad_norm": 8.840059280395508, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.6992790699005127, "logits/rejected": -1.4166303873062134, "logps/chosen": -105.02391052246094, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -116.71864318847656, "loss": 1.1727, "margin_dpo/margin_mean": 14.278076171875, "margin_dpo/margin_std": 23.18692398071289, "step": 184 }, { "KL/chosen_KL_mean": -19.155048370361328, "KL/mean": -27.704193115234375, "KL/rejected_KL_mean": -36.25333786010742, "KL/std": 20.947872161865234, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.020087789744138718, "fcm_dpo/delta": 0.006636530160903931, "fcm_dpo/margin": 17.098295211791992, "fcm_dpo/q_t": 0.42038074135780334, "grad_norm": 9.853805541992188, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.0958902835845947, "logits/rejected": -1.4640566110610962, "logps/chosen": -113.17646789550781, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -117.71002197265625, "loss": 1.1277, "margin_dpo/margin_mean": 17.09829330444336, "margin_dpo/margin_std": 24.217342376708984, "step": 185 }, { "KL/chosen_KL_mean": -23.893291473388672, "KL/mean": -33.23994445800781, "KL/rejected_KL_mean": -42.58659744262695, "KL/std": 20.446767807006836, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.019921161234378815, "fcm_dpo/delta": -0.023484906181693077, "fcm_dpo/margin": 18.69329833984375, "fcm_dpo/q_t": 0.41210034489631653, "grad_norm": 10.560308456420898, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.3005034923553467, "logits/rejected": -1.3738181591033936, "logps/chosen": -138.08497619628906, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -135.92572021484375, "loss": 1.0954, "margin_dpo/margin_mean": 18.69329833984375, "margin_dpo/margin_std": 22.276782989501953, "step": 186 }, { "KL/chosen_KL_mean": -20.269794464111328, "KL/mean": -29.33165740966797, "KL/rejected_KL_mean": -38.393524169921875, "KL/std": 22.841121673583984, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.01997283287346363, "fcm_dpo/delta": -0.012561598792672157, "fcm_dpo/margin": 18.12372589111328, "fcm_dpo/q_t": 0.41604238748550415, "grad_norm": 9.364534378051758, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.75636887550354, "logits/rejected": -1.5200233459472656, "logps/chosen": -97.13934326171875, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -126.5584716796875, "loss": 1.1157, "margin_dpo/margin_mean": 18.12372589111328, "margin_dpo/margin_std": 25.00098991394043, "step": 187 }, { "KL/chosen_KL_mean": -15.605022430419922, "KL/mean": -25.85400390625, "KL/rejected_KL_mean": -36.10298156738281, "KL/std": 22.341522216796875, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.019676636904478073, "fcm_dpo/delta": -0.05594448745250702, "fcm_dpo/margin": 20.497961044311523, "fcm_dpo/q_t": 0.40378692746162415, "grad_norm": 9.791643142700195, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -1.9299322366714478, "logits/rejected": -1.6849679946899414, "logps/chosen": -94.79981994628906, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -132.931884765625, "loss": 1.0672, "margin_dpo/margin_mean": 20.497962951660156, "margin_dpo/margin_std": 21.268678665161133, "step": 188 }, { "KL/chosen_KL_mean": -20.888153076171875, "KL/mean": -31.25075340270996, "KL/rejected_KL_mean": -41.61336135864258, "KL/std": 22.307193756103516, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.019603168591856956, "fcm_dpo/delta": -0.05884230509400368, "fcm_dpo/margin": 20.725202560424805, "fcm_dpo/q_t": 0.40643423795700073, "grad_norm": 8.789103507995605, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.729618787765503, "logits/rejected": -1.0735197067260742, "logps/chosen": -104.78095245361328, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -121.07925415039062, "loss": 1.0769, "margin_dpo/margin_mean": 20.725202560424805, "margin_dpo/margin_std": 24.944263458251953, "step": 189 }, { "KL/chosen_KL_mean": -16.83340072631836, "KL/mean": -29.461353302001953, "KL/rejected_KL_mean": -42.08930206298828, "KL/std": 24.82187271118164, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.019053995609283447, "fcm_dpo/delta": -0.13788223266601562, "fcm_dpo/margin": 25.255908966064453, "fcm_dpo/q_t": 0.3865183889865875, "grad_norm": 9.341219902038574, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.9872088432312012, "logits/rejected": -1.3992767333984375, "logps/chosen": -120.27487182617188, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -151.3043212890625, "loss": 1.0091, "margin_dpo/margin_mean": 25.255908966064453, "margin_dpo/margin_std": 23.661903381347656, "step": 190 }, { "KL/chosen_KL_mean": -16.035720825195312, "KL/mean": -30.437110900878906, "KL/rejected_KL_mean": -44.8385009765625, "KL/std": 27.08008575439453, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.018422584980726242, "fcm_dpo/delta": -0.19050511717796326, "fcm_dpo/margin": 28.802780151367188, "fcm_dpo/q_t": 0.37702763080596924, "grad_norm": 10.233390808105469, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.5960912704467773, "logits/rejected": -1.2894951105117798, "logps/chosen": -115.82123565673828, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -160.83877563476562, "loss": 0.9835, "margin_dpo/margin_mean": 28.802780151367188, "margin_dpo/margin_std": 27.767189025878906, "step": 191 }, { "KL/chosen_KL_mean": -25.509445190429688, "KL/mean": -35.51164627075195, "KL/rejected_KL_mean": -45.51384353637695, "KL/std": 21.59726333618164, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.018161922693252563, "fcm_dpo/delta": -0.013847222551703453, "fcm_dpo/margin": 20.004404067993164, "fcm_dpo/q_t": 0.4151713252067566, "grad_norm": 10.27914810180664, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.6618646383285522, "logits/rejected": -1.4124455451965332, "logps/chosen": -126.36976623535156, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -154.95884704589844, "loss": 1.1049, "margin_dpo/margin_mean": 20.00440216064453, "margin_dpo/margin_std": 25.240753173828125, "step": 192 }, { "KL/chosen_KL_mean": -21.33209991455078, "KL/mean": -29.49224853515625, "KL/rejected_KL_mean": -37.652400970458984, "KL/std": 22.941984176635742, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.01834990456700325, "fcm_dpo/delta": 0.05184069275856018, "fcm_dpo/margin": 16.320295333862305, "fcm_dpo/q_t": 0.4301118850708008, "grad_norm": 9.776859283447266, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.2619965076446533, "logits/rejected": -1.6929523944854736, "logps/chosen": -114.27041625976562, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -120.76158142089844, "loss": 1.1724, "margin_dpo/margin_mean": 16.320295333862305, "margin_dpo/margin_std": 27.766874313354492, "step": 193 }, { "KL/chosen_KL_mean": -21.669654846191406, "KL/mean": -31.181705474853516, "KL/rejected_KL_mean": -40.69375228881836, "KL/std": 23.45128059387207, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.01840157061815262, "fcm_dpo/delta": -0.00021520070731639862, "fcm_dpo/margin": 19.024097442626953, "fcm_dpo/q_t": 0.4182431399822235, "grad_norm": 8.357000350952148, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.3383378982543945, "logits/rejected": -1.952518105506897, "logps/chosen": -85.46122741699219, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -109.42041015625, "loss": 1.1164, "margin_dpo/margin_mean": 19.024097442626953, "margin_dpo/margin_std": 24.894207000732422, "step": 194 }, { "KL/chosen_KL_mean": -20.86920928955078, "KL/mean": -33.66464614868164, "KL/rejected_KL_mean": -46.460086822509766, "KL/std": 23.956514358520508, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.018078096210956573, "fcm_dpo/delta": -0.11793249845504761, "fcm_dpo/margin": 25.59087562561035, "fcm_dpo/q_t": 0.39135146141052246, "grad_norm": 8.686942100524902, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.402970314025879, "logits/rejected": -1.290055751800537, "logps/chosen": -104.75466918945312, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -150.46133422851562, "loss": 1.0343, "margin_dpo/margin_mean": 25.59087371826172, "margin_dpo/margin_std": 27.434768676757812, "step": 195 }, { "KL/chosen_KL_mean": -24.407806396484375, "KL/mean": -36.319236755371094, "KL/rejected_KL_mean": -48.23065948486328, "KL/std": 25.65939712524414, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.01772380620241165, "fcm_dpo/delta": -0.07542221993207932, "fcm_dpo/margin": 23.822856903076172, "fcm_dpo/q_t": 0.39912787079811096, "grad_norm": 8.463820457458496, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.3679383993148804, "logits/rejected": -0.9158716797828674, "logps/chosen": -113.80917358398438, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -131.5780029296875, "loss": 1.0481, "margin_dpo/margin_mean": 23.822856903076172, "margin_dpo/margin_std": 23.1375789642334, "step": 196 }, { "KL/chosen_KL_mean": -24.443706512451172, "KL/mean": -31.78158950805664, "KL/rejected_KL_mean": -39.11947250366211, "KL/std": 20.455331802368164, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.01789541356265545, "fcm_dpo/delta": 0.08972719311714172, "fcm_dpo/margin": 14.675762176513672, "fcm_dpo/q_t": 0.43748152256011963, "grad_norm": 8.625123977661133, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.629509687423706, "logits/rejected": -1.306290864944458, "logps/chosen": -111.28440856933594, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -121.91236877441406, "loss": 1.177, "margin_dpo/margin_mean": 14.675764083862305, "margin_dpo/margin_std": 21.743072509765625, "step": 197 }, { "KL/chosen_KL_mean": -28.128276824951172, "KL/mean": -40.21057891845703, "KL/rejected_KL_mean": -52.292884826660156, "KL/std": 24.536640167236328, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.017764516174793243, "fcm_dpo/delta": -0.08279521763324738, "fcm_dpo/margin": 24.16460418701172, "fcm_dpo/q_t": 0.39930862188339233, "grad_norm": 10.126265525817871, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7988700866699219, "logits/rejected": -1.6343863010406494, "logps/chosen": -113.17652130126953, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -156.39654541015625, "loss": 1.0495, "margin_dpo/margin_mean": 24.16460418701172, "margin_dpo/margin_std": 25.479705810546875, "step": 198 }, { "KL/chosen_KL_mean": -27.560272216796875, "KL/mean": -37.38878631591797, "KL/rejected_KL_mean": -47.2172966003418, "KL/std": 23.87226104736328, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.017704369500279427, "fcm_dpo/delta": 0.0020342878997325897, "fcm_dpo/margin": 19.657024383544922, "fcm_dpo/q_t": 0.4181429445743561, "grad_norm": 8.895153045654297, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.7182517051696777, "logits/rejected": -1.4607136249542236, "logps/chosen": -109.34696960449219, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -138.00807189941406, "loss": 1.1162, "margin_dpo/margin_mean": 19.657024383544922, "margin_dpo/margin_std": 25.292282104492188, "step": 199 }, { "KL/chosen_KL_mean": -27.058204650878906, "KL/mean": -38.62773132324219, "KL/rejected_KL_mean": -50.1972541809082, "KL/std": 25.503803253173828, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.01746981032192707, "fcm_dpo/delta": -0.05706308037042618, "fcm_dpo/margin": 23.139041900634766, "fcm_dpo/q_t": 0.40605437755584717, "grad_norm": 9.052345275878906, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.796336054801941, "logits/rejected": -1.25880765914917, "logps/chosen": -111.3262939453125, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -137.16513061523438, "loss": 1.0869, "margin_dpo/margin_mean": 23.139041900634766, "margin_dpo/margin_std": 29.69148063659668, "step": 200 }, { "KL/chosen_KL_mean": -29.230968475341797, "KL/mean": -40.50202941894531, "KL/rejected_KL_mean": -51.77308654785156, "KL/std": 27.897676467895508, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.017420481890439987, "fcm_dpo/delta": -0.04447166994214058, "fcm_dpo/margin": 22.542118072509766, "fcm_dpo/q_t": 0.4089903235435486, "grad_norm": 9.167548179626465, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.4576005935668945, "logits/rejected": -1.076012372970581, "logps/chosen": -137.0034637451172, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -171.56556701660156, "loss": 1.098, "margin_dpo/margin_mean": 22.542118072509766, "margin_dpo/margin_std": 30.548128128051758, "step": 201 }, { "KL/chosen_KL_mean": -27.18734359741211, "KL/mean": -37.6065673828125, "KL/rejected_KL_mean": -48.025787353515625, "KL/std": 22.839736938476562, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.017322150990366936, "fcm_dpo/delta": -0.011388294398784637, "fcm_dpo/margin": 20.83843994140625, "fcm_dpo/q_t": 0.4156076908111572, "grad_norm": 9.339630126953125, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.7586126327514648, "logits/rejected": -1.6345711946487427, "logps/chosen": -103.15979766845703, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -142.48577880859375, "loss": 1.107, "margin_dpo/margin_mean": 20.83843994140625, "margin_dpo/margin_std": 25.86494255065918, "step": 202 }, { "KL/chosen_KL_mean": -24.448932647705078, "KL/mean": -36.246307373046875, "KL/rejected_KL_mean": -48.04369354248047, "KL/std": 23.848981857299805, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.017172353342175484, "fcm_dpo/delta": -0.05746737867593765, "fcm_dpo/margin": 23.594757080078125, "fcm_dpo/q_t": 0.40399494767189026, "grad_norm": 8.5078125, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.9844975471496582, "logits/rejected": -1.5353015661239624, "logps/chosen": -119.41609191894531, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -140.93136596679688, "loss": 1.0652, "margin_dpo/margin_mean": 23.594757080078125, "margin_dpo/margin_std": 24.922542572021484, "step": 203 }, { "KL/chosen_KL_mean": -30.453285217285156, "KL/mean": -40.661800384521484, "KL/rejected_KL_mean": -50.87031555175781, "KL/std": 24.036476135253906, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.017136959359049797, "fcm_dpo/delta": 2.166256308555603e-06, "fcm_dpo/margin": 20.417024612426758, "fcm_dpo/q_t": 0.4184218645095825, "grad_norm": 9.123360633850098, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.9325566291809082, "logits/rejected": -1.271780252456665, "logps/chosen": -146.81048583984375, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -144.26791381835938, "loss": 1.1234, "margin_dpo/margin_mean": 20.41702651977539, "margin_dpo/margin_std": 27.97418785095215, "step": 204 }, { "KL/chosen_KL_mean": -28.645824432373047, "KL/mean": -41.87255859375, "KL/rejected_KL_mean": -55.09930419921875, "KL/std": 26.493154525756836, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.01681210659444332, "fcm_dpo/delta": -0.09942005574703217, "fcm_dpo/margin": 26.453479766845703, "fcm_dpo/q_t": 0.3932115435600281, "grad_norm": 8.39053726196289, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.413464069366455, "logits/rejected": -1.9931704998016357, "logps/chosen": -117.50517272949219, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -146.95379638671875, "loss": 1.0242, "margin_dpo/margin_mean": 26.453479766845703, "margin_dpo/margin_std": 22.464855194091797, "step": 205 }, { "KL/chosen_KL_mean": -28.894817352294922, "KL/mean": -41.09503173828125, "KL/rejected_KL_mean": -53.29524612426758, "KL/std": 25.690738677978516, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.01656845398247242, "fcm_dpo/delta": -0.056915536522865295, "fcm_dpo/margin": 24.400432586669922, "fcm_dpo/q_t": 0.4028353691101074, "grad_norm": 10.149069786071777, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.7761564254760742, "logits/rejected": -1.6355926990509033, "logps/chosen": -105.441162109375, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -156.24838256835938, "loss": 1.0545, "margin_dpo/margin_mean": 24.400432586669922, "margin_dpo/margin_std": 21.84022331237793, "step": 206 }, { "KL/chosen_KL_mean": -30.418777465820312, "KL/mean": -44.15576171875, "KL/rejected_KL_mean": -57.89273452758789, "KL/std": 31.786436080932617, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.016322778537869453, "fcm_dpo/delta": -0.10304415225982666, "fcm_dpo/margin": 27.473966598510742, "fcm_dpo/q_t": 0.39855194091796875, "grad_norm": 9.414647102355957, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.1310389041900635, "logits/rejected": -1.7110702991485596, "logps/chosen": -116.65042114257812, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -148.5478515625, "loss": 1.0667, "margin_dpo/margin_mean": 27.47396469116211, "margin_dpo/margin_std": 36.439064025878906, "step": 207 }, { "KL/chosen_KL_mean": -31.774677276611328, "KL/mean": -45.759666442871094, "KL/rejected_KL_mean": -59.744659423828125, "KL/std": 28.696502685546875, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.016044892370700836, "fcm_dpo/delta": -0.10331732034683228, "fcm_dpo/margin": 27.969987869262695, "fcm_dpo/q_t": 0.39295175671577454, "grad_norm": 9.409134864807129, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.71266508102417, "logits/rejected": -1.3554356098175049, "logps/chosen": -125.93421936035156, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -160.70700073242188, "loss": 1.0307, "margin_dpo/margin_mean": 27.969985961914062, "margin_dpo/margin_std": 26.265716552734375, "step": 208 }, { "KL/chosen_KL_mean": -26.974193572998047, "KL/mean": -39.279212951660156, "KL/rejected_KL_mean": -51.584232330322266, "KL/std": 25.720535278320312, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.015755310654640198, "fcm_dpo/delta": -0.03960152715444565, "fcm_dpo/margin": 24.61003875732422, "fcm_dpo/q_t": 0.4090106189250946, "grad_norm": 8.371949195861816, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.1583290100097656, "logits/rejected": -1.6860164403915405, "logps/chosen": -111.14476013183594, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -139.2037811279297, "loss": 1.0866, "margin_dpo/margin_mean": 24.61003875732422, "margin_dpo/margin_std": 29.178207397460938, "step": 209 }, { "KL/chosen_KL_mean": -30.41338348388672, "KL/mean": -42.22797393798828, "KL/rejected_KL_mean": -54.04256820678711, "KL/std": 27.17868995666504, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.015704315155744553, "fcm_dpo/delta": -0.021946879103779793, "fcm_dpo/margin": 23.629192352294922, "fcm_dpo/q_t": 0.412602961063385, "grad_norm": 7.846146106719971, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.9891390800476074, "logits/rejected": -1.6040234565734863, "logps/chosen": -111.58455657958984, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -138.21734619140625, "loss": 1.093, "margin_dpo/margin_mean": 23.629192352294922, "margin_dpo/margin_std": 27.372804641723633, "step": 210 }, { "KL/chosen_KL_mean": -34.318363189697266, "KL/mean": -45.71916961669922, "KL/rejected_KL_mean": -57.11997604370117, "KL/std": 28.88601303100586, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.015716662630438805, "fcm_dpo/delta": -0.008742053061723709, "fcm_dpo/margin": 22.801612854003906, "fcm_dpo/q_t": 0.4162019193172455, "grad_norm": 9.547245025634766, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.118391275405884, "logits/rejected": -1.4926280975341797, "logps/chosen": -143.03106689453125, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -150.67562866210938, "loss": 1.1092, "margin_dpo/margin_mean": 22.801612854003906, "margin_dpo/margin_std": 29.189624786376953, "step": 211 }, { "KL/chosen_KL_mean": -24.335796356201172, "KL/mean": -41.272132873535156, "KL/rejected_KL_mean": -58.208457946777344, "KL/std": 33.05406188964844, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.015240604057908058, "fcm_dpo/delta": -0.17576824128627777, "fcm_dpo/margin": 33.872657775878906, "fcm_dpo/q_t": 0.3797544538974762, "grad_norm": 8.433538436889648, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.773194670677185, "logits/rejected": -1.3509290218353271, "logps/chosen": -122.69774627685547, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -168.09844970703125, "loss": 0.993, "margin_dpo/margin_mean": 33.872657775878906, "margin_dpo/margin_std": 33.09877395629883, "step": 212 }, { "KL/chosen_KL_mean": -28.530841827392578, "KL/mean": -41.19398880004883, "KL/rejected_KL_mean": -53.857147216796875, "KL/std": 29.989471435546875, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.015032408758997917, "fcm_dpo/delta": -0.032088760286569595, "fcm_dpo/margin": 25.326303482055664, "fcm_dpo/q_t": 0.41069090366363525, "grad_norm": 10.351570129394531, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.910651445388794, "logits/rejected": -1.1096513271331787, "logps/chosen": -136.58615112304688, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -154.0012969970703, "loss": 1.1001, "margin_dpo/margin_mean": 25.326303482055664, "margin_dpo/margin_std": 33.16255569458008, "step": 213 }, { "KL/chosen_KL_mean": -30.47949981689453, "KL/mean": -42.397743225097656, "KL/rejected_KL_mean": -54.31599426269531, "KL/std": 32.016780853271484, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.01503318827599287, "fcm_dpo/delta": -0.008656425401568413, "fcm_dpo/margin": 23.836496353149414, "fcm_dpo/q_t": 0.4176260530948639, "grad_norm": 8.132122993469238, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.8978779315948486, "logits/rejected": -1.7770617008209229, "logps/chosen": -108.28424072265625, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -138.59178161621094, "loss": 1.1257, "margin_dpo/margin_mean": 23.836496353149414, "margin_dpo/margin_std": 35.335105895996094, "step": 214 }, { "KL/chosen_KL_mean": -26.29001235961914, "KL/mean": -42.500282287597656, "KL/rejected_KL_mean": -58.71055221557617, "KL/std": 31.544925689697266, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.01472446322441101, "fcm_dpo/delta": -0.13370631635189056, "fcm_dpo/margin": 32.420528411865234, "fcm_dpo/q_t": 0.3880866765975952, "grad_norm": 8.11875057220459, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.8437135219573975, "logits/rejected": -1.377397060394287, "logps/chosen": -111.4039077758789, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -155.5720672607422, "loss": 1.0135, "margin_dpo/margin_mean": 32.4205322265625, "margin_dpo/margin_std": 31.36174774169922, "step": 215 }, { "KL/chosen_KL_mean": -28.477867126464844, "KL/mean": -42.53142166137695, "KL/rejected_KL_mean": -56.584983825683594, "KL/std": 28.2762451171875, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.014540246687829494, "fcm_dpo/delta": -0.061248570680618286, "fcm_dpo/margin": 28.107112884521484, "fcm_dpo/q_t": 0.40374505519866943, "grad_norm": 7.674285888671875, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.8793258666992188, "logits/rejected": -1.4757745265960693, "logps/chosen": -109.07017517089844, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -138.00482177734375, "loss": 1.0628, "margin_dpo/margin_mean": 28.107114791870117, "margin_dpo/margin_std": 29.217130661010742, "step": 216 }, { "KL/chosen_KL_mean": -34.71879959106445, "KL/mean": -46.206825256347656, "KL/rejected_KL_mean": -57.69485092163086, "KL/std": 29.593860626220703, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.01449724193662405, "fcm_dpo/delta": 0.017428692430257797, "fcm_dpo/margin": 22.976051330566406, "fcm_dpo/q_t": 0.4231359362602234, "grad_norm": 9.380043983459473, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.0253257751464844, "logits/rejected": -1.7200343608856201, "logps/chosen": -128.19137573242188, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -161.18338012695312, "loss": 1.1374, "margin_dpo/margin_mean": 22.976051330566406, "margin_dpo/margin_std": 34.023555755615234, "step": 217 }, { "KL/chosen_KL_mean": -33.97194290161133, "KL/mean": -47.88216781616211, "KL/rejected_KL_mean": -61.792388916015625, "KL/std": 29.65252113342285, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.014343704096972942, "fcm_dpo/delta": -0.05133969709277153, "fcm_dpo/margin": 27.82044219970703, "fcm_dpo/q_t": 0.4072377383708954, "grad_norm": 8.150956153869629, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.430210828781128, "logits/rejected": -1.0498476028442383, "logps/chosen": -123.03077697753906, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -156.0991973876953, "loss": 1.0889, "margin_dpo/margin_mean": 27.82044219970703, "margin_dpo/margin_std": 35.81314468383789, "step": 218 }, { "KL/chosen_KL_mean": -30.526222229003906, "KL/mean": -42.99448013305664, "KL/rejected_KL_mean": -55.462745666503906, "KL/std": 34.59358215332031, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.014394482597708702, "fcm_dpo/delta": -0.009764101356267929, "fcm_dpo/margin": 24.936519622802734, "fcm_dpo/q_t": 0.41670283675193787, "grad_norm": 8.936561584472656, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.7695119380950928, "logits/rejected": -1.3072192668914795, "logps/chosen": -131.5335693359375, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -147.93069458007812, "loss": 1.1268, "margin_dpo/margin_mean": 24.9365234375, "margin_dpo/margin_std": 36.82815170288086, "step": 219 }, { "KL/chosen_KL_mean": -31.51385498046875, "KL/mean": -43.24660110473633, "KL/rejected_KL_mean": -54.979347229003906, "KL/std": 28.736997604370117, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.014350855723023415, "fcm_dpo/delta": 0.013698277994990349, "fcm_dpo/margin": 23.465492248535156, "fcm_dpo/q_t": 0.4201405644416809, "grad_norm": 9.493952751159668, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.3907630443572998, "logits/rejected": -1.0383857488632202, "logps/chosen": -127.05107116699219, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -149.286376953125, "loss": 1.1181, "margin_dpo/margin_mean": 23.465492248535156, "margin_dpo/margin_std": 28.320018768310547, "step": 220 }, { "KL/chosen_KL_mean": -29.50562286376953, "KL/mean": -47.63861083984375, "KL/rejected_KL_mean": -65.7716064453125, "KL/std": 35.77064514160156, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.01407864410430193, "fcm_dpo/delta": -0.16892023384571075, "fcm_dpo/margin": 36.26597213745117, "fcm_dpo/q_t": 0.38023486733436584, "grad_norm": 9.257247924804688, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.087489128112793, "logits/rejected": -0.9529032111167908, "logps/chosen": -112.3504867553711, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -176.58340454101562, "loss": 0.9898, "margin_dpo/margin_mean": 36.26597595214844, "margin_dpo/margin_std": 31.472869873046875, "step": 221 }, { "KL/chosen_KL_mean": -37.84270477294922, "KL/mean": -52.554176330566406, "KL/rejected_KL_mean": -67.2656478881836, "KL/std": 34.573875427246094, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.013807877898216248, "fcm_dpo/delta": -0.05872175842523575, "fcm_dpo/margin": 29.422935485839844, "fcm_dpo/q_t": 0.4059075713157654, "grad_norm": 9.1558256149292, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.8030542135238647, "logits/rejected": -1.36126708984375, "logps/chosen": -132.98468017578125, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -174.070068359375, "loss": 1.0956, "margin_dpo/margin_mean": 29.422935485839844, "margin_dpo/margin_std": 40.27967834472656, "step": 222 }, { "KL/chosen_KL_mean": -36.833885192871094, "KL/mean": -47.14706039428711, "KL/rejected_KL_mean": -57.46023941040039, "KL/std": 30.355260848999023, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.013833274133503437, "fcm_dpo/delta": 0.06662175804376602, "fcm_dpo/margin": 20.626359939575195, "fcm_dpo/q_t": 0.4332098662853241, "grad_norm": 8.861348152160645, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.1876916885375977, "logits/rejected": -1.7986429929733276, "logps/chosen": -122.40899658203125, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -143.91262817382812, "loss": 1.1726, "margin_dpo/margin_mean": 20.626359939575195, "margin_dpo/margin_std": 33.39556121826172, "step": 223 }, { "KL/chosen_KL_mean": -33.471038818359375, "KL/mean": -48.493560791015625, "KL/rejected_KL_mean": -63.516090393066406, "KL/std": 33.67667770385742, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.01375051960349083, "fcm_dpo/delta": -0.06592772901058197, "fcm_dpo/margin": 30.0450496673584, "fcm_dpo/q_t": 0.4035870432853699, "grad_norm": 8.649076461791992, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.7948837280273438, "logits/rejected": -1.3338199853897095, "logps/chosen": -116.19483947753906, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -146.11148071289062, "loss": 1.071, "margin_dpo/margin_mean": 30.04505157470703, "margin_dpo/margin_std": 35.4399299621582, "step": 224 }, { "KL/chosen_KL_mean": -37.99123001098633, "KL/mean": -53.26079559326172, "KL/rejected_KL_mean": -68.5303726196289, "KL/std": 35.27033233642578, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.013620607554912567, "fcm_dpo/delta": -0.06881606578826904, "fcm_dpo/margin": 30.539138793945312, "fcm_dpo/q_t": 0.40416258573532104, "grad_norm": 9.299295425415039, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.242441177368164, "logits/rejected": -1.7839462757110596, "logps/chosen": -133.66891479492188, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -173.62725830078125, "loss": 1.0806, "margin_dpo/margin_mean": 30.539138793945312, "margin_dpo/margin_std": 39.71565628051758, "step": 225 }, { "KL/chosen_KL_mean": -31.292259216308594, "KL/mean": -46.150108337402344, "KL/rejected_KL_mean": -61.007957458496094, "KL/std": 34.519813537597656, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.013364087790250778, "fcm_dpo/delta": -0.04978608340024948, "fcm_dpo/margin": 29.7156982421875, "fcm_dpo/q_t": 0.4059722423553467, "grad_norm": 7.772035121917725, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.812057614326477, "logits/rejected": -1.268571138381958, "logps/chosen": -124.75318908691406, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -147.709716796875, "loss": 1.0698, "margin_dpo/margin_mean": 29.7156982421875, "margin_dpo/margin_std": 30.66361427307129, "step": 226 }, { "KL/chosen_KL_mean": -42.13101577758789, "KL/mean": -57.506622314453125, "KL/rejected_KL_mean": -72.88223266601562, "KL/std": 33.71291732788086, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.013278959318995476, "fcm_dpo/delta": -0.06080767512321472, "fcm_dpo/margin": 30.75121307373047, "fcm_dpo/q_t": 0.40536749362945557, "grad_norm": 8.616753578186035, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.71928071975708, "logits/rejected": -1.3788871765136719, "logps/chosen": -134.94528198242188, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -177.61917114257812, "loss": 1.0853, "margin_dpo/margin_mean": 30.7512149810791, "margin_dpo/margin_std": 39.896018981933594, "step": 227 }, { "KL/chosen_KL_mean": -39.772804260253906, "KL/mean": -51.37720489501953, "KL/rejected_KL_mean": -62.98160171508789, "KL/std": 31.6168212890625, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.013308518566191196, "fcm_dpo/delta": 0.04245460033416748, "fcm_dpo/margin": 23.20879554748535, "fcm_dpo/q_t": 0.427301824092865, "grad_norm": 8.827247619628906, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -2.0117979049682617, "logits/rejected": -1.5868074893951416, "logps/chosen": -133.8099365234375, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -159.00311279296875, "loss": 1.156, "margin_dpo/margin_mean": 23.20879554748535, "margin_dpo/margin_std": 35.843109130859375, "step": 228 }, { "KL/chosen_KL_mean": -35.311256408691406, "KL/mean": -54.43161392211914, "KL/rejected_KL_mean": -73.55198669433594, "KL/std": 38.19732666015625, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.013043212704360485, "fcm_dpo/delta": -0.15654200315475464, "fcm_dpo/margin": 38.24072265625, "fcm_dpo/q_t": 0.3854658007621765, "grad_norm": 8.729072570800781, "learning_rate": 4.212490049118951e-07, "logits/chosen": -2.0918474197387695, "logits/rejected": -1.484615683555603, "logps/chosen": -130.888916015625, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -162.72576904296875, "loss": 1.0166, "margin_dpo/margin_mean": 38.240726470947266, "margin_dpo/margin_std": 42.202667236328125, "step": 229 }, { "KL/chosen_KL_mean": -36.62153244018555, "KL/mean": -56.36896896362305, "KL/rejected_KL_mean": -76.11640930175781, "KL/std": 37.815494537353516, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.012633368372917175, "fcm_dpo/delta": -0.15681388974189758, "fcm_dpo/margin": 39.49488067626953, "fcm_dpo/q_t": 0.3824649453163147, "grad_norm": 9.821617126464844, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8113195896148682, "logits/rejected": -1.7552611827850342, "logps/chosen": -100.33515930175781, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -168.02511596679688, "loss": 1.0008, "margin_dpo/margin_mean": 39.49488067626953, "margin_dpo/margin_std": 37.69347381591797, "step": 230 }, { "KL/chosen_KL_mean": -40.17181396484375, "KL/mean": -57.41471862792969, "KL/rejected_KL_mean": -74.65763092041016, "KL/std": 36.43785095214844, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.01233719103038311, "fcm_dpo/delta": -0.07946952432394028, "fcm_dpo/margin": 34.4858283996582, "fcm_dpo/q_t": 0.398517906665802, "grad_norm": 7.624197959899902, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.7618632316589355, "logits/rejected": -1.1900835037231445, "logps/chosen": -135.6274871826172, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -155.61331176757812, "loss": 1.0423, "margin_dpo/margin_mean": 34.4858283996582, "margin_dpo/margin_std": 31.462039947509766, "step": 231 }, { "KL/chosen_KL_mean": -42.85033416748047, "KL/mean": -56.94342041015625, "KL/rejected_KL_mean": -71.03651428222656, "KL/std": 33.77946472167969, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.012334372848272324, "fcm_dpo/delta": 0.002419854048639536, "fcm_dpo/margin": 28.186185836791992, "fcm_dpo/q_t": 0.41752952337265015, "grad_norm": 8.830424308776855, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.6419687271118164, "logits/rejected": -1.162764549255371, "logps/chosen": -139.74758911132812, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -160.80113220214844, "loss": 1.1099, "margin_dpo/margin_mean": 28.18618392944336, "margin_dpo/margin_std": 33.479896545410156, "step": 232 }, { "KL/chosen_KL_mean": -44.2665901184082, "KL/mean": -64.33935546875, "KL/rejected_KL_mean": -84.41211700439453, "KL/std": 42.893245697021484, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.012069778516888618, "fcm_dpo/delta": -0.14162592589855194, "fcm_dpo/margin": 40.145530700683594, "fcm_dpo/q_t": 0.38662242889404297, "grad_norm": 9.039531707763672, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.754048466682434, "logits/rejected": -1.5573011636734009, "logps/chosen": -133.31690979003906, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -197.17129516601562, "loss": 1.0178, "margin_dpo/margin_mean": 40.145530700683594, "margin_dpo/margin_std": 41.57579040527344, "step": 233 }, { "KL/chosen_KL_mean": -43.98397445678711, "KL/mean": -65.35726928710938, "KL/rejected_KL_mean": -86.7305679321289, "KL/std": 40.90184783935547, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.011643161997199059, "fcm_dpo/delta": -0.1573936939239502, "fcm_dpo/margin": 42.74659729003906, "fcm_dpo/q_t": 0.38444846868515015, "grad_norm": 8.06415843963623, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.3136980533599854, "logits/rejected": -1.250430703163147, "logps/chosen": -118.30233764648438, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -193.1181640625, "loss": 1.0142, "margin_dpo/margin_mean": 42.74659729003906, "margin_dpo/margin_std": 44.79154586791992, "step": 234 }, { "KL/chosen_KL_mean": -52.94430923461914, "KL/mean": -69.05386352539062, "KL/rejected_KL_mean": -85.16340637207031, "KL/std": 36.924381256103516, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.011597890406847, "fcm_dpo/delta": -0.024592220783233643, "fcm_dpo/margin": 32.21910095214844, "fcm_dpo/q_t": 0.4112706184387207, "grad_norm": 8.169129371643066, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.726823329925537, "logits/rejected": -1.17887544631958, "logps/chosen": -151.16134643554688, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -182.41018676757812, "loss": 1.0863, "margin_dpo/margin_mean": 32.21910095214844, "margin_dpo/margin_std": 34.93455123901367, "step": 235 }, { "KL/chosen_KL_mean": -44.233360290527344, "KL/mean": -65.32077026367188, "KL/rejected_KL_mean": -86.40818786621094, "KL/std": 46.384605407714844, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.011357173323631287, "fcm_dpo/delta": -0.13531382381916046, "fcm_dpo/margin": 42.17481994628906, "fcm_dpo/q_t": 0.3926989436149597, "grad_norm": 8.882852554321289, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.4498162269592285, "logits/rejected": -1.4573298692703247, "logps/chosen": -123.07109069824219, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -195.47161865234375, "loss": 1.0338, "margin_dpo/margin_mean": 42.17481994628906, "margin_dpo/margin_std": 50.18202209472656, "step": 236 }, { "KL/chosen_KL_mean": -53.58562088012695, "KL/mean": -68.68730163574219, "KL/rejected_KL_mean": -83.78899383544922, "KL/std": 41.11155319213867, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.011279432103037834, "fcm_dpo/delta": 0.009648699313402176, "fcm_dpo/margin": 30.203369140625, "fcm_dpo/q_t": 0.42372262477874756, "grad_norm": 8.094720840454102, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.6386216878890991, "logits/rejected": -1.2964560985565186, "logps/chosen": -138.79690551757812, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -183.69898986816406, "loss": 1.1572, "margin_dpo/margin_mean": 30.203365325927734, "margin_dpo/margin_std": 53.178489685058594, "step": 237 }, { "KL/chosen_KL_mean": -58.39180374145508, "KL/mean": -76.21945190429688, "KL/rejected_KL_mean": -94.04711151123047, "KL/std": 42.43247985839844, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.011184731498360634, "fcm_dpo/delta": -0.05090335011482239, "fcm_dpo/margin": 35.655303955078125, "fcm_dpo/q_t": 0.40746504068374634, "grad_norm": 7.672429084777832, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.1974157094955444, "logits/rejected": -1.0516974925994873, "logps/chosen": -137.123046875, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -189.46551513671875, "loss": 1.0831, "margin_dpo/margin_mean": 35.655303955078125, "margin_dpo/margin_std": 44.10497283935547, "step": 238 }, { "KL/chosen_KL_mean": -53.029911041259766, "KL/mean": -69.59077453613281, "KL/rejected_KL_mean": -86.15162658691406, "KL/std": 37.914459228515625, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.01107599027454853, "fcm_dpo/delta": -0.018232613801956177, "fcm_dpo/margin": 33.12171936035156, "fcm_dpo/q_t": 0.41399288177490234, "grad_norm": 8.440138816833496, "learning_rate": 4.116778689174514e-07, "logits/chosen": -2.122427225112915, "logits/rejected": -1.6653714179992676, "logps/chosen": -145.630859375, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -186.6693115234375, "loss": 1.1076, "margin_dpo/margin_mean": 33.1217155456543, "margin_dpo/margin_std": 42.01616668701172, "step": 239 }, { "KL/chosen_KL_mean": -55.19479751586914, "KL/mean": -69.77578735351562, "KL/rejected_KL_mean": -84.35678100585938, "KL/std": 38.52021026611328, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.011148332618176937, "fcm_dpo/delta": 0.02570110186934471, "fcm_dpo/margin": 29.161975860595703, "fcm_dpo/q_t": 0.42362308502197266, "grad_norm": 9.383548736572266, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.094161033630371, "logits/rejected": -1.7294588088989258, "logps/chosen": -141.3545684814453, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -164.81243896484375, "loss": 1.1386, "margin_dpo/margin_mean": 29.161975860595703, "margin_dpo/margin_std": 41.29236602783203, "step": 240 }, { "KL/chosen_KL_mean": -51.1285285949707, "KL/mean": -77.21342468261719, "KL/rejected_KL_mean": -103.29833984375, "KL/std": 43.50746536254883, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.010860471986234188, "fcm_dpo/delta": -0.2289850115776062, "fcm_dpo/margin": 52.16979217529297, "fcm_dpo/q_t": 0.36884522438049316, "grad_norm": 8.320960998535156, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.6405019760131836, "logits/rejected": -0.9418003559112549, "logps/chosen": -152.17446899414062, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -197.34768676757812, "loss": 0.9634, "margin_dpo/margin_mean": 52.169795989990234, "margin_dpo/margin_std": 48.7486572265625, "step": 241 }, { "KL/chosen_KL_mean": -60.5725212097168, "KL/mean": -79.31365966796875, "KL/rejected_KL_mean": -98.05479431152344, "KL/std": 39.51665496826172, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.01063258945941925, "fcm_dpo/delta": -0.05055946484208107, "fcm_dpo/margin": 37.48226547241211, "fcm_dpo/q_t": 0.40575456619262695, "grad_norm": 8.59409236907959, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.8268046379089355, "logits/rejected": -1.1545276641845703, "logps/chosen": -156.24517822265625, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -188.7093505859375, "loss": 1.0801, "margin_dpo/margin_mean": 37.482269287109375, "margin_dpo/margin_std": 44.081146240234375, "step": 242 }, { "KL/chosen_KL_mean": -60.232826232910156, "KL/mean": -78.897216796875, "KL/rejected_KL_mean": -97.56159973144531, "KL/std": 42.3499870300293, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.01057718601077795, "fcm_dpo/delta": -0.04736047983169556, "fcm_dpo/margin": 37.32877731323242, "fcm_dpo/q_t": 0.4088801145553589, "grad_norm": 8.20076847076416, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.9562158584594727, "logits/rejected": -1.1337732076644897, "logps/chosen": -158.26422119140625, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -180.74966430664062, "loss": 1.1026, "margin_dpo/margin_mean": 37.32877731323242, "margin_dpo/margin_std": 50.74687194824219, "step": 243 }, { "KL/chosen_KL_mean": -62.4527587890625, "KL/mean": -82.29202270507812, "KL/rejected_KL_mean": -102.13128662109375, "KL/std": 46.70545196533203, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.010361001826822758, "fcm_dpo/delta": -0.06388510018587112, "fcm_dpo/margin": 39.67852783203125, "fcm_dpo/q_t": 0.4037293791770935, "grad_norm": 7.9408745765686035, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.140453577041626, "logits/rejected": -1.8101768493652344, "logps/chosen": -151.3466796875, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -204.70407104492188, "loss": 1.0696, "margin_dpo/margin_mean": 39.67852783203125, "margin_dpo/margin_std": 45.904029846191406, "step": 244 }, { "KL/chosen_KL_mean": -53.98135757446289, "KL/mean": -74.64305877685547, "KL/rejected_KL_mean": -95.30475616455078, "KL/std": 42.58116149902344, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.010242684744298458, "fcm_dpo/delta": -0.07643130421638489, "fcm_dpo/margin": 41.323394775390625, "fcm_dpo/q_t": 0.40049731731414795, "grad_norm": 7.461879253387451, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.390965461730957, "logits/rejected": -1.5658562183380127, "logps/chosen": -158.17535400390625, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -187.96121215820312, "loss": 1.0556, "margin_dpo/margin_mean": 41.323394775390625, "margin_dpo/margin_std": 44.226966857910156, "step": 245 }, { "KL/chosen_KL_mean": -69.74679565429688, "KL/mean": -86.37682342529297, "KL/rejected_KL_mean": -103.00685119628906, "KL/std": 42.20500183105469, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.01017204113304615, "fcm_dpo/delta": 0.011909859254956245, "fcm_dpo/margin": 33.26005554199219, "fcm_dpo/q_t": 0.4210430383682251, "grad_norm": 7.631123065948486, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.8603770732879639, "logits/rejected": -1.3118857145309448, "logps/chosen": -172.96585083007812, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -193.9990692138672, "loss": 1.1284, "margin_dpo/margin_mean": 33.26005554199219, "margin_dpo/margin_std": 45.74205017089844, "step": 246 }, { "KL/chosen_KL_mean": -63.50765609741211, "KL/mean": -90.86774444580078, "KL/rejected_KL_mean": -118.22782897949219, "KL/std": 47.160003662109375, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.009868226945400238, "fcm_dpo/delta": -0.20203447341918945, "fcm_dpo/margin": 54.720184326171875, "fcm_dpo/q_t": 0.37525835633277893, "grad_norm": 7.962800979614258, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.3738598823547363, "logits/rejected": -2.2060155868530273, "logps/chosen": -150.5020294189453, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -229.56585693359375, "loss": 0.9825, "margin_dpo/margin_mean": 54.72018814086914, "margin_dpo/margin_std": 53.59483337402344, "step": 247 }, { "KL/chosen_KL_mean": -73.56758117675781, "KL/mean": -91.55992126464844, "KL/rejected_KL_mean": -109.55225372314453, "KL/std": 41.55329513549805, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.00982740055769682, "fcm_dpo/delta": -0.003903580829501152, "fcm_dpo/margin": 35.98468017578125, "fcm_dpo/q_t": 0.4163702130317688, "grad_norm": 8.251521110534668, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.7888939380645752, "logits/rejected": -1.7674400806427002, "logps/chosen": -148.3531036376953, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -207.8291473388672, "loss": 1.1061, "margin_dpo/margin_mean": 35.98468017578125, "margin_dpo/margin_std": 42.647071838378906, "step": 248 }, { "KL/chosen_KL_mean": -81.31320190429688, "KL/mean": -97.83648681640625, "KL/rejected_KL_mean": -114.35978698730469, "KL/std": 45.07139587402344, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.009803349152207375, "fcm_dpo/delta": 0.02673913910984993, "fcm_dpo/margin": 33.046600341796875, "fcm_dpo/q_t": 0.4246631860733032, "grad_norm": 9.317215919494629, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.291053295135498, "logits/rejected": -1.755770206451416, "logps/chosen": -177.01699829101562, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -201.50625610351562, "loss": 1.1509, "margin_dpo/margin_mean": 33.046600341796875, "margin_dpo/margin_std": 51.68198776245117, "step": 249 }, { "KL/chosen_KL_mean": -72.22760009765625, "KL/mean": -91.04170227050781, "KL/rejected_KL_mean": -109.85580444335938, "KL/std": 45.4390869140625, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.00978006236255169, "fcm_dpo/delta": -0.019183654338121414, "fcm_dpo/margin": 37.62819290161133, "fcm_dpo/q_t": 0.4131908118724823, "grad_norm": 8.462065696716309, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9290469884872437, "logits/rejected": -0.9755983352661133, "logps/chosen": -186.27981567382812, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -190.9434814453125, "loss": 1.1105, "margin_dpo/margin_mean": 37.628196716308594, "margin_dpo/margin_std": 48.89054870605469, "step": 250 }, { "KL/chosen_KL_mean": -72.8731460571289, "KL/mean": -89.16265106201172, "KL/rejected_KL_mean": -105.45216369628906, "KL/std": 42.354312896728516, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.00974578782916069, "fcm_dpo/delta": -0.0349036306142807, "fcm_dpo/margin": 32.57902145385742, "fcm_dpo/q_t": 0.4261459708213806, "grad_norm": 9.198213577270508, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.7753320932388306, "logits/rejected": -1.05862557888031, "logps/chosen": -170.58444213867188, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -188.97958374023438, "loss": 1.1547, "margin_dpo/margin_mean": 32.579017639160156, "margin_dpo/margin_std": 50.877471923828125, "step": 251 }, { "KL/chosen_KL_mean": -73.0737533569336, "KL/mean": -89.46995544433594, "KL/rejected_KL_mean": -105.86616516113281, "KL/std": 45.36990737915039, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.00980469398200512, "fcm_dpo/delta": 0.029322709888219833, "fcm_dpo/margin": 32.79240798950195, "fcm_dpo/q_t": 0.42421919107437134, "grad_norm": 8.770072937011719, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.0768699645996094, "logits/rejected": -1.5317813158035278, "logps/chosen": -182.0598602294922, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -197.430419921875, "loss": 1.1452, "margin_dpo/margin_mean": 32.79241180419922, "margin_dpo/margin_std": 48.01593780517578, "step": 252 }, { "KL/chosen_KL_mean": -80.67778015136719, "KL/mean": -95.97103118896484, "KL/rejected_KL_mean": -111.26428985595703, "KL/std": 48.85212707519531, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.009779621846973896, "fcm_dpo/delta": -0.02401706948876381, "fcm_dpo/margin": 30.586502075195312, "fcm_dpo/q_t": 0.4285731315612793, "grad_norm": 8.90047550201416, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.195802688598633, "logits/rejected": -1.99476158618927, "logps/chosen": -180.89407348632812, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -216.9409942626953, "loss": 1.1532, "margin_dpo/margin_mean": 30.586502075195312, "margin_dpo/margin_std": 42.584999084472656, "step": 253 }, { "KL/chosen_KL_mean": -77.85879516601562, "KL/mean": -93.82087707519531, "KL/rejected_KL_mean": -109.78294372558594, "KL/std": 46.420310974121094, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.009777981787919998, "fcm_dpo/delta": 0.03894376382231712, "fcm_dpo/margin": 31.92413902282715, "fcm_dpo/q_t": 0.42624717950820923, "grad_norm": 8.705557823181152, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.2558200359344482, "logits/rejected": -1.745460867881775, "logps/chosen": -176.593994140625, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -203.5211944580078, "loss": 1.1571, "margin_dpo/margin_mean": 31.92414093017578, "margin_dpo/margin_std": 49.869537353515625, "step": 254 }, { "KL/chosen_KL_mean": -78.10671997070312, "KL/mean": -103.15672302246094, "KL/rejected_KL_mean": -128.20669555664062, "KL/std": 52.41816329956055, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.00963975302875042, "fcm_dpo/delta": -0.1395847350358963, "fcm_dpo/margin": 50.09998321533203, "fcm_dpo/q_t": 0.38697904348373413, "grad_norm": 8.583830833435059, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.0513083934783936, "logits/rejected": -1.7173068523406982, "logps/chosen": -168.48928833007812, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -225.282958984375, "loss": 1.022, "margin_dpo/margin_mean": 50.09998321533203, "margin_dpo/margin_std": 53.19436264038086, "step": 255 }, { "KL/chosen_KL_mean": -77.16539001464844, "KL/mean": -98.7599105834961, "KL/rejected_KL_mean": -120.35444641113281, "KL/std": 54.49156951904297, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.00949383806437254, "fcm_dpo/delta": -0.06255720555782318, "fcm_dpo/margin": 43.189056396484375, "fcm_dpo/q_t": 0.4063280522823334, "grad_norm": 9.156143188476562, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.8236719369888306, "logits/rejected": -1.484926462173462, "logps/chosen": -165.91819763183594, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -218.8482666015625, "loss": 1.1049, "margin_dpo/margin_mean": 43.189056396484375, "margin_dpo/margin_std": 64.36235046386719, "step": 256 }, { "KL/chosen_KL_mean": -78.42533874511719, "KL/mean": -100.62461853027344, "KL/rejected_KL_mean": -122.82391357421875, "KL/std": 55.874961853027344, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.009275725111365318, "fcm_dpo/delta": -0.06631788611412048, "fcm_dpo/margin": 44.39857482910156, "fcm_dpo/q_t": 0.4040672183036804, "grad_norm": 8.101411819458008, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.8312363624572754, "logits/rejected": -1.4220190048217773, "logps/chosen": -161.22885131835938, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -208.69171142578125, "loss": 1.0813, "margin_dpo/margin_mean": 44.39857482910156, "margin_dpo/margin_std": 54.09214782714844, "step": 257 }, { "KL/chosen_KL_mean": -81.6573715209961, "KL/mean": -101.72740173339844, "KL/rejected_KL_mean": -121.79743957519531, "KL/std": 53.121337890625, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.009335431270301342, "fcm_dpo/delta": -0.026201337575912476, "fcm_dpo/margin": 40.14006805419922, "fcm_dpo/q_t": 0.41180992126464844, "grad_norm": 8.934864044189453, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.305637836456299, "logits/rejected": -1.958158254623413, "logps/chosen": -171.84512329101562, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -225.26812744140625, "loss": 1.1038, "margin_dpo/margin_mean": 40.14006805419922, "margin_dpo/margin_std": 51.063636779785156, "step": 258 }, { "KL/chosen_KL_mean": -88.03573608398438, "KL/mean": -105.28238677978516, "KL/rejected_KL_mean": -122.52903747558594, "KL/std": 53.65234375, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.009195668622851372, "fcm_dpo/delta": -0.04010157659649849, "fcm_dpo/margin": 34.49330139160156, "fcm_dpo/q_t": 0.42525291442871094, "grad_norm": 8.587601661682129, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.1821030378341675, "logits/rejected": -0.9213433265686035, "logps/chosen": -178.80828857421875, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -217.11720275878906, "loss": 1.1696, "margin_dpo/margin_mean": 34.4932975769043, "margin_dpo/margin_std": 60.53424835205078, "step": 259 }, { "KL/chosen_KL_mean": -90.14788818359375, "KL/mean": -104.93034362792969, "KL/rejected_KL_mean": -119.71278381347656, "KL/std": 54.45276641845703, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.009298183023929596, "fcm_dpo/delta": 0.07725410163402557, "fcm_dpo/margin": 29.564899444580078, "fcm_dpo/q_t": 0.4359358549118042, "grad_norm": 9.859212875366211, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.7534310817718506, "logits/rejected": -1.7877947092056274, "logps/chosen": -165.74058532714844, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -220.55831909179688, "loss": 1.2023, "margin_dpo/margin_mean": 29.56490135192871, "margin_dpo/margin_std": 59.08306121826172, "step": 260 }, { "KL/chosen_KL_mean": -69.9984130859375, "KL/mean": -95.72554016113281, "KL/rejected_KL_mean": -121.45268249511719, "KL/std": 54.64764404296875, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0091160349547863, "fcm_dpo/delta": -0.12618738412857056, "fcm_dpo/margin": 51.454261779785156, "fcm_dpo/q_t": 0.3902401924133301, "grad_norm": 8.02148723602295, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.033284902572632, "logits/rejected": -1.6670571565628052, "logps/chosen": -164.25332641601562, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -215.1096649169922, "loss": 1.0275, "margin_dpo/margin_mean": 51.45426559448242, "margin_dpo/margin_std": 53.26464080810547, "step": 261 }, { "KL/chosen_KL_mean": -76.27377319335938, "KL/mean": -100.29241943359375, "KL/rejected_KL_mean": -124.31106567382812, "KL/std": 57.56914520263672, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.008960478007793427, "fcm_dpo/delta": -0.08460726588964462, "fcm_dpo/margin": 48.03728485107422, "fcm_dpo/q_t": 0.39956721663475037, "grad_norm": 8.049735069274902, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.874765157699585, "logits/rejected": -1.348569393157959, "logps/chosen": -161.54107666015625, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -215.13714599609375, "loss": 1.0653, "margin_dpo/margin_mean": 48.03728485107422, "margin_dpo/margin_std": 57.9429931640625, "step": 262 }, { "KL/chosen_KL_mean": -61.731109619140625, "KL/mean": -94.33734893798828, "KL/rejected_KL_mean": -126.94358825683594, "KL/std": 62.55636978149414, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.008670233190059662, "fcm_dpo/delta": -0.22799602150917053, "fcm_dpo/margin": 65.21249389648438, "fcm_dpo/q_t": 0.3690844178199768, "grad_norm": 7.819310665130615, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.0960869789123535, "logits/rejected": -1.5390976667404175, "logps/chosen": -157.489013671875, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -228.77735900878906, "loss": 0.977, "margin_dpo/margin_mean": 65.21249389648438, "margin_dpo/margin_std": 67.24493408203125, "step": 263 }, { "KL/chosen_KL_mean": -71.68956756591797, "KL/mean": -99.31726837158203, "KL/rejected_KL_mean": -126.94497680664062, "KL/std": 58.96384811401367, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.00839165784418583, "fcm_dpo/delta": -0.11942412704229355, "fcm_dpo/margin": 55.25541305541992, "fcm_dpo/q_t": 0.39137500524520874, "grad_norm": 7.736931324005127, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.073894500732422, "logits/rejected": -1.7873640060424805, "logps/chosen": -162.31222534179688, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -227.27052307128906, "loss": 1.0357, "margin_dpo/margin_mean": 55.255409240722656, "margin_dpo/margin_std": 60.297393798828125, "step": 264 }, { "KL/chosen_KL_mean": -75.60216522216797, "KL/mean": -99.61631774902344, "KL/rejected_KL_mean": -123.63047790527344, "KL/std": 60.28556823730469, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.008259125053882599, "fcm_dpo/delta": -0.049103088676929474, "fcm_dpo/margin": 48.02831268310547, "fcm_dpo/q_t": 0.40852242708206177, "grad_norm": 8.11116886138916, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.058133602142334, "logits/rejected": -1.648982286453247, "logps/chosen": -167.9744873046875, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -218.258056640625, "loss": 1.0985, "margin_dpo/margin_mean": 48.02831268310547, "margin_dpo/margin_std": 65.46186828613281, "step": 265 }, { "KL/chosen_KL_mean": -87.20603942871094, "KL/mean": -109.06365966796875, "KL/rejected_KL_mean": -130.92127990722656, "KL/std": 58.391517639160156, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.008243671618402004, "fcm_dpo/delta": -0.010835991241037846, "fcm_dpo/margin": 43.715240478515625, "fcm_dpo/q_t": 0.41649821400642395, "grad_norm": 9.543272018432617, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.2219648361206055, "logits/rejected": -1.381723165512085, "logps/chosen": -189.40606689453125, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -205.28768920898438, "loss": 1.1221, "margin_dpo/margin_mean": 43.715240478515625, "margin_dpo/margin_std": 62.26610565185547, "step": 266 }, { "KL/chosen_KL_mean": -78.04536437988281, "KL/mean": -103.5601806640625, "KL/rejected_KL_mean": -129.07498168945312, "KL/std": 58.69158172607422, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.008139451965689659, "fcm_dpo/delta": -0.06869681179523468, "fcm_dpo/margin": 51.02960205078125, "fcm_dpo/q_t": 0.40274274349212646, "grad_norm": 8.273987770080566, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.9093389511108398, "logits/rejected": -1.2947481870651245, "logps/chosen": -182.33135986328125, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -225.06216430664062, "loss": 1.0662, "margin_dpo/margin_mean": 51.029605865478516, "margin_dpo/margin_std": 57.44816970825195, "step": 267 }, { "KL/chosen_KL_mean": -76.75350189208984, "KL/mean": -107.02985382080078, "KL/rejected_KL_mean": -137.30621337890625, "KL/std": 58.934120178222656, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.007939955219626427, "fcm_dpo/delta": -0.13837280869483948, "fcm_dpo/margin": 60.552703857421875, "fcm_dpo/q_t": 0.38638773560523987, "grad_norm": 8.477631568908691, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.7901979684829712, "logits/rejected": -1.6429294347763062, "logps/chosen": -163.44972229003906, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -246.498046875, "loss": 1.0144, "margin_dpo/margin_mean": 60.552696228027344, "margin_dpo/margin_std": 58.729339599609375, "step": 268 }, { "KL/chosen_KL_mean": -82.17233276367188, "KL/mean": -111.282958984375, "KL/rejected_KL_mean": -140.39361572265625, "KL/std": 63.10334014892578, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.00780849251896143, "fcm_dpo/delta": -0.10945925116539001, "fcm_dpo/margin": 58.221282958984375, "fcm_dpo/q_t": 0.3931369185447693, "grad_norm": 7.845900058746338, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.842146635055542, "logits/rejected": -1.3906536102294922, "logps/chosen": -187.14413452148438, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -252.8700714111328, "loss": 1.0346, "margin_dpo/margin_mean": 58.221282958984375, "margin_dpo/margin_std": 59.99555969238281, "step": 269 }, { "KL/chosen_KL_mean": -92.02018737792969, "KL/mean": -116.09602355957031, "KL/rejected_KL_mean": -140.171875, "KL/std": 61.78753662109375, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.007742383051663637, "fcm_dpo/delta": -0.023804346099495888, "fcm_dpo/margin": 48.151695251464844, "fcm_dpo/q_t": 0.4129055440425873, "grad_norm": 8.701428413391113, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.497821807861328, "logits/rejected": -2.090519428253174, "logps/chosen": -193.09402465820312, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -257.9247741699219, "loss": 1.1197, "margin_dpo/margin_mean": 48.151695251464844, "margin_dpo/margin_std": 70.40263366699219, "step": 270 }, { "KL/chosen_KL_mean": -75.57098388671875, "KL/mean": -105.97521209716797, "KL/rejected_KL_mean": -136.37945556640625, "KL/std": 58.266265869140625, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.007567855529487133, "fcm_dpo/delta": -0.11570172756910324, "fcm_dpo/margin": 60.80847930908203, "fcm_dpo/q_t": 0.3909156620502472, "grad_norm": 9.043657302856445, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.4472119808197021, "logits/rejected": -0.9879046678543091, "logps/chosen": -172.2955780029297, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -234.90386962890625, "loss": 1.0392, "margin_dpo/margin_mean": 60.80847930908203, "margin_dpo/margin_std": 66.67576599121094, "step": 271 }, { "KL/chosen_KL_mean": -64.77155303955078, "KL/mean": -93.96470642089844, "KL/rejected_KL_mean": -123.15787506103516, "KL/std": 64.14971923828125, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.007434091530740261, "fcm_dpo/delta": -0.08788064867258072, "fcm_dpo/margin": 58.38630676269531, "fcm_dpo/q_t": 0.3989737629890442, "grad_norm": 7.788511276245117, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.263198733329773, "logits/rejected": -1.1549503803253174, "logps/chosen": -145.748779296875, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -231.71322631835938, "loss": 1.0625, "margin_dpo/margin_mean": 58.38630676269531, "margin_dpo/margin_std": 69.96501159667969, "step": 272 }, { "KL/chosen_KL_mean": -87.41521453857422, "KL/mean": -117.14788055419922, "KL/rejected_KL_mean": -146.88055419921875, "KL/std": 65.6871337890625, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.007257508113980293, "fcm_dpo/delta": -0.0867895856499672, "fcm_dpo/margin": 59.46532440185547, "fcm_dpo/q_t": 0.3989468216896057, "grad_norm": 7.807210922241211, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.4599825143814087, "logits/rejected": -1.0383872985839844, "logps/chosen": -179.6398162841797, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -244.2436065673828, "loss": 1.074, "margin_dpo/margin_mean": 59.46532440185547, "margin_dpo/margin_std": 74.86640930175781, "step": 273 }, { "KL/chosen_KL_mean": -89.73797607421875, "KL/mean": -112.59333038330078, "KL/rejected_KL_mean": -135.4486846923828, "KL/std": 64.27874755859375, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.007278014440089464, "fcm_dpo/delta": 0.017851950600743294, "fcm_dpo/margin": 45.710716247558594, "fcm_dpo/q_t": 0.42170146107673645, "grad_norm": 11.719466209411621, "learning_rate": 3.75e-07, "logits/chosen": -1.9187718629837036, "logits/rejected": -1.5246264934539795, "logps/chosen": -171.99887084960938, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -219.32568359375, "loss": 1.1263, "margin_dpo/margin_mean": 45.710716247558594, "margin_dpo/margin_std": 59.490509033203125, "step": 274 }, { "KL/chosen_KL_mean": -76.06962585449219, "KL/mean": -102.16270446777344, "KL/rejected_KL_mean": -128.25576782226562, "KL/std": 63.2301025390625, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.007302380166947842, "fcm_dpo/delta": -0.032672151923179626, "fcm_dpo/margin": 52.18614196777344, "fcm_dpo/q_t": 0.4110149145126343, "grad_norm": 8.589164733886719, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.2456319332122803, "logits/rejected": -1.9309704303741455, "logps/chosen": -155.756591796875, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -227.00672912597656, "loss": 1.1172, "margin_dpo/margin_mean": 52.1861457824707, "margin_dpo/margin_std": 75.81049346923828, "step": 275 }, { "KL/chosen_KL_mean": -89.36299133300781, "KL/mean": -117.51942443847656, "KL/rejected_KL_mean": -145.67587280273438, "KL/std": 64.07549285888672, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.007150116842240095, "fcm_dpo/delta": -0.055920906364917755, "fcm_dpo/margin": 56.312904357910156, "fcm_dpo/q_t": 0.4065488576889038, "grad_norm": 9.051939010620117, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.9714518785476685, "logits/rejected": -1.3804383277893066, "logps/chosen": -175.90269470214844, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -223.52981567382812, "loss": 1.0911, "margin_dpo/margin_mean": 56.312904357910156, "margin_dpo/margin_std": 73.4899673461914, "step": 276 }, { "KL/chosen_KL_mean": -97.05482482910156, "KL/mean": -117.39617919921875, "KL/rejected_KL_mean": -137.73753356933594, "KL/std": 62.248390197753906, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0072125717997550964, "fcm_dpo/delta": 0.058265142142772675, "fcm_dpo/margin": 40.682701110839844, "fcm_dpo/q_t": 0.4313299357891083, "grad_norm": 8.221242904663086, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.027783155441284, "logits/rejected": -1.6102209091186523, "logps/chosen": -181.2989501953125, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -236.57174682617188, "loss": 1.1673, "margin_dpo/margin_mean": 40.682701110839844, "margin_dpo/margin_std": 64.49189758300781, "step": 277 }, { "KL/chosen_KL_mean": -92.74212646484375, "KL/mean": -118.56793975830078, "KL/rejected_KL_mean": -144.39376831054688, "KL/std": 66.21127319335938, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.007218712940812111, "fcm_dpo/delta": -0.023841749876737595, "fcm_dpo/margin": 51.65162658691406, "fcm_dpo/q_t": 0.41184836626052856, "grad_norm": 10.928853988647461, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.0978050231933594, "logits/rejected": -1.8817291259765625, "logps/chosen": -175.17401123046875, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -242.25067138671875, "loss": 1.1052, "margin_dpo/margin_mean": 51.65163040161133, "margin_dpo/margin_std": 67.08634948730469, "step": 278 }, { "KL/chosen_KL_mean": -94.55284881591797, "KL/mean": -127.48660278320312, "KL/rejected_KL_mean": -160.42034912109375, "KL/std": 71.2509765625, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.007068542763590813, "fcm_dpo/delta": -0.12187275290489197, "fcm_dpo/margin": 65.86748504638672, "fcm_dpo/q_t": 0.3911688029766083, "grad_norm": 9.932906150817871, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.6829710006713867, "logits/rejected": -1.4655585289001465, "logps/chosen": -179.72085571289062, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -254.5469970703125, "loss": 1.0331, "margin_dpo/margin_mean": 65.86749267578125, "margin_dpo/margin_std": 71.17918395996094, "step": 279 }, { "KL/chosen_KL_mean": -98.19157409667969, "KL/mean": -130.76605224609375, "KL/rejected_KL_mean": -163.3404998779297, "KL/std": 64.81028747558594, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.006948621012270451, "fcm_dpo/delta": -0.10743968188762665, "fcm_dpo/margin": 65.14891052246094, "fcm_dpo/q_t": 0.3933781385421753, "grad_norm": 8.063514709472656, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.7786375284194946, "logits/rejected": -1.5813426971435547, "logps/chosen": -184.04798889160156, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -267.4591064453125, "loss": 1.0378, "margin_dpo/margin_mean": 65.14891052246094, "margin_dpo/margin_std": 68.23074340820312, "step": 280 }, { "KL/chosen_KL_mean": -105.43119812011719, "KL/mean": -137.31155395507812, "KL/rejected_KL_mean": -169.19189453125, "KL/std": 74.85796356201172, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.006794154644012451, "fcm_dpo/delta": -0.08726413547992706, "fcm_dpo/margin": 63.76069259643555, "fcm_dpo/q_t": 0.39906129240989685, "grad_norm": 6.492411136627197, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.1459834575653076, "logits/rejected": -1.4104011058807373, "logps/chosen": -191.98202514648438, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -257.820556640625, "loss": 1.0612, "margin_dpo/margin_mean": 63.76069641113281, "margin_dpo/margin_std": 75.564453125, "step": 281 }, { "KL/chosen_KL_mean": -102.40495300292969, "KL/mean": -142.89710998535156, "KL/rejected_KL_mean": -183.3892822265625, "KL/std": 72.93167114257812, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.006580991670489311, "fcm_dpo/delta": -0.19345536828041077, "fcm_dpo/margin": 80.98432922363281, "fcm_dpo/q_t": 0.3745262026786804, "grad_norm": 8.32663345336914, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.8054569959640503, "logits/rejected": -1.423262357711792, "logps/chosen": -207.51007080078125, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -286.2426452636719, "loss": 0.9762, "margin_dpo/margin_mean": 80.98432922363281, "margin_dpo/margin_std": 73.477294921875, "step": 282 }, { "KL/chosen_KL_mean": -119.07147216796875, "KL/mean": -147.0518798828125, "KL/rejected_KL_mean": -175.03225708007812, "KL/std": 72.02287292480469, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.006506140343844891, "fcm_dpo/delta": -0.014625257812440395, "fcm_dpo/margin": 55.96080017089844, "fcm_dpo/q_t": 0.41502153873443604, "grad_norm": 9.234307289123535, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.283174514770508, "logits/rejected": -1.9532768726348877, "logps/chosen": -214.1240692138672, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -278.5768127441406, "loss": 1.1198, "margin_dpo/margin_mean": 55.96080017089844, "margin_dpo/margin_std": 80.27281951904297, "step": 283 }, { "KL/chosen_KL_mean": -108.46876525878906, "KL/mean": -144.15835571289062, "KL/rejected_KL_mean": -179.8479461669922, "KL/std": 70.65374755859375, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.006373896263539791, "fcm_dpo/delta": -0.11061577498912811, "fcm_dpo/margin": 71.37918090820312, "fcm_dpo/q_t": 0.3913913071155548, "grad_norm": 8.803728103637695, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.5290446281433105, "logits/rejected": -1.1741125583648682, "logps/chosen": -196.13523864746094, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -278.5989990234375, "loss": 1.0218, "margin_dpo/margin_mean": 71.37918090820312, "margin_dpo/margin_std": 63.9495849609375, "step": 284 }, { "KL/chosen_KL_mean": -121.41789245605469, "KL/mean": -146.36827087402344, "KL/rejected_KL_mean": -171.3186492919922, "KL/std": 65.79862976074219, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.00640258751809597, "fcm_dpo/delta": 0.03137678653001785, "fcm_dpo/margin": 49.90076446533203, "fcm_dpo/q_t": 0.42366236448287964, "grad_norm": 9.169021606445312, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.3047409057617188, "logits/rejected": -2.021254062652588, "logps/chosen": -215.85092163085938, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -275.3905944824219, "loss": 1.1317, "margin_dpo/margin_mean": 49.90076446533203, "margin_dpo/margin_std": 62.88381576538086, "step": 285 }, { "KL/chosen_KL_mean": -125.97285461425781, "KL/mean": -160.09835815429688, "KL/rejected_KL_mean": -194.22390747070312, "KL/std": 69.73236846923828, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.006313244346529245, "fcm_dpo/delta": -0.08464810252189636, "fcm_dpo/margin": 68.25103759765625, "fcm_dpo/q_t": 0.398033082485199, "grad_norm": 7.874244213104248, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.978327751159668, "logits/rejected": -1.5154426097869873, "logps/chosen": -229.69325256347656, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -290.4816589355469, "loss": 1.0517, "margin_dpo/margin_mean": 68.25103759765625, "margin_dpo/margin_std": 72.51144409179688, "step": 286 }, { "KL/chosen_KL_mean": -144.39927673339844, "KL/mean": -163.60888671875, "KL/rejected_KL_mean": -182.8184814453125, "KL/std": 62.65721893310547, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.006301576271653175, "fcm_dpo/delta": 0.013595999218523502, "fcm_dpo/margin": 38.41919708251953, "fcm_dpo/q_t": 0.4436153173446655, "grad_norm": 7.332032680511475, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.0325276851654053, "logits/rejected": -1.4102911949157715, "logps/chosen": -238.28915405273438, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -266.1521301269531, "loss": 1.2131, "margin_dpo/margin_mean": 38.4192008972168, "margin_dpo/margin_std": 73.53314208984375, "step": 287 }, { "KL/chosen_KL_mean": -136.67356872558594, "KL/mean": -171.930908203125, "KL/rejected_KL_mean": -207.18824768066406, "KL/std": 70.92202758789062, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.006213480606675148, "fcm_dpo/delta": -0.09239435940980911, "fcm_dpo/margin": 70.51467895507812, "fcm_dpo/q_t": 0.3980831801891327, "grad_norm": 7.061956882476807, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.8384209871292114, "logits/rejected": -1.4234199523925781, "logps/chosen": -224.82958984375, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -300.47021484375, "loss": 1.0523, "margin_dpo/margin_mean": 70.51467895507812, "margin_dpo/margin_std": 79.67691802978516, "step": 288 }, { "KL/chosen_KL_mean": -132.38162231445312, "KL/mean": -167.97216796875, "KL/rejected_KL_mean": -203.5626983642578, "KL/std": 74.10955810546875, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.006123432889580727, "fcm_dpo/delta": -0.08969120681285858, "fcm_dpo/margin": 71.18107604980469, "fcm_dpo/q_t": 0.3999096155166626, "grad_norm": 6.66898250579834, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.0703577995300293, "logits/rejected": -1.869124174118042, "logps/chosen": -207.77455139160156, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -296.71697998046875, "loss": 1.0612, "margin_dpo/margin_mean": 71.18107604980469, "margin_dpo/margin_std": 85.80131530761719, "step": 289 }, { "KL/chosen_KL_mean": -142.53533935546875, "KL/mean": -179.62734985351562, "KL/rejected_KL_mean": -216.71937561035156, "KL/std": 78.79547882080078, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.006027707830071449, "fcm_dpo/delta": -0.10179469734430313, "fcm_dpo/margin": 74.18403625488281, "fcm_dpo/q_t": 0.3968440890312195, "grad_norm": 7.589460372924805, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.1362847089767456, "logits/rejected": -1.1871017217636108, "logps/chosen": -230.57733154296875, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -339.9315185546875, "loss": 1.0497, "margin_dpo/margin_mean": 74.18403625488281, "margin_dpo/margin_std": 83.59091186523438, "step": 290 }, { "KL/chosen_KL_mean": -134.89630126953125, "KL/mean": -166.83445739746094, "KL/rejected_KL_mean": -198.77261352539062, "KL/std": 74.81896209716797, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.0059422957710921764, "fcm_dpo/delta": -0.03073292225599289, "fcm_dpo/margin": 63.876304626464844, "fcm_dpo/q_t": 0.4100034236907959, "grad_norm": 8.976141929626465, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.2341737747192383, "logits/rejected": -1.7614436149597168, "logps/chosen": -228.99154663085938, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -294.82269287109375, "loss": 1.0903, "margin_dpo/margin_mean": 63.876304626464844, "margin_dpo/margin_std": 75.36356353759766, "step": 291 }, { "KL/chosen_KL_mean": -139.16436767578125, "KL/mean": -171.6376190185547, "KL/rejected_KL_mean": -204.11087036132812, "KL/std": 72.63249206542969, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0058939517475664616, "fcm_dpo/delta": -0.03410562127828598, "fcm_dpo/margin": 64.9465103149414, "fcm_dpo/q_t": 0.4090948700904846, "grad_norm": 7.8288397789001465, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0692238807678223, "logits/rejected": -1.6382479667663574, "logps/chosen": -227.41477966308594, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -300.52850341796875, "loss": 1.078, "margin_dpo/margin_mean": 64.94651794433594, "margin_dpo/margin_std": 68.33169555664062, "step": 292 }, { "KL/chosen_KL_mean": -139.09616088867188, "KL/mean": -165.2956085205078, "KL/rejected_KL_mean": -191.49505615234375, "KL/std": 76.19816589355469, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.005939970724284649, "fcm_dpo/delta": 0.03963159769773483, "fcm_dpo/margin": 52.3989143371582, "fcm_dpo/q_t": 0.4280659556388855, "grad_norm": 7.193743705749512, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.3785829544067383, "logits/rejected": -1.8113512992858887, "logps/chosen": -226.4727020263672, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -277.2508544921875, "loss": 1.155, "margin_dpo/margin_mean": 52.3989143371582, "margin_dpo/margin_std": 81.24638366699219, "step": 293 }, { "KL/chosen_KL_mean": -140.63800048828125, "KL/mean": -162.30995178222656, "KL/rejected_KL_mean": -183.98191833496094, "KL/std": 71.9407730102539, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.005945391952991486, "fcm_dpo/delta": 0.022729923948645592, "fcm_dpo/margin": 43.343910217285156, "fcm_dpo/q_t": 0.4399092197418213, "grad_norm": 7.377453804016113, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.691917896270752, "logits/rejected": -1.5277135372161865, "logps/chosen": -214.1459503173828, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -272.0706787109375, "loss": 1.2079, "margin_dpo/margin_mean": 43.343910217285156, "margin_dpo/margin_std": 84.58929443359375, "step": 294 }, { "KL/chosen_KL_mean": -133.11514282226562, "KL/mean": -166.41610717773438, "KL/rejected_KL_mean": -199.71707153320312, "KL/std": 74.42646789550781, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.005899349227547646, "fcm_dpo/delta": -0.04503979533910751, "fcm_dpo/margin": 66.60189819335938, "fcm_dpo/q_t": 0.4079926013946533, "grad_norm": 7.999210357666016, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.7744061946868896, "logits/rejected": -1.6204097270965576, "logps/chosen": -210.88064575195312, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -293.9643249511719, "loss": 1.0854, "margin_dpo/margin_mean": 66.6019058227539, "margin_dpo/margin_std": 80.74388122558594, "step": 295 }, { "KL/chosen_KL_mean": -143.1065673828125, "KL/mean": -178.4542236328125, "KL/rejected_KL_mean": -213.80186462402344, "KL/std": 74.54656982421875, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.00583041924983263, "fcm_dpo/delta": -0.06524956971406937, "fcm_dpo/margin": 70.69529724121094, "fcm_dpo/q_t": 0.4021443724632263, "grad_norm": 10.98575496673584, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.4445385932922363, "logits/rejected": -1.9510250091552734, "logps/chosen": -240.96298217773438, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -314.6181640625, "loss": 1.0626, "margin_dpo/margin_mean": 70.69529724121094, "margin_dpo/margin_std": 75.31361389160156, "step": 296 }, { "KL/chosen_KL_mean": -143.8113250732422, "KL/mean": -173.7741241455078, "KL/rejected_KL_mean": -203.73695373535156, "KL/std": 75.07388305664062, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.005838936660438776, "fcm_dpo/delta": 4.1915103793144226e-05, "fcm_dpo/margin": 59.92561721801758, "fcm_dpo/q_t": 0.41857969760894775, "grad_norm": 9.835691452026367, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2443082332611084, "logits/rejected": -0.9373881816864014, "logps/chosen": -232.37716674804688, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -305.29351806640625, "loss": 1.1188, "margin_dpo/margin_mean": 59.925621032714844, "margin_dpo/margin_std": 80.53132629394531, "step": 297 }, { "KL/chosen_KL_mean": -120.86933135986328, "KL/mean": -159.34036254882812, "KL/rejected_KL_mean": -197.8114013671875, "KL/std": 81.53010559082031, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0057587334886193275, "fcm_dpo/delta": -0.09728903323411942, "fcm_dpo/margin": 76.94205474853516, "fcm_dpo/q_t": 0.39709043502807617, "grad_norm": 10.381818771362305, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.0968682765960693, "logits/rejected": -2.0784752368927, "logps/chosen": -215.74977111816406, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -320.1224365234375, "loss": 1.0454, "margin_dpo/margin_mean": 76.94206237792969, "margin_dpo/margin_std": 83.4140625, "step": 298 }, { "KL/chosen_KL_mean": -125.24130249023438, "KL/mean": -163.216552734375, "KL/rejected_KL_mean": -201.1917724609375, "KL/std": 78.64089965820312, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.005645422264933586, "fcm_dpo/delta": -0.08232633769512177, "fcm_dpo/margin": 75.95047760009766, "fcm_dpo/q_t": 0.39906740188598633, "grad_norm": 9.80588150024414, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.2221968173980713, "logits/rejected": -1.0041632652282715, "logps/chosen": -205.649658203125, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -290.72894287109375, "loss": 1.0534, "margin_dpo/margin_mean": 75.95048522949219, "margin_dpo/margin_std": 82.01759338378906, "step": 299 }, { "KL/chosen_KL_mean": -131.5859375, "KL/mean": -163.2269287109375, "KL/rejected_KL_mean": -194.867919921875, "KL/std": 81.92149353027344, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.005599002353847027, "fcm_dpo/delta": -0.004758972674608231, "fcm_dpo/margin": 63.281986236572266, "fcm_dpo/q_t": 0.41694116592407227, "grad_norm": 7.522899150848389, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.5384433269500732, "logits/rejected": -1.325578212738037, "logps/chosen": -219.74484252929688, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -295.80712890625, "loss": 1.123, "margin_dpo/margin_mean": 63.281986236572266, "margin_dpo/margin_std": 88.60911560058594, "step": 300 }, { "KL/chosen_KL_mean": -121.26895904541016, "KL/mean": -160.43284606933594, "KL/rejected_KL_mean": -199.59674072265625, "KL/std": 80.12481689453125, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.005521929357200861, "fcm_dpo/delta": -0.08699830621480942, "fcm_dpo/margin": 78.32777404785156, "fcm_dpo/q_t": 0.3988070487976074, "grad_norm": 8.654906272888184, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7620010375976562, "logits/rejected": -1.3327397108078003, "logps/chosen": -206.77386474609375, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -298.5135803222656, "loss": 1.0591, "margin_dpo/margin_mean": 78.32777404785156, "margin_dpo/margin_std": 90.33263397216797, "step": 301 }, { "KL/chosen_KL_mean": -115.97718811035156, "KL/mean": -150.0538330078125, "KL/rejected_KL_mean": -184.1304931640625, "KL/std": 82.38044738769531, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.005491076502948999, "fcm_dpo/delta": -0.025276431813836098, "fcm_dpo/margin": 68.15330505371094, "fcm_dpo/q_t": 0.4112234115600586, "grad_norm": 7.553112506866455, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3962509632110596, "logits/rejected": -1.2226777076721191, "logps/chosen": -195.42759704589844, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -287.1895751953125, "loss": 1.094, "margin_dpo/margin_mean": 68.15330505371094, "margin_dpo/margin_std": 80.92733764648438, "step": 302 }, { "KL/chosen_KL_mean": -109.59970092773438, "KL/mean": -146.57974243164062, "KL/rejected_KL_mean": -183.559814453125, "KL/std": 76.71343994140625, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.0054594725370407104, "fcm_dpo/delta": -0.05602101981639862, "fcm_dpo/margin": 73.9601058959961, "fcm_dpo/q_t": 0.4042169153690338, "grad_norm": 7.521291255950928, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.9082120656967163, "logits/rejected": -1.4497567415237427, "logps/chosen": -205.197998046875, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -275.6243896484375, "loss": 1.072, "margin_dpo/margin_mean": 73.96011352539062, "margin_dpo/margin_std": 82.29890441894531, "step": 303 }, { "KL/chosen_KL_mean": -109.38790130615234, "KL/mean": -147.53933715820312, "KL/rejected_KL_mean": -185.69076538085938, "KL/std": 79.2987060546875, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.005382024683058262, "fcm_dpo/delta": -0.06325141340494156, "fcm_dpo/margin": 76.30288696289062, "fcm_dpo/q_t": 0.402679443359375, "grad_norm": 8.192179679870605, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.042022943496704, "logits/rejected": -1.61529541015625, "logps/chosen": -208.84104919433594, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -283.7730407714844, "loss": 1.0642, "margin_dpo/margin_mean": 76.30288696289062, "margin_dpo/margin_std": 82.73768615722656, "step": 304 }, { "KL/chosen_KL_mean": -120.17296600341797, "KL/mean": -156.2691650390625, "KL/rejected_KL_mean": -192.3653564453125, "KL/std": 82.83584594726562, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0053099375218153, "fcm_dpo/delta": -0.035309046506881714, "fcm_dpo/margin": 72.19239807128906, "fcm_dpo/q_t": 0.40936869382858276, "grad_norm": 7.166097164154053, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.0199639797210693, "logits/rejected": -1.5237998962402344, "logps/chosen": -223.43637084960938, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -290.1943359375, "loss": 1.0971, "margin_dpo/margin_mean": 72.19239807128906, "margin_dpo/margin_std": 90.10063171386719, "step": 305 }, { "KL/chosen_KL_mean": -107.980224609375, "KL/mean": -150.15087890625, "KL/rejected_KL_mean": -192.32150268554688, "KL/std": 78.54069519042969, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0052381884306669235, "fcm_dpo/delta": -0.09636321663856506, "fcm_dpo/margin": 84.34127807617188, "fcm_dpo/q_t": 0.393859326839447, "grad_norm": 8.889698028564453, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.250764846801758, "logits/rejected": -1.6948184967041016, "logps/chosen": -208.2840576171875, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -284.5523681640625, "loss": 1.0281, "margin_dpo/margin_mean": 84.34127807617188, "margin_dpo/margin_std": 72.96005249023438, "step": 306 }, { "KL/chosen_KL_mean": -112.49227905273438, "KL/mean": -144.38507080078125, "KL/rejected_KL_mean": -176.27783203125, "KL/std": 79.01229858398438, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.005221827886998653, "fcm_dpo/delta": 0.017367538064718246, "fcm_dpo/margin": 63.785552978515625, "fcm_dpo/q_t": 0.4220791757106781, "grad_norm": 7.942570686340332, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.2056474685668945, "logits/rejected": -1.8721623420715332, "logps/chosen": -209.1420440673828, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -269.6264343261719, "loss": 1.1393, "margin_dpo/margin_mean": 63.785552978515625, "margin_dpo/margin_std": 94.03587341308594, "step": 307 }, { "KL/chosen_KL_mean": -117.96734619140625, "KL/mean": -149.69699096679688, "KL/rejected_KL_mean": -181.4266357421875, "KL/std": 80.64797973632812, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.005229164846241474, "fcm_dpo/delta": 0.0183115191757679, "fcm_dpo/margin": 63.45930099487305, "fcm_dpo/q_t": 0.42274364829063416, "grad_norm": 9.462555885314941, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7089744806289673, "logits/rejected": -1.4242830276489258, "logps/chosen": -203.6920166015625, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -267.8498229980469, "loss": 1.1442, "margin_dpo/margin_mean": 63.45929718017578, "margin_dpo/margin_std": 94.71066284179688, "step": 308 }, { "KL/chosen_KL_mean": -132.37985229492188, "KL/mean": -161.18807983398438, "KL/rejected_KL_mean": -189.99630737304688, "KL/std": 86.82447814941406, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.005300990305840969, "fcm_dpo/delta": 0.04594644904136658, "fcm_dpo/margin": 57.61646270751953, "fcm_dpo/q_t": 0.4282302260398865, "grad_norm": 10.732254028320312, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.5848394632339478, "logits/rejected": -1.1438573598861694, "logps/chosen": -215.53131103515625, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -276.58233642578125, "loss": 1.1578, "margin_dpo/margin_mean": 57.61646270751953, "margin_dpo/margin_std": 87.20914459228516, "step": 309 }, { "KL/chosen_KL_mean": -121.56426239013672, "KL/mean": -153.26654052734375, "KL/rejected_KL_mean": -184.96881103515625, "KL/std": 78.15449523925781, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.005317146889865398, "fcm_dpo/delta": 0.01331715751439333, "fcm_dpo/margin": 63.404541015625, "fcm_dpo/q_t": 0.4203488826751709, "grad_norm": 8.176267623901367, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2380831241607666, "logits/rejected": -1.82277250289917, "logps/chosen": -211.30224609375, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -272.0396423339844, "loss": 1.1237, "margin_dpo/margin_mean": 63.404544830322266, "margin_dpo/margin_std": 82.62731170654297, "step": 310 }, { "KL/chosen_KL_mean": -140.05174255371094, "KL/mean": -180.84841918945312, "KL/rejected_KL_mean": -221.64508056640625, "KL/std": 79.79002380371094, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.00526207173243165, "fcm_dpo/delta": -0.08287054300308228, "fcm_dpo/margin": 81.59333801269531, "fcm_dpo/q_t": 0.3986778259277344, "grad_norm": 8.520589828491211, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.4038853645324707, "logits/rejected": -0.6459282636642456, "logps/chosen": -228.57818603515625, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -303.9111633300781, "loss": 1.0454, "margin_dpo/margin_mean": 81.59333801269531, "margin_dpo/margin_std": 81.71646881103516, "step": 311 }, { "KL/chosen_KL_mean": -135.35687255859375, "KL/mean": -169.5532684326172, "KL/rejected_KL_mean": -203.74966430664062, "KL/std": 87.6471176147461, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.005243558436632156, "fcm_dpo/delta": -0.009057698771357536, "fcm_dpo/margin": 68.39279174804688, "fcm_dpo/q_t": 0.41516172885894775, "grad_norm": 8.971466064453125, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.5237551927566528, "logits/rejected": -0.9923006296157837, "logps/chosen": -215.757568359375, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -286.1968078613281, "loss": 1.1066, "margin_dpo/margin_mean": 68.39279174804688, "margin_dpo/margin_std": 84.198486328125, "step": 312 }, { "KL/chosen_KL_mean": -130.0532684326172, "KL/mean": -163.177001953125, "KL/rejected_KL_mean": -196.30075073242188, "KL/std": 85.82874298095703, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0052446769550442696, "fcm_dpo/delta": 0.0024643726646900177, "fcm_dpo/margin": 66.24748229980469, "fcm_dpo/q_t": 0.41846150159835815, "grad_norm": 9.055826187133789, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.4679150581359863, "logits/rejected": -2.069694757461548, "logps/chosen": -214.19517517089844, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -289.72760009765625, "loss": 1.1259, "margin_dpo/margin_mean": 66.24748229980469, "margin_dpo/margin_std": 92.85113525390625, "step": 313 }, { "KL/chosen_KL_mean": -132.26490783691406, "KL/mean": -164.83261108398438, "KL/rejected_KL_mean": -197.40032958984375, "KL/std": 86.49197387695312, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.005224254913628101, "fcm_dpo/delta": 0.009947361424565315, "fcm_dpo/margin": 65.13542938232422, "fcm_dpo/q_t": 0.4206373691558838, "grad_norm": 9.236519813537598, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -2.272495746612549, "logits/rejected": -2.063737630844116, "logps/chosen": -221.0746307373047, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -301.55926513671875, "loss": 1.1265, "margin_dpo/margin_mean": 65.13543701171875, "margin_dpo/margin_std": 88.77565002441406, "step": 314 }, { "KL/chosen_KL_mean": -128.1319580078125, "KL/mean": -161.8524169921875, "KL/rejected_KL_mean": -195.5728759765625, "KL/std": 90.7603530883789, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.005234315991401672, "fcm_dpo/delta": -0.003129352815449238, "fcm_dpo/margin": 67.44091796875, "fcm_dpo/q_t": 0.41832107305526733, "grad_norm": 8.824543952941895, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.295661449432373, "logits/rejected": -1.1203477382659912, "logps/chosen": -208.9256591796875, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -287.7099914550781, "loss": 1.1364, "margin_dpo/margin_mean": 67.44091796875, "margin_dpo/margin_std": 105.59107971191406, "step": 315 }, { "KL/chosen_KL_mean": -130.52944946289062, "KL/mean": -167.23947143554688, "KL/rejected_KL_mean": -203.9495086669922, "KL/std": 89.46089172363281, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.005216202698647976, "fcm_dpo/delta": -0.03427394479513168, "fcm_dpo/margin": 73.42005920410156, "fcm_dpo/q_t": 0.4108394682407379, "grad_norm": 8.77436637878418, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.8393409252166748, "logits/rejected": -1.8386553525924683, "logps/chosen": -214.7136993408203, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -309.64031982421875, "loss": 1.0961, "margin_dpo/margin_mean": 73.42005920410156, "margin_dpo/margin_std": 94.26622009277344, "step": 316 }, { "KL/chosen_KL_mean": -135.73959350585938, "KL/mean": -180.4974365234375, "KL/rejected_KL_mean": -225.25526428222656, "KL/std": 98.4647216796875, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0051298439502716064, "fcm_dpo/delta": -0.11430003494024277, "fcm_dpo/margin": 89.51567077636719, "fcm_dpo/q_t": 0.3936837911605835, "grad_norm": 10.071937561035156, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.9707934856414795, "logits/rejected": -1.4971979856491089, "logps/chosen": -223.61309814453125, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -333.52044677734375, "loss": 1.049, "margin_dpo/margin_mean": 89.51567077636719, "margin_dpo/margin_std": 105.60486602783203, "step": 317 }, { "KL/chosen_KL_mean": -132.0321044921875, "KL/mean": -171.69967651367188, "KL/rejected_KL_mean": -211.36729431152344, "KL/std": 82.54495239257812, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.005039385985583067, "fcm_dpo/delta": -0.05195074528455734, "fcm_dpo/margin": 79.33518981933594, "fcm_dpo/q_t": 0.40571972727775574, "grad_norm": 9.464632987976074, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.2192163467407227, "logits/rejected": -1.5020105838775635, "logps/chosen": -230.2376251220703, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -299.50360107421875, "loss": 1.0768, "margin_dpo/margin_mean": 79.3351821899414, "margin_dpo/margin_std": 92.03431701660156, "step": 318 }, { "KL/chosen_KL_mean": -114.55751037597656, "KL/mean": -158.569580078125, "KL/rejected_KL_mean": -202.5816650390625, "KL/std": 86.30485534667969, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.004956516902893782, "fcm_dpo/delta": -0.09041348099708557, "fcm_dpo/margin": 88.0241470336914, "fcm_dpo/q_t": 0.3974972069263458, "grad_norm": 9.210772514343262, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.8791106939315796, "logits/rejected": -1.811793327331543, "logps/chosen": -197.06088256835938, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -321.23236083984375, "loss": 1.0522, "margin_dpo/margin_mean": 88.02415466308594, "margin_dpo/margin_std": 97.95591735839844, "step": 319 }, { "KL/chosen_KL_mean": -116.28398132324219, "KL/mean": -152.45193481445312, "KL/rejected_KL_mean": -188.619873046875, "KL/std": 87.54345703125, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.004928593523800373, "fcm_dpo/delta": -0.006812551058828831, "fcm_dpo/margin": 72.33589172363281, "fcm_dpo/q_t": 0.41576850414276123, "grad_norm": 8.092971801757812, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.1446752548217773, "logits/rejected": -1.9597063064575195, "logps/chosen": -203.42120361328125, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -291.843994140625, "loss": 1.1034, "margin_dpo/margin_mean": 72.33589172363281, "margin_dpo/margin_std": 85.44863891601562, "step": 320 }, { "KL/chosen_KL_mean": -131.13351440429688, "KL/mean": -172.87588500976562, "KL/rejected_KL_mean": -214.6182861328125, "KL/std": 90.00694274902344, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.004875160753726959, "fcm_dpo/delta": -0.05981387570500374, "fcm_dpo/margin": 83.4847640991211, "fcm_dpo/q_t": 0.402859628200531, "grad_norm": 7.976510524749756, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.9396759271621704, "logits/rejected": -1.5500634908676147, "logps/chosen": -224.23818969726562, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -312.08642578125, "loss": 1.0668, "margin_dpo/margin_mean": 83.48475646972656, "margin_dpo/margin_std": 90.00255584716797, "step": 321 }, { "KL/chosen_KL_mean": -122.41797637939453, "KL/mean": -167.44923400878906, "KL/rejected_KL_mean": -212.48048400878906, "KL/std": 88.81075286865234, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.004785523284226656, "fcm_dpo/delta": -0.08585826307535172, "fcm_dpo/margin": 90.06251525878906, "fcm_dpo/q_t": 0.39811965823173523, "grad_norm": 12.793742179870605, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.6938512325286865, "logits/rejected": -1.334911584854126, "logps/chosen": -214.6108856201172, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -313.37371826171875, "loss": 1.0534, "margin_dpo/margin_mean": 90.06251525878906, "margin_dpo/margin_std": 95.52256774902344, "step": 322 }, { "KL/chosen_KL_mean": -142.1260986328125, "KL/mean": -178.97390747070312, "KL/rejected_KL_mean": -215.82174682617188, "KL/std": 89.62907409667969, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.00478294538334012, "fcm_dpo/delta": -0.0026323823258280754, "fcm_dpo/margin": 73.69564056396484, "fcm_dpo/q_t": 0.4175586402416229, "grad_norm": 9.697562217712402, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.9654765129089355, "logits/rejected": -1.5841050148010254, "logps/chosen": -222.398193359375, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -296.1250915527344, "loss": 1.1133, "margin_dpo/margin_mean": 73.69564056396484, "margin_dpo/margin_std": 94.10107421875, "step": 323 }, { "KL/chosen_KL_mean": -141.6981201171875, "KL/mean": -180.31112670898438, "KL/rejected_KL_mean": -218.92413330078125, "KL/std": 90.01133728027344, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.004782845266163349, "fcm_dpo/delta": -0.020116854459047318, "fcm_dpo/margin": 77.22601318359375, "fcm_dpo/q_t": 0.41204482316970825, "grad_norm": 8.20111083984375, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7900323867797852, "logits/rejected": -1.151607632637024, "logps/chosen": -237.42747497558594, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -313.0933837890625, "loss": 1.092, "margin_dpo/margin_mean": 77.22601318359375, "margin_dpo/margin_std": 86.63177490234375, "step": 324 }, { "KL/chosen_KL_mean": -128.5873565673828, "KL/mean": -171.26742553710938, "KL/rejected_KL_mean": -213.94744873046875, "KL/std": 85.61288452148438, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.004733443260192871, "fcm_dpo/delta": -0.056300222873687744, "fcm_dpo/margin": 85.360107421875, "fcm_dpo/q_t": 0.4038904905319214, "grad_norm": 7.329254627227783, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.8335036039352417, "logits/rejected": -1.1025006771087646, "logps/chosen": -227.29421997070312, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -290.3555603027344, "loss": 1.0665, "margin_dpo/margin_mean": 85.360107421875, "margin_dpo/margin_std": 90.13819122314453, "step": 325 }, { "KL/chosen_KL_mean": -143.34722900390625, "KL/mean": -181.1986541748047, "KL/rejected_KL_mean": -219.05007934570312, "KL/std": 84.99134826660156, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.004710469860583544, "fcm_dpo/delta": -0.006847780663520098, "fcm_dpo/margin": 75.7028579711914, "fcm_dpo/q_t": 0.415196031332016, "grad_norm": 7.008384704589844, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.394050121307373, "logits/rejected": -1.7127606868743896, "logps/chosen": -248.008544921875, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -305.8754577636719, "loss": 1.1062, "margin_dpo/margin_mean": 75.70285034179688, "margin_dpo/margin_std": 91.78878784179688, "step": 326 }, { "KL/chosen_KL_mean": -137.825927734375, "KL/mean": -174.58151245117188, "KL/rejected_KL_mean": -211.33706665039062, "KL/std": 78.23960876464844, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.004702937789261341, "fcm_dpo/delta": 0.004330374300479889, "fcm_dpo/margin": 73.51114654541016, "fcm_dpo/q_t": 0.4186253547668457, "grad_norm": 8.016932487487793, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.631654143333435, "logits/rejected": -1.0333083868026733, "logps/chosen": -245.26617431640625, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -328.39031982421875, "loss": 1.1123, "margin_dpo/margin_mean": 73.51115417480469, "margin_dpo/margin_std": 89.34968566894531, "step": 327 }, { "KL/chosen_KL_mean": -134.64797973632812, "KL/mean": -182.54779052734375, "KL/rejected_KL_mean": -230.44760131835938, "KL/std": 92.60899353027344, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.004648599773645401, "fcm_dpo/delta": -0.09972397238016129, "fcm_dpo/margin": 95.79962921142578, "fcm_dpo/q_t": 0.39395570755004883, "grad_norm": 8.996434211730957, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.7301642894744873, "logits/rejected": -1.4325016736984253, "logps/chosen": -221.2586669921875, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -327.8494873046875, "loss": 1.031, "margin_dpo/margin_mean": 95.79962158203125, "margin_dpo/margin_std": 89.64315795898438, "step": 328 }, { "KL/chosen_KL_mean": -139.93014526367188, "KL/mean": -177.82614135742188, "KL/rejected_KL_mean": -215.72215270996094, "KL/std": 89.52311706542969, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.004606503993272781, "fcm_dpo/delta": 0.000652313232421875, "fcm_dpo/margin": 75.7919921875, "fcm_dpo/q_t": 0.41795897483825684, "grad_norm": 9.214507102966309, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.4491586685180664, "logits/rejected": -0.9439181089401245, "logps/chosen": -240.543212890625, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -320.67584228515625, "loss": 1.1158, "margin_dpo/margin_mean": 75.7919921875, "margin_dpo/margin_std": 96.83283996582031, "step": 329 }, { "KL/chosen_KL_mean": -118.43016052246094, "KL/mean": -159.012939453125, "KL/rejected_KL_mean": -199.59573364257812, "KL/std": 87.15354919433594, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.004601151682436466, "fcm_dpo/delta": -0.02441122569143772, "fcm_dpo/margin": 81.16558837890625, "fcm_dpo/q_t": 0.4098995327949524, "grad_norm": 7.671301364898682, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.7106246948242188, "logits/rejected": -1.6309400796890259, "logps/chosen": -195.3885955810547, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -292.4106750488281, "loss": 1.0793, "margin_dpo/margin_mean": 81.16558074951172, "margin_dpo/margin_std": 77.7132568359375, "step": 330 }, { "KL/chosen_KL_mean": -127.84239196777344, "KL/mean": -173.0701446533203, "KL/rejected_KL_mean": -218.2978973388672, "KL/std": 85.23104858398438, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.004568981006741524, "fcm_dpo/delta": -0.06600473076105118, "fcm_dpo/margin": 90.45550537109375, "fcm_dpo/q_t": 0.4014698565006256, "grad_norm": 8.936184883117676, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.7125235795974731, "logits/rejected": -1.4007236957550049, "logps/chosen": -210.9018096923828, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -312.42901611328125, "loss": 1.0535, "margin_dpo/margin_mean": 90.45550537109375, "margin_dpo/margin_std": 86.68510437011719, "step": 331 }, { "KL/chosen_KL_mean": -136.75198364257812, "KL/mean": -167.49658203125, "KL/rejected_KL_mean": -198.2412109375, "KL/std": 93.63404846191406, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.004591916687786579, "fcm_dpo/delta": 0.06951860338449478, "fcm_dpo/margin": 61.489219665527344, "fcm_dpo/q_t": 0.4330301284790039, "grad_norm": 7.543329238891602, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.0127501487731934, "logits/rejected": -1.576089859008789, "logps/chosen": -218.68287658691406, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -281.08709716796875, "loss": 1.1665, "margin_dpo/margin_mean": 61.489219665527344, "margin_dpo/margin_std": 91.65592956542969, "step": 332 }, { "KL/chosen_KL_mean": -156.16439819335938, "KL/mean": -200.26217651367188, "KL/rejected_KL_mean": -244.3599395751953, "KL/std": 98.88365173339844, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.004544874653220177, "fcm_dpo/delta": -0.053686223924160004, "fcm_dpo/margin": 88.195556640625, "fcm_dpo/q_t": 0.4060261845588684, "grad_norm": 7.277347087860107, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5413854122161865, "logits/rejected": -1.1999908685684204, "logps/chosen": -244.6969757080078, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -346.8259582519531, "loss": 1.0825, "margin_dpo/margin_mean": 88.195556640625, "margin_dpo/margin_std": 106.23516845703125, "step": 333 }, { "KL/chosen_KL_mean": -142.4854736328125, "KL/mean": -181.5789794921875, "KL/rejected_KL_mean": -220.67250061035156, "KL/std": 93.3104248046875, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.004527302458882332, "fcm_dpo/delta": -0.0047046951949596405, "fcm_dpo/margin": 78.18701934814453, "fcm_dpo/q_t": 0.41592222452163696, "grad_norm": 8.40447998046875, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.4532335996627808, "logits/rejected": -1.1595230102539062, "logps/chosen": -240.24325561523438, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -320.6783447265625, "loss": 1.1198, "margin_dpo/margin_mean": 78.18702697753906, "margin_dpo/margin_std": 104.3531494140625, "step": 334 }, { "KL/chosen_KL_mean": -141.8665313720703, "KL/mean": -174.9089813232422, "KL/rejected_KL_mean": -207.951416015625, "KL/std": 92.25128936767578, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.0045838141813874245, "fcm_dpo/delta": 0.0485406331717968, "fcm_dpo/margin": 66.08487701416016, "fcm_dpo/q_t": 0.4286586046218872, "grad_norm": 11.441469192504883, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.3494784832000732, "logits/rejected": -1.9265847206115723, "logps/chosen": -238.24256896972656, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -302.87542724609375, "loss": 1.1589, "margin_dpo/margin_mean": 66.08488464355469, "margin_dpo/margin_std": 102.01634216308594, "step": 335 }, { "KL/chosen_KL_mean": -144.83770751953125, "KL/mean": -187.82679748535156, "KL/rejected_KL_mean": -230.81588745117188, "KL/std": 94.79708862304688, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.004573004320263863, "fcm_dpo/delta": -0.044939134269952774, "fcm_dpo/margin": 85.97821044921875, "fcm_dpo/q_t": 0.4074280858039856, "grad_norm": 9.390266418457031, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.6410980224609375, "logits/rejected": -1.492825984954834, "logps/chosen": -233.36721801757812, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -340.3716735839844, "loss": 1.081, "margin_dpo/margin_mean": 85.97820281982422, "margin_dpo/margin_std": 100.87501525878906, "step": 336 }, { "KL/chosen_KL_mean": -142.86288452148438, "KL/mean": -183.94622802734375, "KL/rejected_KL_mean": -225.02957153320312, "KL/std": 95.77134704589844, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.004542349372059107, "fcm_dpo/delta": -0.024131158366799355, "fcm_dpo/margin": 82.16670227050781, "fcm_dpo/q_t": 0.4130394160747528, "grad_norm": 8.075562477111816, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.4820823669433594, "logits/rejected": -2.2052319049835205, "logps/chosen": -240.17526245117188, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -335.1609191894531, "loss": 1.1103, "margin_dpo/margin_mean": 82.16669464111328, "margin_dpo/margin_std": 113.35244750976562, "step": 337 }, { "KL/chosen_KL_mean": -147.4730682373047, "KL/mean": -185.25320434570312, "KL/rejected_KL_mean": -223.0333251953125, "KL/std": 96.79815673828125, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.004532797262072563, "fcm_dpo/delta": 0.007759366650134325, "fcm_dpo/margin": 75.56024169921875, "fcm_dpo/q_t": 0.4184812009334564, "grad_norm": 8.348485946655273, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7938966751098633, "logits/rejected": -1.6404742002487183, "logps/chosen": -231.10317993164062, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -316.7256774902344, "loss": 1.1183, "margin_dpo/margin_mean": 75.56024932861328, "margin_dpo/margin_std": 95.74981689453125, "step": 338 }, { "KL/chosen_KL_mean": -157.57089233398438, "KL/mean": -197.72833251953125, "KL/rejected_KL_mean": -237.88577270507812, "KL/std": 96.09046936035156, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.00455496646463871, "fcm_dpo/delta": -0.017107654362916946, "fcm_dpo/margin": 80.31488037109375, "fcm_dpo/q_t": 0.41380810737609863, "grad_norm": 7.204192638397217, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.8053903579711914, "logits/rejected": -2.368513822555542, "logps/chosen": -257.4227294921875, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -346.96014404296875, "loss": 1.114, "margin_dpo/margin_mean": 80.31488800048828, "margin_dpo/margin_std": 106.92840576171875, "step": 339 }, { "KL/chosen_KL_mean": -167.9724578857422, "KL/mean": -203.09127807617188, "KL/rejected_KL_mean": -238.2100830078125, "KL/std": 88.54473876953125, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0045451694168150425, "fcm_dpo/delta": 0.03176550939679146, "fcm_dpo/margin": 70.23762512207031, "fcm_dpo/q_t": 0.4243122637271881, "grad_norm": 9.05514907836914, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.2953407764434814, "logits/rejected": -0.7998344898223877, "logps/chosen": -252.9464111328125, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -317.01043701171875, "loss": 1.1384, "margin_dpo/margin_mean": 70.23762512207031, "margin_dpo/margin_std": 97.00969696044922, "step": 340 }, { "KL/chosen_KL_mean": -175.78150939941406, "KL/mean": -209.64222717285156, "KL/rejected_KL_mean": -243.50296020507812, "KL/std": 93.2364273071289, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.004570466466248035, "fcm_dpo/delta": 0.04177205264568329, "fcm_dpo/margin": 67.72142791748047, "fcm_dpo/q_t": 0.42781922221183777, "grad_norm": 8.385396003723145, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.433375358581543, "logits/rejected": -1.186025857925415, "logps/chosen": -261.0455017089844, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -349.58892822265625, "loss": 1.1604, "margin_dpo/margin_mean": 67.72142028808594, "margin_dpo/margin_std": 108.01136779785156, "step": 341 }, { "KL/chosen_KL_mean": -140.46286010742188, "KL/mean": -189.66281127929688, "KL/rejected_KL_mean": -238.86276245117188, "KL/std": 92.84190368652344, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.004501305520534515, "fcm_dpo/delta": -0.09819352626800537, "fcm_dpo/margin": 98.39988708496094, "fcm_dpo/q_t": 0.39403313398361206, "grad_norm": 7.104044437408447, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.2284777164459229, "logits/rejected": -0.4461154341697693, "logps/chosen": -241.50146484375, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -337.76129150390625, "loss": 1.0437, "margin_dpo/margin_mean": 98.39988708496094, "margin_dpo/margin_std": 99.81686401367188, "step": 342 }, { "KL/chosen_KL_mean": -152.66610717773438, "KL/mean": -194.0093536376953, "KL/rejected_KL_mean": -235.35260009765625, "KL/std": 97.07315063476562, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.004509078338742256, "fcm_dpo/delta": -0.024203289300203323, "fcm_dpo/margin": 82.68648529052734, "fcm_dpo/q_t": 0.41204655170440674, "grad_norm": 8.925413131713867, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.0024397373199463, "logits/rejected": -1.8941776752471924, "logps/chosen": -236.41346740722656, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -351.93170166015625, "loss": 1.1016, "margin_dpo/margin_mean": 82.68648529052734, "margin_dpo/margin_std": 102.0950927734375, "step": 343 }, { "KL/chosen_KL_mean": -152.0487823486328, "KL/mean": -195.95028686523438, "KL/rejected_KL_mean": -239.85177612304688, "KL/std": 103.51048278808594, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.004470549989491701, "fcm_dpo/delta": -0.04451553151011467, "fcm_dpo/margin": 87.80299377441406, "fcm_dpo/q_t": 0.40822768211364746, "grad_norm": 8.708608627319336, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.5757203102111816, "logits/rejected": -1.327228307723999, "logps/chosen": -230.30276489257812, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -335.28045654296875, "loss": 1.0912, "margin_dpo/margin_mean": 87.80299377441406, "margin_dpo/margin_std": 109.5052490234375, "step": 344 }, { "KL/chosen_KL_mean": -158.05563354492188, "KL/mean": -203.9475555419922, "KL/rejected_KL_mean": -249.83946228027344, "KL/std": 103.03012084960938, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.004397682845592499, "fcm_dpo/delta": -0.055972710251808167, "fcm_dpo/margin": 91.7838134765625, "fcm_dpo/q_t": 0.40520304441452026, "grad_norm": 9.607245445251465, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.1408592462539673, "logits/rejected": -0.8284875750541687, "logps/chosen": -256.1351318359375, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -353.8742370605469, "loss": 1.0751, "margin_dpo/margin_mean": 91.7838134765625, "margin_dpo/margin_std": 107.06498718261719, "step": 345 }, { "KL/chosen_KL_mean": -175.51129150390625, "KL/mean": -211.39675903320312, "KL/rejected_KL_mean": -247.2822265625, "KL/std": 95.68539428710938, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.004410427995026112, "fcm_dpo/delta": 0.0345645546913147, "fcm_dpo/margin": 71.77096557617188, "fcm_dpo/q_t": 0.42600908875465393, "grad_norm": 7.184473037719727, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.8902559280395508, "logits/rejected": -1.5827922821044922, "logps/chosen": -256.19476318359375, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -333.1748352050781, "loss": 1.1501, "margin_dpo/margin_mean": 71.77096557617188, "margin_dpo/margin_std": 109.42692565917969, "step": 346 }, { "KL/chosen_KL_mean": -166.05958557128906, "KL/mean": -200.03662109375, "KL/rejected_KL_mean": -234.01365661621094, "KL/std": 95.46206665039062, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.004466019570827484, "fcm_dpo/delta": 0.04760833829641342, "fcm_dpo/margin": 67.95407104492188, "fcm_dpo/q_t": 0.42903193831443787, "grad_norm": 8.011259078979492, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.8780324459075928, "logits/rejected": -1.559518575668335, "logps/chosen": -259.73895263671875, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -331.55615234375, "loss": 1.1581, "margin_dpo/margin_mean": 67.95407104492188, "margin_dpo/margin_std": 103.873779296875, "step": 347 }, { "KL/chosen_KL_mean": -142.58551025390625, "KL/mean": -181.6016082763672, "KL/rejected_KL_mean": -220.6177215576172, "KL/std": 89.42021179199219, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.004397670738399029, "fcm_dpo/delta": -0.07311216741800308, "fcm_dpo/margin": 78.03221893310547, "fcm_dpo/q_t": 0.41759008169174194, "grad_norm": 7.273556232452393, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.0765607357025146, "logits/rejected": -1.8099424839019775, "logps/chosen": -232.83775329589844, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -316.38800048828125, "loss": 1.1169, "margin_dpo/margin_mean": 78.03221893310547, "margin_dpo/margin_std": 92.1607666015625, "step": 348 }, { "KL/chosen_KL_mean": -154.73739624023438, "KL/mean": -191.4396209716797, "KL/rejected_KL_mean": -228.14183044433594, "KL/std": 98.86541748046875, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.004429451189935207, "fcm_dpo/delta": 0.025370419025421143, "fcm_dpo/margin": 73.40444946289062, "fcm_dpo/q_t": 0.4238123297691345, "grad_norm": 9.079095840454102, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.884617567062378, "logits/rejected": -1.655585527420044, "logps/chosen": -246.79547119140625, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -332.10711669921875, "loss": 1.1383, "margin_dpo/margin_mean": 73.4044418334961, "margin_dpo/margin_std": 102.65304565429688, "step": 349 }, { "KL/chosen_KL_mean": -163.68724060058594, "KL/mean": -194.37875366210938, "KL/rejected_KL_mean": -225.07028198242188, "KL/std": 90.88029479980469, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.0044674137607216835, "fcm_dpo/delta": 0.07797056436538696, "fcm_dpo/margin": 61.383033752441406, "fcm_dpo/q_t": 0.4354139566421509, "grad_norm": 8.79728889465332, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.6342201232910156, "logits/rejected": -1.399592638015747, "logps/chosen": -250.74143981933594, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -320.7922058105469, "loss": 1.1794, "margin_dpo/margin_mean": 61.383033752441406, "margin_dpo/margin_std": 99.87274169921875, "step": 350 }, { "KL/chosen_KL_mean": -164.76138305664062, "KL/mean": -208.447509765625, "KL/rejected_KL_mean": -252.1336669921875, "KL/std": 101.17079162597656, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.004466908983886242, "fcm_dpo/delta": -0.04189909249544144, "fcm_dpo/margin": 87.37226867675781, "fcm_dpo/q_t": 0.4080358147621155, "grad_norm": 7.720744609832764, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.4825446605682373, "logits/rejected": -1.1766374111175537, "logps/chosen": -253.930419921875, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -360.43170166015625, "loss": 1.0903, "margin_dpo/margin_mean": 87.37226867675781, "margin_dpo/margin_std": 109.10665893554688, "step": 351 }, { "KL/chosen_KL_mean": -151.11154174804688, "KL/mean": -191.86483764648438, "KL/rejected_KL_mean": -232.6181640625, "KL/std": 99.81370544433594, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.004444379825145006, "fcm_dpo/delta": -0.012705039232969284, "fcm_dpo/margin": 81.50660705566406, "fcm_dpo/q_t": 0.41604626178741455, "grad_norm": 7.999653339385986, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.8044812679290771, "logits/rejected": -1.2461645603179932, "logps/chosen": -247.92300415039062, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -317.0229797363281, "loss": 1.1136, "margin_dpo/margin_mean": 81.50660705566406, "margin_dpo/margin_std": 109.28038024902344, "step": 352 }, { "KL/chosen_KL_mean": -172.9018096923828, "KL/mean": -198.9266815185547, "KL/rejected_KL_mean": -224.95156860351562, "KL/std": 88.31737518310547, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.004464528523385525, "fcm_dpo/delta": 0.026705393567681313, "fcm_dpo/margin": 52.04975509643555, "fcm_dpo/q_t": 0.4446486234664917, "grad_norm": 8.96254825592041, "learning_rate": 2.794480701395219e-07, "logits/chosen": -1.9929308891296387, "logits/rejected": -1.5952637195587158, "logps/chosen": -258.2642822265625, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -312.3188781738281, "loss": 1.2171, "margin_dpo/margin_mean": 52.04975128173828, "margin_dpo/margin_std": 99.78175354003906, "step": 353 }, { "KL/chosen_KL_mean": -150.95286560058594, "KL/mean": -192.36380004882812, "KL/rejected_KL_mean": -233.7747802734375, "KL/std": 89.59739685058594, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0044593652710318565, "fcm_dpo/delta": -0.020106252282857895, "fcm_dpo/margin": 82.8218994140625, "fcm_dpo/q_t": 0.41196611523628235, "grad_norm": 7.897119998931885, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.746599555015564, "logits/rejected": -1.267164945602417, "logps/chosen": -246.7540283203125, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -328.64093017578125, "loss": 1.0841, "margin_dpo/margin_mean": 82.8218994140625, "margin_dpo/margin_std": 83.67002868652344, "step": 354 }, { "KL/chosen_KL_mean": -148.2082061767578, "KL/mean": -186.98193359375, "KL/rejected_KL_mean": -225.75564575195312, "KL/std": 91.2845687866211, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.004437734838575125, "fcm_dpo/delta": 0.005921484902501106, "fcm_dpo/margin": 77.54745483398438, "fcm_dpo/q_t": 0.4183248281478882, "grad_norm": 8.873579978942871, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.3574156761169434, "logits/rejected": -1.8649243116378784, "logps/chosen": -234.39376831054688, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -314.21722412109375, "loss": 1.119, "margin_dpo/margin_mean": 77.54743957519531, "margin_dpo/margin_std": 97.47804260253906, "step": 355 }, { "KL/chosen_KL_mean": -174.98983764648438, "KL/mean": -205.19667053222656, "KL/rejected_KL_mean": -235.40353393554688, "KL/std": 90.18392944335938, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.004516772460192442, "fcm_dpo/delta": 0.07893040776252747, "fcm_dpo/margin": 60.41371154785156, "fcm_dpo/q_t": 0.4358311891555786, "grad_norm": 8.098246574401855, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.843580722808838, "logits/rejected": -1.3756204843521118, "logps/chosen": -265.80792236328125, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -317.5308532714844, "loss": 1.1851, "margin_dpo/margin_mean": 60.41371154785156, "margin_dpo/margin_std": 103.48470306396484, "step": 356 }, { "KL/chosen_KL_mean": -147.1997833251953, "KL/mean": -194.81085205078125, "KL/rejected_KL_mean": -242.42193603515625, "KL/std": 98.35533142089844, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.004445340950042009, "fcm_dpo/delta": -0.07763983309268951, "fcm_dpo/margin": 95.22216033935547, "fcm_dpo/q_t": 0.3991672694683075, "grad_norm": 7.922153472900391, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.9797848463058472, "logits/rejected": -1.612494945526123, "logps/chosen": -234.70452880859375, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -334.67132568359375, "loss": 1.05, "margin_dpo/margin_mean": 95.22215270996094, "margin_dpo/margin_std": 92.40196228027344, "step": 357 }, { "KL/chosen_KL_mean": -157.17820739746094, "KL/mean": -193.06558227539062, "KL/rejected_KL_mean": -228.95298767089844, "KL/std": 91.91741180419922, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.004470685496926308, "fcm_dpo/delta": 0.0300931204110384, "fcm_dpo/margin": 71.7747802734375, "fcm_dpo/q_t": 0.42519044876098633, "grad_norm": 7.872157096862793, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.354710578918457, "logits/rejected": -0.8659044504165649, "logps/chosen": -252.23367309570312, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -325.4291076660156, "loss": 1.137, "margin_dpo/margin_mean": 71.7747802734375, "margin_dpo/margin_std": 98.74140930175781, "step": 358 }, { "KL/chosen_KL_mean": -175.88662719726562, "KL/mean": -209.33206176757812, "KL/rejected_KL_mean": -242.77749633789062, "KL/std": 86.73615264892578, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.00449500884860754, "fcm_dpo/delta": 0.05070550739765167, "fcm_dpo/margin": 66.89087677001953, "fcm_dpo/q_t": 0.4295535683631897, "grad_norm": 8.551814079284668, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4017055034637451, "logits/rejected": -0.8144794702529907, "logps/chosen": -263.25994873046875, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -331.54937744140625, "loss": 1.1593, "margin_dpo/margin_mean": 66.89087677001953, "margin_dpo/margin_std": 100.73869323730469, "step": 359 }, { "KL/chosen_KL_mean": -160.8542938232422, "KL/mean": -203.28257751464844, "KL/rejected_KL_mean": -245.71087646484375, "KL/std": 87.56802368164062, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.004498669877648354, "fcm_dpo/delta": -0.03305763006210327, "fcm_dpo/margin": 84.8565673828125, "fcm_dpo/q_t": 0.40884873270988464, "grad_norm": 10.741021156311035, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.697143316268921, "logits/rejected": -1.3575119972229004, "logps/chosen": -236.07774353027344, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -321.80487060546875, "loss": 1.0818, "margin_dpo/margin_mean": 84.8565673828125, "margin_dpo/margin_std": 92.08477783203125, "step": 360 }, { "KL/chosen_KL_mean": -164.08226013183594, "KL/mean": -201.622314453125, "KL/rejected_KL_mean": -239.162353515625, "KL/std": 98.81802368164062, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.004505776800215244, "fcm_dpo/delta": 0.012107008136808872, "fcm_dpo/margin": 75.08009338378906, "fcm_dpo/q_t": 0.42112597823143005, "grad_norm": 9.12513256072998, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.678523302078247, "logits/rejected": -1.4353785514831543, "logps/chosen": -246.03558349609375, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -335.841552734375, "loss": 1.1294, "margin_dpo/margin_mean": 75.0801010131836, "margin_dpo/margin_std": 104.97282409667969, "step": 361 }, { "KL/chosen_KL_mean": -171.29351806640625, "KL/mean": -210.39791870117188, "KL/rejected_KL_mean": -249.50233459472656, "KL/std": 94.54135131835938, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.004496132023632526, "fcm_dpo/delta": -0.001788012683391571, "fcm_dpo/margin": 78.20882415771484, "fcm_dpo/q_t": 0.4168422818183899, "grad_norm": 8.504530906677246, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.8733630180358887, "logits/rejected": -1.6744513511657715, "logps/chosen": -260.63323974609375, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -360.0043029785156, "loss": 1.1165, "margin_dpo/margin_mean": 78.20882415771484, "margin_dpo/margin_std": 101.45787811279297, "step": 362 }, { "KL/chosen_KL_mean": -159.7242889404297, "KL/mean": -210.65365600585938, "KL/rejected_KL_mean": -261.5830078125, "KL/std": 98.67031860351562, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.004449563100934029, "fcm_dpo/delta": -0.10800551623106003, "fcm_dpo/margin": 101.85870361328125, "fcm_dpo/q_t": 0.3936868906021118, "grad_norm": 9.499687194824219, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.6440894603729248, "logits/rejected": -1.1991972923278809, "logps/chosen": -240.49929809570312, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -341.01568603515625, "loss": 1.0368, "margin_dpo/margin_mean": 101.85870361328125, "margin_dpo/margin_std": 106.08396911621094, "step": 363 }, { "KL/chosen_KL_mean": -156.6092071533203, "KL/mean": -197.97207641601562, "KL/rejected_KL_mean": -239.33493041992188, "KL/std": 92.05769348144531, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.004421957768499851, "fcm_dpo/delta": -0.016931187361478806, "fcm_dpo/margin": 82.7257080078125, "fcm_dpo/q_t": 0.41261452436447144, "grad_norm": 10.41594409942627, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.071493625640869, "logits/rejected": -1.9315736293792725, "logps/chosen": -233.13880920410156, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -336.89666748046875, "loss": 1.0938, "margin_dpo/margin_mean": 82.7257080078125, "margin_dpo/margin_std": 88.74366760253906, "step": 364 }, { "KL/chosen_KL_mean": -153.4305877685547, "KL/mean": -198.31124877929688, "KL/rejected_KL_mean": -243.19192504882812, "KL/std": 98.00442504882812, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0043796999379992485, "fcm_dpo/delta": -0.044980794191360474, "fcm_dpo/margin": 89.76132202148438, "fcm_dpo/q_t": 0.4085640609264374, "grad_norm": 8.354164123535156, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7870889902114868, "logits/rejected": -1.6665809154510498, "logps/chosen": -233.97547912597656, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -342.51507568359375, "loss": 1.0889, "margin_dpo/margin_mean": 89.76132202148438, "margin_dpo/margin_std": 111.82694244384766, "step": 365 }, { "KL/chosen_KL_mean": -156.5340576171875, "KL/mean": -193.3250732421875, "KL/rejected_KL_mean": -230.1160888671875, "KL/std": 101.41372680664062, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.004372448194772005, "fcm_dpo/delta": 0.02921198680996895, "fcm_dpo/margin": 73.58204650878906, "fcm_dpo/q_t": 0.42546021938323975, "grad_norm": 7.948990345001221, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.75642728805542, "logits/rejected": -1.7944540977478027, "logps/chosen": -228.77212524414062, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -321.7879333496094, "loss": 1.1344, "margin_dpo/margin_mean": 73.58204650878906, "margin_dpo/margin_std": 99.62442016601562, "step": 366 }, { "KL/chosen_KL_mean": -141.9380340576172, "KL/mean": -186.29696655273438, "KL/rejected_KL_mean": -230.65589904785156, "KL/std": 89.26113891601562, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.004357443191111088, "fcm_dpo/delta": -0.038040321320295334, "fcm_dpo/margin": 88.71786499023438, "fcm_dpo/q_t": 0.4080609083175659, "grad_norm": 7.594677448272705, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.8096764087677002, "logits/rejected": -1.460798740386963, "logps/chosen": -227.36004638671875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -314.1737060546875, "loss": 1.0747, "margin_dpo/margin_mean": 88.71786499023438, "margin_dpo/margin_std": 92.11600494384766, "step": 367 }, { "KL/chosen_KL_mean": -131.95626831054688, "KL/mean": -181.1734619140625, "KL/rejected_KL_mean": -230.39065551757812, "KL/std": 98.35110473632812, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.004312264267355204, "fcm_dpo/delta": -0.07775908708572388, "fcm_dpo/margin": 98.43437194824219, "fcm_dpo/q_t": 0.3988504111766815, "grad_norm": 8.7396240234375, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.6174986362457275, "logits/rejected": -1.586951732635498, "logps/chosen": -205.79739379882812, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -335.0859375, "loss": 1.0527, "margin_dpo/margin_mean": 98.43437194824219, "margin_dpo/margin_std": 99.77456665039062, "step": 368 }, { "KL/chosen_KL_mean": -147.7201690673828, "KL/mean": -193.41433715820312, "KL/rejected_KL_mean": -239.10850524902344, "KL/std": 91.31127166748047, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.004260452464222908, "fcm_dpo/delta": -0.0409361869096756, "fcm_dpo/margin": 91.38835144042969, "fcm_dpo/q_t": 0.40746843814849854, "grad_norm": 8.168294906616211, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.8798190355300903, "logits/rejected": -1.5230786800384521, "logps/chosen": -246.18212890625, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -338.2652282714844, "loss": 1.0717, "margin_dpo/margin_mean": 91.38835144042969, "margin_dpo/margin_std": 92.75527954101562, "step": 369 }, { "KL/chosen_KL_mean": -165.1224365234375, "KL/mean": -195.47073364257812, "KL/rejected_KL_mean": -225.81903076171875, "KL/std": 87.06025695800781, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.004305828828364611, "fcm_dpo/delta": 0.09096013009548187, "fcm_dpo/margin": 60.69655227661133, "fcm_dpo/q_t": 0.437950998544693, "grad_norm": 7.858801364898682, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.8448810577392578, "logits/rejected": -1.6459017992019653, "logps/chosen": -261.898193359375, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -328.5631408691406, "loss": 1.1885, "margin_dpo/margin_mean": 60.696556091308594, "margin_dpo/margin_std": 101.66566467285156, "step": 370 }, { "KL/chosen_KL_mean": -157.01593017578125, "KL/mean": -191.376220703125, "KL/rejected_KL_mean": -225.73654174804688, "KL/std": 88.26775360107422, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.00436741579324007, "fcm_dpo/delta": 0.051135219633579254, "fcm_dpo/margin": 68.72058868408203, "fcm_dpo/q_t": 0.4285876154899597, "grad_norm": 10.809579849243164, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.7317743301391602, "logits/rejected": -1.358191728591919, "logps/chosen": -260.0061340332031, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -331.087158203125, "loss": 1.1529, "margin_dpo/margin_mean": 68.72059631347656, "margin_dpo/margin_std": 97.08773803710938, "step": 371 }, { "KL/chosen_KL_mean": -133.7889404296875, "KL/mean": -184.45782470703125, "KL/rejected_KL_mean": -235.12672424316406, "KL/std": 102.141357421875, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.004312992561608553, "fcm_dpo/delta": -0.090976282954216, "fcm_dpo/margin": 101.33776092529297, "fcm_dpo/q_t": 0.3972133994102478, "grad_norm": 6.875663757324219, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.978712797164917, "logits/rejected": -1.3536436557769775, "logps/chosen": -224.815185546875, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -320.7369384765625, "loss": 1.0458, "margin_dpo/margin_mean": 101.33776092529297, "margin_dpo/margin_std": 105.66319274902344, "step": 372 }, { "KL/chosen_KL_mean": -155.0839080810547, "KL/mean": -192.56491088867188, "KL/rejected_KL_mean": -230.0458984375, "KL/std": 98.05553436279297, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.004302730783820152, "fcm_dpo/delta": 0.028382688760757446, "fcm_dpo/margin": 74.9620132446289, "fcm_dpo/q_t": 0.42395642399787903, "grad_norm": 9.8591947555542, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.0524075031280518, "logits/rejected": -1.5218886137008667, "logps/chosen": -255.156982421875, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -321.1622314453125, "loss": 1.1397, "margin_dpo/margin_mean": 74.96202087402344, "margin_dpo/margin_std": 106.03819274902344, "step": 373 }, { "KL/chosen_KL_mean": -134.38662719726562, "KL/mean": -176.3747100830078, "KL/rejected_KL_mean": -218.36279296875, "KL/std": 92.99945068359375, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.004297175444662571, "fcm_dpo/delta": -0.011367235332727432, "fcm_dpo/margin": 83.9761734008789, "fcm_dpo/q_t": 0.41439878940582275, "grad_norm": 7.321608543395996, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.2530415058135986, "logits/rejected": -1.86615788936615, "logps/chosen": -232.39895629882812, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -319.11505126953125, "loss": 1.0939, "margin_dpo/margin_mean": 83.9761734008789, "margin_dpo/margin_std": 90.40438079833984, "step": 374 }, { "KL/chosen_KL_mean": -136.63845825195312, "KL/mean": -184.3040313720703, "KL/rejected_KL_mean": -231.9696044921875, "KL/std": 89.56062316894531, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.004243217408657074, "fcm_dpo/delta": -0.0577210858464241, "fcm_dpo/margin": 95.33116149902344, "fcm_dpo/q_t": 0.40326279401779175, "grad_norm": 8.373222351074219, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.5865750312805176, "logits/rejected": -1.3510526418685913, "logps/chosen": -221.03472900390625, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -327.6884765625, "loss": 1.064, "margin_dpo/margin_mean": 95.33116912841797, "margin_dpo/margin_std": 95.51966857910156, "step": 375 }, { "KL/chosen_KL_mean": -132.41268920898438, "KL/mean": -184.82916259765625, "KL/rejected_KL_mean": -237.24563598632812, "KL/std": 103.84638214111328, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.00419023958966136, "fcm_dpo/delta": -0.09350718557834625, "fcm_dpo/margin": 104.83293914794922, "fcm_dpo/q_t": 0.39633071422576904, "grad_norm": 8.26854133605957, "learning_rate": 2.5e-07, "logits/chosen": -1.5308723449707031, "logits/rejected": -1.2040519714355469, "logps/chosen": -228.150390625, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -349.21893310546875, "loss": 1.0458, "margin_dpo/margin_mean": 104.83294677734375, "margin_dpo/margin_std": 111.3342056274414, "step": 376 }, { "KL/chosen_KL_mean": -139.83132934570312, "KL/mean": -194.31912231445312, "KL/rejected_KL_mean": -248.806884765625, "KL/std": 103.25735473632812, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004099456127732992, "fcm_dpo/delta": -0.10177389532327652, "fcm_dpo/margin": 108.97557067871094, "fcm_dpo/q_t": 0.39344173669815063, "grad_norm": 7.18393611907959, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.3841384649276733, "logits/rejected": -1.2685284614562988, "logps/chosen": -218.75828552246094, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -354.705810546875, "loss": 1.0306, "margin_dpo/margin_mean": 108.97557067871094, "margin_dpo/margin_std": 102.01395416259766, "step": 377 }, { "KL/chosen_KL_mean": -136.4365234375, "KL/mean": -179.31790161132812, "KL/rejected_KL_mean": -222.19927978515625, "KL/std": 97.60221099853516, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004105842672288418, "fcm_dpo/delta": -0.0025288276374340057, "fcm_dpo/margin": 85.76274871826172, "fcm_dpo/q_t": 0.4156009554862976, "grad_norm": 8.440792083740234, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.9750723838806152, "logits/rejected": -1.6964304447174072, "logps/chosen": -211.80892944335938, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -315.90338134765625, "loss": 1.1073, "margin_dpo/margin_mean": 85.76274871826172, "margin_dpo/margin_std": 100.08932495117188, "step": 378 }, { "KL/chosen_KL_mean": -160.43218994140625, "KL/mean": -198.27078247070312, "KL/rejected_KL_mean": -236.109375, "KL/std": 101.27043151855469, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004110678099095821, "fcm_dpo/delta": 0.04018237441778183, "fcm_dpo/margin": 75.67716979980469, "fcm_dpo/q_t": 0.42755788564682007, "grad_norm": 8.520137786865234, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.7568016052246094, "logits/rejected": -1.6881129741668701, "logps/chosen": -249.28787231445312, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -357.4905090332031, "loss": 1.1541, "margin_dpo/margin_mean": 75.67716979980469, "margin_dpo/margin_std": 116.45631408691406, "step": 379 }, { "KL/chosen_KL_mean": -175.89535522460938, "KL/mean": -222.82350158691406, "KL/rejected_KL_mean": -269.75164794921875, "KL/std": 104.11702728271484, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004098528064787388, "fcm_dpo/delta": -0.03607076406478882, "fcm_dpo/margin": 93.85630798339844, "fcm_dpo/q_t": 0.41047269105911255, "grad_norm": 11.029603004455566, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.3340685367584229, "logits/rejected": -1.3096084594726562, "logps/chosen": -255.7285614013672, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -380.49639892578125, "loss": 1.0927, "margin_dpo/margin_mean": 93.85630798339844, "margin_dpo/margin_std": 118.42643737792969, "step": 380 }, { "KL/chosen_KL_mean": -154.68821716308594, "KL/mean": -195.2127227783203, "KL/rejected_KL_mean": -235.73724365234375, "KL/std": 100.25949096679688, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004105043597519398, "fcm_dpo/delta": 0.01787124201655388, "fcm_dpo/margin": 81.04903411865234, "fcm_dpo/q_t": 0.4209110140800476, "grad_norm": 8.374640464782715, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.7547436952590942, "logits/rejected": -1.204688549041748, "logps/chosen": -241.25503540039062, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -320.15399169921875, "loss": 1.1268, "margin_dpo/margin_mean": 81.04903411865234, "margin_dpo/margin_std": 106.22268676757812, "step": 381 }, { "KL/chosen_KL_mean": -151.63711547851562, "KL/mean": -198.02391052246094, "KL/rejected_KL_mean": -244.41070556640625, "KL/std": 97.43571472167969, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004070096183568239, "fcm_dpo/delta": -0.02924763597548008, "fcm_dpo/margin": 92.77359008789062, "fcm_dpo/q_t": 0.410100519657135, "grad_norm": 8.416715621948242, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.9263428449630737, "logits/rejected": -1.6657785177230835, "logps/chosen": -256.09893798828125, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -359.012939453125, "loss": 1.0858, "margin_dpo/margin_mean": 92.77359008789062, "margin_dpo/margin_std": 100.39553833007812, "step": 382 }, { "KL/chosen_KL_mean": -165.34024047851562, "KL/mean": -204.68692016601562, "KL/rejected_KL_mean": -244.0336151123047, "KL/std": 101.41728210449219, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0040931906551122665, "fcm_dpo/delta": 0.028809238225221634, "fcm_dpo/margin": 78.69338989257812, "fcm_dpo/q_t": 0.4236026406288147, "grad_norm": 8.115076065063477, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.6301320791244507, "logits/rejected": -1.5083725452423096, "logps/chosen": -247.456298828125, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -342.810791015625, "loss": 1.1319, "margin_dpo/margin_mean": 78.6933822631836, "margin_dpo/margin_std": 100.37464904785156, "step": 383 }, { "KL/chosen_KL_mean": -178.33604431152344, "KL/mean": -216.15756225585938, "KL/rejected_KL_mean": -253.9790802001953, "KL/std": 104.4081802368164, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.004094203468412161, "fcm_dpo/delta": -0.0297089833766222, "fcm_dpo/margin": 75.64303588867188, "fcm_dpo/q_t": 0.42648106813430786, "grad_norm": 10.644187927246094, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.6609258651733398, "logits/rejected": -1.4353476762771606, "logps/chosen": -258.9212951660156, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -345.4269714355469, "loss": 1.161, "margin_dpo/margin_mean": 75.6430435180664, "margin_dpo/margin_std": 118.46755981445312, "step": 384 }, { "KL/chosen_KL_mean": -148.5069580078125, "KL/mean": -198.20492553710938, "KL/rejected_KL_mean": -247.90284729003906, "KL/std": 94.09307861328125, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004031536169350147, "fcm_dpo/delta": -0.05348537117242813, "fcm_dpo/margin": 99.3958969116211, "fcm_dpo/q_t": 0.40525779128074646, "grad_norm": 9.974431991577148, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.233247756958008, "logits/rejected": -1.6282093524932861, "logps/chosen": -241.26712036132812, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -332.36285400390625, "loss": 1.0725, "margin_dpo/margin_mean": 99.3958969116211, "margin_dpo/margin_std": 108.68608856201172, "step": 385 }, { "KL/chosen_KL_mean": -155.0706787109375, "KL/mean": -203.181884765625, "KL/rejected_KL_mean": -251.29310607910156, "KL/std": 96.112548828125, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.004016058519482613, "fcm_dpo/delta": -0.03789763152599335, "fcm_dpo/margin": 96.222412109375, "fcm_dpo/q_t": 0.4072575569152832, "grad_norm": 6.829093933105469, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.827270269393921, "logits/rejected": -1.54856276512146, "logps/chosen": -240.42965698242188, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -348.28375244140625, "loss": 1.0801, "margin_dpo/margin_mean": 96.22239685058594, "margin_dpo/margin_std": 105.21244812011719, "step": 386 }, { "KL/chosen_KL_mean": -180.8468017578125, "KL/mean": -223.14968872070312, "KL/rejected_KL_mean": -265.4525451660156, "KL/std": 93.22897338867188, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004020377527922392, "fcm_dpo/delta": 0.010135759599506855, "fcm_dpo/margin": 84.60575103759766, "fcm_dpo/q_t": 0.41888856887817383, "grad_norm": 9.812148094177246, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.5025627613067627, "logits/rejected": -1.8269617557525635, "logps/chosen": -283.5349426269531, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -364.881103515625, "loss": 1.1111, "margin_dpo/margin_mean": 84.60575866699219, "margin_dpo/margin_std": 96.38554382324219, "step": 387 }, { "KL/chosen_KL_mean": -181.57606506347656, "KL/mean": -229.15017700195312, "KL/rejected_KL_mean": -276.7242736816406, "KL/std": 104.29414367675781, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.003994767554104328, "fcm_dpo/delta": -0.03131572902202606, "fcm_dpo/margin": 95.14820861816406, "fcm_dpo/q_t": 0.40958231687545776, "grad_norm": 7.980307102203369, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.728763461112976, "logits/rejected": -1.6144022941589355, "logps/chosen": -265.16131591796875, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -386.82464599609375, "loss": 1.0859, "margin_dpo/margin_mean": 95.14820861816406, "margin_dpo/margin_std": 107.2002944946289, "step": 388 }, { "KL/chosen_KL_mean": -188.09542846679688, "KL/mean": -227.7694854736328, "KL/rejected_KL_mean": -267.44354248046875, "KL/std": 105.17201232910156, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004008483607321978, "fcm_dpo/delta": 0.03299577906727791, "fcm_dpo/margin": 79.34814453125, "fcm_dpo/q_t": 0.4250856637954712, "grad_norm": 10.585310935974121, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.381779193878174, "logits/rejected": -1.9329383373260498, "logps/chosen": -277.9176940917969, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -352.2973327636719, "loss": 1.1431, "margin_dpo/margin_mean": 79.34815216064453, "margin_dpo/margin_std": 113.85603332519531, "step": 389 }, { "KL/chosen_KL_mean": -185.89500427246094, "KL/mean": -225.18771362304688, "KL/rejected_KL_mean": -264.48040771484375, "KL/std": 103.37911987304688, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004021891392767429, "fcm_dpo/delta": 0.03484056517481804, "fcm_dpo/margin": 78.58541870117188, "fcm_dpo/q_t": 0.4270336627960205, "grad_norm": 7.8028178215026855, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.535722017288208, "logits/rejected": -1.3289077281951904, "logps/chosen": -263.42742919921875, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -357.0643310546875, "loss": 1.1507, "margin_dpo/margin_mean": 78.58541870117188, "margin_dpo/margin_std": 119.06599426269531, "step": 390 }, { "KL/chosen_KL_mean": -169.44627380371094, "KL/mean": -211.35220336914062, "KL/rejected_KL_mean": -253.25811767578125, "KL/std": 99.0418701171875, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004042004235088825, "fcm_dpo/delta": 0.011515261605381966, "fcm_dpo/margin": 83.81183624267578, "fcm_dpo/q_t": 0.4196796417236328, "grad_norm": 7.85827112197876, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.7961448431015015, "logits/rejected": -1.3397352695465088, "logps/chosen": -257.67919921875, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -346.26568603515625, "loss": 1.1201, "margin_dpo/margin_mean": 83.81183624267578, "margin_dpo/margin_std": 104.46539306640625, "step": 391 }, { "KL/chosen_KL_mean": -170.4093017578125, "KL/mean": -222.81796264648438, "KL/rejected_KL_mean": -275.22662353515625, "KL/std": 111.94932556152344, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004021737724542618, "fcm_dpo/delta": -0.07464998960494995, "fcm_dpo/margin": 104.81730651855469, "fcm_dpo/q_t": 0.4004719853401184, "grad_norm": 8.354537010192871, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.5231242179870605, "logits/rejected": -1.4072794914245605, "logps/chosen": -258.1537170410156, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -383.7973327636719, "loss": 1.0648, "margin_dpo/margin_mean": 104.81730651855469, "margin_dpo/margin_std": 120.1443099975586, "step": 392 }, { "KL/chosen_KL_mean": -151.05247497558594, "KL/mean": -197.68392944335938, "KL/rejected_KL_mean": -244.31536865234375, "KL/std": 104.566162109375, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.003979050554335117, "fcm_dpo/delta": -0.021936770528554916, "fcm_dpo/margin": 93.26287841796875, "fcm_dpo/q_t": 0.4117734432220459, "grad_norm": 7.0718302726745605, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.8781790733337402, "logits/rejected": -1.5648589134216309, "logps/chosen": -227.61630249023438, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -324.04156494140625, "loss": 1.0893, "margin_dpo/margin_mean": 93.26287841796875, "margin_dpo/margin_std": 102.81459045410156, "step": 393 }, { "KL/chosen_KL_mean": -161.4943084716797, "KL/mean": -212.4337158203125, "KL/rejected_KL_mean": -263.37310791015625, "KL/std": 100.27935791015625, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.003945984877645969, "fcm_dpo/delta": -0.05418417975306511, "fcm_dpo/margin": 101.87882995605469, "fcm_dpo/q_t": 0.403766930103302, "grad_norm": 7.967808723449707, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.5946886539459229, "logits/rejected": -1.3650068044662476, "logps/chosen": -246.19296264648438, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -360.83087158203125, "loss": 1.0614, "margin_dpo/margin_mean": 101.87882995605469, "margin_dpo/margin_std": 99.60018920898438, "step": 394 }, { "KL/chosen_KL_mean": -167.8800048828125, "KL/mean": -206.48492431640625, "KL/rejected_KL_mean": -245.08984375, "KL/std": 104.74335479736328, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0039463527500629425, "fcm_dpo/delta": 0.04659000784158707, "fcm_dpo/margin": 77.20983123779297, "fcm_dpo/q_t": 0.4302087426185608, "grad_norm": 8.083805084228516, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5791773796081543, "logits/rejected": -2.1813063621520996, "logps/chosen": -256.76763916015625, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -336.1353759765625, "loss": 1.1802, "margin_dpo/margin_mean": 77.20983123779297, "margin_dpo/margin_std": 141.38632202148438, "step": 395 }, { "KL/chosen_KL_mean": -177.1423797607422, "KL/mean": -212.0301971435547, "KL/rejected_KL_mean": -246.91802978515625, "KL/std": 92.68482971191406, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004013544879853725, "fcm_dpo/delta": 0.07198107242584229, "fcm_dpo/margin": 69.77566528320312, "fcm_dpo/q_t": 0.43396133184432983, "grad_norm": 9.847525596618652, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8035774230957031, "logits/rejected": -1.3044931888580322, "logps/chosen": -279.61370849609375, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -340.15283203125, "loss": 1.1731, "margin_dpo/margin_mean": 69.77566528320312, "margin_dpo/margin_std": 110.20846557617188, "step": 396 }, { "KL/chosen_KL_mean": -158.19989013671875, "KL/mean": -220.1042938232422, "KL/rejected_KL_mean": -282.0086669921875, "KL/std": 107.79715728759766, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.003937765024602413, "fcm_dpo/delta": -0.14481693506240845, "fcm_dpo/margin": 123.80879211425781, "fcm_dpo/q_t": 0.38519424200057983, "grad_norm": 7.437973976135254, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.266530990600586, "logits/rejected": -1.6693720817565918, "logps/chosen": -259.24432373046875, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -390.85986328125, "loss": 1.0182, "margin_dpo/margin_mean": 123.80879211425781, "margin_dpo/margin_std": 129.30946350097656, "step": 397 }, { "KL/chosen_KL_mean": -167.50302124023438, "KL/mean": -212.44491577148438, "KL/rejected_KL_mean": -257.3868408203125, "KL/std": 105.8741683959961, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.003908536862581968, "fcm_dpo/delta": -0.0013920245692133904, "fcm_dpo/margin": 89.8837890625, "fcm_dpo/q_t": 0.4176360070705414, "grad_norm": 7.324161052703857, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -1.873284935951233, "logits/rejected": -1.5841383934020996, "logps/chosen": -262.55224609375, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -371.3924560546875, "loss": 1.1224, "margin_dpo/margin_mean": 89.88379669189453, "margin_dpo/margin_std": 125.20209503173828, "step": 398 }, { "KL/chosen_KL_mean": -145.56097412109375, "KL/mean": -202.2679901123047, "KL/rejected_KL_mean": -258.9749755859375, "KL/std": 95.35153198242188, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0038644778542220592, "fcm_dpo/delta": -0.09224853664636612, "fcm_dpo/margin": 113.41400146484375, "fcm_dpo/q_t": 0.3953678607940674, "grad_norm": 7.809300899505615, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.088022470474243, "logits/rejected": -1.284350872039795, "logps/chosen": -242.19952392578125, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -342.51141357421875, "loss": 1.0337, "margin_dpo/margin_mean": 113.41400146484375, "margin_dpo/margin_std": 103.82345581054688, "step": 399 }, { "KL/chosen_KL_mean": -152.52459716796875, "KL/mean": -209.85552978515625, "KL/rejected_KL_mean": -267.1864929199219, "KL/std": 104.63770294189453, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0037916162982583046, "fcm_dpo/delta": -0.0885932594537735, "fcm_dpo/margin": 114.66188049316406, "fcm_dpo/q_t": 0.3972344994544983, "grad_norm": 11.571727752685547, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.1714072227478027, "logits/rejected": -1.689143419265747, "logps/chosen": -257.8277587890625, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -375.842041015625, "loss": 1.0494, "margin_dpo/margin_mean": 114.66187286376953, "margin_dpo/margin_std": 120.82101440429688, "step": 400 }, { "KL/chosen_KL_mean": -168.75015258789062, "KL/mean": -204.2123565673828, "KL/rejected_KL_mean": -239.674560546875, "KL/std": 107.67204284667969, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0037846756167709827, "fcm_dpo/delta": 0.014278288930654526, "fcm_dpo/margin": 70.9244384765625, "fcm_dpo/q_t": 0.4385364353656769, "grad_norm": 9.941385269165039, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.502549171447754, "logits/rejected": -1.2594444751739502, "logps/chosen": -253.039306640625, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -330.61767578125, "loss": 1.1956, "margin_dpo/margin_mean": 70.9244384765625, "margin_dpo/margin_std": 132.94927978515625, "step": 401 }, { "KL/chosen_KL_mean": -147.35812377929688, "KL/mean": -215.51458740234375, "KL/rejected_KL_mean": -283.6710205078125, "KL/std": 114.91386413574219, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0037005767226219177, "fcm_dpo/delta": -0.16242989897727966, "fcm_dpo/margin": 136.31292724609375, "fcm_dpo/q_t": 0.3793047070503235, "grad_norm": 6.992890357971191, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.8717849254608154, "logits/rejected": -1.4021966457366943, "logps/chosen": -230.9512481689453, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -371.4813232421875, "loss": 0.9867, "margin_dpo/margin_mean": 136.31292724609375, "margin_dpo/margin_std": 113.2481460571289, "step": 402 }, { "KL/chosen_KL_mean": -152.427734375, "KL/mean": -202.84359741210938, "KL/rejected_KL_mean": -253.25946044921875, "KL/std": 109.49195861816406, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.003646942786872387, "fcm_dpo/delta": -0.01850617304444313, "fcm_dpo/margin": 100.83173370361328, "fcm_dpo/q_t": 0.41269397735595703, "grad_norm": 8.075706481933594, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.188070774078369, "logits/rejected": -1.784805178642273, "logps/chosen": -226.18081665039062, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -337.1795959472656, "loss": 1.0966, "margin_dpo/margin_mean": 100.83173370361328, "margin_dpo/margin_std": 117.73521423339844, "step": 403 }, { "KL/chosen_KL_mean": -180.85617065429688, "KL/mean": -224.353759765625, "KL/rejected_KL_mean": -267.851318359375, "KL/std": 105.67926025390625, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.003660098183900118, "fcm_dpo/delta": 0.03260389715433121, "fcm_dpo/margin": 86.99517822265625, "fcm_dpo/q_t": 0.4252658784389496, "grad_norm": 11.873234748840332, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.9928500652313232, "logits/rejected": -1.6546742916107178, "logps/chosen": -260.5323486328125, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -352.1318359375, "loss": 1.1475, "margin_dpo/margin_mean": 86.99517822265625, "margin_dpo/margin_std": 130.10366821289062, "step": 404 }, { "KL/chosen_KL_mean": -184.93276977539062, "KL/mean": -228.37274169921875, "KL/rejected_KL_mean": -271.8127136230469, "KL/std": 100.00123596191406, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0037048289086669683, "fcm_dpo/delta": 0.028606494888663292, "fcm_dpo/margin": 86.87989807128906, "fcm_dpo/q_t": 0.4237843155860901, "grad_norm": 7.516399383544922, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.6626975536346436, "logits/rejected": -1.1925535202026367, "logps/chosen": -281.39471435546875, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -364.68341064453125, "loss": 1.1288, "margin_dpo/margin_mean": 86.87989807128906, "margin_dpo/margin_std": 106.55793762207031, "step": 405 }, { "KL/chosen_KL_mean": -175.93710327148438, "KL/mean": -215.526123046875, "KL/rejected_KL_mean": -255.11514282226562, "KL/std": 106.20571899414062, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.003726581810042262, "fcm_dpo/delta": 0.05660545080900192, "fcm_dpo/margin": 79.17803955078125, "fcm_dpo/q_t": 0.43050628900527954, "grad_norm": 12.613397598266602, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.9329559803009033, "logits/rejected": -1.7667988538742065, "logps/chosen": -254.07106018066406, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -353.39874267578125, "loss": 1.1649, "margin_dpo/margin_mean": 79.17803955078125, "margin_dpo/margin_std": 124.655029296875, "step": 406 }, { "KL/chosen_KL_mean": -183.22247314453125, "KL/mean": -217.94737243652344, "KL/rejected_KL_mean": -252.67227172851562, "KL/std": 104.20257568359375, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.003787833033129573, "fcm_dpo/delta": 0.08927379548549652, "fcm_dpo/margin": 69.44979858398438, "fcm_dpo/q_t": 0.437732070684433, "grad_norm": 8.527554512023926, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.2251386642456055, "logits/rejected": -1.7114545106887817, "logps/chosen": -267.23529052734375, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -335.45330810546875, "loss": 1.1832, "margin_dpo/margin_mean": 69.44979858398438, "margin_dpo/margin_std": 111.4359130859375, "step": 407 }, { "KL/chosen_KL_mean": -175.09222412109375, "KL/mean": -209.7032012939453, "KL/rejected_KL_mean": -244.31417846679688, "KL/std": 100.23177337646484, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0038507296703755856, "fcm_dpo/delta": 0.08578319847583771, "fcm_dpo/margin": 69.221923828125, "fcm_dpo/q_t": 0.437787801027298, "grad_norm": 8.303929328918457, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.1538326740264893, "logits/rejected": -1.5209341049194336, "logps/chosen": -279.5539855957031, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -340.68634033203125, "loss": 1.193, "margin_dpo/margin_mean": 69.221923828125, "margin_dpo/margin_std": 124.9068603515625, "step": 408 }, { "KL/chosen_KL_mean": -186.27786254882812, "KL/mean": -223.03982543945312, "KL/rejected_KL_mean": -259.80181884765625, "KL/std": 102.34248352050781, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.003907359205186367, "fcm_dpo/delta": 0.06457874178886414, "fcm_dpo/margin": 73.52392578125, "fcm_dpo/q_t": 0.43123966455459595, "grad_norm": 7.843317031860352, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5169367790222168, "logits/rejected": -1.0774972438812256, "logps/chosen": -283.9461669921875, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -349.84765625, "loss": 1.1679, "margin_dpo/margin_mean": 73.52392578125, "margin_dpo/margin_std": 112.20512390136719, "step": 409 }, { "KL/chosen_KL_mean": -180.58078002929688, "KL/mean": -225.36758422851562, "KL/rejected_KL_mean": -270.15435791015625, "KL/std": 108.14596557617188, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.00391848711296916, "fcm_dpo/delta": -0.0010329487267881632, "fcm_dpo/margin": 89.57362365722656, "fcm_dpo/q_t": 0.41610169410705566, "grad_norm": 7.709005355834961, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.737133264541626, "logits/rejected": -1.607742428779602, "logps/chosen": -257.0500183105469, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -358.7950134277344, "loss": 1.1099, "margin_dpo/margin_mean": 89.57362365722656, "margin_dpo/margin_std": 109.42320251464844, "step": 410 }, { "KL/chosen_KL_mean": -193.09078979492188, "KL/mean": -234.73931884765625, "KL/rejected_KL_mean": -276.3878479003906, "KL/std": 101.26686096191406, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.003922713920474052, "fcm_dpo/delta": 0.023937596008181572, "fcm_dpo/margin": 83.29708099365234, "fcm_dpo/q_t": 0.42355746030807495, "grad_norm": 7.625581741333008, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.2998390197753906, "logits/rejected": -1.8921958208084106, "logps/chosen": -280.257080078125, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -363.48388671875, "loss": 1.1328, "margin_dpo/margin_mean": 83.29707336425781, "margin_dpo/margin_std": 111.31565856933594, "step": 411 }, { "KL/chosen_KL_mean": -183.2465057373047, "KL/mean": -224.58810424804688, "KL/rejected_KL_mean": -265.9296875, "KL/std": 101.71357727050781, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.003945737145841122, "fcm_dpo/delta": 0.02452954463660717, "fcm_dpo/margin": 82.68319702148438, "fcm_dpo/q_t": 0.4244285225868225, "grad_norm": 9.23458194732666, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.2358553409576416, "logits/rejected": -2.018293857574463, "logps/chosen": -262.19384765625, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -372.0352478027344, "loss": 1.1501, "margin_dpo/margin_mean": 82.6832046508789, "margin_dpo/margin_std": 130.78961181640625, "step": 412 }, { "KL/chosen_KL_mean": -196.4401397705078, "KL/mean": -241.51571655273438, "KL/rejected_KL_mean": -286.59124755859375, "KL/std": 107.83294677734375, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.003944039344787598, "fcm_dpo/delta": -0.005924813449382782, "fcm_dpo/margin": 90.15110778808594, "fcm_dpo/q_t": 0.41585713624954224, "grad_norm": 8.79367446899414, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.0845298767089844, "logits/rejected": -0.8215593695640564, "logps/chosen": -292.1348571777344, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -394.5521240234375, "loss": 1.1244, "margin_dpo/margin_mean": 90.15110778808594, "margin_dpo/margin_std": 129.85516357421875, "step": 413 }, { "KL/chosen_KL_mean": -180.77798461914062, "KL/mean": -223.57940673828125, "KL/rejected_KL_mean": -266.3808288574219, "KL/std": 104.20537567138672, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.003965448122471571, "fcm_dpo/delta": 0.010859481059014797, "fcm_dpo/margin": 85.60285949707031, "fcm_dpo/q_t": 0.42091506719589233, "grad_norm": 11.118621826171875, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.3789873123168945, "logits/rejected": -1.0088872909545898, "logps/chosen": -269.0546569824219, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -359.2508850097656, "loss": 1.1415, "margin_dpo/margin_mean": 85.60285949707031, "margin_dpo/margin_std": 133.12649536132812, "step": 414 }, { "KL/chosen_KL_mean": -197.00869750976562, "KL/mean": -232.78240966796875, "KL/rejected_KL_mean": -268.55609130859375, "KL/std": 95.05973052978516, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.004002832341939211, "fcm_dpo/delta": 0.06544608622789383, "fcm_dpo/margin": 71.5473861694336, "fcm_dpo/q_t": 0.4327963590621948, "grad_norm": 9.240524291992188, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.3790526390075684, "logits/rejected": -0.989001989364624, "logps/chosen": -305.92462158203125, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -376.0274658203125, "loss": 1.1735, "margin_dpo/margin_mean": 71.5473861694336, "margin_dpo/margin_std": 116.76353454589844, "step": 415 }, { "KL/chosen_KL_mean": -197.80267333984375, "KL/mean": -247.04115295410156, "KL/rejected_KL_mean": -296.27960205078125, "KL/std": 109.75790405273438, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004011586774140596, "fcm_dpo/delta": -0.04743169993162155, "fcm_dpo/margin": 98.47695922851562, "fcm_dpo/q_t": 0.4071429967880249, "grad_norm": 8.366456031799316, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.796558141708374, "logits/rejected": -1.2916998863220215, "logps/chosen": -291.2015380859375, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -393.95257568359375, "loss": 1.0852, "margin_dpo/margin_mean": 98.47695922851562, "margin_dpo/margin_std": 115.0363998413086, "step": 416 }, { "KL/chosen_KL_mean": -183.39358520507812, "KL/mean": -228.7921600341797, "KL/rejected_KL_mean": -274.1907653808594, "KL/std": 101.90353393554688, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.003973241895437241, "fcm_dpo/delta": -0.011168424040079117, "fcm_dpo/margin": 90.79716491699219, "fcm_dpo/q_t": 0.4143015146255493, "grad_norm": 8.294706344604492, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.195577383041382, "logits/rejected": -1.2798810005187988, "logps/chosen": -266.92889404296875, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -348.6325988769531, "loss": 1.099, "margin_dpo/margin_mean": 90.79716491699219, "margin_dpo/margin_std": 104.03199768066406, "step": 417 }, { "KL/chosen_KL_mean": -175.8104248046875, "KL/mean": -233.44410705566406, "KL/rejected_KL_mean": -291.0777893066406, "KL/std": 109.60391235351562, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.003935807850211859, "fcm_dpo/delta": -0.1090674102306366, "fcm_dpo/margin": 115.26734924316406, "fcm_dpo/q_t": 0.3929750919342041, "grad_norm": 10.284289360046387, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.144583225250244, "logits/rejected": -1.484584093093872, "logps/chosen": -284.0319519042969, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -402.94244384765625, "loss": 1.0332, "margin_dpo/margin_mean": 115.26734924316406, "margin_dpo/margin_std": 108.538818359375, "step": 418 }, { "KL/chosen_KL_mean": -183.52664184570312, "KL/mean": -233.3314208984375, "KL/rejected_KL_mean": -283.13616943359375, "KL/std": 97.23074340820312, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0038753452245146036, "fcm_dpo/delta": -0.037635207176208496, "fcm_dpo/margin": 99.60953521728516, "fcm_dpo/q_t": 0.4084605872631073, "grad_norm": 10.36099910736084, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.3793437480926514, "logits/rejected": -0.9275814294815063, "logps/chosen": -280.7099304199219, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -381.32147216796875, "loss": 1.0804, "margin_dpo/margin_mean": 99.60954284667969, "margin_dpo/margin_std": 107.86277770996094, "step": 419 }, { "KL/chosen_KL_mean": -187.4990692138672, "KL/mean": -232.35643005371094, "KL/rejected_KL_mean": -277.21380615234375, "KL/std": 103.1019058227539, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0038651241920888424, "fcm_dpo/delta": 0.0031954199075698853, "fcm_dpo/margin": 89.71470642089844, "fcm_dpo/q_t": 0.41847360134124756, "grad_norm": 13.283466339111328, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -1.998119831085205, "logits/rejected": -0.8518587946891785, "logps/chosen": -301.80755615234375, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -352.8973388671875, "loss": 1.1209, "margin_dpo/margin_mean": 89.71470642089844, "margin_dpo/margin_std": 119.61029052734375, "step": 420 }, { "KL/chosen_KL_mean": -177.02723693847656, "KL/mean": -227.31399536132812, "KL/rejected_KL_mean": -277.60076904296875, "KL/std": 106.48191833496094, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0038296410348266363, "fcm_dpo/delta": -0.03664024546742439, "fcm_dpo/margin": 100.57356262207031, "fcm_dpo/q_t": 0.40858137607574463, "grad_norm": 7.2389936447143555, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8705718517303467, "logits/rejected": -1.49953293800354, "logps/chosen": -262.9071044921875, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -373.937255859375, "loss": 1.0865, "margin_dpo/margin_mean": 100.57355499267578, "margin_dpo/margin_std": 117.15936279296875, "step": 421 }, { "KL/chosen_KL_mean": -175.9891357421875, "KL/mean": -223.4783172607422, "KL/rejected_KL_mean": -270.9674987792969, "KL/std": 98.25015258789062, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.003822949482128024, "fcm_dpo/delta": -0.013597086071968079, "fcm_dpo/margin": 94.9783935546875, "fcm_dpo/q_t": 0.4140799641609192, "grad_norm": 11.520052909851074, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.4299509525299072, "logits/rejected": -2.069840431213379, "logps/chosen": -276.4697265625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -365.375732421875, "loss": 1.0985, "margin_dpo/margin_mean": 94.9783935546875, "margin_dpo/margin_std": 110.69526672363281, "step": 422 }, { "KL/chosen_KL_mean": -163.5093536376953, "KL/mean": -212.1278076171875, "KL/rejected_KL_mean": -260.7463073730469, "KL/std": 117.63165283203125, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0038179787807166576, "fcm_dpo/delta": -0.02226075902581215, "fcm_dpo/margin": 97.2369384765625, "fcm_dpo/q_t": 0.4138193130493164, "grad_norm": 9.015412330627441, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.8029694557189941, "logits/rejected": -1.6822130680084229, "logps/chosen": -241.95928955078125, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -352.7928466796875, "loss": 1.1122, "margin_dpo/margin_mean": 97.23694610595703, "margin_dpo/margin_std": 133.85911560058594, "step": 423 }, { "KL/chosen_KL_mean": -184.1868896484375, "KL/mean": -230.88516235351562, "KL/rejected_KL_mean": -277.58343505859375, "KL/std": 104.78347778320312, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.003795037977397442, "fcm_dpo/delta": -0.004614821169525385, "fcm_dpo/margin": 93.39657592773438, "fcm_dpo/q_t": 0.4154837727546692, "grad_norm": 7.995769500732422, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7678401470184326, "logits/rejected": -1.177971601486206, "logps/chosen": -271.8292236328125, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -349.9490966796875, "loss": 1.108, "margin_dpo/margin_mean": 93.39657592773438, "margin_dpo/margin_std": 113.71613311767578, "step": 424 }, { "KL/chosen_KL_mean": -191.52781677246094, "KL/mean": -223.8763427734375, "KL/rejected_KL_mean": -256.22491455078125, "KL/std": 108.74922180175781, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0038230956997722387, "fcm_dpo/delta": 0.03539080172777176, "fcm_dpo/margin": 64.69710540771484, "fcm_dpo/q_t": 0.4435163736343384, "grad_norm": 9.21104621887207, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.5999715328216553, "logits/rejected": -1.5364484786987305, "logps/chosen": -270.047607421875, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -358.97357177734375, "loss": 1.2192, "margin_dpo/margin_mean": 64.69709777832031, "margin_dpo/margin_std": 136.83717346191406, "step": 425 }, { "KL/chosen_KL_mean": -163.87127685546875, "KL/mean": -210.96530151367188, "KL/rejected_KL_mean": -258.0593566894531, "KL/std": 99.31695556640625, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0038258689455688, "fcm_dpo/delta": -0.010823998600244522, "fcm_dpo/margin": 94.1880874633789, "fcm_dpo/q_t": 0.41460585594177246, "grad_norm": 9.517433166503906, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.7382333278656006, "logits/rejected": -0.9990628957748413, "logps/chosen": -272.3770751953125, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -346.3594055175781, "loss": 1.1067, "margin_dpo/margin_mean": 94.18807983398438, "margin_dpo/margin_std": 116.87905883789062, "step": 426 }, { "KL/chosen_KL_mean": -135.41111755371094, "KL/mean": -178.46217346191406, "KL/rejected_KL_mean": -221.5132293701172, "KL/std": 94.29625701904297, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0038299639709293842, "fcm_dpo/delta": 0.020889844745397568, "fcm_dpo/margin": 86.10211181640625, "fcm_dpo/q_t": 0.4214085042476654, "grad_norm": 7.767801761627197, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.5477523803710938, "logits/rejected": -1.9314658641815186, "logps/chosen": -234.53158569335938, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -307.23785400390625, "loss": 1.1205, "margin_dpo/margin_mean": 86.10211181640625, "margin_dpo/margin_std": 102.25273132324219, "step": 427 }, { "KL/chosen_KL_mean": -192.54261779785156, "KL/mean": -229.836181640625, "KL/rejected_KL_mean": -267.12969970703125, "KL/std": 106.76009368896484, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0038251096848398447, "fcm_dpo/delta": -0.0074324896559119225, "fcm_dpo/margin": 74.58710479736328, "fcm_dpo/q_t": 0.43267184495925903, "grad_norm": 9.2195463180542, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.4976836442947388, "logits/rejected": -1.2292969226837158, "logps/chosen": -298.5118713378906, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -376.2319030761719, "loss": 1.1722, "margin_dpo/margin_mean": 74.58710479736328, "margin_dpo/margin_std": 119.04150390625, "step": 428 }, { "KL/chosen_KL_mean": -177.10830688476562, "KL/mean": -224.55239868164062, "KL/rejected_KL_mean": -271.9964904785156, "KL/std": 111.43148803710938, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0038392606656998396, "fcm_dpo/delta": -0.015476591885089874, "fcm_dpo/margin": 94.88818359375, "fcm_dpo/q_t": 0.41491901874542236, "grad_norm": 7.4318647384643555, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9383342266082764, "logits/rejected": -1.222325086593628, "logps/chosen": -290.6531677246094, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -370.2384948730469, "loss": 1.1146, "margin_dpo/margin_mean": 94.88818359375, "margin_dpo/margin_std": 126.68368530273438, "step": 429 }, { "KL/chosen_KL_mean": -167.74647521972656, "KL/mean": -204.64425659179688, "KL/rejected_KL_mean": -241.54208374023438, "KL/std": 103.25558471679688, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.003861828241497278, "fcm_dpo/delta": 0.06659203767776489, "fcm_dpo/margin": 73.79560852050781, "fcm_dpo/q_t": 0.43332409858703613, "grad_norm": 8.275336265563965, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4013278484344482, "logits/rejected": -1.3678839206695557, "logps/chosen": -259.06585693359375, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -351.6517333984375, "loss": 1.1739, "margin_dpo/margin_mean": 73.79560852050781, "margin_dpo/margin_std": 120.43782043457031, "step": 430 }, { "KL/chosen_KL_mean": -169.80471801757812, "KL/mean": -213.78955078125, "KL/rejected_KL_mean": -257.77435302734375, "KL/std": 106.26868438720703, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0038659481797367334, "fcm_dpo/delta": 0.010248349979519844, "fcm_dpo/margin": 87.96966552734375, "fcm_dpo/q_t": 0.4197179675102234, "grad_norm": 9.150556564331055, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8818788528442383, "logits/rejected": -1.4968676567077637, "logps/chosen": -262.9859313964844, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -355.9066162109375, "loss": 1.128, "margin_dpo/margin_mean": 87.96966552734375, "margin_dpo/margin_std": 121.50164794921875, "step": 431 }, { "KL/chosen_KL_mean": -148.65780639648438, "KL/mean": -201.59130859375, "KL/rejected_KL_mean": -254.5247802734375, "KL/std": 109.869873046875, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.003848678432404995, "fcm_dpo/delta": -0.05987904593348503, "fcm_dpo/margin": 105.86697387695312, "fcm_dpo/q_t": 0.40297171473503113, "grad_norm": 8.10593318939209, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.3893712759017944, "logits/rejected": -0.8268561363220215, "logps/chosen": -253.09259033203125, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -359.61431884765625, "loss": 1.0623, "margin_dpo/margin_mean": 105.86697387695312, "margin_dpo/margin_std": 106.12940979003906, "step": 432 }, { "KL/chosen_KL_mean": -179.00732421875, "KL/mean": -220.34580993652344, "KL/rejected_KL_mean": -261.6842956542969, "KL/std": 104.43914031982422, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0038591313641518354, "fcm_dpo/delta": 0.03153546154499054, "fcm_dpo/margin": 82.67697143554688, "fcm_dpo/q_t": 0.4249793291091919, "grad_norm": 8.180280685424805, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.8158376216888428, "logits/rejected": -1.5065380334854126, "logps/chosen": -268.8505554199219, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -363.41778564453125, "loss": 1.1441, "margin_dpo/margin_mean": 82.67697143554688, "margin_dpo/margin_std": 118.45199584960938, "step": 433 }, { "KL/chosen_KL_mean": -154.00941467285156, "KL/mean": -201.30569458007812, "KL/rejected_KL_mean": -248.60198974609375, "KL/std": 115.76327514648438, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0038457319606095552, "fcm_dpo/delta": -0.014301536604762077, "fcm_dpo/margin": 94.59257507324219, "fcm_dpo/q_t": 0.41680121421813965, "grad_norm": 10.543797492980957, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.3249669075012207, "logits/rejected": -1.6370041370391846, "logps/chosen": -251.70193481445312, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -332.69329833984375, "loss": 1.1171, "margin_dpo/margin_mean": 94.59257507324219, "margin_dpo/margin_std": 135.2604522705078, "step": 434 }, { "KL/chosen_KL_mean": -150.43032836914062, "KL/mean": -199.70208740234375, "KL/rejected_KL_mean": -248.97384643554688, "KL/std": 107.50237274169922, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0038271979428827763, "fcm_dpo/delta": -0.028201783075928688, "fcm_dpo/margin": 98.54354095458984, "fcm_dpo/q_t": 0.41166508197784424, "grad_norm": 8.967982292175293, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.4550814628601074, "logits/rejected": -2.01522159576416, "logps/chosen": -236.60223388671875, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -342.7255554199219, "loss": 1.0944, "margin_dpo/margin_mean": 98.54354858398438, "margin_dpo/margin_std": 120.45396423339844, "step": 435 }, { "KL/chosen_KL_mean": -126.9485855102539, "KL/mean": -176.78211975097656, "KL/rejected_KL_mean": -226.6156463623047, "KL/std": 90.35716247558594, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.00379771925508976, "fcm_dpo/delta": -0.02967016212642193, "fcm_dpo/margin": 99.66706848144531, "fcm_dpo/q_t": 0.4091498553752899, "grad_norm": 9.417986869812012, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.7652595043182373, "logits/rejected": -0.9049780368804932, "logps/chosen": -232.73568725585938, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -315.2403564453125, "loss": 1.0742, "margin_dpo/margin_mean": 99.66706848144531, "margin_dpo/margin_std": 94.41795349121094, "step": 436 }, { "KL/chosen_KL_mean": -147.28018188476562, "KL/mean": -188.18582153320312, "KL/rejected_KL_mean": -229.0914306640625, "KL/std": 97.37655639648438, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.003821802092716098, "fcm_dpo/delta": 0.038460828363895416, "fcm_dpo/margin": 81.81124114990234, "fcm_dpo/q_t": 0.4246496856212616, "grad_norm": 9.778956413269043, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -2.0191614627838135, "logits/rejected": -1.3685252666473389, "logps/chosen": -243.3422393798828, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -318.1036376953125, "loss": 1.1317, "margin_dpo/margin_mean": 81.81124114990234, "margin_dpo/margin_std": 97.37457275390625, "step": 437 }, { "KL/chosen_KL_mean": -152.15206909179688, "KL/mean": -193.69491577148438, "KL/rejected_KL_mean": -235.2377166748047, "KL/std": 98.26370239257812, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0038298247382044792, "fcm_dpo/delta": 0.03270462155342102, "fcm_dpo/margin": 83.08563232421875, "fcm_dpo/q_t": 0.4242904782295227, "grad_norm": 8.528343200683594, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.0063064098358154, "logits/rejected": -1.1786775588989258, "logps/chosen": -254.00746154785156, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -324.68536376953125, "loss": 1.1347, "margin_dpo/margin_mean": 83.08562469482422, "margin_dpo/margin_std": 106.33834838867188, "step": 438 }, { "KL/chosen_KL_mean": -162.20938110351562, "KL/mean": -203.87869262695312, "KL/rejected_KL_mean": -245.54798889160156, "KL/std": 106.83394622802734, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.003817687975242734, "fcm_dpo/delta": -0.041722387075424194, "fcm_dpo/margin": 83.3386001586914, "fcm_dpo/q_t": 0.42422032356262207, "grad_norm": 8.626630783081055, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.4528486728668213, "logits/rejected": -1.214032530784607, "logps/chosen": -243.96502685546875, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -336.13433837890625, "loss": 1.1345, "margin_dpo/margin_mean": 83.3386001586914, "margin_dpo/margin_std": 104.8135986328125, "step": 439 }, { "KL/chosen_KL_mean": -146.45263671875, "KL/mean": -198.04237365722656, "KL/rejected_KL_mean": -249.63209533691406, "KL/std": 107.7030258178711, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0037775232922285795, "fcm_dpo/delta": -0.04193960130214691, "fcm_dpo/margin": 103.17948150634766, "fcm_dpo/q_t": 0.4082227945327759, "grad_norm": 6.765104293823242, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.527088165283203, "logits/rejected": -2.0390355587005615, "logps/chosen": -252.09371948242188, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -353.0331115722656, "loss": 1.0834, "margin_dpo/margin_mean": 103.17948913574219, "margin_dpo/margin_std": 118.86466979980469, "step": 440 }, { "KL/chosen_KL_mean": -189.3421630859375, "KL/mean": -232.15728759765625, "KL/rejected_KL_mean": -274.9723815917969, "KL/std": 102.21826934814453, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0037958736065775156, "fcm_dpo/delta": 0.025798771530389786, "fcm_dpo/margin": 85.63023376464844, "fcm_dpo/q_t": 0.4241536259651184, "grad_norm": 7.976558208465576, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.7173223495483398, "logits/rejected": -1.3602298498153687, "logps/chosen": -280.8717041015625, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -378.591796875, "loss": 1.1313, "margin_dpo/margin_mean": 85.63023376464844, "margin_dpo/margin_std": 113.97724151611328, "step": 441 }, { "KL/chosen_KL_mean": -183.38720703125, "KL/mean": -238.9866943359375, "KL/rejected_KL_mean": -294.586181640625, "KL/std": 107.71421813964844, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0037655127234756947, "fcm_dpo/delta": -0.07171474397182465, "fcm_dpo/margin": 111.19898223876953, "fcm_dpo/q_t": 0.4002749025821686, "grad_norm": 10.563261985778809, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.4691431522369385, "logits/rejected": -0.9836171269416809, "logps/chosen": -268.1647644042969, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -378.41033935546875, "loss": 1.0517, "margin_dpo/margin_mean": 111.19898223876953, "margin_dpo/margin_std": 110.26953125, "step": 442 }, { "KL/chosen_KL_mean": -180.15948486328125, "KL/mean": -213.82052612304688, "KL/rejected_KL_mean": -247.48159790039062, "KL/std": 105.76519775390625, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0037657800130546093, "fcm_dpo/delta": 0.01940564066171646, "fcm_dpo/margin": 67.32211303710938, "fcm_dpo/q_t": 0.4395884573459625, "grad_norm": 9.714847564697266, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -2.0454277992248535, "logits/rejected": -1.5061851739883423, "logps/chosen": -282.80877685546875, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -340.5196533203125, "loss": 1.1864, "margin_dpo/margin_mean": 67.32211303710938, "margin_dpo/margin_std": 103.1595687866211, "step": 443 }, { "KL/chosen_KL_mean": -184.04476928710938, "KL/mean": -225.06973266601562, "KL/rejected_KL_mean": -266.0947265625, "KL/std": 103.16741943359375, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.003791303373873234, "fcm_dpo/delta": 0.04013078659772873, "fcm_dpo/margin": 82.04994201660156, "fcm_dpo/q_t": 0.4268607497215271, "grad_norm": 10.210365295410156, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6695352792739868, "logits/rejected": -1.5052481889724731, "logps/chosen": -271.9644775390625, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -369.41815185546875, "loss": 1.1489, "margin_dpo/margin_mean": 82.04994201660156, "margin_dpo/margin_std": 120.0274658203125, "step": 444 }, { "KL/chosen_KL_mean": -161.8876953125, "KL/mean": -209.44468688964844, "KL/rejected_KL_mean": -257.001708984375, "KL/std": 100.81484985351562, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.003779977560043335, "fcm_dpo/delta": -0.010067718103528023, "fcm_dpo/margin": 95.11400604248047, "fcm_dpo/q_t": 0.4143327474594116, "grad_norm": 7.515559196472168, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -2.063795804977417, "logits/rejected": -1.6561250686645508, "logps/chosen": -263.2885437011719, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -356.03961181640625, "loss": 1.0985, "margin_dpo/margin_mean": 95.11400604248047, "margin_dpo/margin_std": 106.87651062011719, "step": 445 }, { "KL/chosen_KL_mean": -155.4358673095703, "KL/mean": -197.40676879882812, "KL/rejected_KL_mean": -239.37767028808594, "KL/std": 105.1705551147461, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.003812289796769619, "fcm_dpo/delta": 0.03087405301630497, "fcm_dpo/margin": 83.9417953491211, "fcm_dpo/q_t": 0.42391306161880493, "grad_norm": 11.083151817321777, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.7831169366836548, "logits/rejected": -1.6436889171600342, "logps/chosen": -242.85821533203125, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -346.07843017578125, "loss": 1.1289, "margin_dpo/margin_mean": 83.9417953491211, "margin_dpo/margin_std": 102.57586669921875, "step": 446 }, { "KL/chosen_KL_mean": -151.77920532226562, "KL/mean": -208.04087829589844, "KL/rejected_KL_mean": -264.30255126953125, "KL/std": 110.34654235839844, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0037684570997953415, "fcm_dpo/delta": -0.07731570303440094, "fcm_dpo/margin": 112.52333068847656, "fcm_dpo/q_t": 0.4001156985759735, "grad_norm": 7.701168060302734, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.1190688610076904, "logits/rejected": -2.0753798484802246, "logps/chosen": -235.39447021484375, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -369.2149658203125, "loss": 1.0616, "margin_dpo/margin_mean": 112.52333068847656, "margin_dpo/margin_std": 126.94161987304688, "step": 447 }, { "KL/chosen_KL_mean": -165.8087158203125, "KL/mean": -219.55224609375, "KL/rejected_KL_mean": -273.2958068847656, "KL/std": 115.209716796875, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0037383572198450565, "fcm_dpo/delta": -0.05410553142428398, "fcm_dpo/margin": 107.48709106445312, "fcm_dpo/q_t": 0.4059299826622009, "grad_norm": 8.979249954223633, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.6701862812042236, "logits/rejected": -1.3999063968658447, "logps/chosen": -258.3844909667969, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -380.985595703125, "loss": 1.0872, "margin_dpo/margin_mean": 107.48709106445312, "margin_dpo/margin_std": 135.12059020996094, "step": 448 }, { "KL/chosen_KL_mean": -163.68035888671875, "KL/mean": -218.75225830078125, "KL/rejected_KL_mean": -273.82415771484375, "KL/std": 107.4521255493164, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0036798424553126097, "fcm_dpo/delta": -0.05767472833395004, "fcm_dpo/margin": 110.14380645751953, "fcm_dpo/q_t": 0.40398430824279785, "grad_norm": 8.117274284362793, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.5961836576461792, "logits/rejected": -1.1621379852294922, "logps/chosen": -266.07928466796875, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -368.9730224609375, "loss": 1.0711, "margin_dpo/margin_mean": 110.14380645751953, "margin_dpo/margin_std": 123.95384979248047, "step": 449 }, { "KL/chosen_KL_mean": -146.38343811035156, "KL/mean": -198.30047607421875, "KL/rejected_KL_mean": -250.21755981445312, "KL/std": 109.4210205078125, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0036505977623164654, "fcm_dpo/delta": -0.030246740207076073, "fcm_dpo/margin": 103.83409881591797, "fcm_dpo/q_t": 0.4097868502140045, "grad_norm": 8.272851943969727, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.2635793685913086, "logits/rejected": -2.070467472076416, "logps/chosen": -233.3762969970703, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -358.74957275390625, "loss": 1.088, "margin_dpo/margin_mean": 103.83409881591797, "margin_dpo/margin_std": 118.95993041992188, "step": 450 }, { "KL/chosen_KL_mean": -165.2799072265625, "KL/mean": -206.35755920410156, "KL/rejected_KL_mean": -247.43519592285156, "KL/std": 108.06063842773438, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.003680099733173847, "fcm_dpo/delta": 0.049043335020542145, "fcm_dpo/margin": 82.15528869628906, "fcm_dpo/q_t": 0.4294675886631012, "grad_norm": 11.538263320922852, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.465996742248535, "logits/rejected": -1.9689087867736816, "logps/chosen": -252.0911865234375, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -327.290771484375, "loss": 1.1662, "margin_dpo/margin_mean": 82.15528869628906, "margin_dpo/margin_std": 135.2615203857422, "step": 451 }, { "KL/chosen_KL_mean": -144.2205810546875, "KL/mean": -190.93099975585938, "KL/rejected_KL_mean": -237.64141845703125, "KL/std": 103.01483154296875, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.003666744101792574, "fcm_dpo/delta": 0.00705854594707489, "fcm_dpo/margin": 93.42083740234375, "fcm_dpo/q_t": 0.41813337802886963, "grad_norm": 7.525914192199707, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7754335403442383, "logits/rejected": -1.720882773399353, "logps/chosen": -223.6002197265625, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -341.3568115234375, "loss": 1.1075, "margin_dpo/margin_mean": 93.42083740234375, "margin_dpo/margin_std": 97.0105972290039, "step": 452 }, { "KL/chosen_KL_mean": -174.63427734375, "KL/mean": -231.02047729492188, "KL/rejected_KL_mean": -287.40667724609375, "KL/std": 105.40919494628906, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0036578248254954815, "fcm_dpo/delta": -0.06514446437358856, "fcm_dpo/margin": 112.77238464355469, "fcm_dpo/q_t": 0.4024232029914856, "grad_norm": 10.383268356323242, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.8302236795425415, "logits/rejected": -1.3534992933273315, "logps/chosen": -262.3294677734375, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -377.46490478515625, "loss": 1.0585, "margin_dpo/margin_mean": 112.77238464355469, "margin_dpo/margin_std": 115.07199096679688, "step": 453 }, { "KL/chosen_KL_mean": -181.7522430419922, "KL/mean": -227.40805053710938, "KL/rejected_KL_mean": -273.0638427734375, "KL/std": 108.24821472167969, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0036656316369771957, "fcm_dpo/delta": 0.015384722501039505, "fcm_dpo/margin": 91.31158447265625, "fcm_dpo/q_t": 0.4220200181007385, "grad_norm": 8.089325904846191, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.025735855102539, "logits/rejected": -1.5811426639556885, "logps/chosen": -271.3184814453125, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -365.98486328125, "loss": 1.1449, "margin_dpo/margin_mean": 91.31158447265625, "margin_dpo/margin_std": 141.1439208984375, "step": 454 }, { "KL/chosen_KL_mean": -171.80966186523438, "KL/mean": -215.33624267578125, "KL/rejected_KL_mean": -258.86279296875, "KL/std": 107.06873321533203, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0036802366375923157, "fcm_dpo/delta": 0.03034432977437973, "fcm_dpo/margin": 87.05314636230469, "fcm_dpo/q_t": 0.42438793182373047, "grad_norm": 9.099287986755371, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8217436075210571, "logits/rejected": -1.7857723236083984, "logps/chosen": -249.4396514892578, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -377.84075927734375, "loss": 1.1415, "margin_dpo/margin_mean": 87.05314636230469, "margin_dpo/margin_std": 122.5904541015625, "step": 455 }, { "KL/chosen_KL_mean": -175.7742919921875, "KL/mean": -219.4339599609375, "KL/rejected_KL_mean": -263.0936279296875, "KL/std": 108.19908142089844, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0036958770360797644, "fcm_dpo/delta": 0.02810784988105297, "fcm_dpo/margin": 87.31932067871094, "fcm_dpo/q_t": 0.4234347343444824, "grad_norm": 10.236311912536621, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.092703342437744, "logits/rejected": -1.9648008346557617, "logps/chosen": -265.39117431640625, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -372.6533203125, "loss": 1.1292, "margin_dpo/margin_mean": 87.31932067871094, "margin_dpo/margin_std": 109.97627258300781, "step": 456 }, { "KL/chosen_KL_mean": -142.38052368164062, "KL/mean": -199.77947998046875, "KL/rejected_KL_mean": -257.17840576171875, "KL/std": 108.38119506835938, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.003633399959653616, "fcm_dpo/delta": -0.07133542001247406, "fcm_dpo/margin": 114.7978744506836, "fcm_dpo/q_t": 0.3996906280517578, "grad_norm": 8.830941200256348, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.1353631019592285, "logits/rejected": -1.8211092948913574, "logps/chosen": -229.70220947265625, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -347.94500732421875, "loss": 1.0463, "margin_dpo/margin_mean": 114.7978744506836, "margin_dpo/margin_std": 98.3472900390625, "step": 457 }, { "KL/chosen_KL_mean": -166.15895080566406, "KL/mean": -207.7908935546875, "KL/rejected_KL_mean": -249.4228515625, "KL/std": 106.99250030517578, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0036754917819052935, "fcm_dpo/delta": 0.045183245092630386, "fcm_dpo/margin": 83.26388549804688, "fcm_dpo/q_t": 0.42884039878845215, "grad_norm": 8.978321075439453, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5232584476470947, "logits/rejected": -1.2834289073944092, "logps/chosen": -259.67938232421875, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -352.79180908203125, "loss": 1.1551, "margin_dpo/margin_mean": 83.26388549804688, "margin_dpo/margin_std": 126.62335205078125, "step": 458 }, { "KL/chosen_KL_mean": -156.6053466796875, "KL/mean": -208.68515014648438, "KL/rejected_KL_mean": -260.76495361328125, "KL/std": 111.9120101928711, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0036452244967222214, "fcm_dpo/delta": -0.03129339963197708, "fcm_dpo/margin": 104.15959930419922, "fcm_dpo/q_t": 0.41059359908103943, "grad_norm": 6.5856709480285645, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.7647117376327515, "logits/rejected": -1.519968867301941, "logps/chosen": -232.29354858398438, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -352.9354248046875, "loss": 1.0893, "margin_dpo/margin_mean": 104.15960693359375, "margin_dpo/margin_std": 120.40950012207031, "step": 459 }, { "KL/chosen_KL_mean": -161.26702880859375, "KL/mean": -209.5472412109375, "KL/rejected_KL_mean": -257.82745361328125, "KL/std": 108.91401672363281, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.003641176503151655, "fcm_dpo/delta": -0.0018613841384649277, "fcm_dpo/margin": 96.5604248046875, "fcm_dpo/q_t": 0.41706520318984985, "grad_norm": 9.79261302947998, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0515353679656982, "logits/rejected": -2.0813822746276855, "logps/chosen": -242.38491821289062, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -368.13983154296875, "loss": 1.1161, "margin_dpo/margin_mean": 96.56043243408203, "margin_dpo/margin_std": 124.95089721679688, "step": 460 }, { "KL/chosen_KL_mean": -198.6721954345703, "KL/mean": -238.40460205078125, "KL/rejected_KL_mean": -278.1369934082031, "KL/std": 95.63264465332031, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0036772601306438446, "fcm_dpo/delta": 0.05956210941076279, "fcm_dpo/margin": 79.46479034423828, "fcm_dpo/q_t": 0.43117132782936096, "grad_norm": 7.454066753387451, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.3705121278762817, "logits/rejected": -1.173903465270996, "logps/chosen": -280.2557373046875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -372.01409912109375, "loss": 1.1595, "margin_dpo/margin_mean": 79.46479034423828, "margin_dpo/margin_std": 115.21940612792969, "step": 461 }, { "KL/chosen_KL_mean": -151.77906799316406, "KL/mean": -201.67929077148438, "KL/rejected_KL_mean": -251.57952880859375, "KL/std": 102.97764587402344, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.003690198063850403, "fcm_dpo/delta": -0.018989909440279007, "fcm_dpo/margin": 99.80046081542969, "fcm_dpo/q_t": 0.4126705527305603, "grad_norm": 7.87039041519165, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.154907703399658, "logits/rejected": -1.6468193531036377, "logps/chosen": -245.9776153564453, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -337.2111511230469, "loss": 1.0918, "margin_dpo/margin_mean": 99.80046081542969, "margin_dpo/margin_std": 111.1092529296875, "step": 462 }, { "KL/chosen_KL_mean": -147.24838256835938, "KL/mean": -207.97576904296875, "KL/rejected_KL_mean": -268.703125, "KL/std": 111.36909484863281, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.003649400779977441, "fcm_dpo/delta": -0.09765061736106873, "fcm_dpo/margin": 121.45474243164062, "fcm_dpo/q_t": 0.3944533169269562, "grad_norm": 11.575797080993652, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.7691724300384521, "logits/rejected": -1.3399362564086914, "logps/chosen": -233.17312622070312, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -365.60498046875, "loss": 1.0419, "margin_dpo/margin_mean": 121.45475006103516, "margin_dpo/margin_std": 119.29830932617188, "step": 463 }, { "KL/chosen_KL_mean": -180.07803344726562, "KL/mean": -229.7659454345703, "KL/rejected_KL_mean": -279.453857421875, "KL/std": 110.3133773803711, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.003598831593990326, "fcm_dpo/delta": -0.008031496778130531, "fcm_dpo/margin": 99.37580108642578, "fcm_dpo/q_t": 0.4156453013420105, "grad_norm": 11.15394401550293, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.7389912605285645, "logits/rejected": -1.821057677268982, "logps/chosen": -259.7672424316406, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -386.74615478515625, "loss": 1.1094, "margin_dpo/margin_mean": 99.37579345703125, "margin_dpo/margin_std": 126.01393127441406, "step": 464 }, { "KL/chosen_KL_mean": -171.55410766601562, "KL/mean": -228.99664306640625, "KL/rejected_KL_mean": -286.439208984375, "KL/std": 119.34693908691406, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.003568315412849188, "fcm_dpo/delta": -0.06258530914783478, "fcm_dpo/margin": 114.8851089477539, "fcm_dpo/q_t": 0.40214622020721436, "grad_norm": 7.759310722351074, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6805726289749146, "logits/rejected": -1.382225751876831, "logps/chosen": -263.414306640625, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -405.14923095703125, "loss": 1.057, "margin_dpo/margin_mean": 114.88511657714844, "margin_dpo/margin_std": 113.51573181152344, "step": 465 }, { "KL/chosen_KL_mean": -152.35321044921875, "KL/mean": -210.81861877441406, "KL/rejected_KL_mean": -269.28399658203125, "KL/std": 113.54776000976562, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.003523090621456504, "fcm_dpo/delta": -0.06469067931175232, "fcm_dpo/margin": 116.9307861328125, "fcm_dpo/q_t": 0.4020484685897827, "grad_norm": 6.367717266082764, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.4648833274841309, "logits/rejected": -1.1439869403839111, "logps/chosen": -237.05462646484375, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -365.3348388671875, "loss": 1.0586, "margin_dpo/margin_mean": 116.9307861328125, "margin_dpo/margin_std": 119.91029357910156, "step": 466 }, { "KL/chosen_KL_mean": -198.1439208984375, "KL/mean": -246.212158203125, "KL/rejected_KL_mean": -294.2803955078125, "KL/std": 118.82621765136719, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0035075186751782894, "fcm_dpo/delta": 0.012982618063688278, "fcm_dpo/margin": 96.13646697998047, "fcm_dpo/q_t": 0.4204251766204834, "grad_norm": 11.237042427062988, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.2293894290924072, "logits/rejected": -1.6436560153961182, "logps/chosen": -307.4422607421875, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -403.1240234375, "loss": 1.1331, "margin_dpo/margin_mean": 96.13645935058594, "margin_dpo/margin_std": 135.48971557617188, "step": 467 }, { "KL/chosen_KL_mean": -152.12576293945312, "KL/mean": -197.616455078125, "KL/rejected_KL_mean": -243.10714721679688, "KL/std": 116.39295196533203, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0035359710454940796, "fcm_dpo/delta": 0.029242999851703644, "fcm_dpo/margin": 90.98137664794922, "fcm_dpo/q_t": 0.42515265941619873, "grad_norm": 8.713449478149414, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.9283758401870728, "logits/rejected": -1.726074457168579, "logps/chosen": -250.44741821289062, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -349.78759765625, "loss": 1.1444, "margin_dpo/margin_mean": 90.98137664794922, "margin_dpo/margin_std": 135.01336669921875, "step": 468 }, { "KL/chosen_KL_mean": -187.17120361328125, "KL/mean": -242.8747100830078, "KL/rejected_KL_mean": -298.5782165527344, "KL/std": 116.49641418457031, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0035055140033364296, "fcm_dpo/delta": -0.042820751667022705, "fcm_dpo/margin": 111.40699768066406, "fcm_dpo/q_t": 0.4076286256313324, "grad_norm": 8.942715644836426, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.292907238006592, "logits/rejected": -1.880362629890442, "logps/chosen": -283.9354248046875, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -408.1732177734375, "loss": 1.0816, "margin_dpo/margin_mean": 111.40699768066406, "margin_dpo/margin_std": 125.92008972167969, "step": 469 }, { "KL/chosen_KL_mean": -171.951904296875, "KL/mean": -223.20053100585938, "KL/rejected_KL_mean": -274.44915771484375, "KL/std": 107.92627716064453, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0035076348576694727, "fcm_dpo/delta": -0.009884364902973175, "fcm_dpo/margin": 102.49728393554688, "fcm_dpo/q_t": 0.414161741733551, "grad_norm": 8.459217071533203, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.6088606119155884, "logits/rejected": -1.1879987716674805, "logps/chosen": -255.44854736328125, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -362.9349670410156, "loss": 1.0941, "margin_dpo/margin_mean": 102.49727630615234, "margin_dpo/margin_std": 109.22308349609375, "step": 470 }, { "KL/chosen_KL_mean": -196.32427978515625, "KL/mean": -244.75840759277344, "KL/rejected_KL_mean": -293.1925354003906, "KL/std": 107.41836547851562, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0035120295360684395, "fcm_dpo/delta": 0.010141678154468536, "fcm_dpo/margin": 96.8682632446289, "fcm_dpo/q_t": 0.4189138412475586, "grad_norm": 8.103554725646973, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.4588942527770996, "logits/rejected": -1.0298177003860474, "logps/chosen": -284.798583984375, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -383.67425537109375, "loss": 1.1096, "margin_dpo/margin_mean": 96.86827087402344, "margin_dpo/margin_std": 108.68827056884766, "step": 471 }, { "KL/chosen_KL_mean": -186.0305633544922, "KL/mean": -242.62794494628906, "KL/rejected_KL_mean": -299.225341796875, "KL/std": 108.45075988769531, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0034829839132726192, "fcm_dpo/delta": -0.04625207185745239, "fcm_dpo/margin": 113.19477844238281, "fcm_dpo/q_t": 0.4067213535308838, "grad_norm": 7.5819807052612305, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.317960739135742, "logits/rejected": -1.6368966102600098, "logps/chosen": -311.2642822265625, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -420.2788391113281, "loss": 1.0705, "margin_dpo/margin_mean": 113.19477844238281, "margin_dpo/margin_std": 116.70756530761719, "step": 472 }, { "KL/chosen_KL_mean": -210.21804809570312, "KL/mean": -245.76239013671875, "KL/rejected_KL_mean": -281.3067321777344, "KL/std": 109.60829162597656, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.003538992255926132, "fcm_dpo/delta": 0.10080716013908386, "fcm_dpo/margin": 71.08870697021484, "fcm_dpo/q_t": 0.44156894087791443, "grad_norm": 8.930468559265137, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8683528900146484, "logits/rejected": -1.3826295137405396, "logps/chosen": -305.82940673828125, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -369.4578857421875, "loss": 1.1996, "margin_dpo/margin_mean": 71.08869934082031, "margin_dpo/margin_std": 128.37716674804688, "step": 473 }, { "KL/chosen_KL_mean": -195.58636474609375, "KL/mean": -232.35910034179688, "KL/rejected_KL_mean": -269.1318664550781, "KL/std": 103.88653564453125, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0035479464568197727, "fcm_dpo/delta": -0.013695323839783669, "fcm_dpo/margin": 73.54548645019531, "fcm_dpo/q_t": 0.43837958574295044, "grad_norm": 12.680495262145996, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.551560401916504, "logits/rejected": -1.3643872737884521, "logps/chosen": -277.06610107421875, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -365.59747314453125, "loss": 1.1971, "margin_dpo/margin_mean": 73.54548645019531, "margin_dpo/margin_std": 128.28639221191406, "step": 474 }, { "KL/chosen_KL_mean": -148.80575561523438, "KL/mean": -205.51531982421875, "KL/rejected_KL_mean": -262.2248840332031, "KL/std": 110.92727661132812, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.003518566256389022, "fcm_dpo/delta": -0.051087651401758194, "fcm_dpo/margin": 113.41912841796875, "fcm_dpo/q_t": 0.4059932827949524, "grad_norm": 7.3847575187683105, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -1.9918149709701538, "logits/rejected": -1.939777135848999, "logps/chosen": -223.00173950195312, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -356.91729736328125, "loss": 1.0747, "margin_dpo/margin_mean": 113.41913604736328, "margin_dpo/margin_std": 128.20233154296875, "step": 475 }, { "KL/chosen_KL_mean": -172.1573486328125, "KL/mean": -222.3309326171875, "KL/rejected_KL_mean": -272.5045166015625, "KL/std": 113.27568054199219, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.003498323494568467, "fcm_dpo/delta": -0.0011973697692155838, "fcm_dpo/margin": 100.34715270996094, "fcm_dpo/q_t": 0.4174845516681671, "grad_norm": 8.773711204528809, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.9956138134002686, "logits/rejected": -1.6234877109527588, "logps/chosen": -244.12844848632812, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -352.7667541503906, "loss": 1.1104, "margin_dpo/margin_mean": 100.34715270996094, "margin_dpo/margin_std": 123.28178405761719, "step": 476 }, { "KL/chosen_KL_mean": -173.17788696289062, "KL/mean": -222.74859619140625, "KL/rejected_KL_mean": -272.3193054199219, "KL/std": 98.89154052734375, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.003510025329887867, "fcm_dpo/delta": 0.0020699170418083668, "fcm_dpo/margin": 99.14140319824219, "fcm_dpo/q_t": 0.4171368181705475, "grad_norm": 8.441904067993164, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.18198299407959, "logits/rejected": -1.2648335695266724, "logps/chosen": -278.1834411621094, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -354.1977233886719, "loss": 1.1136, "margin_dpo/margin_mean": 99.14141082763672, "margin_dpo/margin_std": 123.56255340576172, "step": 477 }, { "KL/chosen_KL_mean": -168.641845703125, "KL/mean": -220.2840576171875, "KL/rejected_KL_mean": -271.92626953125, "KL/std": 107.52497100830078, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.003499750280752778, "fcm_dpo/delta": -0.011910448782145977, "fcm_dpo/margin": 103.28440856933594, "fcm_dpo/q_t": 0.41406986117362976, "grad_norm": 8.295984268188477, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.7215707302093506, "logits/rejected": -1.548275113105774, "logps/chosen": -245.4300537109375, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -362.3662109375, "loss": 1.0994, "margin_dpo/margin_mean": 103.28441619873047, "margin_dpo/margin_std": 119.56977081298828, "step": 478 }, { "KL/chosen_KL_mean": -170.8726806640625, "KL/mean": -223.28994750976562, "KL/rejected_KL_mean": -275.70721435546875, "KL/std": 109.33702850341797, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0035019279457628727, "fcm_dpo/delta": -0.01795099675655365, "fcm_dpo/margin": 104.83454132080078, "fcm_dpo/q_t": 0.4122762084007263, "grad_norm": 11.120959281921387, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.5654895305633545, "logits/rejected": -1.4247112274169922, "logps/chosen": -252.2349853515625, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -376.79833984375, "loss": 1.0918, "margin_dpo/margin_mean": 104.83454895019531, "margin_dpo/margin_std": 113.38084411621094, "step": 479 }, { "KL/chosen_KL_mean": -198.81619262695312, "KL/mean": -239.9024658203125, "KL/rejected_KL_mean": -280.98876953125, "KL/std": 104.21759033203125, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0035206791944801807, "fcm_dpo/delta": 0.062481313943862915, "fcm_dpo/margin": 82.17259216308594, "fcm_dpo/q_t": 0.43133682012557983, "grad_norm": 8.71274185180664, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.8375613689422607, "logits/rejected": -1.1707818508148193, "logps/chosen": -295.590087890625, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -367.39349365234375, "loss": 1.157, "margin_dpo/margin_mean": 82.17259216308594, "margin_dpo/margin_std": 114.38607788085938, "step": 480 }, { "KL/chosen_KL_mean": -193.21163940429688, "KL/mean": -241.49343872070312, "KL/rejected_KL_mean": -289.77520751953125, "KL/std": 108.70539855957031, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0035333973355591297, "fcm_dpo/delta": 0.009111843071877956, "fcm_dpo/margin": 96.5635986328125, "fcm_dpo/q_t": 0.4189502000808716, "grad_norm": 10.80313491821289, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.7878085374832153, "logits/rejected": -1.3433544635772705, "logps/chosen": -284.8818054199219, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -388.4701232910156, "loss": 1.1175, "margin_dpo/margin_mean": 96.5635986328125, "margin_dpo/margin_std": 120.6896743774414, "step": 481 }, { "KL/chosen_KL_mean": -188.6186065673828, "KL/mean": -234.6708984375, "KL/rejected_KL_mean": -280.7231750488281, "KL/std": 107.70341491699219, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0035418274346739054, "fcm_dpo/delta": 0.024530591443181038, "fcm_dpo/margin": 92.10456848144531, "fcm_dpo/q_t": 0.42222094535827637, "grad_norm": 10.734329223632812, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.4199113845825195, "logits/rejected": -1.0575703382492065, "logps/chosen": -287.13873291015625, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -375.5526123046875, "loss": 1.1316, "margin_dpo/margin_mean": 92.10456848144531, "margin_dpo/margin_std": 121.29742431640625, "step": 482 }, { "KL/chosen_KL_mean": -193.5730438232422, "KL/mean": -243.23512268066406, "KL/rejected_KL_mean": -292.897216796875, "KL/std": 112.748779296875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0035694304388016462, "fcm_dpo/delta": -0.005171734839677811, "fcm_dpo/margin": 99.32415008544922, "fcm_dpo/q_t": 0.41537487506866455, "grad_norm": 6.910060405731201, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.9406828880310059, "logits/rejected": -1.2891228199005127, "logps/chosen": -300.691650390625, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -394.01220703125, "loss": 1.1039, "margin_dpo/margin_mean": 99.32415008544922, "margin_dpo/margin_std": 112.52894592285156, "step": 483 }, { "KL/chosen_KL_mean": -176.06654357910156, "KL/mean": -231.6673583984375, "KL/rejected_KL_mean": -287.2681579589844, "KL/std": 114.54606628417969, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.003536177333444357, "fcm_dpo/delta": -0.045017048716545105, "fcm_dpo/margin": 111.20159912109375, "fcm_dpo/q_t": 0.4065644145011902, "grad_norm": 7.739297866821289, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.305288791656494, "logits/rejected": -1.822198748588562, "logps/chosen": -263.04644775390625, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -377.9918212890625, "loss": 1.0794, "margin_dpo/margin_mean": 111.20160675048828, "margin_dpo/margin_std": 125.97113037109375, "step": 484 }, { "KL/chosen_KL_mean": -168.18959045410156, "KL/mean": -228.34178161621094, "KL/rejected_KL_mean": -288.49395751953125, "KL/std": 119.8036880493164, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.003482174128293991, "fcm_dpo/delta": -0.07192133367061615, "fcm_dpo/margin": 120.30439758300781, "fcm_dpo/q_t": 0.4006262421607971, "grad_norm": 7.9969072341918945, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.7569878101348877, "logits/rejected": -1.3441483974456787, "logps/chosen": -262.24835205078125, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -397.05694580078125, "loss": 1.0536, "margin_dpo/margin_mean": 120.30438995361328, "margin_dpo/margin_std": 122.30110931396484, "step": 485 }, { "KL/chosen_KL_mean": -192.80593872070312, "KL/mean": -245.52272033691406, "KL/rejected_KL_mean": -298.239501953125, "KL/std": 115.28067016601562, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.003468230366706848, "fcm_dpo/delta": -0.016331855207681656, "fcm_dpo/margin": 105.43354797363281, "fcm_dpo/q_t": 0.4130376875400543, "grad_norm": 9.43345832824707, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.9576878547668457, "logits/rejected": -1.2499067783355713, "logps/chosen": -286.55181884765625, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -396.31011962890625, "loss": 1.0988, "margin_dpo/margin_mean": 105.43354797363281, "margin_dpo/margin_std": 123.76641082763672, "step": 486 }, { "KL/chosen_KL_mean": -186.0725860595703, "KL/mean": -233.72265625, "KL/rejected_KL_mean": -281.37274169921875, "KL/std": 109.3471908569336, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0034674457274377346, "fcm_dpo/delta": 0.020222168415784836, "fcm_dpo/margin": 95.3001480102539, "fcm_dpo/q_t": 0.4214191138744354, "grad_norm": 8.363853454589844, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.1798763275146484, "logits/rejected": -1.8729960918426514, "logps/chosen": -274.10614013671875, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -379.8448486328125, "loss": 1.123, "margin_dpo/margin_mean": 95.30015563964844, "margin_dpo/margin_std": 117.59741973876953, "step": 487 }, { "KL/chosen_KL_mean": -167.96810913085938, "KL/mean": -221.46859741210938, "KL/rejected_KL_mean": -274.9690856933594, "KL/std": 116.1239013671875, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0034652245230972767, "fcm_dpo/delta": -0.021593431010842323, "fcm_dpo/margin": 107.00098419189453, "fcm_dpo/q_t": 0.4109659194946289, "grad_norm": 7.904480457305908, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.5642039775848389, "logits/rejected": -1.2998894453048706, "logps/chosen": -252.75775146484375, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -365.2425537109375, "loss": 1.0821, "margin_dpo/margin_mean": 107.0009765625, "margin_dpo/margin_std": 103.531494140625, "step": 488 }, { "KL/chosen_KL_mean": -190.6567840576172, "KL/mean": -238.8407440185547, "KL/rejected_KL_mean": -287.02471923828125, "KL/std": 106.14044189453125, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.00346622196957469, "fcm_dpo/delta": 0.01651071570813656, "fcm_dpo/margin": 96.367919921875, "fcm_dpo/q_t": 0.42081743478775024, "grad_norm": 8.367411613464355, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6423286199569702, "logits/rejected": -1.1554198265075684, "logps/chosen": -281.1260986328125, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -373.4223327636719, "loss": 1.1203, "margin_dpo/margin_mean": 96.367919921875, "margin_dpo/margin_std": 118.590087890625, "step": 489 }, { "KL/chosen_KL_mean": -190.39193725585938, "KL/mean": -235.7205810546875, "KL/rejected_KL_mean": -281.0491943359375, "KL/std": 114.899658203125, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0034541257191449404, "fcm_dpo/delta": -0.03985903412103653, "fcm_dpo/margin": 90.65727233886719, "fcm_dpo/q_t": 0.42698535323143005, "grad_norm": 7.405566215515137, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -1.9694141149520874, "logits/rejected": -2.0555520057678223, "logps/chosen": -275.7120666503906, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -397.0430603027344, "loss": 1.1672, "margin_dpo/margin_mean": 90.65727233886719, "margin_dpo/margin_std": 152.73873901367188, "step": 490 }, { "KL/chosen_KL_mean": -193.72019958496094, "KL/mean": -235.68002319335938, "KL/rejected_KL_mean": -277.63983154296875, "KL/std": 117.0687255859375, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0034703616984188557, "fcm_dpo/delta": 0.06055054813623428, "fcm_dpo/margin": 83.91963958740234, "fcm_dpo/q_t": 0.4307933449745178, "grad_norm": 9.638858795166016, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8824546337127686, "logits/rejected": -1.3976110219955444, "logps/chosen": -279.86370849609375, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -358.3193054199219, "loss": 1.1575, "margin_dpo/margin_mean": 83.91963958740234, "margin_dpo/margin_std": 119.05918884277344, "step": 491 }, { "KL/chosen_KL_mean": -181.82156372070312, "KL/mean": -215.8145751953125, "KL/rejected_KL_mean": -249.8076171875, "KL/std": 115.58494567871094, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.003513314528390765, "fcm_dpo/delta": 0.04437117651104927, "fcm_dpo/margin": 67.98603057861328, "fcm_dpo/q_t": 0.4439983665943146, "grad_norm": 11.910463333129883, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.9611644744873047, "logits/rejected": -1.5738914012908936, "logps/chosen": -262.92913818359375, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -330.5596008300781, "loss": 1.2237, "margin_dpo/margin_mean": 67.98603057861328, "margin_dpo/margin_std": 144.552734375, "step": 492 }, { "KL/chosen_KL_mean": -180.19873046875, "KL/mean": -224.85617065429688, "KL/rejected_KL_mean": -269.51361083984375, "KL/std": 103.72267150878906, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.003519045189023018, "fcm_dpo/delta": 0.036693867295980453, "fcm_dpo/margin": 89.31488800048828, "fcm_dpo/q_t": 0.42500752210617065, "grad_norm": 15.132545471191406, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.4117212295532227, "logits/rejected": -1.684227466583252, "logps/chosen": -292.40606689453125, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -363.12078857421875, "loss": 1.136, "margin_dpo/margin_mean": 89.31488037109375, "margin_dpo/margin_std": 112.1026382446289, "step": 493 }, { "KL/chosen_KL_mean": -174.93557739257812, "KL/mean": -233.77862548828125, "KL/rejected_KL_mean": -292.6217041015625, "KL/std": 115.109130859375, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.003489833790808916, "fcm_dpo/delta": -0.06400926411151886, "fcm_dpo/margin": 117.68609619140625, "fcm_dpo/q_t": 0.4025000035762787, "grad_norm": 8.709985733032227, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.3965866565704346, "logits/rejected": -1.3232474327087402, "logps/chosen": -248.05047607421875, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -384.7846984863281, "loss": 1.0652, "margin_dpo/margin_mean": 117.68610382080078, "margin_dpo/margin_std": 126.31973266601562, "step": 494 }, { "KL/chosen_KL_mean": -179.59649658203125, "KL/mean": -230.99942016601562, "KL/rejected_KL_mean": -282.4023742675781, "KL/std": 109.49714660644531, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0034876042045652866, "fcm_dpo/delta": -0.00887532252818346, "fcm_dpo/margin": 102.80587005615234, "fcm_dpo/q_t": 0.41548794507980347, "grad_norm": 10.19632625579834, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.1634316444396973, "logits/rejected": -1.6924965381622314, "logps/chosen": -279.1168212890625, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -380.333251953125, "loss": 1.1098, "margin_dpo/margin_mean": 102.80587768554688, "margin_dpo/margin_std": 132.11651611328125, "step": 495 }, { "KL/chosen_KL_mean": -200.29039001464844, "KL/mean": -244.1722869873047, "KL/rejected_KL_mean": -288.0542297363281, "KL/std": 109.90963745117188, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0035173906944692135, "fcm_dpo/delta": 0.04244166612625122, "fcm_dpo/margin": 87.7638168334961, "fcm_dpo/q_t": 0.4261983036994934, "grad_norm": 8.163657188415527, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.06965970993042, "logits/rejected": -1.4145108461380005, "logps/chosen": -308.14715576171875, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -380.82476806640625, "loss": 1.1393, "margin_dpo/margin_mean": 87.76380920410156, "margin_dpo/margin_std": 111.0191650390625, "step": 496 }, { "KL/chosen_KL_mean": -212.7672119140625, "KL/mean": -241.07339477539062, "KL/rejected_KL_mean": -269.37957763671875, "KL/std": 111.04718017578125, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0035550547763705254, "fcm_dpo/delta": 0.05280128866434097, "fcm_dpo/margin": 56.61235809326172, "fcm_dpo/q_t": 0.4525056481361389, "grad_norm": 7.859197616577148, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.7544639110565186, "logits/rejected": -1.3077800273895264, "logps/chosen": -304.8504638671875, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -351.17462158203125, "loss": 1.2402, "margin_dpo/margin_mean": 56.61235427856445, "margin_dpo/margin_std": 119.74966430664062, "step": 497 }, { "KL/chosen_KL_mean": -182.74752807617188, "KL/mean": -229.97174072265625, "KL/rejected_KL_mean": -277.19598388671875, "KL/std": 109.8856201171875, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0035611032508313656, "fcm_dpo/delta": 0.014137128368020058, "fcm_dpo/margin": 94.44844055175781, "fcm_dpo/q_t": 0.4199615716934204, "grad_norm": 11.034602165222168, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.8116369247436523, "logits/rejected": -1.3151178359985352, "logps/chosen": -280.94189453125, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -367.8834228515625, "loss": 1.1241, "margin_dpo/margin_mean": 94.44844055175781, "margin_dpo/margin_std": 123.05360412597656, "step": 498 }, { "KL/chosen_KL_mean": -189.39154052734375, "KL/mean": -239.59115600585938, "KL/rejected_KL_mean": -289.790771484375, "KL/std": 118.07666015625, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0035636252723634243, "fcm_dpo/delta": -0.008080375380814075, "fcm_dpo/margin": 100.39924621582031, "fcm_dpo/q_t": 0.4155081510543823, "grad_norm": 8.29495620727539, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.8839493989944458, "logits/rejected": -1.8339301347732544, "logps/chosen": -270.43145751953125, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -396.71246337890625, "loss": 1.1083, "margin_dpo/margin_mean": 100.39924621582031, "margin_dpo/margin_std": 126.0709457397461, "step": 499 }, { "KL/chosen_KL_mean": -211.02285766601562, "KL/mean": -253.43365478515625, "KL/rejected_KL_mean": -295.84442138671875, "KL/std": 113.74246215820312, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0035803706850856543, "fcm_dpo/delta": 0.04778574779629707, "fcm_dpo/margin": 84.82154846191406, "fcm_dpo/q_t": 0.4291677176952362, "grad_norm": 10.582213401794434, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.5011379718780518, "logits/rejected": -1.3272111415863037, "logps/chosen": -300.27093505859375, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -396.254638671875, "loss": 1.1563, "margin_dpo/margin_mean": 84.82154846191406, "margin_dpo/margin_std": 128.5194854736328, "step": 500 }, { "KL/chosen_KL_mean": -185.23126220703125, "KL/mean": -232.45306396484375, "KL/rejected_KL_mean": -279.67486572265625, "KL/std": 122.60513305664062, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.003619695082306862, "fcm_dpo/delta": 0.007628034800291061, "fcm_dpo/margin": 94.443603515625, "fcm_dpo/q_t": 0.42015451192855835, "grad_norm": 8.294617652893066, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.5849406719207764, "logits/rejected": -1.1501197814941406, "logps/chosen": -275.82818603515625, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -364.9721374511719, "loss": 1.1368, "margin_dpo/margin_mean": 94.44361114501953, "margin_dpo/margin_std": 138.31069946289062, "step": 501 }, { "KL/chosen_KL_mean": -185.39065551757812, "KL/mean": -227.544921875, "KL/rejected_KL_mean": -269.69921875, "KL/std": 102.24156951904297, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.003635676112025976, "fcm_dpo/delta": 0.04445258527994156, "fcm_dpo/margin": 84.30854797363281, "fcm_dpo/q_t": 0.4273098409175873, "grad_norm": 9.682639122009277, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9149703979492188, "logits/rejected": -1.1831245422363281, "logps/chosen": -288.86138916015625, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -358.74334716796875, "loss": 1.1551, "margin_dpo/margin_mean": 84.30854797363281, "margin_dpo/margin_std": 126.0497055053711, "step": 502 }, { "KL/chosen_KL_mean": -184.37777709960938, "KL/mean": -231.67709350585938, "KL/rejected_KL_mean": -278.9764099121094, "KL/std": 109.58749389648438, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.003647264325991273, "fcm_dpo/delta": 0.004856102168560028, "fcm_dpo/margin": 94.59862518310547, "fcm_dpo/q_t": 0.4183366000652313, "grad_norm": 9.243871688842773, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7151260375976562, "logits/rejected": -1.2164381742477417, "logps/chosen": -278.1077880859375, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -366.58538818359375, "loss": 1.1192, "margin_dpo/margin_mean": 94.59861755371094, "margin_dpo/margin_std": 121.35775756835938, "step": 503 }, { "KL/chosen_KL_mean": -188.73452758789062, "KL/mean": -249.68174743652344, "KL/rejected_KL_mean": -310.62896728515625, "KL/std": 114.59334564208984, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0035885274410247803, "fcm_dpo/delta": -0.0913601666688919, "fcm_dpo/margin": 121.8944091796875, "fcm_dpo/q_t": 0.395521879196167, "grad_norm": 8.927103996276855, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.1576266288757324, "logits/rejected": -1.8315094709396362, "logps/chosen": -281.0263977050781, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -419.6665954589844, "loss": 1.0375, "margin_dpo/margin_mean": 121.89440155029297, "margin_dpo/margin_std": 116.02076721191406, "step": 504 }, { "KL/chosen_KL_mean": -205.61614990234375, "KL/mean": -249.38954162597656, "KL/rejected_KL_mean": -293.16290283203125, "KL/std": 113.5660400390625, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0035868994891643524, "fcm_dpo/delta": 0.03715867921710014, "fcm_dpo/margin": 87.54676818847656, "fcm_dpo/q_t": 0.4253244698047638, "grad_norm": 8.561593055725098, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.545558452606201, "logits/rejected": -2.0049493312835693, "logps/chosen": -311.8930969238281, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -387.3690185546875, "loss": 1.135, "margin_dpo/margin_mean": 87.54676818847656, "margin_dpo/margin_std": 110.25186920166016, "step": 505 }, { "KL/chosen_KL_mean": -199.0520782470703, "KL/mean": -248.78738403320312, "KL/rejected_KL_mean": -298.522705078125, "KL/std": 106.77815246582031, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0035802177153527737, "fcm_dpo/delta": -0.006594102829694748, "fcm_dpo/margin": 99.47065734863281, "fcm_dpo/q_t": 0.4144606590270996, "grad_norm": 7.981509685516357, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9839035272598267, "logits/rejected": -1.6269145011901855, "logps/chosen": -280.492919921875, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -384.1871032714844, "loss": 1.1016, "margin_dpo/margin_mean": 99.47065734863281, "margin_dpo/margin_std": 112.22975158691406, "step": 506 }, { "KL/chosen_KL_mean": -207.85507202148438, "KL/mean": -257.60333251953125, "KL/rejected_KL_mean": -307.35162353515625, "KL/std": 112.45246887207031, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0035867104306817055, "fcm_dpo/delta": -0.007119990419596434, "fcm_dpo/margin": 99.49655151367188, "fcm_dpo/q_t": 0.4162372946739197, "grad_norm": 8.50875186920166, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.0565733909606934, "logits/rejected": -1.6098239421844482, "logps/chosen": -307.28204345703125, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -409.6045837402344, "loss": 1.1165, "margin_dpo/margin_mean": 99.49655151367188, "margin_dpo/margin_std": 132.78350830078125, "step": 507 }, { "KL/chosen_KL_mean": -200.7310333251953, "KL/mean": -242.9747772216797, "KL/rejected_KL_mean": -285.218505859375, "KL/std": 120.6661376953125, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0036112265661358833, "fcm_dpo/delta": 0.046299755573272705, "fcm_dpo/margin": 84.48748779296875, "fcm_dpo/q_t": 0.4291643798351288, "grad_norm": 14.380181312561035, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.931424617767334, "logits/rejected": -1.4405816793441772, "logps/chosen": -298.64666748046875, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -389.016357421875, "loss": 1.1685, "margin_dpo/margin_mean": 84.48748779296875, "margin_dpo/margin_std": 141.1713409423828, "step": 508 }, { "KL/chosen_KL_mean": -187.82366943359375, "KL/mean": -226.3107452392578, "KL/rejected_KL_mean": -264.7978210449219, "KL/std": 93.97451782226562, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0036603715270757675, "fcm_dpo/delta": 0.07014653086662292, "fcm_dpo/margin": 76.97415161132812, "fcm_dpo/q_t": 0.43269461393356323, "grad_norm": 13.2564115524292, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.1071910858154297, "logits/rejected": -1.4545390605926514, "logps/chosen": -288.55352783203125, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -351.74114990234375, "loss": 1.176, "margin_dpo/margin_mean": 76.97415161132812, "margin_dpo/margin_std": 125.49037170410156, "step": 509 }, { "KL/chosen_KL_mean": -220.6427001953125, "KL/mean": -268.06878662109375, "KL/rejected_KL_mean": -315.49493408203125, "KL/std": 113.48007202148438, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0036751613952219486, "fcm_dpo/delta": 0.0014247912913560867, "fcm_dpo/margin": 94.85222625732422, "fcm_dpo/q_t": 0.41720956563949585, "grad_norm": 10.144198417663574, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.6430813074111938, "logits/rejected": -1.4643030166625977, "logps/chosen": -309.81207275390625, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -423.414306640625, "loss": 1.1106, "margin_dpo/margin_mean": 94.85222625732422, "margin_dpo/margin_std": 113.00849914550781, "step": 510 }, { "KL/chosen_KL_mean": -202.2946319580078, "KL/mean": -252.87405395507812, "KL/rejected_KL_mean": -303.4534912109375, "KL/std": 111.47917175292969, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.003665823955088854, "fcm_dpo/delta": -0.021657807752490044, "fcm_dpo/margin": 101.15885925292969, "fcm_dpo/q_t": 0.4129549264907837, "grad_norm": 10.130990028381348, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.184415340423584, "logits/rejected": -1.6210508346557617, "logps/chosen": -292.04443359375, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -401.1847839355469, "loss": 1.1006, "margin_dpo/margin_mean": 101.15885925292969, "margin_dpo/margin_std": 126.12850952148438, "step": 511 }, { "KL/chosen_KL_mean": -192.6759033203125, "KL/mean": -241.7565460205078, "KL/rejected_KL_mean": -290.83721923828125, "KL/std": 109.43336486816406, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0036659478209912777, "fcm_dpo/delta": -0.010581165552139282, "fcm_dpo/margin": 98.16130828857422, "fcm_dpo/q_t": 0.4148399233818054, "grad_norm": 12.596420288085938, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.9905426502227783, "logits/rejected": -1.5659904479980469, "logps/chosen": -278.4110412597656, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -378.46209716796875, "loss": 1.1153, "margin_dpo/margin_mean": 98.16130828857422, "margin_dpo/margin_std": 130.37428283691406, "step": 512 }, { "KL/chosen_KL_mean": -204.2006378173828, "KL/mean": -245.07667541503906, "KL/rejected_KL_mean": -285.9527282714844, "KL/std": 105.23699951171875, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.003662682604044676, "fcm_dpo/delta": 0.05204736068844795, "fcm_dpo/margin": 81.75205993652344, "fcm_dpo/q_t": 0.4285077452659607, "grad_norm": 10.188175201416016, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.2897233963012695, "logits/rejected": -1.5949485301971436, "logps/chosen": -300.3911437988281, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -368.8704833984375, "loss": 1.1587, "margin_dpo/margin_mean": 81.75205993652344, "margin_dpo/margin_std": 120.81651306152344, "step": 513 }, { "KL/chosen_KL_mean": -197.57208251953125, "KL/mean": -242.10592651367188, "KL/rejected_KL_mean": -286.63970947265625, "KL/std": 105.97879791259766, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.00369897298514843, "fcm_dpo/delta": 0.021228918805718422, "fcm_dpo/margin": 89.06764221191406, "fcm_dpo/q_t": 0.42292487621307373, "grad_norm": 10.486163139343262, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.5472581386566162, "logits/rejected": -0.9609041213989258, "logps/chosen": -295.3957214355469, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -372.0095520019531, "loss": 1.136, "margin_dpo/margin_mean": 89.06763458251953, "margin_dpo/margin_std": 126.61221313476562, "step": 514 }, { "KL/chosen_KL_mean": -189.39907836914062, "KL/mean": -234.1923828125, "KL/rejected_KL_mean": -278.9857177734375, "KL/std": 109.69906616210938, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0037112319841980934, "fcm_dpo/delta": 0.01812698133289814, "fcm_dpo/margin": 89.58662414550781, "fcm_dpo/q_t": 0.42138558626174927, "grad_norm": 10.124283790588379, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.7971962690353394, "logits/rejected": -1.5676167011260986, "logps/chosen": -271.3326416015625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -372.562255859375, "loss": 1.1303, "margin_dpo/margin_mean": 89.58663940429688, "margin_dpo/margin_std": 120.45381927490234, "step": 515 }, { "KL/chosen_KL_mean": -194.1672821044922, "KL/mean": -237.17877197265625, "KL/rejected_KL_mean": -280.19024658203125, "KL/std": 126.10204315185547, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.003736538579687476, "fcm_dpo/delta": 0.029438240453600883, "fcm_dpo/margin": 86.02294158935547, "fcm_dpo/q_t": 0.42609429359436035, "grad_norm": 12.550764083862305, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.2853751182556152, "logits/rejected": -2.117717742919922, "logps/chosen": -291.25897216796875, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -398.7469177246094, "loss": 1.1641, "margin_dpo/margin_mean": 86.02294158935547, "margin_dpo/margin_std": 149.06100463867188, "step": 516 }, { "KL/chosen_KL_mean": -207.3135986328125, "KL/mean": -259.28863525390625, "KL/rejected_KL_mean": -311.263671875, "KL/std": 112.59333801269531, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.003694976679980755, "fcm_dpo/delta": -0.0363662987947464, "fcm_dpo/margin": 103.9500732421875, "fcm_dpo/q_t": 0.4096449613571167, "grad_norm": 8.650229454040527, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.8742021322250366, "logits/rejected": -1.4494314193725586, "logps/chosen": -300.50018310546875, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -406.8564453125, "loss": 1.0959, "margin_dpo/margin_mean": 103.9500732421875, "margin_dpo/margin_std": 127.87490844726562, "step": 517 }, { "KL/chosen_KL_mean": -216.13909912109375, "KL/mean": -255.55569458007812, "KL/rejected_KL_mean": -294.9722900390625, "KL/std": 105.98251342773438, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0037395060062408447, "fcm_dpo/delta": 0.056903403252363205, "fcm_dpo/margin": 78.83317565917969, "fcm_dpo/q_t": 0.43136492371559143, "grad_norm": 9.171182632446289, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.8862206935882568, "logits/rejected": -1.5587050914764404, "logps/chosen": -303.16046142578125, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -386.21685791015625, "loss": 1.1719, "margin_dpo/margin_mean": 78.83317565917969, "margin_dpo/margin_std": 132.766845703125, "step": 518 }, { "KL/chosen_KL_mean": -200.68801879882812, "KL/mean": -242.96278381347656, "KL/rejected_KL_mean": -285.2375793457031, "KL/std": 99.03076934814453, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.003764968365430832, "fcm_dpo/delta": 0.032718490809202194, "fcm_dpo/margin": 84.54957580566406, "fcm_dpo/q_t": 0.42471984028816223, "grad_norm": 9.85444450378418, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.973852276802063, "logits/rejected": -1.6058682203292847, "logps/chosen": -301.37554931640625, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -394.26800537109375, "loss": 1.1307, "margin_dpo/margin_mean": 84.54957580566406, "margin_dpo/margin_std": 104.6343765258789, "step": 519 }, { "KL/chosen_KL_mean": -187.18389892578125, "KL/mean": -237.69288635253906, "KL/rejected_KL_mean": -288.20184326171875, "KL/std": 111.96141052246094, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0037539214827120304, "fcm_dpo/delta": -0.03046102076768875, "fcm_dpo/margin": 101.01795959472656, "fcm_dpo/q_t": 0.4096372127532959, "grad_norm": 8.610065460205078, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.4602785110473633, "logits/rejected": -1.018876314163208, "logps/chosen": -285.9615478515625, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -395.1475830078125, "loss": 1.0844, "margin_dpo/margin_mean": 101.01795959472656, "margin_dpo/margin_std": 110.69432067871094, "step": 520 }, { "KL/chosen_KL_mean": -186.480224609375, "KL/mean": -238.56015014648438, "KL/rejected_KL_mean": -290.6400451660156, "KL/std": 104.96720886230469, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.003745339810848236, "fcm_dpo/delta": -0.04189406707882881, "fcm_dpo/margin": 104.15984344482422, "fcm_dpo/q_t": 0.40725165605545044, "grad_norm": 9.40650463104248, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9044756889343262, "logits/rejected": -1.5660072565078735, "logps/chosen": -272.831298828125, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -381.0332336425781, "loss": 1.0746, "margin_dpo/margin_mean": 104.15984344482422, "margin_dpo/margin_std": 108.37776184082031, "step": 521 }, { "KL/chosen_KL_mean": -196.66424560546875, "KL/mean": -242.25320434570312, "KL/rejected_KL_mean": -287.8421630859375, "KL/std": 114.36814880371094, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0037341518327593803, "fcm_dpo/delta": 0.009792602621018887, "fcm_dpo/margin": 91.17794036865234, "fcm_dpo/q_t": 0.419525146484375, "grad_norm": 9.686522483825684, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.7731294631958008, "logits/rejected": -1.4124106168746948, "logps/chosen": -272.67230224609375, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -362.283447265625, "loss": 1.1236, "margin_dpo/margin_mean": 91.17794799804688, "margin_dpo/margin_std": 120.81549072265625, "step": 522 }, { "KL/chosen_KL_mean": -216.45358276367188, "KL/mean": -250.39422607421875, "KL/rejected_KL_mean": -284.3348693847656, "KL/std": 113.65496826171875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.003780013881623745, "fcm_dpo/delta": 0.09594298899173737, "fcm_dpo/margin": 67.88130187988281, "fcm_dpo/q_t": 0.4405025243759155, "grad_norm": 10.478557586669922, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.0826165676116943, "logits/rejected": -1.3967843055725098, "logps/chosen": -312.9498596191406, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -375.35015869140625, "loss": 1.2059, "margin_dpo/margin_mean": 67.88130950927734, "margin_dpo/margin_std": 132.08935546875, "step": 523 }, { "KL/chosen_KL_mean": -197.23512268066406, "KL/mean": -245.64877319335938, "KL/rejected_KL_mean": -294.0623779296875, "KL/std": 112.85758972167969, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.003788803005591035, "fcm_dpo/delta": -0.017533954232931137, "fcm_dpo/margin": 96.8272933959961, "fcm_dpo/q_t": 0.413879930973053, "grad_norm": 11.646639823913574, "learning_rate": 7.781828926091535e-08, "logits/chosen": -1.964921474456787, "logits/rejected": -1.2433466911315918, "logps/chosen": -300.640625, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -381.9725341796875, "loss": 1.1018, "margin_dpo/margin_mean": 96.82728576660156, "margin_dpo/margin_std": 120.0452880859375, "step": 524 }, { "KL/chosen_KL_mean": -185.51718139648438, "KL/mean": -241.59011840820312, "KL/rejected_KL_mean": -297.66302490234375, "KL/std": 113.54193878173828, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0037190313450992107, "fcm_dpo/delta": -0.0718376487493515, "fcm_dpo/margin": 112.1458511352539, "fcm_dpo/q_t": 0.40325284004211426, "grad_norm": 9.65093994140625, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.4775643348693848, "logits/rejected": -1.3898857831954956, "logps/chosen": -269.72369384765625, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -403.92462158203125, "loss": 1.0737, "margin_dpo/margin_mean": 112.14584350585938, "margin_dpo/margin_std": 131.68031311035156, "step": 525 }, { "KL/chosen_KL_mean": -191.0419464111328, "KL/mean": -240.35336303710938, "KL/rejected_KL_mean": -289.664794921875, "KL/std": 106.99334716796875, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0037255685310810804, "fcm_dpo/delta": -0.018087774515151978, "fcm_dpo/margin": 98.62283325195312, "fcm_dpo/q_t": 0.41339293122291565, "grad_norm": 7.776037693023682, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.793632984161377, "logits/rejected": -1.1487914323806763, "logps/chosen": -293.89605712890625, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -383.31622314453125, "loss": 1.1057, "margin_dpo/margin_mean": 98.62283325195312, "margin_dpo/margin_std": 126.7509536743164, "step": 526 }, { "KL/chosen_KL_mean": -194.12429809570312, "KL/mean": -244.64202880859375, "KL/rejected_KL_mean": -295.1597900390625, "KL/std": 112.13742065429688, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0037054414860904217, "fcm_dpo/delta": -0.025346871465444565, "fcm_dpo/margin": 101.03549194335938, "fcm_dpo/q_t": 0.4100729525089264, "grad_norm": 8.990571975708008, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.045637369155884, "logits/rejected": -2.223583221435547, "logps/chosen": -290.88970947265625, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -380.55499267578125, "loss": 1.0791, "margin_dpo/margin_mean": 101.03549194335938, "margin_dpo/margin_std": 99.27645874023438, "step": 527 }, { "KL/chosen_KL_mean": -181.04397583007812, "KL/mean": -230.51739501953125, "KL/rejected_KL_mean": -279.99078369140625, "KL/std": 114.39199829101562, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.003698178566992283, "fcm_dpo/delta": -0.01654074341058731, "fcm_dpo/margin": 98.94681549072266, "fcm_dpo/q_t": 0.41398417949676514, "grad_norm": 8.755908012390137, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.263066530227661, "logits/rejected": -1.761635184288025, "logps/chosen": -275.2088928222656, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -380.36309814453125, "loss": 1.1095, "margin_dpo/margin_mean": 98.94680786132812, "margin_dpo/margin_std": 131.02001953125, "step": 528 }, { "KL/chosen_KL_mean": -209.95843505859375, "KL/mean": -255.12881469726562, "KL/rejected_KL_mean": -300.2991943359375, "KL/std": 112.90772247314453, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0037063632626086473, "fcm_dpo/delta": 0.01557399332523346, "fcm_dpo/margin": 90.34076690673828, "fcm_dpo/q_t": 0.42203935980796814, "grad_norm": 10.391129493713379, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.617582082748413, "logits/rejected": -1.4403666257858276, "logps/chosen": -290.7352294921875, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -397.4903564453125, "loss": 1.1387, "margin_dpo/margin_mean": 90.34076690673828, "margin_dpo/margin_std": 134.39183044433594, "step": 529 }, { "KL/chosen_KL_mean": -192.8917236328125, "KL/mean": -237.88470458984375, "KL/rejected_KL_mean": -282.877685546875, "KL/std": 108.81076049804688, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0037014430854469538, "fcm_dpo/delta": 0.01740468479692936, "fcm_dpo/margin": 89.98597717285156, "fcm_dpo/q_t": 0.4211847186088562, "grad_norm": 9.386820793151855, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.3750388622283936, "logits/rejected": -1.0489709377288818, "logps/chosen": -272.6674499511719, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -374.93157958984375, "loss": 1.1179, "margin_dpo/margin_mean": 89.98597717285156, "margin_dpo/margin_std": 105.92375183105469, "step": 530 }, { "KL/chosen_KL_mean": -185.37269592285156, "KL/mean": -242.11434936523438, "KL/rejected_KL_mean": -298.8559875488281, "KL/std": 115.90069580078125, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.003671992104500532, "fcm_dpo/delta": -0.0697207897901535, "fcm_dpo/margin": 113.48330688476562, "fcm_dpo/q_t": 0.40041667222976685, "grad_norm": 11.281847953796387, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.9988183975219727, "logits/rejected": -1.4149906635284424, "logps/chosen": -264.69189453125, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -376.9643249511719, "loss": 1.0555, "margin_dpo/margin_mean": 113.48330688476562, "margin_dpo/margin_std": 115.69195556640625, "step": 531 }, { "KL/chosen_KL_mean": -200.35659790039062, "KL/mean": -261.4466552734375, "KL/rejected_KL_mean": -322.53668212890625, "KL/std": 110.38591003417969, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.003618501592427492, "fcm_dpo/delta": -0.09625843912363052, "fcm_dpo/margin": 122.18009185791016, "fcm_dpo/q_t": 0.395268052816391, "grad_norm": 10.523765563964844, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.6976091861724854, "logits/rejected": -1.0748405456542969, "logps/chosen": -301.9935302734375, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -415.882080078125, "loss": 1.0411, "margin_dpo/margin_mean": 122.18008422851562, "margin_dpo/margin_std": 123.33810424804688, "step": 532 }, { "KL/chosen_KL_mean": -193.70330810546875, "KL/mean": -229.02333068847656, "KL/rejected_KL_mean": -264.3433532714844, "KL/std": 116.5345230102539, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.003648084355518222, "fcm_dpo/delta": 0.09463018923997879, "fcm_dpo/margin": 70.64005279541016, "fcm_dpo/q_t": 0.44041889905929565, "grad_norm": 10.332172393798828, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.5457515716552734, "logits/rejected": -1.9624764919281006, "logps/chosen": -284.449951171875, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -347.23187255859375, "loss": 1.1953, "margin_dpo/margin_mean": 70.64005279541016, "margin_dpo/margin_std": 125.44705200195312, "step": 533 }, { "KL/chosen_KL_mean": -172.7174530029297, "KL/mean": -224.91893005371094, "KL/rejected_KL_mean": -277.12042236328125, "KL/std": 109.21537780761719, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0036494785454124212, "fcm_dpo/delta": -0.03223882615566254, "fcm_dpo/margin": 104.40293884277344, "fcm_dpo/q_t": 0.4089839458465576, "grad_norm": 9.242754936218262, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.9426603317260742, "logits/rejected": -1.5452160835266113, "logps/chosen": -259.3482971191406, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -361.99237060546875, "loss": 1.0793, "margin_dpo/margin_mean": 104.40294647216797, "margin_dpo/margin_std": 108.64551544189453, "step": 534 }, { "KL/chosen_KL_mean": -173.71408081054688, "KL/mean": -229.5833282470703, "KL/rejected_KL_mean": -285.4525451660156, "KL/std": 110.26399230957031, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.003621132578700781, "fcm_dpo/delta": -0.05698984116315842, "fcm_dpo/margin": 111.73844909667969, "fcm_dpo/q_t": 0.4046723246574402, "grad_norm": 7.128327369689941, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.9593716859817505, "logits/rejected": -1.7793586254119873, "logps/chosen": -264.0594787597656, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -386.23052978515625, "loss": 1.0724, "margin_dpo/margin_mean": 111.73844909667969, "margin_dpo/margin_std": 125.63351440429688, "step": 535 }, { "KL/chosen_KL_mean": -171.966796875, "KL/mean": -227.51123046875, "KL/rejected_KL_mean": -283.0556640625, "KL/std": 111.04515075683594, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.003565022721886635, "fcm_dpo/delta": -0.04809026047587395, "fcm_dpo/margin": 111.08885955810547, "fcm_dpo/q_t": 0.4055427014827728, "grad_norm": 8.142410278320312, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.7423102855682373, "logits/rejected": -1.4878580570220947, "logps/chosen": -250.92637634277344, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -376.1434631347656, "loss": 1.071, "margin_dpo/margin_mean": 111.0888671875, "margin_dpo/margin_std": 116.97557067871094, "step": 536 }, { "KL/chosen_KL_mean": -151.9663848876953, "KL/mean": -208.97288513183594, "KL/rejected_KL_mean": -265.9793701171875, "KL/std": 113.79142761230469, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0035508633591234684, "fcm_dpo/delta": -0.057404760271310806, "fcm_dpo/margin": 114.01297760009766, "fcm_dpo/q_t": 0.4033905863761902, "grad_norm": 8.943068504333496, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.430394172668457, "logits/rejected": -0.7311882972717285, "logps/chosen": -238.67578125, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -344.8870544433594, "loss": 1.0623, "margin_dpo/margin_mean": 114.01298522949219, "margin_dpo/margin_std": 111.31535339355469, "step": 537 }, { "KL/chosen_KL_mean": -207.32974243164062, "KL/mean": -254.31741333007812, "KL/rejected_KL_mean": -301.30511474609375, "KL/std": 102.00543212890625, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0035327780060470104, "fcm_dpo/delta": 0.01861899346113205, "fcm_dpo/margin": 93.97532653808594, "fcm_dpo/q_t": 0.42088937759399414, "grad_norm": 8.07778549194336, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.6889193058013916, "logits/rejected": -1.1383318901062012, "logps/chosen": -305.77191162109375, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -392.3997497558594, "loss": 1.118, "margin_dpo/margin_mean": 93.97533416748047, "margin_dpo/margin_std": 109.98233795166016, "step": 538 }, { "KL/chosen_KL_mean": -180.59934997558594, "KL/mean": -230.15231323242188, "KL/rejected_KL_mean": -279.70526123046875, "KL/std": 112.00794982910156, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0035359340254217386, "fcm_dpo/delta": -0.00044883578084409237, "fcm_dpo/margin": 99.1059341430664, "fcm_dpo/q_t": 0.41694220900535583, "grad_norm": 7.9984331130981445, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.3405637741088867, "logits/rejected": -1.8550949096679688, "logps/chosen": -271.5314636230469, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -377.1507568359375, "loss": 1.115, "margin_dpo/margin_mean": 99.1059341430664, "margin_dpo/margin_std": 127.63572692871094, "step": 539 }, { "KL/chosen_KL_mean": -194.36886596679688, "KL/mean": -255.1884765625, "KL/rejected_KL_mean": -316.008056640625, "KL/std": 123.82765197753906, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0035065882839262486, "fcm_dpo/delta": -0.07994043827056885, "fcm_dpo/margin": 121.63917541503906, "fcm_dpo/q_t": 0.4010760188102722, "grad_norm": 9.597548484802246, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.8456082344055176, "logits/rejected": -1.6051323413848877, "logps/chosen": -302.23876953125, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -437.40155029296875, "loss": 1.0668, "margin_dpo/margin_mean": 121.63917541503906, "margin_dpo/margin_std": 145.95077514648438, "step": 540 }, { "KL/chosen_KL_mean": -214.39366149902344, "KL/mean": -251.995361328125, "KL/rejected_KL_mean": -289.59710693359375, "KL/std": 101.08285522460938, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.003471862059086561, "fcm_dpo/delta": -0.010158803313970566, "fcm_dpo/margin": 75.20344543457031, "fcm_dpo/q_t": 0.43791496753692627, "grad_norm": 8.656953811645508, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.3250484466552734, "logits/rejected": -1.8622653484344482, "logps/chosen": -320.9611511230469, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -398.6634521484375, "loss": 1.1814, "margin_dpo/margin_mean": 75.20345306396484, "margin_dpo/margin_std": 109.61821746826172, "step": 541 }, { "KL/chosen_KL_mean": -182.25750732421875, "KL/mean": -239.80133056640625, "KL/rejected_KL_mean": -297.3451232910156, "KL/std": 113.83293914794922, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0034466434735804796, "fcm_dpo/delta": -0.04859615117311478, "fcm_dpo/margin": 115.08761596679688, "fcm_dpo/q_t": 0.4053635597229004, "grad_norm": 9.179239273071289, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.0284407138824463, "logits/rejected": -1.9017888307571411, "logps/chosen": -269.6268005371094, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -400.16546630859375, "loss": 1.0722, "margin_dpo/margin_mean": 115.08763122558594, "margin_dpo/margin_std": 123.78321838378906, "step": 542 }, { "KL/chosen_KL_mean": -208.16015625, "KL/mean": -238.977783203125, "KL/rejected_KL_mean": -269.79541015625, "KL/std": 121.44940185546875, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0034658510703593493, "fcm_dpo/delta": 0.039931923151016235, "fcm_dpo/margin": 61.635276794433594, "fcm_dpo/q_t": 0.45024219155311584, "grad_norm": 11.537575721740723, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.9039433002471924, "logits/rejected": -1.9526920318603516, "logps/chosen": -295.312744140625, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -381.3271484375, "loss": 1.249, "margin_dpo/margin_mean": 61.63527297973633, "margin_dpo/margin_std": 149.325439453125, "step": 543 }, { "KL/chosen_KL_mean": -218.96533203125, "KL/mean": -257.943603515625, "KL/rejected_KL_mean": -296.9219055175781, "KL/std": 113.99932098388672, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.003504591528326273, "fcm_dpo/delta": 0.07899578660726547, "fcm_dpo/margin": 77.95660400390625, "fcm_dpo/q_t": 0.43580859899520874, "grad_norm": 8.389333724975586, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.619746208190918, "logits/rejected": -1.528540849685669, "logps/chosen": -305.11224365234375, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -403.3158264160156, "loss": 1.174, "margin_dpo/margin_mean": 77.95660400390625, "margin_dpo/margin_std": 120.2328109741211, "step": 544 }, { "KL/chosen_KL_mean": -191.79345703125, "KL/mean": -240.0297393798828, "KL/rejected_KL_mean": -288.2660217285156, "KL/std": 113.57461547851562, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.003531014546751976, "fcm_dpo/delta": 0.009631103835999966, "fcm_dpo/margin": 96.47254943847656, "fcm_dpo/q_t": 0.42000478506088257, "grad_norm": 8.303099632263184, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.6626473665237427, "logits/rejected": -1.2465167045593262, "logps/chosen": -288.21771240234375, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -389.00006103515625, "loss": 1.1339, "margin_dpo/margin_mean": 96.47254943847656, "margin_dpo/margin_std": 140.88851928710938, "step": 545 }, { "KL/chosen_KL_mean": -198.90142822265625, "KL/mean": -241.52761840820312, "KL/rejected_KL_mean": -284.15380859375, "KL/std": 114.06819152832031, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0035481934901326895, "fcm_dpo/delta": 0.049016885459423065, "fcm_dpo/margin": 85.25238037109375, "fcm_dpo/q_t": 0.427976131439209, "grad_norm": 8.463441848754883, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.5044810771942139, "logits/rejected": -1.1506500244140625, "logps/chosen": -284.92254638671875, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -371.5264587402344, "loss": 1.1553, "margin_dpo/margin_mean": 85.25237274169922, "margin_dpo/margin_std": 126.45768737792969, "step": 546 }, { "KL/chosen_KL_mean": -192.7201385498047, "KL/mean": -244.76358032226562, "KL/rejected_KL_mean": -296.8070068359375, "KL/std": 111.87278747558594, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.003559216856956482, "fcm_dpo/delta": -0.02130865678191185, "fcm_dpo/margin": 104.08687591552734, "fcm_dpo/q_t": 0.4111068844795227, "grad_norm": 8.546212196350098, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.9306116104125977, "logits/rejected": -1.4740939140319824, "logps/chosen": -293.37115478515625, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -394.71875, "loss": 1.0814, "margin_dpo/margin_mean": 104.08687591552734, "margin_dpo/margin_std": 100.65296936035156, "step": 547 }, { "KL/chosen_KL_mean": -187.52847290039062, "KL/mean": -230.36521911621094, "KL/rejected_KL_mean": -273.20196533203125, "KL/std": 114.19947814941406, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0035731415264308453, "fcm_dpo/delta": 0.04523646831512451, "fcm_dpo/margin": 85.67349243164062, "fcm_dpo/q_t": 0.42822831869125366, "grad_norm": 10.519240379333496, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.3426060676574707, "logits/rejected": -1.9913389682769775, "logps/chosen": -283.55859375, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -371.61517333984375, "loss": 1.1672, "margin_dpo/margin_mean": 85.6734848022461, "margin_dpo/margin_std": 143.60479736328125, "step": 548 }, { "KL/chosen_KL_mean": -179.11123657226562, "KL/mean": -222.1546630859375, "KL/rejected_KL_mean": -265.1980895996094, "KL/std": 108.54437255859375, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.003615580964833498, "fcm_dpo/delta": 0.03949017822742462, "fcm_dpo/margin": 86.08686065673828, "fcm_dpo/q_t": 0.42630475759506226, "grad_norm": 9.976551055908203, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.7082140445709229, "logits/rejected": -1.2845286130905151, "logps/chosen": -262.56219482421875, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -346.57830810546875, "loss": 1.159, "margin_dpo/margin_mean": 86.08686065673828, "margin_dpo/margin_std": 134.32659912109375, "step": 549 }, { "KL/chosen_KL_mean": -198.07762145996094, "KL/mean": -240.52317810058594, "KL/rejected_KL_mean": -282.96868896484375, "KL/std": 102.56906127929688, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0036357841454446316, "fcm_dpo/delta": 0.04258155822753906, "fcm_dpo/margin": 84.89111328125, "fcm_dpo/q_t": 0.4270329475402832, "grad_norm": 8.363508224487305, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.585087776184082, "logits/rejected": -1.3738791942596436, "logps/chosen": -287.6719665527344, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -384.2842102050781, "loss": 1.1429, "margin_dpo/margin_mean": 84.89111328125, "margin_dpo/margin_std": 112.36885070800781, "step": 550 }, { "KL/chosen_KL_mean": -177.17831420898438, "KL/mean": -229.80349731445312, "KL/rejected_KL_mean": -282.4287109375, "KL/std": 117.92803192138672, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.003607895690947771, "fcm_dpo/delta": -0.03133741766214371, "fcm_dpo/margin": 105.25039672851562, "fcm_dpo/q_t": 0.4109448194503784, "grad_norm": 9.829827308654785, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.5682897567749023, "logits/rejected": -1.4281201362609863, "logps/chosen": -272.2604675292969, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -400.1479797363281, "loss": 1.0879, "margin_dpo/margin_mean": 105.25040435791016, "margin_dpo/margin_std": 120.77340698242188, "step": 551 }, { "KL/chosen_KL_mean": -168.04107666015625, "KL/mean": -242.57498168945312, "KL/rejected_KL_mean": -317.10894775390625, "KL/std": 126.97428894042969, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0034962629433721304, "fcm_dpo/delta": -0.1826794147491455, "fcm_dpo/margin": 149.06784057617188, "fcm_dpo/q_t": 0.3760908842086792, "grad_norm": 10.586475372314453, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.0521278381347656, "logits/rejected": -1.6936209201812744, "logps/chosen": -269.45025634765625, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -427.9473876953125, "loss": 0.9852, "margin_dpo/margin_mean": 149.06784057617188, "margin_dpo/margin_std": 132.89865112304688, "step": 552 }, { "KL/chosen_KL_mean": -181.4647216796875, "KL/mean": -226.1086883544922, "KL/rejected_KL_mean": -270.75262451171875, "KL/std": 117.89599609375, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.003511346410959959, "fcm_dpo/delta": 0.03755655884742737, "fcm_dpo/margin": 89.28793334960938, "fcm_dpo/q_t": 0.42544877529144287, "grad_norm": 8.715890884399414, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.4146729707717896, "logits/rejected": -1.0797001123428345, "logps/chosen": -279.4285888671875, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -374.329833984375, "loss": 1.1455, "margin_dpo/margin_mean": 89.28793334960938, "margin_dpo/margin_std": 126.4961166381836, "step": 553 }, { "KL/chosen_KL_mean": -190.81781005859375, "KL/mean": -240.1916961669922, "KL/rejected_KL_mean": -289.5655517578125, "KL/std": 118.36160278320312, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0035250496584922075, "fcm_dpo/delta": 0.0017596334218978882, "fcm_dpo/margin": 98.74777221679688, "fcm_dpo/q_t": 0.4185582399368286, "grad_norm": 8.441490173339844, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.9050065279006958, "logits/rejected": -1.1337482929229736, "logps/chosen": -294.504638671875, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -382.55767822265625, "loss": 1.1374, "margin_dpo/margin_mean": 98.74777221679688, "margin_dpo/margin_std": 151.89859008789062, "step": 554 }, { "KL/chosen_KL_mean": -198.46168518066406, "KL/mean": -243.5849151611328, "KL/rejected_KL_mean": -288.7081298828125, "KL/std": 122.31109619140625, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.003531009890139103, "fcm_dpo/delta": 0.03237007558345795, "fcm_dpo/margin": 90.2464828491211, "fcm_dpo/q_t": 0.42589229345321655, "grad_norm": 8.39963436126709, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.793553113937378, "logits/rejected": -1.7043761014938354, "logps/chosen": -284.6653137207031, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -403.25537109375, "loss": 1.1532, "margin_dpo/margin_mean": 90.24647521972656, "margin_dpo/margin_std": 142.6334228515625, "step": 555 }, { "KL/chosen_KL_mean": -180.72763061523438, "KL/mean": -229.08200073242188, "KL/rejected_KL_mean": -277.4363708496094, "KL/std": 107.62989807128906, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.00353134423494339, "fcm_dpo/delta": 0.008648969233036041, "fcm_dpo/margin": 96.708740234375, "fcm_dpo/q_t": 0.4190796911716461, "grad_norm": 10.821390151977539, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.5066819190979004, "logits/rejected": -1.20830237865448, "logps/chosen": -269.131103515625, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -384.07550048828125, "loss": 1.1191, "margin_dpo/margin_mean": 96.708740234375, "margin_dpo/margin_std": 122.39891052246094, "step": 556 }, { "KL/chosen_KL_mean": -193.26211547851562, "KL/mean": -249.51931762695312, "KL/rejected_KL_mean": -305.7765197753906, "KL/std": 125.55513000488281, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.0035052881576120853, "fcm_dpo/delta": -0.046675145626068115, "fcm_dpo/margin": 112.514404296875, "fcm_dpo/q_t": 0.40768754482269287, "grad_norm": 7.855673789978027, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.884885549545288, "logits/rejected": -1.7106280326843262, "logps/chosen": -292.09735107421875, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -421.4254150390625, "loss": 1.0868, "margin_dpo/margin_mean": 112.514404296875, "margin_dpo/margin_std": 138.13348388671875, "step": 557 }, { "KL/chosen_KL_mean": -189.36181640625, "KL/mean": -250.08526611328125, "KL/rejected_KL_mean": -310.8087463378906, "KL/std": 108.59098815917969, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.003463957691565156, "fcm_dpo/delta": -0.07404932379722595, "fcm_dpo/margin": 121.44691467285156, "fcm_dpo/q_t": 0.399513304233551, "grad_norm": 10.023776054382324, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3280816078186035, "logits/rejected": -1.7179542779922485, "logps/chosen": -295.3736572265625, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -419.2000732421875, "loss": 1.0461, "margin_dpo/margin_mean": 121.44691467285156, "margin_dpo/margin_std": 111.65013122558594, "step": 558 }, { "KL/chosen_KL_mean": -207.67294311523438, "KL/mean": -254.62962341308594, "KL/rejected_KL_mean": -301.5863037109375, "KL/std": 111.31138610839844, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0034738099202513695, "fcm_dpo/delta": 0.024532420560717583, "fcm_dpo/margin": 93.913330078125, "fcm_dpo/q_t": 0.42217856645584106, "grad_norm": 9.603667259216309, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.3610217571258545, "logits/rejected": -1.8903889656066895, "logps/chosen": -308.0976867675781, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -401.9989929199219, "loss": 1.1332, "margin_dpo/margin_mean": 93.913330078125, "margin_dpo/margin_std": 126.29349517822266, "step": 559 }, { "KL/chosen_KL_mean": -194.4308624267578, "KL/mean": -241.85711669921875, "KL/rejected_KL_mean": -289.28338623046875, "KL/std": 108.63676452636719, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0034833569079637527, "fcm_dpo/delta": 0.02026817388832569, "fcm_dpo/margin": 94.85253143310547, "fcm_dpo/q_t": 0.4217734932899475, "grad_norm": 10.023667335510254, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.5179170370101929, "logits/rejected": -1.0840182304382324, "logps/chosen": -282.0663757324219, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -376.66326904296875, "loss": 1.1292, "margin_dpo/margin_mean": 94.85253143310547, "margin_dpo/margin_std": 125.89270782470703, "step": 560 }, { "KL/chosen_KL_mean": -204.8238525390625, "KL/mean": -248.489990234375, "KL/rejected_KL_mean": -292.1561279296875, "KL/std": 106.87451934814453, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.003518040757626295, "fcm_dpo/delta": 0.04399743303656578, "fcm_dpo/margin": 87.332275390625, "fcm_dpo/q_t": 0.4266801178455353, "grad_norm": 8.021707534790039, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.9129680395126343, "logits/rejected": -1.8870296478271484, "logps/chosen": -279.70526123046875, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -398.2040710449219, "loss": 1.1343, "margin_dpo/margin_mean": 87.332275390625, "margin_dpo/margin_std": 103.2053451538086, "step": 561 }, { "KL/chosen_KL_mean": -186.30548095703125, "KL/mean": -230.82940673828125, "KL/rejected_KL_mean": -275.3533630371094, "KL/std": 108.56792449951172, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0035289404913783073, "fcm_dpo/delta": 0.036836374551057816, "fcm_dpo/margin": 89.04788208007812, "fcm_dpo/q_t": 0.4263342618942261, "grad_norm": 9.825562477111816, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.063478469848633, "logits/rejected": -1.605584979057312, "logps/chosen": -286.96868896484375, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -377.59423828125, "loss": 1.1509, "margin_dpo/margin_mean": 89.04788208007812, "margin_dpo/margin_std": 134.69857788085938, "step": 562 }, { "KL/chosen_KL_mean": -197.03280639648438, "KL/mean": -233.1240234375, "KL/rejected_KL_mean": -269.2152099609375, "KL/std": 109.09457397460938, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0035538184456527233, "fcm_dpo/delta": 0.020880797877907753, "fcm_dpo/margin": 72.1823959350586, "fcm_dpo/q_t": 0.4389062821865082, "grad_norm": 9.450041770935059, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.7265961170196533, "logits/rejected": -1.3938815593719482, "logps/chosen": -293.2920227050781, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -370.4202575683594, "loss": 1.1888, "margin_dpo/margin_mean": 72.1823959350586, "margin_dpo/margin_std": 117.22450256347656, "step": 563 }, { "KL/chosen_KL_mean": -193.35025024414062, "KL/mean": -234.18368530273438, "KL/rejected_KL_mean": -275.0171203613281, "KL/std": 109.07282257080078, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.003597792237997055, "fcm_dpo/delta": 0.05773669481277466, "fcm_dpo/margin": 81.66685485839844, "fcm_dpo/q_t": 0.429607093334198, "grad_norm": 11.191895484924316, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.3705523014068604, "logits/rejected": -1.8037471771240234, "logps/chosen": -306.3106689453125, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -374.545166015625, "loss": 1.1457, "margin_dpo/margin_mean": 81.66685485839844, "margin_dpo/margin_std": 99.60574340820312, "step": 564 }, { "KL/chosen_KL_mean": -189.67156982421875, "KL/mean": -240.24447631835938, "KL/rejected_KL_mean": -290.8173522949219, "KL/std": 118.9386978149414, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0035917828790843487, "fcm_dpo/delta": -0.013826126232743263, "fcm_dpo/margin": 101.14576721191406, "fcm_dpo/q_t": 0.4145999848842621, "grad_norm": 8.363946914672852, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.6090872287750244, "logits/rejected": -1.0985816717147827, "logps/chosen": -285.03717041015625, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -383.03472900390625, "loss": 1.1088, "margin_dpo/margin_mean": 101.14576721191406, "margin_dpo/margin_std": 131.41986083984375, "step": 565 }, { "KL/chosen_KL_mean": -197.93115234375, "KL/mean": -246.37832641601562, "KL/rejected_KL_mean": -294.8255310058594, "KL/std": 121.02455139160156, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0036003051791340113, "fcm_dpo/delta": 0.0011118315160274506, "fcm_dpo/margin": 96.89437866210938, "fcm_dpo/q_t": 0.4169330596923828, "grad_norm": 7.455462455749512, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9121630191802979, "logits/rejected": -1.4834628105163574, "logps/chosen": -296.90435791015625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -405.64642333984375, "loss": 1.1133, "margin_dpo/margin_mean": 96.89437866210938, "margin_dpo/margin_std": 119.95796203613281, "step": 566 }, { "KL/chosen_KL_mean": -190.77066040039062, "KL/mean": -233.41404724121094, "KL/rejected_KL_mean": -276.05743408203125, "KL/std": 109.31199645996094, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.0036020888946950436, "fcm_dpo/delta": 0.04397448152303696, "fcm_dpo/margin": 85.2867202758789, "fcm_dpo/q_t": 0.42661383748054504, "grad_norm": 9.104766845703125, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.1696252822875977, "logits/rejected": -1.4840627908706665, "logps/chosen": -283.4742431640625, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -367.28173828125, "loss": 1.1476, "margin_dpo/margin_mean": 85.28672790527344, "margin_dpo/margin_std": 116.36089324951172, "step": 567 }, { "KL/chosen_KL_mean": -181.86331176757812, "KL/mean": -239.2467803955078, "KL/rejected_KL_mean": -296.6302490234375, "KL/std": 115.29779052734375, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0035802796483039856, "fcm_dpo/delta": -0.06387396901845932, "fcm_dpo/margin": 114.76695251464844, "fcm_dpo/q_t": 0.4024001955986023, "grad_norm": 7.9680609703063965, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3012007474899292, "logits/rejected": -1.1734957695007324, "logps/chosen": -254.99557495117188, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -391.5829162597656, "loss": 1.0596, "margin_dpo/margin_mean": 114.76695251464844, "margin_dpo/margin_std": 117.41258239746094, "step": 568 }, { "KL/chosen_KL_mean": -192.97401428222656, "KL/mean": -239.18646240234375, "KL/rejected_KL_mean": -285.39892578125, "KL/std": 117.06562042236328, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0035934888292104006, "fcm_dpo/delta": 0.018431857228279114, "fcm_dpo/margin": 92.42491149902344, "fcm_dpo/q_t": 0.421556293964386, "grad_norm": 12.887516975402832, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.8857532739639282, "logits/rejected": -1.5045015811920166, "logps/chosen": -282.19110107421875, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -371.80999755859375, "loss": 1.1445, "margin_dpo/margin_mean": 92.42491149902344, "margin_dpo/margin_std": 141.7397918701172, "step": 569 }, { "KL/chosen_KL_mean": -172.2167205810547, "KL/mean": -232.86175537109375, "KL/rejected_KL_mean": -293.50677490234375, "KL/std": 110.95115661621094, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0035460367798805237, "fcm_dpo/delta": -0.083748459815979, "fcm_dpo/margin": 121.2900390625, "fcm_dpo/q_t": 0.3970003128051758, "grad_norm": 8.749626159667969, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.227011203765869, "logits/rejected": -1.538137674331665, "logps/chosen": -266.9794921875, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -382.35369873046875, "loss": 1.0424, "margin_dpo/margin_mean": 121.2900390625, "margin_dpo/margin_std": 116.27960205078125, "step": 570 }, { "KL/chosen_KL_mean": -180.41046142578125, "KL/mean": -233.0526580810547, "KL/rejected_KL_mean": -285.69482421875, "KL/std": 108.79902648925781, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.003527991008013487, "fcm_dpo/delta": -0.022305209189653397, "fcm_dpo/margin": 105.28437805175781, "fcm_dpo/q_t": 0.412392795085907, "grad_norm": 11.293088912963867, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7487030029296875, "logits/rejected": -1.5946201086044312, "logps/chosen": -261.23175048828125, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -382.71014404296875, "loss": 1.1008, "margin_dpo/margin_mean": 105.28437805175781, "margin_dpo/margin_std": 131.46383666992188, "step": 571 }, { "KL/chosen_KL_mean": -184.57254028320312, "KL/mean": -237.90609741210938, "KL/rejected_KL_mean": -291.23968505859375, "KL/std": 116.03223419189453, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.003500755177810788, "fcm_dpo/delta": -0.0243685320019722, "fcm_dpo/margin": 106.66716003417969, "fcm_dpo/q_t": 0.4122371971607208, "grad_norm": 7.702200889587402, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.6371374130249023, "logits/rejected": -1.466029405593872, "logps/chosen": -261.95245361328125, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -381.8056640625, "loss": 1.0927, "margin_dpo/margin_mean": 106.66716003417969, "margin_dpo/margin_std": 124.71554565429688, "step": 572 }, { "KL/chosen_KL_mean": -179.2711181640625, "KL/mean": -228.96124267578125, "KL/rejected_KL_mean": -278.6513366699219, "KL/std": 115.18270874023438, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.003517458215355873, "fcm_dpo/delta": -5.8736652135849e-05, "fcm_dpo/margin": 99.38020324707031, "fcm_dpo/q_t": 0.4168274402618408, "grad_norm": 8.358694076538086, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1662466526031494, "logits/rejected": -1.9378020763397217, "logps/chosen": -262.9772033691406, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -369.34881591796875, "loss": 1.1147, "margin_dpo/margin_mean": 99.38020324707031, "margin_dpo/margin_std": 123.52162170410156, "step": 573 }, { "KL/chosen_KL_mean": -189.58619689941406, "KL/mean": -243.55184936523438, "KL/rejected_KL_mean": -297.51751708984375, "KL/std": 114.34774780273438, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0034951698035001755, "fcm_dpo/delta": -0.028393561020493507, "fcm_dpo/margin": 107.93132019042969, "fcm_dpo/q_t": 0.4105871319770813, "grad_norm": 8.607494354248047, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.16677188873291, "logits/rejected": -1.8440101146697998, "logps/chosen": -274.7156982421875, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -396.571533203125, "loss": 1.0936, "margin_dpo/margin_mean": 107.93132019042969, "margin_dpo/margin_std": 128.80502319335938, "step": 574 }, { "KL/chosen_KL_mean": -194.84368896484375, "KL/mean": -244.37933349609375, "KL/rejected_KL_mean": -293.91497802734375, "KL/std": 117.02687072753906, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.003482311498373747, "fcm_dpo/delta": 0.005181839689612389, "fcm_dpo/margin": 99.07133483886719, "fcm_dpo/q_t": 0.4187672436237335, "grad_norm": 9.850213050842285, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.2270342111587524, "logits/rejected": -0.7484307289123535, "logps/chosen": -300.6145324707031, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -394.86444091796875, "loss": 1.1249, "margin_dpo/margin_mean": 99.07133483886719, "margin_dpo/margin_std": 136.86322021484375, "step": 575 }, { "KL/chosen_KL_mean": -193.72335815429688, "KL/mean": -241.3595428466797, "KL/rejected_KL_mean": -288.9957275390625, "KL/std": 109.08699035644531, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.0034968480467796326, "fcm_dpo/delta": 0.01737806759774685, "fcm_dpo/margin": 95.27239990234375, "fcm_dpo/q_t": 0.4210534691810608, "grad_norm": 10.300097465515137, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.029911756515503, "logits/rejected": -1.9733035564422607, "logps/chosen": -274.9278869628906, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -394.4051513671875, "loss": 1.1314, "margin_dpo/margin_mean": 95.27238464355469, "margin_dpo/margin_std": 130.84689331054688, "step": 576 }, { "KL/chosen_KL_mean": -173.93589782714844, "KL/mean": -233.73297119140625, "KL/rejected_KL_mean": -293.530029296875, "KL/std": 117.67340087890625, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0034509035758674145, "fcm_dpo/delta": -0.06581096351146698, "fcm_dpo/margin": 119.59412384033203, "fcm_dpo/q_t": 0.40231502056121826, "grad_norm": 7.52719783782959, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.8654322624206543, "logits/rejected": -1.1553118228912354, "logps/chosen": -275.40008544921875, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -382.85638427734375, "loss": 1.0683, "margin_dpo/margin_mean": 119.59412384033203, "margin_dpo/margin_std": 135.1849822998047, "step": 577 }, { "KL/chosen_KL_mean": -206.7569122314453, "KL/mean": -250.67324829101562, "KL/rejected_KL_mean": -294.589599609375, "KL/std": 112.85346984863281, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0034722588025033474, "fcm_dpo/delta": 0.04645734280347824, "fcm_dpo/margin": 87.8326644897461, "fcm_dpo/q_t": 0.4278610348701477, "grad_norm": 12.85041332244873, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.2432020902633667, "logits/rejected": -0.9998447895050049, "logps/chosen": -284.04931640625, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -382.83660888671875, "loss": 1.1526, "margin_dpo/margin_mean": 87.8326644897461, "margin_dpo/margin_std": 128.7124481201172, "step": 578 }, { "KL/chosen_KL_mean": -186.51910400390625, "KL/mean": -230.61856079101562, "KL/rejected_KL_mean": -274.718017578125, "KL/std": 111.71483612060547, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0034922868944704533, "fcm_dpo/delta": 0.04321115463972092, "fcm_dpo/margin": 88.19894409179688, "fcm_dpo/q_t": 0.42722418904304504, "grad_norm": 7.30455207824707, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7606816291809082, "logits/rejected": -1.4741365909576416, "logps/chosen": -273.8621520996094, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -376.83905029296875, "loss": 1.1434, "margin_dpo/margin_mean": 88.19893646240234, "margin_dpo/margin_std": 113.66670227050781, "step": 579 }, { "KL/chosen_KL_mean": -178.10693359375, "KL/mean": -232.84207153320312, "KL/rejected_KL_mean": -287.5772399902344, "KL/std": 112.4979476928711, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.003483676817268133, "fcm_dpo/delta": -0.032938726246356964, "fcm_dpo/margin": 109.47030639648438, "fcm_dpo/q_t": 0.41085368394851685, "grad_norm": 8.71484088897705, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.2467094659805298, "logits/rejected": -0.9204531908035278, "logps/chosen": -267.0357360839844, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -376.27386474609375, "loss": 1.0956, "margin_dpo/margin_mean": 109.47030639648438, "margin_dpo/margin_std": 137.18463134765625, "step": 580 }, { "KL/chosen_KL_mean": -188.02130126953125, "KL/mean": -237.4720916748047, "KL/rejected_KL_mean": -286.9228820800781, "KL/std": 112.6465835571289, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0034941760823130608, "fcm_dpo/delta": 0.004566298332065344, "fcm_dpo/margin": 98.90158081054688, "fcm_dpo/q_t": 0.4184223413467407, "grad_norm": 9.991008758544922, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.956554889678955, "logits/rejected": -1.609755277633667, "logps/chosen": -280.2445373535156, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -386.31231689453125, "loss": 1.1291, "margin_dpo/margin_mean": 98.90158081054688, "margin_dpo/margin_std": 141.4687042236328, "step": 581 }, { "KL/chosen_KL_mean": -166.77777099609375, "KL/mean": -211.40338134765625, "KL/rejected_KL_mean": -256.02899169921875, "KL/std": 117.9171142578125, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0034729435574263334, "fcm_dpo/delta": -0.05666983872652054, "fcm_dpo/margin": 89.25123596191406, "fcm_dpo/q_t": 0.4276024103164673, "grad_norm": 9.526449203491211, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.988544225692749, "logits/rejected": -1.981550931930542, "logps/chosen": -254.1134033203125, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -371.18341064453125, "loss": 1.1543, "margin_dpo/margin_mean": 89.25123596191406, "margin_dpo/margin_std": 130.63160705566406, "step": 582 }, { "KL/chosen_KL_mean": -198.99697875976562, "KL/mean": -248.833251953125, "KL/rejected_KL_mean": -298.6695251464844, "KL/std": 111.44200134277344, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.003445640206336975, "fcm_dpo/delta": 0.006610171869397163, "fcm_dpo/margin": 99.67256164550781, "fcm_dpo/q_t": 0.41925838589668274, "grad_norm": 10.589168548583984, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4502912759780884, "logits/rejected": -1.2149510383605957, "logps/chosen": -282.71405029296875, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -393.2541809082031, "loss": 1.1228, "margin_dpo/margin_mean": 99.67256164550781, "margin_dpo/margin_std": 132.7218780517578, "step": 583 }, { "KL/chosen_KL_mean": -178.20452880859375, "KL/mean": -231.0072021484375, "KL/rejected_KL_mean": -283.80987548828125, "KL/std": 115.47435760498047, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0034696445800364017, "fcm_dpo/delta": -0.017669547349214554, "fcm_dpo/margin": 105.60533142089844, "fcm_dpo/q_t": 0.4129828214645386, "grad_norm": 7.229574203491211, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9791631698608398, "logits/rejected": -1.52965247631073, "logps/chosen": -272.464111328125, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -388.29254150390625, "loss": 1.1065, "margin_dpo/margin_mean": 105.60533142089844, "margin_dpo/margin_std": 131.051025390625, "step": 584 }, { "KL/chosen_KL_mean": -190.26431274414062, "KL/mean": -248.98995971679688, "KL/rejected_KL_mean": -307.715576171875, "KL/std": 131.8213653564453, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0034261164255440235, "fcm_dpo/delta": -0.05458701774477959, "fcm_dpo/margin": 117.45125579833984, "fcm_dpo/q_t": 0.40634796023368835, "grad_norm": 8.260565757751465, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.785527229309082, "logits/rejected": -1.5055351257324219, "logps/chosen": -275.90106201171875, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -398.00103759765625, "loss": 1.0917, "margin_dpo/margin_mean": 117.45124816894531, "margin_dpo/margin_std": 155.91871643066406, "step": 585 }, { "KL/chosen_KL_mean": -210.00210571289062, "KL/mean": -249.47320556640625, "KL/rejected_KL_mean": -288.94427490234375, "KL/std": 114.77053833007812, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0034480984322726727, "fcm_dpo/delta": 0.08000018447637558, "fcm_dpo/margin": 78.94217681884766, "fcm_dpo/q_t": 0.4351271390914917, "grad_norm": 9.672295570373535, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.7234480381011963, "logits/rejected": -1.4579981565475464, "logps/chosen": -293.6832275390625, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -387.303466796875, "loss": 1.1684, "margin_dpo/margin_mean": 78.94217681884766, "margin_dpo/margin_std": 110.31071472167969, "step": 586 }, { "KL/chosen_KL_mean": -171.4534912109375, "KL/mean": -214.4781494140625, "KL/rejected_KL_mean": -257.5028076171875, "KL/std": 111.19734191894531, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0034924540668725967, "fcm_dpo/delta": 0.05095529183745384, "fcm_dpo/margin": 86.04928588867188, "fcm_dpo/q_t": 0.4286629557609558, "grad_norm": 10.007936477661133, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.3828611373901367, "logits/rejected": -2.103625774383545, "logps/chosen": -249.031005859375, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -346.8231201171875, "loss": 1.1476, "margin_dpo/margin_mean": 86.0492935180664, "margin_dpo/margin_std": 115.8317642211914, "step": 587 }, { "KL/chosen_KL_mean": -185.85910034179688, "KL/mean": -226.25701904296875, "KL/rejected_KL_mean": -266.6549377441406, "KL/std": 113.45948791503906, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.00352849205955863, "fcm_dpo/delta": 0.06687210500240326, "fcm_dpo/margin": 80.79582977294922, "fcm_dpo/q_t": 0.4338369369506836, "grad_norm": 12.756317138671875, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.9066078662872314, "logits/rejected": -1.8726420402526855, "logps/chosen": -269.6633605957031, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -379.77435302734375, "loss": 1.1779, "margin_dpo/margin_mean": 80.79582214355469, "margin_dpo/margin_std": 137.2581787109375, "step": 588 }, { "KL/chosen_KL_mean": -198.03628540039062, "KL/mean": -250.8634796142578, "KL/rejected_KL_mean": -303.690673828125, "KL/std": 111.32929992675781, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.003534066490828991, "fcm_dpo/delta": -0.0242912657558918, "fcm_dpo/margin": 105.65438842773438, "fcm_dpo/q_t": 0.4106478691101074, "grad_norm": 8.175896644592285, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.4705778360366821, "logits/rejected": -1.0720696449279785, "logps/chosen": -294.50128173828125, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -419.72784423828125, "loss": 1.0822, "margin_dpo/margin_mean": 105.65438842773438, "margin_dpo/margin_std": 107.5307846069336, "step": 589 }, { "KL/chosen_KL_mean": -181.79086303710938, "KL/mean": -235.18319702148438, "KL/rejected_KL_mean": -288.5755615234375, "KL/std": 104.24312591552734, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0035255709663033485, "fcm_dpo/delta": -0.027638763189315796, "fcm_dpo/margin": 106.78469848632812, "fcm_dpo/q_t": 0.4104716181755066, "grad_norm": 10.6754789352417, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.879866123199463, "logits/rejected": -1.7387266159057617, "logps/chosen": -270.2187194824219, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -398.7743225097656, "loss": 1.0808, "margin_dpo/margin_mean": 106.78469848632812, "margin_dpo/margin_std": 109.04780578613281, "step": 590 }, { "KL/chosen_KL_mean": -174.86251831054688, "KL/mean": -230.57498168945312, "KL/rejected_KL_mean": -286.2874755859375, "KL/std": 110.97222900390625, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.003481500316411257, "fcm_dpo/delta": -0.03961782902479172, "fcm_dpo/margin": 111.42495727539062, "fcm_dpo/q_t": 0.407271146774292, "grad_norm": 6.739634037017822, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.8778777122497559, "logits/rejected": -1.2612208127975464, "logps/chosen": -262.71221923828125, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -375.8666687011719, "loss": 1.0701, "margin_dpo/margin_mean": 111.42495727539062, "margin_dpo/margin_std": 108.20076751708984, "step": 591 }, { "KL/chosen_KL_mean": -206.6039276123047, "KL/mean": -249.44696044921875, "KL/rejected_KL_mean": -292.28997802734375, "KL/std": 109.39385223388672, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0035089803859591484, "fcm_dpo/delta": 0.05084206163883209, "fcm_dpo/margin": 85.6860580444336, "fcm_dpo/q_t": 0.42888957262039185, "grad_norm": 9.837154388427734, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.9010226726531982, "logits/rejected": -1.5792036056518555, "logps/chosen": -281.57965087890625, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -373.8426208496094, "loss": 1.1654, "margin_dpo/margin_mean": 85.6860580444336, "margin_dpo/margin_std": 139.00949096679688, "step": 592 }, { "KL/chosen_KL_mean": -189.03062438964844, "KL/mean": -226.2881622314453, "KL/rejected_KL_mean": -263.54571533203125, "KL/std": 114.33052062988281, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0035293034743517637, "fcm_dpo/delta": 0.017430134117603302, "fcm_dpo/margin": 74.51504516601562, "fcm_dpo/q_t": 0.4380563497543335, "grad_norm": 13.098954200744629, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8008370399475098, "logits/rejected": -0.49760594964027405, "logps/chosen": -261.2696533203125, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -333.6422119140625, "loss": 1.19, "margin_dpo/margin_mean": 74.51505279541016, "margin_dpo/margin_std": 127.60198974609375, "step": 593 }, { "KL/chosen_KL_mean": -168.15219116210938, "KL/mean": -225.23193359375, "KL/rejected_KL_mean": -282.31170654296875, "KL/std": 110.68601989746094, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.003512836527079344, "fcm_dpo/delta": -0.05319659784436226, "fcm_dpo/margin": 114.15951538085938, "fcm_dpo/q_t": 0.4042590856552124, "grad_norm": 8.783903121948242, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -2.0792417526245117, "logits/rejected": -1.5678362846374512, "logps/chosen": -258.58990478515625, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -374.57928466796875, "loss": 1.0606, "margin_dpo/margin_mean": 114.15950775146484, "margin_dpo/margin_std": 108.0217514038086, "step": 594 }, { "KL/chosen_KL_mean": -182.26873779296875, "KL/mean": -228.85220336914062, "KL/rejected_KL_mean": -275.4356689453125, "KL/std": 111.16075897216797, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0035119852982461452, "fcm_dpo/delta": 0.023417416960000992, "fcm_dpo/margin": 93.16691589355469, "fcm_dpo/q_t": 0.42319855093955994, "grad_norm": 8.954280853271484, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.810857892036438, "logits/rejected": -1.4084001779556274, "logps/chosen": -263.7835693359375, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -361.95611572265625, "loss": 1.1347, "margin_dpo/margin_mean": 93.16691589355469, "margin_dpo/margin_std": 128.27586364746094, "step": 595 }, { "KL/chosen_KL_mean": -189.8021240234375, "KL/mean": -231.7706298828125, "KL/rejected_KL_mean": -273.7391357421875, "KL/std": 110.09567260742188, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0035391761921346188, "fcm_dpo/delta": 0.05451183766126633, "fcm_dpo/margin": 83.93700408935547, "fcm_dpo/q_t": 0.4306198060512543, "grad_norm": 11.341007232666016, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.5115466117858887, "logits/rejected": -0.921109676361084, "logps/chosen": -278.01019287109375, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -359.8328857421875, "loss": 1.1702, "margin_dpo/margin_mean": 83.93701171875, "margin_dpo/margin_std": 139.76132202148438, "step": 596 }, { "KL/chosen_KL_mean": -183.98892211914062, "KL/mean": -240.1314697265625, "KL/rejected_KL_mean": -296.2740478515625, "KL/std": 118.17386627197266, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.003515570890158415, "fcm_dpo/delta": -0.04674527794122696, "fcm_dpo/margin": 112.28511047363281, "fcm_dpo/q_t": 0.4068632125854492, "grad_norm": 8.440449714660645, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.405953049659729, "logits/rejected": -1.2921844720840454, "logps/chosen": -267.57891845703125, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -403.21844482421875, "loss": 1.0779, "margin_dpo/margin_mean": 112.28511047363281, "margin_dpo/margin_std": 127.55223846435547, "step": 597 }, { "KL/chosen_KL_mean": -174.95870971679688, "KL/mean": -222.31350708007812, "KL/rejected_KL_mean": -269.66827392578125, "KL/std": 110.65052032470703, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0035113669000566006, "fcm_dpo/delta": 0.017869286239147186, "fcm_dpo/margin": 94.7095947265625, "fcm_dpo/q_t": 0.42070499062538147, "grad_norm": 9.276159286499023, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -2.055034637451172, "logits/rejected": -1.749863624572754, "logps/chosen": -265.141357421875, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -364.46685791015625, "loss": 1.1297, "margin_dpo/margin_mean": 94.7095947265625, "margin_dpo/margin_std": 125.17028045654297, "step": 598 }, { "KL/chosen_KL_mean": -176.36309814453125, "KL/mean": -218.7890625, "KL/rejected_KL_mean": -261.21502685546875, "KL/std": 114.99496459960938, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.003538079559803009, "fcm_dpo/delta": 0.05116652324795723, "fcm_dpo/margin": 84.85193634033203, "fcm_dpo/q_t": 0.4288262724876404, "grad_norm": 8.322378158569336, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.827032208442688, "logits/rejected": -1.7099120616912842, "logps/chosen": -254.2053680419922, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -350.41265869140625, "loss": 1.1603, "margin_dpo/margin_mean": 84.85194396972656, "margin_dpo/margin_std": 127.90095520019531, "step": 599 }, { "KL/chosen_KL_mean": -199.15533447265625, "KL/mean": -246.34823608398438, "KL/rejected_KL_mean": -293.5411376953125, "KL/std": 108.49746704101562, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.003571701468899846, "fcm_dpo/delta": 0.013317971490323544, "fcm_dpo/margin": 94.38581848144531, "fcm_dpo/q_t": 0.42035117745399475, "grad_norm": 8.531938552856445, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.1949477195739746, "logits/rejected": -1.8387726545333862, "logps/chosen": -291.3431396484375, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -386.4954528808594, "loss": 1.121, "margin_dpo/margin_mean": 94.38581848144531, "margin_dpo/margin_std": 118.37740325927734, "step": 600 }, { "KL/chosen_KL_mean": -199.7097930908203, "KL/mean": -250.49078369140625, "KL/rejected_KL_mean": -301.27178955078125, "KL/std": 111.10797882080078, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0035566266160458326, "fcm_dpo/delta": -0.011806674301624298, "fcm_dpo/margin": 101.56198120117188, "fcm_dpo/q_t": 0.4138578772544861, "grad_norm": 8.219511985778809, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.750211238861084, "logits/rejected": -1.4267208576202393, "logps/chosen": -288.9259338378906, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -401.44232177734375, "loss": 1.0924, "margin_dpo/margin_mean": 101.56198120117188, "margin_dpo/margin_std": 106.52436065673828, "step": 601 }, { "KL/chosen_KL_mean": -158.89047241210938, "KL/mean": -224.17971801757812, "KL/rejected_KL_mean": -289.46893310546875, "KL/std": 120.68541717529297, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.003505142405629158, "fcm_dpo/delta": -0.11284001171588898, "fcm_dpo/margin": 130.57847595214844, "fcm_dpo/q_t": 0.39246562123298645, "grad_norm": 8.576184272766113, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.1350436210632324, "logits/rejected": -1.8765997886657715, "logps/chosen": -238.9480743408203, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -382.66644287109375, "loss": 1.0358, "margin_dpo/margin_mean": 130.57847595214844, "margin_dpo/margin_std": 139.32717895507812, "step": 602 }, { "KL/chosen_KL_mean": -168.6845703125, "KL/mean": -222.65469360351562, "KL/rejected_KL_mean": -276.62481689453125, "KL/std": 109.00338745117188, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0034725465811789036, "fcm_dpo/delta": -0.025786617770791054, "fcm_dpo/margin": 107.94024658203125, "fcm_dpo/q_t": 0.41136181354522705, "grad_norm": 7.347172260284424, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.9519805908203125, "logits/rejected": -1.3582828044891357, "logps/chosen": -270.994140625, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -384.69366455078125, "loss": 1.0918, "margin_dpo/margin_mean": 107.94024658203125, "margin_dpo/margin_std": 126.68817138671875, "step": 603 }, { "KL/chosen_KL_mean": -177.26966857910156, "KL/mean": -232.88235473632812, "KL/rejected_KL_mean": -288.49505615234375, "KL/std": 114.39747619628906, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0034393281675875187, "fcm_dpo/delta": -0.034079670906066895, "fcm_dpo/margin": 111.22539520263672, "fcm_dpo/q_t": 0.408101350069046, "grad_norm": 8.032367706298828, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.5923093557357788, "logits/rejected": -1.4383927583694458, "logps/chosen": -255.44375610351562, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -387.9911804199219, "loss": 1.0806, "margin_dpo/margin_mean": 111.22539520263672, "margin_dpo/margin_std": 118.61466979980469, "step": 604 }, { "KL/chosen_KL_mean": -193.58218383789062, "KL/mean": -239.50128173828125, "KL/rejected_KL_mean": -285.4203796386719, "KL/std": 101.73286437988281, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0034584549721330404, "fcm_dpo/delta": 0.033454686403274536, "fcm_dpo/margin": 91.83821105957031, "fcm_dpo/q_t": 0.4246283769607544, "grad_norm": 8.563535690307617, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.7851614952087402, "logits/rejected": -1.2889310121536255, "logps/chosen": -288.35552978515625, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -392.72528076171875, "loss": 1.1317, "margin_dpo/margin_mean": 91.83821105957031, "margin_dpo/margin_std": 114.91499328613281, "step": 605 }, { "KL/chosen_KL_mean": -186.79183959960938, "KL/mean": -234.07064819335938, "KL/rejected_KL_mean": -281.34942626953125, "KL/std": 111.93463134765625, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.003491360228508711, "fcm_dpo/delta": 0.020074378699064255, "fcm_dpo/margin": 94.55760192871094, "fcm_dpo/q_t": 0.4211965501308441, "grad_norm": 7.629883766174316, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -2.00656795501709, "logits/rejected": -1.6322216987609863, "logps/chosen": -274.54522705078125, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -385.5916442871094, "loss": 1.1271, "margin_dpo/margin_mean": 94.55758666992188, "margin_dpo/margin_std": 119.76522064208984, "step": 606 }, { "KL/chosen_KL_mean": -180.66964721679688, "KL/mean": -234.7469024658203, "KL/rejected_KL_mean": -288.82415771484375, "KL/std": 119.42845153808594, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0034689288586378098, "fcm_dpo/delta": -0.02615378238260746, "fcm_dpo/margin": 108.15451049804688, "fcm_dpo/q_t": 0.4119724631309509, "grad_norm": 11.093561172485352, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.720505714416504, "logits/rejected": -1.1913247108459473, "logps/chosen": -268.9986877441406, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -375.59228515625, "loss": 1.108, "margin_dpo/margin_mean": 108.15451049804688, "margin_dpo/margin_std": 146.97747802734375, "step": 607 }, { "KL/chosen_KL_mean": -164.75753784179688, "KL/mean": -220.54457092285156, "KL/rejected_KL_mean": -276.33160400390625, "KL/std": 117.73294067382812, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.003425049129873514, "fcm_dpo/delta": -0.034237079322338104, "fcm_dpo/margin": 111.57405853271484, "fcm_dpo/q_t": 0.4088108241558075, "grad_norm": 7.610110282897949, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.98862886428833, "logits/rejected": -1.6989576816558838, "logps/chosen": -250.52691650390625, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -376.564453125, "loss": 1.0909, "margin_dpo/margin_mean": 111.57405853271484, "margin_dpo/margin_std": 129.20742797851562, "step": 608 }, { "KL/chosen_KL_mean": -184.87887573242188, "KL/mean": -236.9264373779297, "KL/rejected_KL_mean": -288.9739990234375, "KL/std": 119.30481719970703, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.003421235829591751, "fcm_dpo/delta": -0.006696373224258423, "fcm_dpo/margin": 104.09510803222656, "fcm_dpo/q_t": 0.4160383343696594, "grad_norm": 9.392801284790039, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.255984306335449, "logits/rejected": -1.8387209177017212, "logps/chosen": -277.845458984375, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -384.8921813964844, "loss": 1.111, "margin_dpo/margin_mean": 104.09512329101562, "margin_dpo/margin_std": 126.72396087646484, "step": 609 }, { "KL/chosen_KL_mean": -169.05908203125, "KL/mean": -225.1430206298828, "KL/rejected_KL_mean": -281.2269592285156, "KL/std": 112.95584106445312, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.003409506054595113, "fcm_dpo/delta": -0.033869437873363495, "fcm_dpo/margin": 112.16790008544922, "fcm_dpo/q_t": 0.4083462953567505, "grad_norm": 7.7450337409973145, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.747790813446045, "logits/rejected": -1.207839012145996, "logps/chosen": -263.7593688964844, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -370.91436767578125, "loss": 1.0804, "margin_dpo/margin_mean": 112.16790008544922, "margin_dpo/margin_std": 119.8309326171875, "step": 610 }, { "KL/chosen_KL_mean": -178.201416015625, "KL/mean": -233.3702392578125, "KL/rejected_KL_mean": -288.5390625, "KL/std": 117.07122039794922, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.003410309785977006, "fcm_dpo/delta": -0.027544818818569183, "fcm_dpo/margin": 110.33763122558594, "fcm_dpo/q_t": 0.4108175039291382, "grad_norm": 9.778915405273438, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.9972116947174072, "logits/rejected": -1.6506361961364746, "logps/chosen": -264.9847412109375, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -378.38214111328125, "loss": 1.0861, "margin_dpo/margin_mean": 110.3376235961914, "margin_dpo/margin_std": 119.09321594238281, "step": 611 }, { "KL/chosen_KL_mean": -197.093505859375, "KL/mean": -252.7247314453125, "KL/rejected_KL_mean": -308.35595703125, "KL/std": 110.58013916015625, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.003374893218278885, "fcm_dpo/delta": -0.02653767168521881, "fcm_dpo/margin": 111.26248931884766, "fcm_dpo/q_t": 0.41046568751335144, "grad_norm": 8.606035232543945, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.167975902557373, "logits/rejected": -1.3426400423049927, "logps/chosen": -298.1136474609375, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -402.13897705078125, "loss": 1.0827, "margin_dpo/margin_mean": 111.26249694824219, "margin_dpo/margin_std": 116.78410339355469, "step": 612 }, { "KL/chosen_KL_mean": -157.66094970703125, "KL/mean": -221.19058227539062, "KL/rejected_KL_mean": -284.72015380859375, "KL/std": 113.20948791503906, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0033307780977338552, "fcm_dpo/delta": -0.07658270746469498, "fcm_dpo/margin": 127.05923461914062, "fcm_dpo/q_t": 0.39842790365219116, "grad_norm": 7.72005558013916, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.8735430240631104, "logits/rejected": -1.6777470111846924, "logps/chosen": -246.6495819091797, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -394.7156982421875, "loss": 1.0402, "margin_dpo/margin_mean": 127.05923461914062, "margin_dpo/margin_std": 111.66445922851562, "step": 613 }, { "KL/chosen_KL_mean": -197.67440795898438, "KL/mean": -249.30972290039062, "KL/rejected_KL_mean": -300.94500732421875, "KL/std": 107.53172302246094, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0033229091204702854, "fcm_dpo/delta": 0.0070751081220805645, "fcm_dpo/margin": 103.27064514160156, "fcm_dpo/q_t": 0.4183030128479004, "grad_norm": 9.698579788208008, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.3775895833969116, "logits/rejected": -1.3712102174758911, "logps/chosen": -277.87445068359375, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -410.80743408203125, "loss": 1.1198, "margin_dpo/margin_mean": 103.27063751220703, "margin_dpo/margin_std": 132.76174926757812, "step": 614 }, { "KL/chosen_KL_mean": -184.89938354492188, "KL/mean": -226.29830932617188, "KL/rejected_KL_mean": -267.6972351074219, "KL/std": 108.71145629882812, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0033219167962670326, "fcm_dpo/delta": -0.0152206402271986, "fcm_dpo/margin": 82.79783630371094, "fcm_dpo/q_t": 0.4335615634918213, "grad_norm": 8.082898139953613, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.8914053440093994, "logits/rejected": -1.5034254789352417, "logps/chosen": -285.33465576171875, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -368.8772888183594, "loss": 1.1621, "margin_dpo/margin_mean": 82.79783630371094, "margin_dpo/margin_std": 104.41648864746094, "step": 615 }, { "KL/chosen_KL_mean": -187.347900390625, "KL/mean": -244.44650268554688, "KL/rejected_KL_mean": -301.54510498046875, "KL/std": 108.35932922363281, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0033136396668851376, "fcm_dpo/delta": -0.029664671048521996, "fcm_dpo/margin": 114.19720458984375, "fcm_dpo/q_t": 0.4100320339202881, "grad_norm": 7.696615695953369, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.9807953834533691, "logits/rejected": -1.4996821880340576, "logps/chosen": -279.8408203125, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -383.60577392578125, "loss": 1.087, "margin_dpo/margin_mean": 114.19720458984375, "margin_dpo/margin_std": 127.80918884277344, "step": 616 }, { "KL/chosen_KL_mean": -194.686279296875, "KL/mean": -240.47817993164062, "KL/rejected_KL_mean": -286.2701110839844, "KL/std": 105.96460723876953, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.003319657640531659, "fcm_dpo/delta": 0.047427088022232056, "fcm_dpo/margin": 91.58384704589844, "fcm_dpo/q_t": 0.4273640513420105, "grad_norm": 8.810709953308105, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.610962152481079, "logits/rejected": -0.9745215177536011, "logps/chosen": -285.78326416015625, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -356.68017578125, "loss": 1.1441, "margin_dpo/margin_mean": 91.58384704589844, "margin_dpo/margin_std": 119.66078186035156, "step": 617 }, { "KL/chosen_KL_mean": -196.60098266601562, "KL/mean": -248.43057250976562, "KL/rejected_KL_mean": -300.2601623535156, "KL/std": 117.79489135742188, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0033279480412602425, "fcm_dpo/delta": 0.005192516837269068, "fcm_dpo/margin": 103.65916442871094, "fcm_dpo/q_t": 0.4173547029495239, "grad_norm": 7.446666240692139, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.9240946769714355, "logits/rejected": -1.556665062904358, "logps/chosen": -298.62158203125, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -400.06134033203125, "loss": 1.1165, "margin_dpo/margin_mean": 103.6591567993164, "margin_dpo/margin_std": 129.87933349609375, "step": 618 }, { "KL/chosen_KL_mean": -190.57763671875, "KL/mean": -236.24630737304688, "KL/rejected_KL_mean": -281.91497802734375, "KL/std": 115.95675659179688, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0033538234420120716, "fcm_dpo/delta": 0.04506026208400726, "fcm_dpo/margin": 91.33732604980469, "fcm_dpo/q_t": 0.42864879965782166, "grad_norm": 8.264452934265137, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.5551719665527344, "logits/rejected": -1.197195053100586, "logps/chosen": -280.31903076171875, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -381.81634521484375, "loss": 1.1551, "margin_dpo/margin_mean": 91.33732604980469, "margin_dpo/margin_std": 139.26023864746094, "step": 619 }, { "KL/chosen_KL_mean": -183.00587463378906, "KL/mean": -232.372314453125, "KL/rejected_KL_mean": -281.73876953125, "KL/std": 121.62008666992188, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0033174469135701656, "fcm_dpo/delta": -0.06713143736124039, "fcm_dpo/margin": 98.73289489746094, "fcm_dpo/q_t": 0.42182785272598267, "grad_norm": 8.412545204162598, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.1284539699554443, "logits/rejected": -1.600308895111084, "logps/chosen": -268.13018798828125, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -367.15130615234375, "loss": 1.1369, "margin_dpo/margin_mean": 98.73291015625, "margin_dpo/margin_std": 131.9212646484375, "step": 620 }, { "KL/chosen_KL_mean": -186.82647705078125, "KL/mean": -236.02951049804688, "KL/rejected_KL_mean": -285.2325439453125, "KL/std": 117.50662231445312, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0032968369778245687, "fcm_dpo/delta": -0.06251519918441772, "fcm_dpo/margin": 98.40607452392578, "fcm_dpo/q_t": 0.4228900969028473, "grad_norm": 7.593396186828613, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.935260534286499, "logits/rejected": -1.6728988885879517, "logps/chosen": -276.07489013671875, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -380.69720458984375, "loss": 1.1429, "margin_dpo/margin_mean": 98.40607452392578, "margin_dpo/margin_std": 136.85784912109375, "step": 621 }, { "KL/chosen_KL_mean": -182.4711456298828, "KL/mean": -235.116943359375, "KL/rejected_KL_mean": -287.76275634765625, "KL/std": 105.28482055664062, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.00327374623157084, "fcm_dpo/delta": 0.005449601449072361, "fcm_dpo/margin": 105.29159545898438, "fcm_dpo/q_t": 0.4174577593803406, "grad_norm": 8.840940475463867, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.50095796585083, "logits/rejected": -0.8509475588798523, "logps/chosen": -282.2652587890625, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -378.5909729003906, "loss": 1.1054, "margin_dpo/margin_mean": 105.29159545898438, "margin_dpo/margin_std": 114.52371978759766, "step": 622 }, { "KL/chosen_KL_mean": -190.48562622070312, "KL/mean": -227.74783325195312, "KL/rejected_KL_mean": -265.0100402832031, "KL/std": 113.51170349121094, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0032981170807033777, "fcm_dpo/delta": 0.02784675732254982, "fcm_dpo/margin": 74.52438354492188, "fcm_dpo/q_t": 0.44212615489959717, "grad_norm": 8.414548873901367, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.086088180541992, "logits/rejected": -1.640014410018921, "logps/chosen": -280.9411926269531, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -356.3327941894531, "loss": 1.2074, "margin_dpo/margin_mean": 74.5243911743164, "margin_dpo/margin_std": 139.760498046875, "step": 623 }, { "KL/chosen_KL_mean": -199.1152801513672, "KL/mean": -258.4050598144531, "KL/rejected_KL_mean": -317.6948547363281, "KL/std": 112.46224975585938, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0032780820038169622, "fcm_dpo/delta": -0.04028310626745224, "fcm_dpo/margin": 118.57955169677734, "fcm_dpo/q_t": 0.4076188802719116, "grad_norm": 9.769975662231445, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.217583656311035, "logits/rejected": -1.259132981300354, "logps/chosen": -308.9905090332031, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -422.46807861328125, "loss": 1.0776, "margin_dpo/margin_mean": 118.57955169677734, "margin_dpo/margin_std": 128.98757934570312, "step": 624 }, { "KL/chosen_KL_mean": -205.2417755126953, "KL/mean": -247.3330535888672, "KL/rejected_KL_mean": -289.4243469238281, "KL/std": 110.85861206054688, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.00331327598541975, "fcm_dpo/delta": 0.0729292631149292, "fcm_dpo/margin": 84.18255615234375, "fcm_dpo/q_t": 0.433790385723114, "grad_norm": 7.931212425231934, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.6376514434814453, "logits/rejected": -1.2626535892486572, "logps/chosen": -292.4099426269531, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -381.28582763671875, "loss": 1.1642, "margin_dpo/margin_mean": 84.18255615234375, "margin_dpo/margin_std": 118.72379302978516, "step": 625 }, { "KL/chosen_KL_mean": -193.25192260742188, "KL/mean": -236.1665496826172, "KL/rejected_KL_mean": -279.0811767578125, "KL/std": 123.45663452148438, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0033579072915017605, "fcm_dpo/delta": 0.06342357397079468, "fcm_dpo/margin": 85.82923889160156, "fcm_dpo/q_t": 0.43238839507102966, "grad_norm": 10.342421531677246, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.4004700183868408, "logits/rejected": -0.9666751623153687, "logps/chosen": -297.47613525390625, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -383.2586364746094, "loss": 1.168, "margin_dpo/margin_mean": 85.82923889160156, "margin_dpo/margin_std": 134.68539428710938, "step": 626 }, { "KL/chosen_KL_mean": -171.34739685058594, "KL/mean": -225.79583740234375, "KL/rejected_KL_mean": -280.24432373046875, "KL/std": 126.70186614990234, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0033553021494299173, "fcm_dpo/delta": -0.015962354838848114, "fcm_dpo/margin": 108.89692687988281, "fcm_dpo/q_t": 0.4146357774734497, "grad_norm": 6.9747724533081055, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.5827610492706299, "logits/rejected": -1.7313612699508667, "logps/chosen": -246.28201293945312, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -392.81719970703125, "loss": 1.1065, "margin_dpo/margin_mean": 108.89692687988281, "margin_dpo/margin_std": 140.97955322265625, "step": 627 }, { "KL/chosen_KL_mean": -181.89535522460938, "KL/mean": -236.91354370117188, "KL/rejected_KL_mean": -291.93170166015625, "KL/std": 114.32543182373047, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0033427407033741474, "fcm_dpo/delta": -0.018501149490475655, "fcm_dpo/margin": 110.0363998413086, "fcm_dpo/q_t": 0.41289806365966797, "grad_norm": 8.970799446105957, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.1459946632385254, "logits/rejected": -1.8919672966003418, "logps/chosen": -275.58642578125, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -399.27569580078125, "loss": 1.0978, "margin_dpo/margin_mean": 110.0363998413086, "margin_dpo/margin_std": 130.47315979003906, "step": 628 }, { "KL/chosen_KL_mean": -194.95986938476562, "KL/mean": -245.18460083007812, "KL/rejected_KL_mean": -295.4093017578125, "KL/std": 114.79954528808594, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.003338109701871872, "fcm_dpo/delta": 0.015106620267033577, "fcm_dpo/margin": 100.44944763183594, "fcm_dpo/q_t": 0.41976767778396606, "grad_norm": 8.651371002197266, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.8855013847351074, "logits/rejected": -1.437255620956421, "logps/chosen": -298.19024658203125, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -392.5777282714844, "loss": 1.1202, "margin_dpo/margin_mean": 100.4494400024414, "margin_dpo/margin_std": 122.74364471435547, "step": 629 }, { "KL/chosen_KL_mean": -178.5328369140625, "KL/mean": -210.74102783203125, "KL/rejected_KL_mean": -242.94921875, "KL/std": 106.65604400634766, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0033632481936365366, "fcm_dpo/delta": 0.042040303349494934, "fcm_dpo/margin": 64.4163818359375, "fcm_dpo/q_t": 0.4481634497642517, "grad_norm": 9.585331916809082, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.058631420135498, "logits/rejected": -1.5432552099227905, "logps/chosen": -272.43035888671875, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -332.3236083984375, "loss": 1.2224, "margin_dpo/margin_mean": 64.4163818359375, "margin_dpo/margin_std": 120.99031066894531, "step": 630 }, { "KL/chosen_KL_mean": -175.22048950195312, "KL/mean": -224.36752319335938, "KL/rejected_KL_mean": -273.5145263671875, "KL/std": 106.57066345214844, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.003383931703865528, "fcm_dpo/delta": 0.017979802563786507, "fcm_dpo/margin": 98.29402160644531, "fcm_dpo/q_t": 0.4206445813179016, "grad_norm": 7.689121246337891, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.389888286590576, "logits/rejected": -1.9574506282806396, "logps/chosen": -262.9913330078125, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -362.19696044921875, "loss": 1.1227, "margin_dpo/margin_mean": 98.29402160644531, "margin_dpo/margin_std": 122.55073547363281, "step": 631 }, { "KL/chosen_KL_mean": -172.23092651367188, "KL/mean": -225.507568359375, "KL/rejected_KL_mean": -278.78424072265625, "KL/std": 111.10283660888672, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.003371529746800661, "fcm_dpo/delta": -0.00993361696600914, "fcm_dpo/margin": 106.55332946777344, "fcm_dpo/q_t": 0.4152054786682129, "grad_norm": 10.037825584411621, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.0443451404571533, "logits/rejected": -1.6209609508514404, "logps/chosen": -260.8574523925781, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -370.23516845703125, "loss": 1.1123, "margin_dpo/margin_mean": 106.55332946777344, "margin_dpo/margin_std": 138.3260498046875, "step": 632 }, { "KL/chosen_KL_mean": -213.8040313720703, "KL/mean": -244.79229736328125, "KL/rejected_KL_mean": -275.7805480957031, "KL/std": 113.07044982910156, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.003392538521438837, "fcm_dpo/delta": 0.02862752601504326, "fcm_dpo/margin": 61.97651672363281, "fcm_dpo/q_t": 0.45137518644332886, "grad_norm": 9.838850021362305, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.1569337844848633, "logits/rejected": -1.9679250717163086, "logps/chosen": -295.178466796875, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -374.40625, "loss": 1.2526, "margin_dpo/margin_mean": 61.97651672363281, "margin_dpo/margin_std": 152.80958557128906, "step": 633 }, { "KL/chosen_KL_mean": -196.92962646484375, "KL/mean": -237.3447265625, "KL/rejected_KL_mean": -277.7598571777344, "KL/std": 102.51593780517578, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0033950500655919313, "fcm_dpo/delta": -0.021208835765719414, "fcm_dpo/margin": 80.83023071289062, "fcm_dpo/q_t": 0.43417683243751526, "grad_norm": 8.893980026245117, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.449609637260437, "logits/rejected": -1.2803399562835693, "logps/chosen": -293.052490234375, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -390.607666015625, "loss": 1.1652, "margin_dpo/margin_mean": 80.83023071289062, "margin_dpo/margin_std": 106.03016662597656, "step": 634 }, { "KL/chosen_KL_mean": -206.82827758789062, "KL/mean": -238.01840209960938, "KL/rejected_KL_mean": -269.2085266113281, "KL/std": 113.16014099121094, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0033980892039835453, "fcm_dpo/delta": 0.030133865773677826, "fcm_dpo/margin": 62.38026809692383, "fcm_dpo/q_t": 0.4501435160636902, "grad_norm": 10.942432403564453, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.7033997774124146, "logits/rejected": -1.156761646270752, "logps/chosen": -305.51239013671875, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -359.10772705078125, "loss": 1.2391, "margin_dpo/margin_mean": 62.38026809692383, "margin_dpo/margin_std": 137.23117065429688, "step": 635 }, { "KL/chosen_KL_mean": -183.26348876953125, "KL/mean": -237.536865234375, "KL/rejected_KL_mean": -291.81024169921875, "KL/std": 114.73563385009766, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.003402278758585453, "fcm_dpo/delta": -0.020052166655659676, "fcm_dpo/margin": 108.54673767089844, "fcm_dpo/q_t": 0.41197529435157776, "grad_norm": 8.395978927612305, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.4909863471984863, "logits/rejected": -1.0097061395645142, "logps/chosen": -273.68145751953125, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -379.5171203613281, "loss": 1.0875, "margin_dpo/margin_mean": 108.5467300415039, "margin_dpo/margin_std": 114.74783325195312, "step": 636 }, { "KL/chosen_KL_mean": -183.60906982421875, "KL/mean": -241.54922485351562, "KL/rejected_KL_mean": -299.4893798828125, "KL/std": 127.90811920166016, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0033770897425711155, "fcm_dpo/delta": -0.042999621480703354, "fcm_dpo/margin": 115.88031005859375, "fcm_dpo/q_t": 0.40660184621810913, "grad_norm": 7.0350518226623535, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.8941807746887207, "logits/rejected": -1.7915160655975342, "logps/chosen": -266.05877685546875, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -403.5179748535156, "loss": 1.0737, "margin_dpo/margin_mean": 115.88031005859375, "margin_dpo/margin_std": 121.09880065917969, "step": 637 }, { "KL/chosen_KL_mean": -174.25863647460938, "KL/mean": -229.5906524658203, "KL/rejected_KL_mean": -284.92266845703125, "KL/std": 113.49605560302734, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0033685294911265373, "fcm_dpo/delta": -0.02394915744662285, "fcm_dpo/margin": 110.6640625, "fcm_dpo/q_t": 0.4106457233428955, "grad_norm": 7.5097832679748535, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.6552932262420654, "logits/rejected": -0.9881856441497803, "logps/chosen": -266.18359375, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -371.209716796875, "loss": 1.0819, "margin_dpo/margin_mean": 110.66405487060547, "margin_dpo/margin_std": 107.62484741210938, "step": 638 }, { "KL/chosen_KL_mean": -230.69219970703125, "KL/mean": -256.1324462890625, "KL/rejected_KL_mean": -281.57269287109375, "KL/std": 104.67341613769531, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.003371472004801035, "fcm_dpo/delta": 0.06506194174289703, "fcm_dpo/margin": 50.880489349365234, "fcm_dpo/q_t": 0.4589598774909973, "grad_norm": 8.417521476745605, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.5421725511550903, "logits/rejected": -0.8017259240150452, "logps/chosen": -335.2197570800781, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -363.05303955078125, "loss": 1.2695, "margin_dpo/margin_mean": 50.880489349365234, "margin_dpo/margin_std": 130.41525268554688, "step": 639 }, { "KL/chosen_KL_mean": -194.89564514160156, "KL/mean": -245.41465759277344, "KL/rejected_KL_mean": -295.93365478515625, "KL/std": 114.26948547363281, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.003370284801349044, "fcm_dpo/delta": -0.06860696524381638, "fcm_dpo/margin": 101.03800201416016, "fcm_dpo/q_t": 0.418256551027298, "grad_norm": 12.029590606689453, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.6921995878219604, "logits/rejected": -1.1684958934783936, "logps/chosen": -292.7809143066406, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -408.638671875, "loss": 1.1162, "margin_dpo/margin_mean": 101.03800201416016, "margin_dpo/margin_std": 114.1785659790039, "step": 640 }, { "KL/chosen_KL_mean": -179.6422882080078, "KL/mean": -231.93545532226562, "KL/rejected_KL_mean": -284.2286376953125, "KL/std": 132.43328857421875, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0033488920889794827, "fcm_dpo/delta": -0.00026359735056757927, "fcm_dpo/margin": 104.58635711669922, "fcm_dpo/q_t": 0.41941431164741516, "grad_norm": 7.756344795227051, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.218348741531372, "logits/rejected": -1.6709359884262085, "logps/chosen": -276.08795166015625, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -379.36431884765625, "loss": 1.1231, "margin_dpo/margin_mean": 104.58636474609375, "margin_dpo/margin_std": 148.23236083984375, "step": 641 }, { "KL/chosen_KL_mean": -198.4072265625, "KL/mean": -247.98988342285156, "KL/rejected_KL_mean": -297.57257080078125, "KL/std": 114.50184631347656, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.003363373689353466, "fcm_dpo/delta": 0.016884401440620422, "fcm_dpo/margin": 99.16535949707031, "fcm_dpo/q_t": 0.4226464629173279, "grad_norm": 10.66854190826416, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.6215360164642334, "logits/rejected": -1.2953722476959229, "logps/chosen": -299.16705322265625, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -412.28021240234375, "loss": 1.135, "margin_dpo/margin_mean": 99.16535949707031, "margin_dpo/margin_std": 143.51361083984375, "step": 642 }, { "KL/chosen_KL_mean": -186.45486450195312, "KL/mean": -233.1005096435547, "KL/rejected_KL_mean": -279.74615478515625, "KL/std": 105.79261779785156, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0033337250351905823, "fcm_dpo/delta": -0.03673406317830086, "fcm_dpo/margin": 93.29129791259766, "fcm_dpo/q_t": 0.4252333641052246, "grad_norm": 10.577420234680176, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.091465950012207, "logits/rejected": -1.7177904844284058, "logps/chosen": -271.1985168457031, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -374.0645751953125, "loss": 1.1425, "margin_dpo/margin_mean": 93.29129791259766, "margin_dpo/margin_std": 122.46355438232422, "step": 643 }, { "KL/chosen_KL_mean": -193.88006591796875, "KL/mean": -232.27322387695312, "KL/rejected_KL_mean": -270.6663818359375, "KL/std": 107.37822723388672, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0033802520483732224, "fcm_dpo/delta": 0.09285460412502289, "fcm_dpo/margin": 76.78633117675781, "fcm_dpo/q_t": 0.43831491470336914, "grad_norm": 9.980169296264648, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.983435869216919, "logits/rejected": -1.7255079746246338, "logps/chosen": -279.20281982421875, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -365.2750244140625, "loss": 1.1818, "margin_dpo/margin_mean": 76.78633117675781, "margin_dpo/margin_std": 117.65728759765625, "step": 644 }, { "KL/chosen_KL_mean": -204.02987670898438, "KL/mean": -229.01377868652344, "KL/rejected_KL_mean": -253.9976806640625, "KL/std": 115.7428207397461, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0033962137531489134, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.967796325683594, "fcm_dpo/q_t": 0.45964038372039795, "grad_norm": 8.852198600769043, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.4156619310379028, "logits/rejected": -1.250929832458496, "logps/chosen": -286.6201171875, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -357.0014343261719, "loss": 1.2738, "margin_dpo/margin_mean": 49.967796325683594, "margin_dpo/margin_std": 132.48777770996094, "step": 645 }, { "KL/chosen_KL_mean": -181.0863494873047, "KL/mean": -230.622314453125, "KL/rejected_KL_mean": -280.15826416015625, "KL/std": 100.90652465820312, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.003392031416296959, "fcm_dpo/delta": 0.014247369021177292, "fcm_dpo/margin": 99.07190704345703, "fcm_dpo/q_t": 0.4195610284805298, "grad_norm": 7.840439796447754, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.8174148797988892, "logits/rejected": -1.2180248498916626, "logps/chosen": -272.27838134765625, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -363.93658447265625, "loss": 1.1135, "margin_dpo/margin_mean": 99.07191467285156, "margin_dpo/margin_std": 110.73246002197266, "step": 646 }, { "KL/chosen_KL_mean": -194.686767578125, "KL/mean": -238.63629150390625, "KL/rejected_KL_mean": -282.58587646484375, "KL/std": 121.48390197753906, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.003431747667491436, "fcm_dpo/delta": 0.04985031485557556, "fcm_dpo/margin": 87.89909362792969, "fcm_dpo/q_t": 0.4292941689491272, "grad_norm": 9.614469528198242, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.491310954093933, "logits/rejected": -1.3551092147827148, "logps/chosen": -283.5312194824219, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -382.0841979980469, "loss": 1.1587, "margin_dpo/margin_mean": 87.89909362792969, "margin_dpo/margin_std": 134.92454528808594, "step": 647 }, { "KL/chosen_KL_mean": -183.70068359375, "KL/mean": -243.515625, "KL/rejected_KL_mean": -303.33056640625, "KL/std": 125.16532897949219, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0034156262408941984, "fcm_dpo/delta": -0.06106676161289215, "fcm_dpo/margin": 119.62989044189453, "fcm_dpo/q_t": 0.403150737285614, "grad_norm": 8.753889083862305, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.9093115329742432, "logits/rejected": -1.8384679555892944, "logps/chosen": -271.5487976074219, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -408.0006408691406, "loss": 1.069, "margin_dpo/margin_mean": 119.62989807128906, "margin_dpo/margin_std": 133.58096313476562, "step": 648 }, { "KL/chosen_KL_mean": -167.79345703125, "KL/mean": -219.1807098388672, "KL/rejected_KL_mean": -270.56793212890625, "KL/std": 109.2806167602539, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0033863845746964216, "fcm_dpo/delta": 0.0017535127699375153, "fcm_dpo/margin": 102.77452850341797, "fcm_dpo/q_t": 0.4166712760925293, "grad_norm": 10.005664825439453, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.359224796295166, "logits/rejected": -1.95405912399292, "logps/chosen": -257.4859619140625, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -359.2745361328125, "loss": 1.1016, "margin_dpo/margin_mean": 102.77452087402344, "margin_dpo/margin_std": 106.79210662841797, "step": 649 }, { "KL/chosen_KL_mean": -199.00250244140625, "KL/mean": -246.427734375, "KL/rejected_KL_mean": -293.8529968261719, "KL/std": 118.55265808105469, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0034151775762438774, "fcm_dpo/delta": 0.026918306946754456, "fcm_dpo/margin": 94.8504638671875, "fcm_dpo/q_t": 0.4238082766532898, "grad_norm": 8.381464004516602, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.2775838375091553, "logits/rejected": -1.0781590938568115, "logps/chosen": -288.93310546875, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -396.4658203125, "loss": 1.1433, "margin_dpo/margin_mean": 94.8504638671875, "margin_dpo/margin_std": 141.0338134765625, "step": 650 }, { "KL/chosen_KL_mean": -164.063232421875, "KL/mean": -222.91213989257812, "KL/rejected_KL_mean": -281.760986328125, "KL/std": 113.97966003417969, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.003390092169865966, "fcm_dpo/delta": -0.05107622221112251, "fcm_dpo/margin": 117.69775390625, "fcm_dpo/q_t": 0.4041847288608551, "grad_norm": 8.188891410827637, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.2825565338134766, "logits/rejected": -2.110431671142578, "logps/chosen": -243.2505645751953, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -387.6943359375, "loss": 1.0622, "margin_dpo/margin_mean": 117.69775390625, "margin_dpo/margin_std": 112.99480438232422, "step": 651 }, { "KL/chosen_KL_mean": -172.09812927246094, "KL/mean": -220.95535278320312, "KL/rejected_KL_mean": -269.8125915527344, "KL/std": 105.91816711425781, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0033991290256381035, "fcm_dpo/delta": 0.01835181564092636, "fcm_dpo/margin": 97.71446228027344, "fcm_dpo/q_t": 0.42043372988700867, "grad_norm": 8.7688627243042, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.52700138092041, "logits/rejected": -2.2368173599243164, "logps/chosen": -273.88836669921875, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -386.1371765136719, "loss": 1.1105, "margin_dpo/margin_mean": 97.71446228027344, "margin_dpo/margin_std": 101.66691589355469, "step": 652 }, { "KL/chosen_KL_mean": -196.95474243164062, "KL/mean": -244.91351318359375, "KL/rejected_KL_mean": -292.872314453125, "KL/std": 116.63587951660156, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.003411289071664214, "fcm_dpo/delta": 0.023516448214650154, "fcm_dpo/margin": 95.91755676269531, "fcm_dpo/q_t": 0.42121195793151855, "grad_norm": 8.550946235656738, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.1376895904541016, "logits/rejected": -1.883497953414917, "logps/chosen": -290.308349609375, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -401.99554443359375, "loss": 1.1232, "margin_dpo/margin_mean": 95.91755676269531, "margin_dpo/margin_std": 114.58612060546875, "step": 653 }, { "KL/chosen_KL_mean": -198.67144775390625, "KL/mean": -239.67886352539062, "KL/rejected_KL_mean": -280.686279296875, "KL/std": 111.10236358642578, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.003446391085162759, "fcm_dpo/delta": 0.06931611895561218, "fcm_dpo/margin": 82.01478576660156, "fcm_dpo/q_t": 0.43463659286499023, "grad_norm": 11.125301361083984, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.113518238067627, "logits/rejected": -1.928418517112732, "logps/chosen": -287.7869873046875, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -385.606201171875, "loss": 1.1793, "margin_dpo/margin_mean": 82.0147933959961, "margin_dpo/margin_std": 140.49981689453125, "step": 654 }, { "KL/chosen_KL_mean": -180.68470764160156, "KL/mean": -230.079833984375, "KL/rejected_KL_mean": -279.4749755859375, "KL/std": 120.37229919433594, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0034662706311792135, "fcm_dpo/delta": 0.00781136704608798, "fcm_dpo/margin": 98.79026794433594, "fcm_dpo/q_t": 0.4186277687549591, "grad_norm": 8.333544731140137, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.036299705505371, "logits/rejected": -2.0389158725738525, "logps/chosen": -261.7208251953125, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -382.2773132324219, "loss": 1.1223, "margin_dpo/margin_mean": 98.79026794433594, "margin_dpo/margin_std": 129.9178924560547, "step": 655 }, { "KL/chosen_KL_mean": -182.9242401123047, "KL/mean": -245.66119384765625, "KL/rejected_KL_mean": -308.3981628417969, "KL/std": 121.33467102050781, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0034199710935354233, "fcm_dpo/delta": -0.08272607624530792, "fcm_dpo/margin": 125.47395324707031, "fcm_dpo/q_t": 0.3984416127204895, "grad_norm": 8.948976516723633, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.443798542022705, "logits/rejected": -1.5136582851409912, "logps/chosen": -268.24957275390625, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -426.73681640625, "loss": 1.0458, "margin_dpo/margin_mean": 125.47395324707031, "margin_dpo/margin_std": 125.4962387084961, "step": 656 }, { "KL/chosen_KL_mean": -198.75411987304688, "KL/mean": -245.6455078125, "KL/rejected_KL_mean": -292.5368957519531, "KL/std": 112.54963684082031, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0034095942974090576, "fcm_dpo/delta": 0.031024159863591194, "fcm_dpo/margin": 93.78279113769531, "fcm_dpo/q_t": 0.42510542273521423, "grad_norm": 9.351618766784668, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9671616554260254, "logits/rejected": -1.8029563426971436, "logps/chosen": -278.95184326171875, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -395.1181945800781, "loss": 1.1455, "margin_dpo/margin_mean": 93.78279113769531, "margin_dpo/margin_std": 136.26573181152344, "step": 657 }, { "KL/chosen_KL_mean": -193.1478729248047, "KL/mean": -236.691650390625, "KL/rejected_KL_mean": -280.23541259765625, "KL/std": 105.69900512695312, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.003463080618530512, "fcm_dpo/delta": 0.04966277256608009, "fcm_dpo/margin": 87.0875473022461, "fcm_dpo/q_t": 0.4281567633152008, "grad_norm": 7.675615310668945, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.938429594039917, "logits/rejected": -1.4662251472473145, "logps/chosen": -287.78240966796875, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -381.8670654296875, "loss": 1.1452, "margin_dpo/margin_mean": 87.0875473022461, "margin_dpo/margin_std": 112.936767578125, "step": 658 }, { "KL/chosen_KL_mean": -177.2928466796875, "KL/mean": -229.27943420410156, "KL/rejected_KL_mean": -281.2659912109375, "KL/std": 113.2333755493164, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0034470444079488516, "fcm_dpo/delta": -0.00896776095032692, "fcm_dpo/margin": 103.97315216064453, "fcm_dpo/q_t": 0.4160118103027344, "grad_norm": 7.848608016967773, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.080031156539917, "logits/rejected": -1.1078513860702515, "logps/chosen": -289.8487243652344, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -375.1876220703125, "loss": 1.1232, "margin_dpo/margin_mean": 103.97315979003906, "margin_dpo/margin_std": 146.57887268066406, "step": 659 }, { "KL/chosen_KL_mean": -186.3700714111328, "KL/mean": -239.69699096679688, "KL/rejected_KL_mean": -293.02392578125, "KL/std": 109.93087768554688, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0034475158900022507, "fcm_dpo/delta": -0.01836588606238365, "fcm_dpo/margin": 106.65380859375, "fcm_dpo/q_t": 0.4126740097999573, "grad_norm": 10.8320951461792, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8358134031295776, "logits/rejected": -1.2109220027923584, "logps/chosen": -279.74749755859375, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -374.4187316894531, "loss": 1.0931, "margin_dpo/margin_mean": 106.65381622314453, "margin_dpo/margin_std": 120.64234924316406, "step": 660 }, { "KL/chosen_KL_mean": -183.50411987304688, "KL/mean": -243.52642822265625, "KL/rejected_KL_mean": -303.54876708984375, "KL/std": 124.10574340820312, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0034184735268354416, "fcm_dpo/delta": -0.06290622055530548, "fcm_dpo/margin": 120.04463958740234, "fcm_dpo/q_t": 0.4035378098487854, "grad_norm": 8.871769905090332, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.7172073125839233, "logits/rejected": -1.197739839553833, "logps/chosen": -271.3592834472656, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -395.95208740234375, "loss": 1.0714, "margin_dpo/margin_mean": 120.04463195800781, "margin_dpo/margin_std": 137.99220275878906, "step": 661 }, { "KL/chosen_KL_mean": -179.6253662109375, "KL/mean": -234.7665252685547, "KL/rejected_KL_mean": -289.9076843261719, "KL/std": 107.95916748046875, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0033934656530618668, "fcm_dpo/delta": -0.025188688188791275, "fcm_dpo/margin": 110.28230285644531, "fcm_dpo/q_t": 0.4107075333595276, "grad_norm": 9.718920707702637, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.832409143447876, "logits/rejected": -1.2703156471252441, "logps/chosen": -282.4051513671875, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -385.13299560546875, "loss": 1.0839, "margin_dpo/margin_mean": 110.28230285644531, "margin_dpo/margin_std": 114.79473114013672, "step": 662 }, { "KL/chosen_KL_mean": -180.96388244628906, "KL/mean": -231.06790161132812, "KL/rejected_KL_mean": -281.17193603515625, "KL/std": 112.36647033691406, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.003394567873328924, "fcm_dpo/delta": 0.010116891004145145, "fcm_dpo/margin": 100.20806121826172, "fcm_dpo/q_t": 0.4195176362991333, "grad_norm": 10.836363792419434, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.313595771789551, "logits/rejected": -2.1035706996917725, "logps/chosen": -277.31048583984375, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -401.6983947753906, "loss": 1.1223, "margin_dpo/margin_mean": 100.20806884765625, "margin_dpo/margin_std": 130.4529571533203, "step": 663 }, { "KL/chosen_KL_mean": -201.48269653320312, "KL/mean": -246.76535034179688, "KL/rejected_KL_mean": -292.04803466796875, "KL/std": 115.85481262207031, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.003418875625357032, "fcm_dpo/delta": 0.04149341210722923, "fcm_dpo/margin": 90.5653305053711, "fcm_dpo/q_t": 0.4263980984687805, "grad_norm": 7.569568634033203, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.6568931341171265, "logits/rejected": -1.4869258403778076, "logps/chosen": -293.32513427734375, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -403.88470458984375, "loss": 1.1392, "margin_dpo/margin_mean": 90.56533813476562, "margin_dpo/margin_std": 114.92953491210938, "step": 664 }, { "KL/chosen_KL_mean": -173.08029174804688, "KL/mean": -222.21023559570312, "KL/rejected_KL_mean": -271.34014892578125, "KL/std": 110.31889343261719, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0034325486049056053, "fcm_dpo/delta": 0.013060073368251324, "fcm_dpo/margin": 98.25990295410156, "fcm_dpo/q_t": 0.41923218965530396, "grad_norm": 7.218274116516113, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.556035041809082, "logits/rejected": -2.1422629356384277, "logps/chosen": -261.2664794921875, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -363.252197265625, "loss": 1.1149, "margin_dpo/margin_mean": 98.25990295410156, "margin_dpo/margin_std": 114.71678161621094, "step": 665 }, { "KL/chosen_KL_mean": -189.00869750976562, "KL/mean": -239.36476135253906, "KL/rejected_KL_mean": -289.7208557128906, "KL/std": 108.73002624511719, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0034239611122757196, "fcm_dpo/delta": 0.005277812480926514, "fcm_dpo/margin": 100.71214294433594, "fcm_dpo/q_t": 0.4187452793121338, "grad_norm": 9.418240547180176, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.62392258644104, "logits/rejected": -1.2454705238342285, "logps/chosen": -289.9832763671875, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -390.97076416015625, "loss": 1.1194, "margin_dpo/margin_mean": 100.71214294433594, "margin_dpo/margin_std": 129.53799438476562, "step": 666 }, { "KL/chosen_KL_mean": -206.49607849121094, "KL/mean": -257.9269104003906, "KL/rejected_KL_mean": -309.35772705078125, "KL/std": 113.43196868896484, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0034272093325853348, "fcm_dpo/delta": -0.0026414282619953156, "fcm_dpo/margin": 102.8616714477539, "fcm_dpo/q_t": 0.41557225584983826, "grad_norm": 8.66822338104248, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.5467379093170166, "logits/rejected": -0.8997913599014282, "logps/chosen": -304.0672607421875, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -402.9425048828125, "loss": 1.0995, "margin_dpo/margin_mean": 102.86167907714844, "margin_dpo/margin_std": 110.91558837890625, "step": 667 }, { "KL/chosen_KL_mean": -184.11416625976562, "KL/mean": -243.88800048828125, "KL/rejected_KL_mean": -303.6618347167969, "KL/std": 120.3576431274414, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0034024794586002827, "fcm_dpo/delta": -0.059132110327482224, "fcm_dpo/margin": 119.54766845703125, "fcm_dpo/q_t": 0.4044184684753418, "grad_norm": 8.401734352111816, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.2256393432617188, "logits/rejected": -1.7026796340942383, "logps/chosen": -286.689208984375, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -412.4795227050781, "loss": 1.0679, "margin_dpo/margin_mean": 119.54766845703125, "margin_dpo/margin_std": 132.2169647216797, "step": 668 }, { "KL/chosen_KL_mean": -167.03665161132812, "KL/mean": -224.61474609375, "KL/rejected_KL_mean": -282.19281005859375, "KL/std": 117.92649841308594, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0033712447620928288, "fcm_dpo/delta": -0.03974897414445877, "fcm_dpo/margin": 115.15614318847656, "fcm_dpo/q_t": 0.4079408645629883, "grad_norm": 8.567197799682617, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.651002049446106, "logits/rejected": -1.2560780048370361, "logps/chosen": -256.53143310546875, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -364.71234130859375, "loss": 1.0793, "margin_dpo/margin_mean": 115.15615844726562, "margin_dpo/margin_std": 127.49119567871094, "step": 669 }, { "KL/chosen_KL_mean": -168.29527282714844, "KL/mean": -231.47268676757812, "KL/rejected_KL_mean": -294.65008544921875, "KL/std": 119.83363342285156, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.003330487059429288, "fcm_dpo/delta": -0.07387222349643707, "fcm_dpo/margin": 126.35479736328125, "fcm_dpo/q_t": 0.3991125226020813, "grad_norm": 6.973134517669678, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.6635534763336182, "logits/rejected": -1.1236180067062378, "logps/chosen": -263.7498779296875, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -396.1829833984375, "loss": 1.0522, "margin_dpo/margin_mean": 126.35478210449219, "margin_dpo/margin_std": 126.76319885253906, "step": 670 }, { "KL/chosen_KL_mean": -204.53890991210938, "KL/mean": -247.5751953125, "KL/rejected_KL_mean": -290.6114807128906, "KL/std": 111.38224792480469, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.003309906693175435, "fcm_dpo/delta": -0.010835696943104267, "fcm_dpo/margin": 86.07257843017578, "fcm_dpo/q_t": 0.43243005871772766, "grad_norm": 8.40960693359375, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.7432224750518799, "logits/rejected": -1.523545265197754, "logps/chosen": -286.66204833984375, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -380.8311767578125, "loss": 1.1647, "margin_dpo/margin_mean": 86.07257843017578, "margin_dpo/margin_std": 125.01029205322266, "step": 671 }, { "KL/chosen_KL_mean": -203.8109130859375, "KL/mean": -241.07012939453125, "KL/rejected_KL_mean": -278.3293762207031, "KL/std": 106.47489929199219, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0033531333319842815, "fcm_dpo/delta": 0.10279587656259537, "fcm_dpo/margin": 74.51846313476562, "fcm_dpo/q_t": 0.44076117873191833, "grad_norm": 11.480619430541992, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.1202077865600586, "logits/rejected": -1.8007152080535889, "logps/chosen": -293.872802734375, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -378.7825927734375, "loss": 1.1899, "margin_dpo/margin_mean": 74.51846313476562, "margin_dpo/margin_std": 113.82307434082031, "step": 672 }, { "KL/chosen_KL_mean": -182.73817443847656, "KL/mean": -224.92620849609375, "KL/rejected_KL_mean": -267.1142578125, "KL/std": 109.61982727050781, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0034145149402320385, "fcm_dpo/delta": 0.06350070238113403, "fcm_dpo/margin": 84.37609100341797, "fcm_dpo/q_t": 0.4315384328365326, "grad_norm": 10.40854263305664, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -2.0433497428894043, "logits/rejected": -1.686403512954712, "logps/chosen": -262.0013122558594, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -358.4635009765625, "loss": 1.155, "margin_dpo/margin_mean": 84.3760986328125, "margin_dpo/margin_std": 112.97915649414062, "step": 673 }, { "KL/chosen_KL_mean": -170.44448852539062, "KL/mean": -231.56442260742188, "KL/rejected_KL_mean": -292.68438720703125, "KL/std": 110.31066131591797, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0033829095773398876, "fcm_dpo/delta": -0.06630389392375946, "fcm_dpo/margin": 122.2398681640625, "fcm_dpo/q_t": 0.401985764503479, "grad_norm": 6.946048259735107, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.542041301727295, "logits/rejected": -1.136229395866394, "logps/chosen": -245.90280151367188, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -368.88800048828125, "loss": 1.0548, "margin_dpo/margin_mean": 122.2398681640625, "margin_dpo/margin_std": 121.80815124511719, "step": 674 }, { "KL/chosen_KL_mean": -186.83975219726562, "KL/mean": -240.33424377441406, "KL/rejected_KL_mean": -293.8287353515625, "KL/std": 119.1572265625, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.003351309336721897, "fcm_dpo/delta": -0.009446181356906891, "fcm_dpo/margin": 106.98898315429688, "fcm_dpo/q_t": 0.41515764594078064, "grad_norm": 9.009278297424316, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.3235104084014893, "logits/rejected": -0.9739023447036743, "logps/chosen": -266.749267578125, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -375.0469970703125, "loss": 1.1193, "margin_dpo/margin_mean": 106.98898315429688, "margin_dpo/margin_std": 145.97933959960938, "step": 675 }, { "KL/chosen_KL_mean": -200.86544799804688, "KL/mean": -253.35006713867188, "KL/rejected_KL_mean": -305.83465576171875, "KL/std": 114.35556030273438, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0033596190623939037, "fcm_dpo/delta": -0.002817225642502308, "fcm_dpo/margin": 104.9692153930664, "fcm_dpo/q_t": 0.4160141944885254, "grad_norm": 7.825042724609375, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.8352282047271729, "logits/rejected": -1.2376707792282104, "logps/chosen": -299.03656005859375, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -400.85931396484375, "loss": 1.1109, "margin_dpo/margin_mean": 104.9692153930664, "margin_dpo/margin_std": 129.09085083007812, "step": 676 }, { "KL/chosen_KL_mean": -181.03103637695312, "KL/mean": -233.12625122070312, "KL/rejected_KL_mean": -285.221435546875, "KL/std": 112.51914978027344, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.003357836976647377, "fcm_dpo/delta": 6.152130663394928e-05, "fcm_dpo/margin": 104.19041442871094, "fcm_dpo/q_t": 0.4163985848426819, "grad_norm": 7.743579387664795, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.60833740234375, "logits/rejected": -0.9272428750991821, "logps/chosen": -272.4103088378906, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -368.0992126464844, "loss": 1.1016, "margin_dpo/margin_mean": 104.19041442871094, "margin_dpo/margin_std": 111.15620422363281, "step": 677 }, { "KL/chosen_KL_mean": -169.87156677246094, "KL/mean": -232.66851806640625, "KL/rejected_KL_mean": -295.4654541015625, "KL/std": 120.97608184814453, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0033405025023967028, "fcm_dpo/delta": -0.07258939743041992, "fcm_dpo/margin": 125.5938949584961, "fcm_dpo/q_t": 0.4000316560268402, "grad_norm": 8.668078422546387, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.5862784385681152, "logits/rejected": -1.2570691108703613, "logps/chosen": -245.51744079589844, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -382.43157958984375, "loss": 1.0523, "margin_dpo/margin_mean": 125.59390258789062, "margin_dpo/margin_std": 123.81230163574219, "step": 678 }, { "KL/chosen_KL_mean": -197.0479736328125, "KL/mean": -248.73818969726562, "KL/rejected_KL_mean": -300.42840576171875, "KL/std": 101.25009155273438, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0033179777674376965, "fcm_dpo/delta": 0.00723117683082819, "fcm_dpo/margin": 103.38043212890625, "fcm_dpo/q_t": 0.4182513356208801, "grad_norm": 7.844883918762207, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.1195738315582275, "logits/rejected": -1.5963364839553833, "logps/chosen": -277.8214111328125, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -383.3069152832031, "loss": 1.1032, "margin_dpo/margin_mean": 103.38043212890625, "margin_dpo/margin_std": 108.51544189453125, "step": 679 }, { "KL/chosen_KL_mean": -197.87289428710938, "KL/mean": -254.29995727539062, "KL/rejected_KL_mean": -310.72698974609375, "KL/std": 119.90037536621094, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0033259219489991665, "fcm_dpo/delta": -0.026780877262353897, "fcm_dpo/margin": 112.85408020019531, "fcm_dpo/q_t": 0.41146570444107056, "grad_norm": 6.325095176696777, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.8234180212020874, "logits/rejected": -1.4846572875976562, "logps/chosen": -305.55584716796875, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -426.8218994140625, "loss": 1.0942, "margin_dpo/margin_mean": 112.85407257080078, "margin_dpo/margin_std": 131.35833740234375, "step": 680 }, { "KL/chosen_KL_mean": -180.9478759765625, "KL/mean": -229.90121459960938, "KL/rejected_KL_mean": -278.85455322265625, "KL/std": 113.19036865234375, "epoch": 1.0, "fcm_dpo/beta": 0.0033210553228855133, "fcm_dpo/delta": 0.02559760771691799, "fcm_dpo/margin": 97.90666198730469, "fcm_dpo/q_t": 0.4230335056781769, "grad_norm": 8.070178985595703, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.2142140865325928, "logits/rejected": -1.9002517461776733, "logps/chosen": -273.95892333984375, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -373.6767272949219, "loss": 1.129, "margin_dpo/margin_mean": 97.90666198730469, "margin_dpo/margin_std": 124.50938415527344, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1279081486045177, "train_runtime": 2022.8903, "train_samples_per_second": 21.552, "train_steps_per_second": 0.337 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }