{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.818790435791016, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.757081985473633, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.021671295166015625, "KL/mean": 0.006854303181171417, "KL/rejected_KL_mean": -0.007961273193359375, "KL/std": 0.30449360609054565, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02963055670261383, "fcm_dpo/q_t": 0.49925968050956726, "grad_norm": 26.02140998840332, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42696934938430786, "logits/rejected": 0.6501704454421997, "logps/chosen": -93.78931427001953, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.23746490478516, "loss": 1.3839, "margin_dpo/margin_mean": 0.02963070571422577, "margin_dpo/margin_std": 0.46567025780677795, "step": 3 }, { "KL/chosen_KL_mean": -0.07135391235351562, "KL/mean": -0.03454618155956268, "KL/rejected_KL_mean": 0.00225830078125, "KL/std": 0.31684693694114685, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07361140847206116, "fcm_dpo/q_t": 0.5018385648727417, "grad_norm": 32.986228942871094, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6463301181793213, "logits/rejected": 0.7778327465057373, "logps/chosen": -87.36382293701172, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.8219985961914, "loss": 1.3941, "margin_dpo/margin_mean": -0.07361075282096863, "margin_dpo/margin_std": 0.42805010080337524, "step": 4 }, { "KL/chosen_KL_mean": -0.018524169921875, "KL/mean": -0.025578178465366364, "KL/rejected_KL_mean": -0.032634735107421875, "KL/std": 0.29510587453842163, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014108225703239441, "fcm_dpo/q_t": 0.4996475577354431, "grad_norm": 35.06546401977539, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.44538697600364685, "logits/rejected": 0.6565948128700256, "logps/chosen": -89.35528564453125, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.78047180175781, "loss": 1.3853, "margin_dpo/margin_mean": 0.014108598232269287, "margin_dpo/margin_std": 0.39123034477233887, "step": 5 }, { "KL/chosen_KL_mean": -0.011829376220703125, "KL/mean": -0.04287449270486832, "KL/rejected_KL_mean": -0.07391738891601562, "KL/std": 0.28067395091056824, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.062087371945381165, "fcm_dpo/q_t": 0.4984477162361145, "grad_norm": 35.97876739501953, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.48503077030181885, "logits/rejected": 0.8014276027679443, "logps/chosen": -97.33330535888672, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.95736694335938, "loss": 1.3806, "margin_dpo/margin_mean": 0.062086790800094604, "margin_dpo/margin_std": 0.4176598787307739, "step": 6 }, { "KL/chosen_KL_mean": -0.02739715576171875, "KL/mean": -0.021956272423267365, "KL/rejected_KL_mean": -0.016521453857421875, "KL/std": 0.3020332455635071, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.010879531502723694, "fcm_dpo/q_t": 0.500271201133728, "grad_norm": 38.315704345703125, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4889507591724396, "logits/rejected": 0.5368040204048157, "logps/chosen": -86.67593383789062, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.63270568847656, "loss": 1.3879, "margin_dpo/margin_mean": -0.010879814624786377, "margin_dpo/margin_std": 0.4372590184211731, "step": 7 }, { "KL/chosen_KL_mean": -0.020076751708984375, "KL/mean": 0.03646580129861832, "KL/rejected_KL_mean": 0.09300994873046875, "KL/std": 0.3312501311302185, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11308695375919342, "fcm_dpo/q_t": 0.502825140953064, "grad_norm": 33.107723236083984, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.42082926630973816, "logits/rejected": 0.7327120304107666, "logps/chosen": -89.94375610351562, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.13502502441406, "loss": 1.3981, "margin_dpo/margin_mean": -0.11308658123016357, "margin_dpo/margin_std": 0.4509584605693817, "step": 8 }, { "KL/chosen_KL_mean": 0.089447021484375, "KL/mean": 0.044904135167598724, "KL/rejected_KL_mean": 0.000370025634765625, "KL/std": 0.34314411878585815, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08906598389148712, "fcm_dpo/q_t": 0.4977736473083496, "grad_norm": 35.907989501953125, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.8717671632766724, "logits/rejected": 1.0636789798736572, "logps/chosen": -103.76768493652344, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.31895446777344, "loss": 1.378, "margin_dpo/margin_mean": 0.08906540274620056, "margin_dpo/margin_std": 0.45465314388275146, "step": 9 }, { "KL/chosen_KL_mean": 0.07085418701171875, "KL/mean": 0.0742531567811966, "KL/rejected_KL_mean": 0.0776519775390625, "KL/std": 0.304537296295166, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006798893213272095, "fcm_dpo/q_t": 0.5001706480979919, "grad_norm": 33.504188537597656, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5113935470581055, "logits/rejected": 0.7241404056549072, "logps/chosen": -76.13351440429688, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.07444763183594, "loss": 1.3875, "margin_dpo/margin_mean": -0.006799072027206421, "margin_dpo/margin_std": 0.44117069244384766, "step": 10 }, { "KL/chosen_KL_mean": 0.025814056396484375, "KL/mean": 0.008111231029033661, "KL/rejected_KL_mean": -0.00959014892578125, "KL/std": 0.2957918643951416, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03539972007274628, "fcm_dpo/q_t": 0.49911487102508545, "grad_norm": 36.8640251159668, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.15076781809329987, "logits/rejected": 0.35913902521133423, "logps/chosen": -82.34068298339844, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.27421569824219, "loss": 1.3832, "margin_dpo/margin_mean": 0.03540100157260895, "margin_dpo/margin_std": 0.4190555214881897, "step": 11 }, { "KL/chosen_KL_mean": 0.048145294189453125, "KL/mean": 0.01866292953491211, "KL/rejected_KL_mean": -0.010822296142578125, "KL/std": 0.3419435918331146, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05896294116973877, "fcm_dpo/q_t": 0.49852585792541504, "grad_norm": 40.02372741699219, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.3993605375289917, "logits/rejected": 0.5276395678520203, "logps/chosen": -99.05735778808594, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.28223419189453, "loss": 1.3811, "margin_dpo/margin_mean": 0.058963656425476074, "margin_dpo/margin_std": 0.5231530070304871, "step": 12 }, { "KL/chosen_KL_mean": 0.06639862060546875, "KL/mean": 0.029717475175857544, "KL/rejected_KL_mean": -0.0069580078125, "KL/std": 0.3149820566177368, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07336193323135376, "fcm_dpo/q_t": 0.49816781282424927, "grad_norm": 41.846954345703125, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.6599808931350708, "logits/rejected": 0.9293898344039917, "logps/chosen": -90.49333190917969, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.69805908203125, "loss": 1.3795, "margin_dpo/margin_mean": 0.07336178421974182, "margin_dpo/margin_std": 0.45284169912338257, "step": 13 }, { "KL/chosen_KL_mean": -0.045650482177734375, "KL/mean": 0.014876686036586761, "KL/rejected_KL_mean": 0.07540130615234375, "KL/std": 0.31531578302383423, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12105651199817657, "fcm_dpo/q_t": 0.5030248165130615, "grad_norm": 36.17204284667969, "learning_rate": 9.420289855072464e-08, "logits/chosen": 1.0401277542114258, "logits/rejected": 1.1336442232131958, "logps/chosen": -99.8728256225586, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.8666000366211, "loss": 1.3989, "margin_dpo/margin_mean": -0.12105602025985718, "margin_dpo/margin_std": 0.4285448491573334, "step": 14 }, { "KL/chosen_KL_mean": 0.005847930908203125, "KL/mean": 0.007467329502105713, "KL/rejected_KL_mean": 0.00908660888671875, "KL/std": 0.3139072060585022, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0032372474670410156, "fcm_dpo/q_t": 0.5000815391540527, "grad_norm": 31.755294799804688, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.34732091426849365, "logits/rejected": 0.5132207870483398, "logps/chosen": -78.90412139892578, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.05325317382812, "loss": 1.3871, "margin_dpo/margin_mean": -0.003237396478652954, "margin_dpo/margin_std": 0.42780041694641113, "step": 15 }, { "KL/chosen_KL_mean": -0.012302398681640625, "KL/mean": -0.013572901487350464, "KL/rejected_KL_mean": -0.014850616455078125, "KL/std": 0.33251744508743286, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0025451183319091797, "fcm_dpo/q_t": 0.4999363422393799, "grad_norm": 34.63500213623047, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.41101276874542236, "logits/rejected": 0.6560854911804199, "logps/chosen": -97.43557739257812, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.61431121826172, "loss": 1.3865, "margin_dpo/margin_mean": 0.0025452077388763428, "margin_dpo/margin_std": 0.40987563133239746, "step": 16 }, { "KL/chosen_KL_mean": 0.0492401123046875, "KL/mean": 0.003294132649898529, "KL/rejected_KL_mean": -0.04265594482421875, "KL/std": 0.28036364912986755, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09189911186695099, "fcm_dpo/q_t": 0.497702956199646, "grad_norm": 35.68550491333008, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5915480852127075, "logits/rejected": 0.8340729475021362, "logps/chosen": -104.3150634765625, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.5103759765625, "loss": 1.3776, "margin_dpo/margin_mean": 0.09189826250076294, "margin_dpo/margin_std": 0.4121086001396179, "step": 17 }, { "KL/chosen_KL_mean": -0.041347503662109375, "KL/mean": -0.025243237614631653, "KL/rejected_KL_mean": -0.00913238525390625, "KL/std": 0.2684563100337982, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03222152590751648, "fcm_dpo/q_t": 0.5008057355880737, "grad_norm": 41.79636001586914, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.29146233201026917, "logits/rejected": 0.6260148286819458, "logps/chosen": -87.13330078125, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.85984802246094, "loss": 1.3899, "margin_dpo/margin_mean": -0.03222121298313141, "margin_dpo/margin_std": 0.3839834928512573, "step": 18 }, { "KL/chosen_KL_mean": -0.015033721923828125, "KL/mean": -0.005023196339607239, "KL/rejected_KL_mean": 0.004985809326171875, "KL/std": 0.28734222054481506, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.020025283098220825, "fcm_dpo/q_t": 0.5005007386207581, "grad_norm": 31.445186614990234, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.5563080310821533, "logits/rejected": 0.8287099003791809, "logps/chosen": -105.88858032226562, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.92524719238281, "loss": 1.3887, "margin_dpo/margin_mean": -0.020025253295898438, "margin_dpo/margin_std": 0.3829053044319153, "step": 19 }, { "KL/chosen_KL_mean": 0.02141571044921875, "KL/mean": 0.017449840903282166, "KL/rejected_KL_mean": 0.0134735107421875, "KL/std": 0.3162112832069397, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007941097021102905, "fcm_dpo/q_t": 0.4998020827770233, "grad_norm": 32.6539306640625, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.13984465599060059, "logits/rejected": 0.4051741659641266, "logps/chosen": -90.73670959472656, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.89884948730469, "loss": 1.386, "margin_dpo/margin_mean": 0.00794130563735962, "margin_dpo/margin_std": 0.4545300006866455, "step": 20 }, { "KL/chosen_KL_mean": 0.0329132080078125, "KL/mean": 0.033201664686203, "KL/rejected_KL_mean": 0.03348541259765625, "KL/std": 0.26948782801628113, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0005737543106079102, "fcm_dpo/q_t": 0.5000141263008118, "grad_norm": 31.44552230834961, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.49360620975494385, "logits/rejected": 0.6663397550582886, "logps/chosen": -80.30054473876953, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.90025329589844, "loss": 1.3867, "margin_dpo/margin_mean": -0.0005732178688049316, "margin_dpo/margin_std": 0.36499908566474915, "step": 21 }, { "KL/chosen_KL_mean": 0.04799652099609375, "KL/mean": 0.026042208075523376, "KL/rejected_KL_mean": 0.00408935546875, "KL/std": 0.3621774911880493, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04389879107475281, "fcm_dpo/q_t": 0.4989026188850403, "grad_norm": 42.828311920166016, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.5307182669639587, "logits/rejected": 0.6452593803405762, "logps/chosen": -95.34730529785156, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.46942901611328, "loss": 1.3825, "margin_dpo/margin_mean": 0.043898940086364746, "margin_dpo/margin_std": 0.46846675872802734, "step": 22 }, { "KL/chosen_KL_mean": -0.030504226684570312, "KL/mean": 0.030004985630512238, "KL/rejected_KL_mean": 0.09052276611328125, "KL/std": 0.31763917207717896, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12102554738521576, "fcm_dpo/q_t": 0.5030245184898376, "grad_norm": 31.820659637451172, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.6594729423522949, "logits/rejected": 0.832984447479248, "logps/chosen": -90.66801452636719, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.50373840332031, "loss": 1.399, "margin_dpo/margin_mean": -0.12102556228637695, "margin_dpo/margin_std": 0.4603731632232666, "step": 23 }, { "KL/chosen_KL_mean": 0.03353118896484375, "KL/mean": 0.017414167523384094, "KL/rejected_KL_mean": 0.0012969970703125, "KL/std": 0.28221040964126587, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03223666548728943, "fcm_dpo/q_t": 0.4991948902606964, "grad_norm": 44.195777893066406, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.779197096824646, "logits/rejected": 0.8072788715362549, "logps/chosen": -69.88375854492188, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.60860443115234, "loss": 1.3835, "margin_dpo/margin_mean": 0.03223720192909241, "margin_dpo/margin_std": 0.40609243512153625, "step": 24 }, { "KL/chosen_KL_mean": -0.01694488525390625, "KL/mean": -0.04884765297174454, "KL/rejected_KL_mean": -0.08075332641601562, "KL/std": 0.2854161858558655, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0638028234243393, "fcm_dpo/q_t": 0.4984061121940613, "grad_norm": 37.070560455322266, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.3408451974391937, "logits/rejected": 0.5028581619262695, "logps/chosen": -80.8424301147461, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.03785705566406, "loss": 1.3804, "margin_dpo/margin_mean": 0.06380262970924377, "margin_dpo/margin_std": 0.41451069712638855, "step": 25 }, { "KL/chosen_KL_mean": -0.013584136962890625, "KL/mean": -0.06058196723461151, "KL/rejected_KL_mean": -0.10757827758789062, "KL/std": 0.30167388916015625, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09399202466011047, "fcm_dpo/q_t": 0.4976511299610138, "grad_norm": 40.933719635009766, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.3206663131713867, "logits/rejected": 0.5289573073387146, "logps/chosen": -88.91475677490234, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.93577575683594, "loss": 1.3774, "margin_dpo/margin_mean": 0.09399241209030151, "margin_dpo/margin_std": 0.4147573709487915, "step": 26 }, { "KL/chosen_KL_mean": 0.0140838623046875, "KL/mean": -0.04431355744600296, "KL/rejected_KL_mean": -0.10271835327148438, "KL/std": 0.31112605333328247, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1167982667684555, "fcm_dpo/q_t": 0.4970807433128357, "grad_norm": 44.049415588378906, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.1291225552558899, "logits/rejected": 0.31685787439346313, "logps/chosen": -77.58192443847656, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.04122924804688, "loss": 1.3751, "margin_dpo/margin_mean": 0.11679837107658386, "margin_dpo/margin_std": 0.4345523416996002, "step": 27 }, { "KL/chosen_KL_mean": 0.08240890502929688, "KL/mean": 0.00828734040260315, "KL/rejected_KL_mean": -0.06583786010742188, "KL/std": 0.34131544828414917, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14824581146240234, "fcm_dpo/q_t": 0.4962955117225647, "grad_norm": 36.52998733520508, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5140984058380127, "logits/rejected": 0.7237167954444885, "logps/chosen": -102.14616394042969, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.02532958984375, "loss": 1.372, "margin_dpo/margin_mean": 0.14824622869491577, "margin_dpo/margin_std": 0.4177868962287903, "step": 28 }, { "KL/chosen_KL_mean": -0.02703094482421875, "KL/mean": -0.051579274237155914, "KL/rejected_KL_mean": -0.07612228393554688, "KL/std": 0.3761771619319916, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04909248650074005, "fcm_dpo/q_t": 0.4987727403640747, "grad_norm": 42.573970794677734, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.3865003287792206, "logits/rejected": 0.6505852341651917, "logps/chosen": -88.67408752441406, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.03623962402344, "loss": 1.382, "margin_dpo/margin_mean": 0.04909384250640869, "margin_dpo/margin_std": 0.49197590351104736, "step": 29 }, { "KL/chosen_KL_mean": -0.00634002685546875, "KL/mean": -0.0467878133058548, "KL/rejected_KL_mean": -0.08724594116210938, "KL/std": 0.3573537766933441, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08089473843574524, "fcm_dpo/q_t": 0.4979800879955292, "grad_norm": 39.17037582397461, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.6145938634872437, "logits/rejected": 0.7312387228012085, "logps/chosen": -88.39472961425781, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.40614318847656, "loss": 1.3787, "margin_dpo/margin_mean": 0.0808950662612915, "margin_dpo/margin_std": 0.43756431341171265, "step": 30 }, { "KL/chosen_KL_mean": 0.023040771484375, "KL/mean": -0.04631167650222778, "KL/rejected_KL_mean": -0.11566543579101562, "KL/std": 0.3610226511955261, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13870501518249512, "fcm_dpo/q_t": 0.49653762578964233, "grad_norm": 31.569486618041992, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.8822535276412964, "logits/rejected": 1.1470757722854614, "logps/chosen": -101.10261535644531, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.97409057617188, "loss": 1.3732, "margin_dpo/margin_mean": 0.13870558142662048, "margin_dpo/margin_std": 0.5249038934707642, "step": 31 }, { "KL/chosen_KL_mean": 0.001262664794921875, "KL/mean": -0.0602106973528862, "KL/rejected_KL_mean": -0.12168502807617188, "KL/std": 0.3004150688648224, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12295179069042206, "fcm_dpo/q_t": 0.4969291090965271, "grad_norm": 35.253387451171875, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.43904367089271545, "logits/rejected": 0.579369068145752, "logps/chosen": -96.61576843261719, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.34605407714844, "loss": 1.3744, "margin_dpo/margin_mean": 0.12295247614383698, "margin_dpo/margin_std": 0.39896121621131897, "step": 32 }, { "KL/chosen_KL_mean": 0.01226806640625, "KL/mean": -0.027182169258594513, "KL/rejected_KL_mean": -0.0666351318359375, "KL/std": 0.2867492437362671, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07890431582927704, "fcm_dpo/q_t": 0.49802812933921814, "grad_norm": 33.62782669067383, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5759651064872742, "logits/rejected": 0.7562488317489624, "logps/chosen": -81.50874328613281, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.87258911132812, "loss": 1.3788, "margin_dpo/margin_mean": 0.07890519499778748, "margin_dpo/margin_std": 0.3877814710140228, "step": 33 }, { "KL/chosen_KL_mean": 0.037372589111328125, "KL/mean": -0.06360594928264618, "KL/rejected_KL_mean": -0.16458511352539062, "KL/std": 0.336570143699646, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20195963978767395, "fcm_dpo/q_t": 0.49495887756347656, "grad_norm": 40.91716003417969, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5438739657402039, "logits/rejected": 0.7167494297027588, "logps/chosen": -77.16466522216797, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.88334655761719, "loss": 1.3668, "margin_dpo/margin_mean": 0.20196029543876648, "margin_dpo/margin_std": 0.4638097286224365, "step": 34 }, { "KL/chosen_KL_mean": 0.0463104248046875, "KL/mean": -0.07932448387145996, "KL/rejected_KL_mean": -0.20496368408203125, "KL/std": 0.352822482585907, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2512739896774292, "fcm_dpo/q_t": 0.4937238395214081, "grad_norm": 42.51215744018555, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.4210290014743805, "logits/rejected": 0.5761604309082031, "logps/chosen": -77.52404022216797, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.39351654052734, "loss": 1.3619, "margin_dpo/margin_mean": 0.2512742280960083, "margin_dpo/margin_std": 0.4886428713798523, "step": 35 }, { "KL/chosen_KL_mean": 0.036334991455078125, "KL/mean": -0.07064832746982574, "KL/rejected_KL_mean": -0.17763519287109375, "KL/std": 0.3771868944168091, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2139652669429779, "fcm_dpo/q_t": 0.4946596920490265, "grad_norm": 32.47846221923828, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.4800012707710266, "logits/rejected": 0.6729844808578491, "logps/chosen": -83.25190734863281, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.93505096435547, "loss": 1.3657, "margin_dpo/margin_mean": 0.21396507322788239, "margin_dpo/margin_std": 0.5167742371559143, "step": 36 }, { "KL/chosen_KL_mean": -0.052364349365234375, "KL/mean": -0.17281019687652588, "KL/rejected_KL_mean": -0.2932586669921875, "KL/std": 0.38918250799179077, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24088743329048157, "fcm_dpo/q_t": 0.4939855933189392, "grad_norm": 36.22098922729492, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.8439332246780396, "logits/rejected": 1.0195629596710205, "logps/chosen": -94.82345581054688, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01052856445312, "loss": 1.3629, "margin_dpo/margin_mean": 0.2408878207206726, "margin_dpo/margin_std": 0.480082243680954, "step": 37 }, { "KL/chosen_KL_mean": 0.01981353759765625, "KL/mean": -0.08537587523460388, "KL/rejected_KL_mean": -0.19055938720703125, "KL/std": 0.3881682753562927, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21037143468856812, "fcm_dpo/q_t": 0.4947461485862732, "grad_norm": 37.194435119628906, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.5247505903244019, "logits/rejected": 0.6633862257003784, "logps/chosen": -75.95869445800781, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.2945785522461, "loss": 1.366, "margin_dpo/margin_mean": 0.21037214994430542, "margin_dpo/margin_std": 0.4869280755519867, "step": 38 }, { "KL/chosen_KL_mean": -0.07168197631835938, "KL/mean": -0.22170394659042358, "KL/rejected_KL_mean": -0.37172698974609375, "KL/std": 0.4155295491218567, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3000420331954956, "fcm_dpo/q_t": 0.4925070106983185, "grad_norm": 36.21068572998047, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.29801425337791443, "logits/rejected": 0.4747072458267212, "logps/chosen": -81.25746154785156, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.56764221191406, "loss": 1.3573, "margin_dpo/margin_mean": 0.30004221200942993, "margin_dpo/margin_std": 0.5511003732681274, "step": 39 }, { "KL/chosen_KL_mean": 0.03951263427734375, "KL/mean": -0.07564333081245422, "KL/rejected_KL_mean": -0.1907958984375, "KL/std": 0.3692399859428406, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23030871152877808, "fcm_dpo/q_t": 0.4942456781864166, "grad_norm": 29.056631088256836, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.574290931224823, "logits/rejected": 0.7680929899215698, "logps/chosen": -83.29306030273438, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.44670867919922, "loss": 1.3639, "margin_dpo/margin_mean": 0.2303086519241333, "margin_dpo/margin_std": 0.42588528990745544, "step": 40 }, { "KL/chosen_KL_mean": -0.033843994140625, "KL/mean": -0.15382085740566254, "KL/rejected_KL_mean": -0.2737998962402344, "KL/std": 0.4298606514930725, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23995599150657654, "fcm_dpo/q_t": 0.4940118193626404, "grad_norm": 33.47755432128906, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.4089720845222473, "logits/rejected": 0.5827963948249817, "logps/chosen": -93.18251037597656, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.35301208496094, "loss": 1.3635, "margin_dpo/margin_mean": 0.23995620012283325, "margin_dpo/margin_std": 0.6230607032775879, "step": 41 }, { "KL/chosen_KL_mean": 0.016384124755859375, "KL/mean": -0.21432140469551086, "KL/rejected_KL_mean": -0.44503021240234375, "KL/std": 0.4542730748653412, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.46140992641448975, "fcm_dpo/q_t": 0.488476425409317, "grad_norm": 41.09767150878906, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.4619252681732178, "logits/rejected": 0.6031174659729004, "logps/chosen": -90.68524169921875, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.30967712402344, "loss": 1.3415, "margin_dpo/margin_mean": 0.4614105522632599, "margin_dpo/margin_std": 0.5325796604156494, "step": 42 }, { "KL/chosen_KL_mean": 0.0055389404296875, "KL/mean": -0.23651106655597687, "KL/rejected_KL_mean": -0.4785652160644531, "KL/std": 0.5331444144248962, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4841013252735138, "fcm_dpo/q_t": 0.4879271984100342, "grad_norm": 37.792755126953125, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.5522860288619995, "logits/rejected": 0.7410375475883484, "logps/chosen": -89.63848876953125, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.430419921875, "loss": 1.3398, "margin_dpo/margin_mean": 0.48410117626190186, "margin_dpo/margin_std": 0.7226862907409668, "step": 43 }, { "KL/chosen_KL_mean": -0.07061004638671875, "KL/mean": -0.2553819417953491, "KL/rejected_KL_mean": -0.4401588439941406, "KL/std": 0.49590742588043213, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3695499300956726, "fcm_dpo/q_t": 0.49077358841896057, "grad_norm": 34.59714889526367, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.5380641222000122, "logits/rejected": 0.6663861274719238, "logps/chosen": -81.94892883300781, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.86178588867188, "loss": 1.3505, "margin_dpo/margin_mean": 0.3695501983165741, "margin_dpo/margin_std": 0.5708489418029785, "step": 44 }, { "KL/chosen_KL_mean": -0.07503509521484375, "KL/mean": -0.17636412382125854, "KL/rejected_KL_mean": -0.2776908874511719, "KL/std": 0.3671168386936188, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20265348255634308, "fcm_dpo/q_t": 0.49493712186813354, "grad_norm": 24.553430557250977, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.7525115013122559, "logits/rejected": 0.8668965101242065, "logps/chosen": -77.41963195800781, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.52543640136719, "loss": 1.3668, "margin_dpo/margin_mean": 0.2026546448469162, "margin_dpo/margin_std": 0.4983961582183838, "step": 45 }, { "KL/chosen_KL_mean": 0.0382080078125, "KL/mean": -0.21678254008293152, "KL/rejected_KL_mean": -0.4717750549316406, "KL/std": 0.6133427619934082, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5099853277206421, "fcm_dpo/q_t": 0.4872693121433258, "grad_norm": 32.94456481933594, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.3620871305465698, "logits/rejected": 0.5980793237686157, "logps/chosen": -90.3026123046875, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.1482925415039, "loss": 1.3369, "margin_dpo/margin_mean": 0.509985089302063, "margin_dpo/margin_std": 0.6254180669784546, "step": 46 }, { "KL/chosen_KL_mean": -0.049037933349609375, "KL/mean": -0.2382146567106247, "KL/rejected_KL_mean": -0.4273948669433594, "KL/std": 0.5305290818214417, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3783544600009918, "fcm_dpo/q_t": 0.49055618047714233, "grad_norm": 34.039939880371094, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.5451183319091797, "logits/rejected": 0.7332047820091248, "logps/chosen": -104.46034240722656, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.68229675292969, "loss": 1.3501, "margin_dpo/margin_mean": 0.3783537745475769, "margin_dpo/margin_std": 0.7154488563537598, "step": 47 }, { "KL/chosen_KL_mean": 0.001861572265625, "KL/mean": -0.27693599462509155, "KL/rejected_KL_mean": -0.5557327270507812, "KL/std": 0.6296478509902954, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5575941801071167, "fcm_dpo/q_t": 0.48609042167663574, "grad_norm": 38.840736389160156, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.2580597996711731, "logits/rejected": 0.4607198238372803, "logps/chosen": -91.0984115600586, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.63630676269531, "loss": 1.3329, "margin_dpo/margin_mean": 0.557594895362854, "margin_dpo/margin_std": 0.803669273853302, "step": 48 }, { "KL/chosen_KL_mean": 0.004302978515625, "KL/mean": -0.38034722208976746, "KL/rejected_KL_mean": -0.7649993896484375, "KL/std": 0.7794817686080933, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7692994475364685, "fcm_dpo/q_t": 0.48082590103149414, "grad_norm": 39.623844146728516, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.2853112816810608, "logits/rejected": 0.777318000793457, "logps/chosen": -92.99937438964844, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.51399230957031, "loss": 1.3129, "margin_dpo/margin_mean": 0.7692991495132446, "margin_dpo/margin_std": 0.8921933174133301, "step": 49 }, { "KL/chosen_KL_mean": -0.15637969970703125, "KL/mean": -0.4420780539512634, "KL/rejected_KL_mean": -0.7277717590332031, "KL/std": 0.7587863206863403, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5713925361633301, "fcm_dpo/q_t": 0.4857657551765442, "grad_norm": 33.39608383178711, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.5616767406463623, "logits/rejected": 0.7326728105545044, "logps/chosen": -94.78318786621094, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.30213165283203, "loss": 1.3322, "margin_dpo/margin_mean": 0.5713920593261719, "margin_dpo/margin_std": 0.9184191226959229, "step": 50 }, { "KL/chosen_KL_mean": -0.0929718017578125, "KL/mean": -0.4101176857948303, "KL/rejected_KL_mean": -0.7272605895996094, "KL/std": 0.8895403146743774, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6342880129814148, "fcm_dpo/q_t": 0.48424389958381653, "grad_norm": 29.867021560668945, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5068886280059814, "logits/rejected": 0.6419796347618103, "logps/chosen": -87.60025024414062, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.19961547851562, "loss": 1.3271, "margin_dpo/margin_mean": 0.6342877149581909, "margin_dpo/margin_std": 1.127030372619629, "step": 51 }, { "KL/chosen_KL_mean": 0.00662994384765625, "KL/mean": -0.42715489864349365, "KL/rejected_KL_mean": -0.8609466552734375, "KL/std": 0.8066141605377197, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8675732612609863, "fcm_dpo/q_t": 0.4783938527107239, "grad_norm": 32.06678009033203, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.32477742433547974, "logits/rejected": 0.5298559665679932, "logps/chosen": -90.6236343383789, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.90000915527344, "loss": 1.3038, "margin_dpo/margin_mean": 0.8675732612609863, "margin_dpo/margin_std": 0.9688670039176941, "step": 52 }, { "KL/chosen_KL_mean": -0.0036773681640625, "KL/mean": -0.5710875391960144, "KL/rejected_KL_mean": -1.1385040283203125, "KL/std": 1.1324098110198975, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1348307132720947, "fcm_dpo/q_t": 0.4719311594963074, "grad_norm": 39.25545120239258, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.1324218213558197, "logits/rejected": 0.24661576747894287, "logps/chosen": -81.58675384521484, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.80003356933594, "loss": 1.2815, "margin_dpo/margin_mean": 1.1348299980163574, "margin_dpo/margin_std": 1.3858613967895508, "step": 53 }, { "KL/chosen_KL_mean": -0.0109710693359375, "KL/mean": -0.5591979622840881, "KL/rejected_KL_mean": -1.107421875, "KL/std": 1.2602225542068481, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.096449375152588, "fcm_dpo/q_t": 0.472777783870697, "grad_norm": 39.99812316894531, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.411001980304718, "logits/rejected": 0.590743899345398, "logps/chosen": -88.921142578125, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.2249755859375, "loss": 1.2838, "margin_dpo/margin_mean": 1.096449613571167, "margin_dpo/margin_std": 1.2867379188537598, "step": 54 }, { "KL/chosen_KL_mean": 0.081878662109375, "KL/mean": -0.5114444494247437, "KL/rejected_KL_mean": -1.1047630310058594, "KL/std": 1.2128612995147705, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1866364479064941, "fcm_dpo/q_t": 0.4706483781337738, "grad_norm": 35.27198028564453, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.07767891883850098, "logits/rejected": 0.3842528164386749, "logps/chosen": -92.37403869628906, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.07569885253906, "loss": 1.2775, "margin_dpo/margin_mean": 1.1866364479064941, "margin_dpo/margin_std": 1.5616772174835205, "step": 55 }, { "KL/chosen_KL_mean": -0.07259750366210938, "KL/mean": -0.604528546333313, "KL/rejected_KL_mean": -1.1364593505859375, "KL/std": 1.149642825126648, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0638604164123535, "fcm_dpo/q_t": 0.47371965646743774, "grad_norm": 33.0576171875, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.26537859439849854, "logits/rejected": 0.6049945950508118, "logps/chosen": -87.30924987792969, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.78199768066406, "loss": 1.2894, "margin_dpo/margin_mean": 1.0638600587844849, "margin_dpo/margin_std": 1.625234842300415, "step": 56 }, { "KL/chosen_KL_mean": -0.04801177978515625, "KL/mean": -0.6234676837921143, "KL/rejected_KL_mean": -1.1989288330078125, "KL/std": 1.2612141370773315, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1509301662445068, "fcm_dpo/q_t": 0.4715151786804199, "grad_norm": 33.56852722167969, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.5257400870323181, "logits/rejected": 0.7529832124710083, "logps/chosen": -98.1987533569336, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.25177001953125, "loss": 1.2809, "margin_dpo/margin_mean": 1.1509311199188232, "margin_dpo/margin_std": 1.5969160795211792, "step": 57 }, { "KL/chosen_KL_mean": 0.2808265686035156, "KL/mean": -0.4764922261238098, "KL/rejected_KL_mean": -1.2338027954101562, "KL/std": 1.4024996757507324, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5146275758743286, "fcm_dpo/q_t": 0.4627363383769989, "grad_norm": 38.13439178466797, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.21389831602573395, "logits/rejected": 0.5381336212158203, "logps/chosen": -99.30014038085938, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.4560775756836, "loss": 1.2493, "margin_dpo/margin_mean": 1.514627456665039, "margin_dpo/margin_std": 1.8252344131469727, "step": 58 }, { "KL/chosen_KL_mean": 0.066650390625, "KL/mean": -0.5715759992599487, "KL/rejected_KL_mean": -1.2098045349121094, "KL/std": 1.1946048736572266, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.276451587677002, "fcm_dpo/q_t": 0.4683869779109955, "grad_norm": 34.11297607421875, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.06465496867895126, "logits/rejected": 0.2516256868839264, "logps/chosen": -89.73567199707031, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.48648071289062, "loss": 1.2683, "margin_dpo/margin_mean": 1.2764509916305542, "margin_dpo/margin_std": 1.4772450923919678, "step": 59 }, { "KL/chosen_KL_mean": 0.07891082763671875, "KL/mean": -0.5114976167678833, "KL/rejected_KL_mean": -1.1019096374511719, "KL/std": 1.2191505432128906, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1808154582977295, "fcm_dpo/q_t": 0.4706842601299286, "grad_norm": 28.810983657836914, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.35787028074264526, "logits/rejected": 0.6250767707824707, "logps/chosen": -95.07679748535156, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.47434997558594, "loss": 1.2764, "margin_dpo/margin_mean": 1.1808154582977295, "margin_dpo/margin_std": 1.3722972869873047, "step": 60 }, { "KL/chosen_KL_mean": -0.19487762451171875, "KL/mean": -0.7509031295776367, "KL/rejected_KL_mean": -1.3069267272949219, "KL/std": 1.331721544265747, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1120473146438599, "fcm_dpo/q_t": 0.4726504385471344, "grad_norm": 28.474720001220703, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.19000256061553955, "logits/rejected": 0.3307693302631378, "logps/chosen": -85.92718505859375, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.27268981933594, "loss": 1.2858, "margin_dpo/margin_mean": 1.1120470762252808, "margin_dpo/margin_std": 1.697868824005127, "step": 61 }, { "KL/chosen_KL_mean": -0.17077255249023438, "KL/mean": -0.7939934730529785, "KL/rejected_KL_mean": -1.4172172546386719, "KL/std": 1.466771125793457, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.246443748474121, "fcm_dpo/q_t": 0.46924084424972534, "grad_norm": 28.240062713623047, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.320276141166687, "logits/rejected": 0.526498556137085, "logps/chosen": -81.80615234375, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.45552825927734, "loss": 1.276, "margin_dpo/margin_mean": 1.2464436292648315, "margin_dpo/margin_std": 1.9024014472961426, "step": 62 }, { "KL/chosen_KL_mean": -0.02742767333984375, "KL/mean": -0.7824211716651917, "KL/rejected_KL_mean": -1.5374183654785156, "KL/std": 1.3532404899597168, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5100005865097046, "fcm_dpo/q_t": 0.46260470151901245, "grad_norm": 35.04413986206055, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.37564387917518616, "logits/rejected": 0.5879080891609192, "logps/chosen": -103.6514892578125, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.94044494628906, "loss": 1.2472, "margin_dpo/margin_mean": 1.5100007057189941, "margin_dpo/margin_std": 1.559769630432129, "step": 63 }, { "KL/chosen_KL_mean": -0.1163482666015625, "KL/mean": -0.9928090572357178, "KL/rejected_KL_mean": -1.8692817687988281, "KL/std": 1.8749275207519531, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.26532095670700073, "fcm_dpo/delta": -0.006150953471660614, "fcm_dpo/margin": 1.7529385089874268, "fcm_dpo/q_t": 0.4092322587966919, "grad_norm": 61.353118896484375, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.8051044940948486, "logits/rejected": 1.0369267463684082, "logps/chosen": -87.11793518066406, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.45475769042969, "loss": 1.0744, "margin_dpo/margin_mean": 1.7529386281967163, "margin_dpo/margin_std": 2.1219122409820557, "step": 64 }, { "KL/chosen_KL_mean": -0.0517578125, "KL/mean": -0.9510102272033691, "KL/rejected_KL_mean": -1.8502655029296875, "KL/std": 1.7075883150100708, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.09062723815441132, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.798502802848816, "fcm_dpo/q_t": 0.4599056839942932, "grad_norm": 35.325599670410156, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.00710725411772728, "logits/rejected": 0.1513298898935318, "logps/chosen": -91.27367401123047, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.18579864501953, "loss": 1.2393, "margin_dpo/margin_mean": 1.798502802848816, "margin_dpo/margin_std": 2.11547589302063, "step": 65 }, { "KL/chosen_KL_mean": -0.2900848388671875, "KL/mean": -0.9452009201049805, "KL/rejected_KL_mean": -1.6003189086914062, "KL/std": 1.5451171398162842, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.09062723815441132, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3102327585220337, "fcm_dpo/q_t": 0.4706510901451111, "grad_norm": 23.608844757080078, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.04125874489545822, "logits/rejected": 0.23813584446907043, "logps/chosen": -84.07431030273438, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.3524169921875, "loss": 1.2782, "margin_dpo/margin_mean": 1.3102333545684814, "margin_dpo/margin_std": 1.800450325012207, "step": 66 }, { "KL/chosen_KL_mean": -0.8150100708007812, "KL/mean": -1.3876750469207764, "KL/rejected_KL_mean": -1.9603271484375, "KL/std": 1.9714303016662598, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.09062723815441132, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1453179121017456, "fcm_dpo/q_t": 0.47469741106033325, "grad_norm": 22.2416934967041, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.3759915828704834, "logits/rejected": 0.6551348567008972, "logps/chosen": -88.48797607421875, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.04707336425781, "loss": 1.2997, "margin_dpo/margin_mean": 1.145317554473877, "margin_dpo/margin_std": 2.6598129272460938, "step": 67 }, { "KL/chosen_KL_mean": -0.22647857666015625, "KL/mean": -1.2726013660430908, "KL/rejected_KL_mean": -2.3187179565429688, "KL/std": 2.2305376529693604, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.09062723815441132, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.092235565185547, "fcm_dpo/q_t": 0.4540542662143707, "grad_norm": 26.329004287719727, "learning_rate": 4.855072463768116e-07, "logits/chosen": 0.13552409410476685, "logits/rejected": 0.4515076279640198, "logps/chosen": -97.30532836914062, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.43744659423828, "loss": 1.2218, "margin_dpo/margin_mean": 2.092235565185547, "margin_dpo/margin_std": 2.8273775577545166, "step": 68 }, { "KL/chosen_KL_mean": -0.2841606140136719, "KL/mean": -1.3829357624053955, "KL/rejected_KL_mean": -2.481708526611328, "KL/std": 2.4181699752807617, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.24354881048202515, "fcm_dpo/delta": 0.09224040806293488, "fcm_dpo/margin": 2.1975462436676025, "fcm_dpo/q_t": 0.4517165422439575, "grad_norm": 48.607547760009766, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.22362498939037323, "logits/rejected": 0.33328506350517273, "logps/chosen": -86.00387573242188, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.9619140625, "loss": 1.0577, "margin_dpo/margin_mean": 2.197547197341919, "margin_dpo/margin_std": 2.963463544845581, "step": 69 }, { "KL/chosen_KL_mean": -0.5632362365722656, "KL/mean": -1.8467820882797241, "KL/rejected_KL_mean": -3.130329132080078, "KL/std": 2.744410991668701, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": -0.29104289412498474, "fcm_dpo/margin": 2.567096710205078, "fcm_dpo/q_t": 0.40120548009872437, "grad_norm": 1.584383249282837, "learning_rate": 5e-07, "logits/chosen": 0.18647563457489014, "logits/rejected": 0.35994869470596313, "logps/chosen": -95.57318115234375, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.34304809570312, "loss": 1.3767, "margin_dpo/margin_mean": 2.567096710205078, "margin_dpo/margin_std": 3.310142755508423, "step": 70 }, { "KL/chosen_KL_mean": -0.001220703125, "KL/mean": -1.790406346321106, "KL/rejected_KL_mean": -3.5795936584472656, "KL/std": 3.415595054626465, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5783724784851074, "fcm_dpo/q_t": 0.4966316521167755, "grad_norm": 1.9741885662078857, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.17271456122398376, "logits/rejected": 0.18850931525230408, "logps/chosen": -90.97857666015625, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.17062377929688, "loss": 1.3729, "margin_dpo/margin_mean": 3.578371524810791, "margin_dpo/margin_std": 3.9362552165985107, "step": 71 }, { "KL/chosen_KL_mean": -0.5382270812988281, "KL/mean": -1.9992274045944214, "KL/rejected_KL_mean": -3.4602317810058594, "KL/std": 2.9428329467773438, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9220080375671387, "fcm_dpo/q_t": 0.4972493648529053, "grad_norm": 1.5540196895599365, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.2700241506099701, "logits/rejected": 0.04623167961835861, "logps/chosen": -98.42974853515625, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.65194702148438, "loss": 1.3754, "margin_dpo/margin_mean": 2.9220082759857178, "margin_dpo/margin_std": 3.337193489074707, "step": 72 }, { "KL/chosen_KL_mean": -0.7025871276855469, "KL/mean": -2.260904312133789, "KL/rejected_KL_mean": -3.8192214965820312, "KL/std": 3.7451374530792236, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1166365146636963, "fcm_dpo/q_t": 0.4970664083957672, "grad_norm": 1.7552045583724976, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.5246434211730957, "logits/rejected": -0.23604914546012878, "logps/chosen": -96.47160339355469, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.75166320800781, "loss": 1.3747, "margin_dpo/margin_mean": 3.116635799407959, "margin_dpo/margin_std": 4.752752304077148, "step": 73 }, { "KL/chosen_KL_mean": -0.92266845703125, "KL/mean": -2.674625873565674, "KL/rejected_KL_mean": -4.426582336425781, "KL/std": 3.788412094116211, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.503913402557373, "fcm_dpo/q_t": 0.49670177698135376, "grad_norm": 1.9134386777877808, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.16614490747451782, "logits/rejected": -0.022616159170866013, "logps/chosen": -79.73106384277344, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.07334899902344, "loss": 1.3732, "margin_dpo/margin_mean": 3.503913640975952, "margin_dpo/margin_std": 4.439431190490723, "step": 74 }, { "KL/chosen_KL_mean": -1.0819587707519531, "KL/mean": -3.323181629180908, "KL/rejected_KL_mean": -5.5644073486328125, "KL/std": 4.536660194396973, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.482450485229492, "fcm_dpo/q_t": 0.49578094482421875, "grad_norm": 2.227281093597412, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.3122856318950653, "logits/rejected": -0.11078165471553802, "logps/chosen": -79.36381530761719, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -120.96752166748047, "loss": 1.3696, "margin_dpo/margin_mean": 4.482450485229492, "margin_dpo/margin_std": 5.358396530151367, "step": 75 }, { "KL/chosen_KL_mean": -1.4160385131835938, "KL/mean": -2.9614415168762207, "KL/rejected_KL_mean": -4.506843566894531, "KL/std": 3.796802043914795, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.0908074378967285, "fcm_dpo/q_t": 0.4970906972885132, "grad_norm": 1.2046599388122559, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.5904031991958618, "logits/rejected": -0.21014755964279175, "logps/chosen": -89.29318237304688, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.22652435302734, "loss": 1.3748, "margin_dpo/margin_mean": 3.0908069610595703, "margin_dpo/margin_std": 4.553689479827881, "step": 76 }, { "KL/chosen_KL_mean": -1.6688575744628906, "KL/mean": -3.3091516494750977, "KL/rejected_KL_mean": -4.9494476318359375, "KL/std": 4.4015607833862305, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.280587673187256, "fcm_dpo/q_t": 0.49691200256347656, "grad_norm": 1.591613531112671, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.4479508399963379, "logits/rejected": -0.10117948800325394, "logps/chosen": -101.36920166015625, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.15521240234375, "loss": 1.3741, "margin_dpo/margin_mean": 3.2805874347686768, "margin_dpo/margin_std": 4.9707818031311035, "step": 77 }, { "KL/chosen_KL_mean": -0.660400390625, "KL/mean": -2.5030629634857178, "KL/rejected_KL_mean": -4.345722198486328, "KL/std": 3.759228467941284, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6853184700012207, "fcm_dpo/q_t": 0.49653103947639465, "grad_norm": 1.546011209487915, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.01144075021147728, "logits/rejected": 0.2444465607404709, "logps/chosen": -90.95710754394531, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.48344421386719, "loss": 1.3725, "margin_dpo/margin_mean": 3.685319423675537, "margin_dpo/margin_std": 4.4554033279418945, "step": 78 }, { "KL/chosen_KL_mean": -0.9380950927734375, "KL/mean": -3.340681552886963, "KL/rejected_KL_mean": -5.7432708740234375, "KL/std": 5.454984188079834, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.80517578125, "fcm_dpo/q_t": 0.4954773187637329, "grad_norm": 1.9817345142364502, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.5340848565101624, "logits/rejected": -0.14208638668060303, "logps/chosen": -87.31642150878906, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.85104370117188, "loss": 1.3684, "margin_dpo/margin_mean": 4.805175304412842, "margin_dpo/margin_std": 6.059697151184082, "step": 79 }, { "KL/chosen_KL_mean": -1.6074600219726562, "KL/mean": -3.508573293685913, "KL/rejected_KL_mean": -5.409687042236328, "KL/std": 4.826504707336426, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.802220344543457, "fcm_dpo/q_t": 0.496421217918396, "grad_norm": 1.729668378829956, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7863693833351135, "logits/rejected": -0.4306637644767761, "logps/chosen": -95.57777404785156, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.98410034179688, "loss": 1.3721, "margin_dpo/margin_mean": 3.8022193908691406, "margin_dpo/margin_std": 5.354310035705566, "step": 80 }, { "KL/chosen_KL_mean": -1.1276664733886719, "KL/mean": -3.8812570571899414, "KL/rejected_KL_mean": -6.634845733642578, "KL/std": 6.046879291534424, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.50717830657959, "fcm_dpo/q_t": 0.49481743574142456, "grad_norm": 2.2538087368011475, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5886618494987488, "logits/rejected": -0.162519633769989, "logps/chosen": -100.95779418945312, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.30731201171875, "loss": 1.3659, "margin_dpo/margin_mean": 5.507179260253906, "margin_dpo/margin_std": 7.166586875915527, "step": 81 }, { "KL/chosen_KL_mean": -1.997894287109375, "KL/mean": -3.919947624206543, "KL/rejected_KL_mean": -5.842006683349609, "KL/std": 4.815462112426758, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.8441152572631836, "fcm_dpo/q_t": 0.4963817000389099, "grad_norm": 1.5970832109451294, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.23490184545516968, "logits/rejected": -0.014727029949426651, "logps/chosen": -85.04387664794922, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.36795806884766, "loss": 1.372, "margin_dpo/margin_mean": 3.8441152572631836, "margin_dpo/margin_std": 5.252106666564941, "step": 82 }, { "KL/chosen_KL_mean": -2.5723953247070312, "KL/mean": -4.73173713684082, "KL/rejected_KL_mean": -6.891075134277344, "KL/std": 5.543157577514648, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.318686008453369, "fcm_dpo/q_t": 0.4959353804588318, "grad_norm": 1.6860783100128174, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.5153559446334839, "logits/rejected": -0.08312012255191803, "logps/chosen": -94.74861145019531, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.6357192993164, "loss": 1.3703, "margin_dpo/margin_mean": 4.318686485290527, "margin_dpo/margin_std": 7.025886058807373, "step": 83 }, { "KL/chosen_KL_mean": -3.2230567932128906, "KL/mean": -4.536263465881348, "KL/rejected_KL_mean": -5.8494720458984375, "KL/std": 4.944561004638672, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.626411199569702, "fcm_dpo/q_t": 0.49752795696258545, "grad_norm": 1.2285547256469727, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.5184240937232971, "logits/rejected": -0.12076182663440704, "logps/chosen": -104.74954986572266, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.27555847167969, "loss": 1.3765, "margin_dpo/margin_mean": 2.6264114379882812, "margin_dpo/margin_std": 5.331443786621094, "step": 84 }, { "KL/chosen_KL_mean": -2.963397979736328, "KL/mean": -4.737985610961914, "KL/rejected_KL_mean": -6.5125732421875, "KL/std": 5.503900527954102, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.549172878265381, "fcm_dpo/q_t": 0.496659517288208, "grad_norm": 1.2035118341445923, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.37125474214553833, "logits/rejected": -0.025124449282884598, "logps/chosen": -99.09077453613281, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.56776428222656, "loss": 1.3731, "margin_dpo/margin_mean": 3.549172878265381, "margin_dpo/margin_std": 6.713469505310059, "step": 85 }, { "KL/chosen_KL_mean": -2.4125328063964844, "KL/mean": -4.714378833770752, "KL/rejected_KL_mean": -7.016231536865234, "KL/std": 6.403882026672363, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.603703498840332, "fcm_dpo/q_t": 0.4956679940223694, "grad_norm": 1.6268105506896973, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.7097886800765991, "logits/rejected": -0.4626527726650238, "logps/chosen": -84.11679077148438, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.6717300415039, "loss": 1.3693, "margin_dpo/margin_mean": 4.603703498840332, "margin_dpo/margin_std": 8.072294235229492, "step": 86 }, { "KL/chosen_KL_mean": -2.7907257080078125, "KL/mean": -5.319161891937256, "KL/rejected_KL_mean": -7.847591400146484, "KL/std": 7.1768364906311035, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.0568623542785645, "fcm_dpo/q_t": 0.4952406585216522, "grad_norm": 1.6947498321533203, "learning_rate": 4.990486745229364e-07, "logits/chosen": -1.037142276763916, "logits/rejected": -0.6610112190246582, "logps/chosen": -95.47669219970703, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.7657699584961, "loss": 1.3676, "margin_dpo/margin_mean": 5.056862831115723, "margin_dpo/margin_std": 8.615457534790039, "step": 87 }, { "KL/chosen_KL_mean": -3.4810218811035156, "KL/mean": -5.526471138000488, "KL/rejected_KL_mean": -7.5719146728515625, "KL/std": 7.0644330978393555, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.090887069702148, "fcm_dpo/q_t": 0.49615001678466797, "grad_norm": 1.5470759868621826, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.4489251375198364, "logits/rejected": -0.2525068521499634, "logps/chosen": -104.24400329589844, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -120.7222900390625, "loss": 1.3712, "margin_dpo/margin_mean": 4.090886116027832, "margin_dpo/margin_std": 7.519907474517822, "step": 88 }, { "KL/chosen_KL_mean": -2.8311233520507812, "KL/mean": -5.087518215179443, "KL/rejected_KL_mean": -7.343914031982422, "KL/std": 6.022047519683838, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.512784957885742, "fcm_dpo/q_t": 0.49575358629226685, "grad_norm": 1.6912108659744263, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.9412289261817932, "logits/rejected": -0.6037502288818359, "logps/chosen": -92.52220153808594, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.91224670410156, "loss": 1.3696, "margin_dpo/margin_mean": 4.512784481048584, "margin_dpo/margin_std": 7.827564239501953, "step": 89 }, { "KL/chosen_KL_mean": -2.6827011108398438, "KL/mean": -5.730555057525635, "KL/rejected_KL_mean": -8.77840805053711, "KL/std": 7.409266471862793, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.095707416534424, "fcm_dpo/q_t": 0.4942643344402313, "grad_norm": 2.5181589126586914, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.5407233238220215, "logits/rejected": -0.4182374179363251, "logps/chosen": -84.06526184082031, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -126.55554962158203, "loss": 1.3638, "margin_dpo/margin_mean": 6.095707416534424, "margin_dpo/margin_std": 8.98596477508545, "step": 90 }, { "KL/chosen_KL_mean": -3.384521484375, "KL/mean": -5.749150276184082, "KL/rejected_KL_mean": -8.113777160644531, "KL/std": 7.1800737380981445, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.729260444641113, "fcm_dpo/q_t": 0.49555009603500366, "grad_norm": 1.486694574356079, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.5244979858398438, "logits/rejected": -0.08317074179649353, "logps/chosen": -100.60639953613281, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -101.09052276611328, "loss": 1.3688, "margin_dpo/margin_mean": 4.729259967803955, "margin_dpo/margin_std": 8.748383522033691, "step": 91 }, { "KL/chosen_KL_mean": -3.409748077392578, "KL/mean": -5.582268714904785, "KL/rejected_KL_mean": -7.754783630371094, "KL/std": 7.148016929626465, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.345035552978516, "fcm_dpo/q_t": 0.4959110915660858, "grad_norm": 1.624102234840393, "learning_rate": 4.984074589033043e-07, "logits/chosen": -1.0049301385879517, "logits/rejected": -0.6252140998840332, "logps/chosen": -87.94002532958984, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -92.256103515625, "loss": 1.3702, "margin_dpo/margin_mean": 4.345035552978516, "margin_dpo/margin_std": 7.736715316772461, "step": 92 }, { "KL/chosen_KL_mean": -3.9985427856445312, "KL/mean": -5.980595588684082, "KL/rejected_KL_mean": -7.962654113769531, "KL/std": 6.8615522384643555, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.964113712310791, "fcm_dpo/q_t": 0.49626949429512024, "grad_norm": 1.2877416610717773, "learning_rate": 4.982595640958425e-07, "logits/chosen": -1.1282920837402344, "logits/rejected": -0.5296691656112671, "logps/chosen": -94.24897766113281, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -92.05686950683594, "loss": 1.3717, "margin_dpo/margin_mean": 3.964114189147949, "margin_dpo/margin_std": 8.367789268493652, "step": 93 }, { "KL/chosen_KL_mean": -4.467403411865234, "KL/mean": -7.737934589385986, "KL/rejected_KL_mean": -11.008464813232422, "KL/std": 8.529043197631836, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.541067600250244, "fcm_dpo/q_t": 0.4938449561595917, "grad_norm": 2.077260971069336, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.913996696472168, "logits/rejected": -0.5888646841049194, "logps/chosen": -96.92162322998047, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -116.25575256347656, "loss": 1.3622, "margin_dpo/margin_mean": 6.541066646575928, "margin_dpo/margin_std": 9.913118362426758, "step": 94 }, { "KL/chosen_KL_mean": -3.6254920959472656, "KL/mean": -6.197024345397949, "KL/rejected_KL_mean": -8.768550872802734, "KL/std": 7.234351634979248, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.143061637878418, "fcm_dpo/q_t": 0.49516090750694275, "grad_norm": 1.5718673467636108, "learning_rate": 4.979441529392784e-07, "logits/chosen": -1.3283727169036865, "logits/rejected": -0.9845603108406067, "logps/chosen": -82.49919891357422, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -92.3597640991211, "loss": 1.3673, "margin_dpo/margin_mean": 5.14306116104126, "margin_dpo/margin_std": 9.044772148132324, "step": 95 }, { "KL/chosen_KL_mean": -3.7330780029296875, "KL/mean": -7.0136637687683105, "KL/rejected_KL_mean": -10.29425048828125, "KL/std": 8.749151229858398, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.561176300048828, "fcm_dpo/q_t": 0.4938267469406128, "grad_norm": 1.9065179824829102, "learning_rate": 4.977766449015534e-07, "logits/chosen": -1.0577038526535034, "logits/rejected": -0.5777291059494019, "logps/chosen": -110.32524108886719, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -112.0622787475586, "loss": 1.3621, "margin_dpo/margin_mean": 6.561176300048828, "margin_dpo/margin_std": 9.848321914672852, "step": 96 }, { "KL/chosen_KL_mean": -5.284553527832031, "KL/mean": -8.113698959350586, "KL/rejected_KL_mean": -10.94283676147461, "KL/std": 8.130651473999023, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.65828275680542, "fcm_dpo/q_t": 0.49467530846595764, "grad_norm": 1.6742347478866577, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.8179422616958618, "logits/rejected": -0.39897704124450684, "logps/chosen": -93.96443176269531, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -95.755126953125, "loss": 1.3654, "margin_dpo/margin_mean": 5.658282279968262, "margin_dpo/margin_std": 8.953571319580078, "step": 97 }, { "KL/chosen_KL_mean": -4.9869537353515625, "KL/mean": -8.53247356414795, "KL/rejected_KL_mean": -12.077991485595703, "KL/std": 8.133942604064941, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.091033935546875, "fcm_dpo/q_t": 0.49332794547080994, "grad_norm": 1.8219529390335083, "learning_rate": 4.974220459770639e-07, "logits/chosen": -1.081231713294983, "logits/rejected": -0.7899430990219116, "logps/chosen": -97.22944641113281, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -113.59747314453125, "loss": 1.3601, "margin_dpo/margin_mean": 7.091033935546875, "margin_dpo/margin_std": 9.309075355529785, "step": 98 }, { "KL/chosen_KL_mean": -3.6861572265625, "KL/mean": -8.01785659790039, "KL/rejected_KL_mean": -12.349563598632812, "KL/std": 10.202335357666016, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.663414001464844, "fcm_dpo/q_t": 0.4918542802333832, "grad_norm": 2.267669677734375, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.9943547248840332, "logits/rejected": -0.7902296185493469, "logps/chosen": -75.87080383300781, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -104.23088073730469, "loss": 1.3546, "margin_dpo/margin_mean": 8.663414001464844, "margin_dpo/margin_std": 13.329263687133789, "step": 99 }, { "KL/chosen_KL_mean": -5.058589935302734, "KL/mean": -7.705911636352539, "KL/rejected_KL_mean": -10.35323715209961, "KL/std": 8.18824577331543, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.294646263122559, "fcm_dpo/q_t": 0.49501800537109375, "grad_norm": 1.2859854698181152, "learning_rate": 4.970413680203148e-07, "logits/chosen": -1.0804595947265625, "logits/rejected": -0.5294402837753296, "logps/chosen": -94.57241821289062, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -91.57036590576172, "loss": 1.3668, "margin_dpo/margin_mean": 5.294646263122559, "margin_dpo/margin_std": 9.070869445800781, "step": 100 }, { "KL/chosen_KL_mean": -7.497413635253906, "KL/mean": -10.147871971130371, "KL/rejected_KL_mean": -12.798324584960938, "KL/std": 11.714496612548828, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.300906181335449, "fcm_dpo/q_t": 0.49501127004623413, "grad_norm": 1.5357290506362915, "learning_rate": 4.968412618365215e-07, "logits/chosen": -1.4907853603363037, "logits/rejected": -1.1910130977630615, "logps/chosen": -101.90060424804688, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -102.70834350585938, "loss": 1.367, "margin_dpo/margin_mean": 5.300906181335449, "margin_dpo/margin_std": 12.57318115234375, "step": 101 }, { "KL/chosen_KL_mean": -8.626598358154297, "KL/mean": -10.696632385253906, "KL/rejected_KL_mean": -12.766666412353516, "KL/std": 11.60372257232666, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.14007568359375, "fcm_dpo/q_t": 0.4961074888706207, "grad_norm": 1.4358313083648682, "learning_rate": 4.966346511559149e-07, "logits/chosen": -1.5681395530700684, "logits/rejected": -1.1441547870635986, "logps/chosen": -100.23602294921875, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -88.53558349609375, "loss": 1.3715, "margin_dpo/margin_mean": 4.14007568359375, "margin_dpo/margin_std": 13.897628784179688, "step": 102 }, { "KL/chosen_KL_mean": -7.87237548828125, "KL/mean": -11.782609939575195, "KL/rejected_KL_mean": -15.692848205566406, "KL/std": 12.265132904052734, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.820472717285156, "fcm_dpo/q_t": 0.49264469742774963, "grad_norm": 1.6421149969100952, "learning_rate": 4.964215414228785e-07, "logits/chosen": -1.4836540222167969, "logits/rejected": -1.0320228338241577, "logps/chosen": -105.27476501464844, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -114.67453002929688, "loss": 1.3578, "margin_dpo/margin_mean": 7.820473670959473, "margin_dpo/margin_std": 14.284269332885742, "step": 103 }, { "KL/chosen_KL_mean": -7.125385284423828, "KL/mean": -11.391769409179688, "KL/rejected_KL_mean": -15.658153533935547, "KL/std": 12.811123847961426, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.532770156860352, "fcm_dpo/q_t": 0.49197977781295776, "grad_norm": 1.623204231262207, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.9859437942504883, "logits/rejected": -1.5753898620605469, "logps/chosen": -101.98175048828125, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -112.68484497070312, "loss": 1.3554, "margin_dpo/margin_mean": 8.532772064208984, "margin_dpo/margin_std": 16.082883834838867, "step": 104 }, { "KL/chosen_KL_mean": -9.007575988769531, "KL/mean": -13.570745468139648, "KL/rejected_KL_mean": -18.133914947509766, "KL/std": 11.895915985107422, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.126335144042969, "fcm_dpo/q_t": 0.49141648411750793, "grad_norm": 1.939829707145691, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.6991173028945923, "logits/rejected": -1.166365146636963, "logps/chosen": -105.76547241210938, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -119.71141052246094, "loss": 1.353, "margin_dpo/margin_mean": 9.126335144042969, "margin_dpo/margin_std": 14.688159942626953, "step": 105 }, { "KL/chosen_KL_mean": -8.91470718383789, "KL/mean": -12.749215126037598, "KL/rejected_KL_mean": -16.583721160888672, "KL/std": 12.132576942443848, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.669011116027832, "fcm_dpo/q_t": 0.4927913546562195, "grad_norm": 1.5854536294937134, "learning_rate": 4.957432749209755e-07, "logits/chosen": -1.3193466663360596, "logits/rejected": -0.8395383358001709, "logps/chosen": -101.35261535644531, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -107.59342193603516, "loss": 1.3586, "margin_dpo/margin_mean": 7.669011116027832, "margin_dpo/margin_std": 16.29126739501953, "step": 106 }, { "KL/chosen_KL_mean": -8.413341522216797, "KL/mean": -13.23744010925293, "KL/rejected_KL_mean": -18.061542510986328, "KL/std": 15.802051544189453, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.648210525512695, "fcm_dpo/q_t": 0.4909575581550598, "grad_norm": 1.8240782022476196, "learning_rate": 4.955042268449307e-07, "logits/chosen": -1.6104557514190674, "logits/rejected": -1.000460147857666, "logps/chosen": -113.53834533691406, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -119.2440414428711, "loss": 1.3518, "margin_dpo/margin_mean": 9.648210525512695, "margin_dpo/margin_std": 19.123430252075195, "step": 107 }, { "KL/chosen_KL_mean": -9.65432357788086, "KL/mean": -14.87826156616211, "KL/rejected_KL_mean": -20.10219955444336, "KL/std": 17.074356079101562, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.447867393493652, "fcm_dpo/q_t": 0.490192174911499, "grad_norm": 2.1940975189208984, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.9372913837432861, "logits/rejected": -1.489933967590332, "logps/chosen": -98.86368560791016, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -121.94004821777344, "loss": 1.3488, "margin_dpo/margin_mean": 10.447867393493652, "margin_dpo/margin_std": 20.054126739501953, "step": 108 }, { "KL/chosen_KL_mean": -8.754364013671875, "KL/mean": -14.503969192504883, "KL/rejected_KL_mean": -20.253578186035156, "KL/std": 14.881446838378906, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.499210357666016, "fcm_dpo/q_t": 0.489197701215744, "grad_norm": 1.8667724132537842, "learning_rate": 4.95006729368358e-07, "logits/chosen": -1.1734018325805664, "logits/rejected": -0.8864021301269531, "logps/chosen": -108.94145202636719, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -123.79096984863281, "loss": 1.3449, "margin_dpo/margin_mean": 11.499211311340332, "margin_dpo/margin_std": 19.957075119018555, "step": 109 }, { "KL/chosen_KL_mean": -11.276626586914062, "KL/mean": -13.929431915283203, "KL/rejected_KL_mean": -16.582237243652344, "KL/std": 14.61277961730957, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.305609703063965, "fcm_dpo/q_t": 0.49501240253448486, "grad_norm": 1.7625371217727661, "learning_rate": 4.947482930773511e-07, "logits/chosen": -1.0906040668487549, "logits/rejected": -0.5630761981010437, "logps/chosen": -118.17234802246094, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -102.39344024658203, "loss": 1.3675, "margin_dpo/margin_mean": 5.305609226226807, "margin_dpo/margin_std": 16.70790672302246, "step": 110 }, { "KL/chosen_KL_mean": -9.685504913330078, "KL/mean": -16.58721923828125, "KL/rejected_KL_mean": -23.488933563232422, "KL/std": 17.328399658203125, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.803428649902344, "fcm_dpo/q_t": 0.4870281219482422, "grad_norm": 1.9659970998764038, "learning_rate": 4.944834074412042e-07, "logits/chosen": -1.9075260162353516, "logits/rejected": -1.5954536199569702, "logps/chosen": -103.01487731933594, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -127.66641998291016, "loss": 1.3367, "margin_dpo/margin_mean": 13.803428649902344, "margin_dpo/margin_std": 21.55770492553711, "step": 111 }, { "KL/chosen_KL_mean": -9.86495590209961, "KL/mean": -12.928884506225586, "KL/rejected_KL_mean": -15.992809295654297, "KL/std": 10.730396270751953, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.127849578857422, "fcm_dpo/q_t": 0.4942377507686615, "grad_norm": 1.4309886693954468, "learning_rate": 4.942120794399002e-07, "logits/chosen": -2.262859344482422, "logits/rejected": -1.6880300045013428, "logps/chosen": -92.61199951171875, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -87.55125427246094, "loss": 1.3642, "margin_dpo/margin_mean": 6.127849578857422, "margin_dpo/margin_std": 14.970535278320312, "step": 112 }, { "KL/chosen_KL_mean": -12.770938873291016, "KL/mean": -16.118534088134766, "KL/rejected_KL_mean": -19.46613311767578, "KL/std": 15.207590103149414, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.695189952850342, "fcm_dpo/q_t": 0.4937092661857605, "grad_norm": 1.8101656436920166, "learning_rate": 4.939343162231841e-07, "logits/chosen": -1.6914732456207275, "logits/rejected": -1.0653377771377563, "logps/chosen": -110.6704330444336, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -103.88572692871094, "loss": 1.3622, "margin_dpo/margin_mean": 6.695189952850342, "margin_dpo/margin_std": 15.89450454711914, "step": 113 }, { "KL/chosen_KL_mean": -12.481117248535156, "KL/mean": -17.519729614257812, "KL/rejected_KL_mean": -22.55834197998047, "KL/std": 15.807758331298828, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.077221870422363, "fcm_dpo/q_t": 0.49053651094436646, "grad_norm": 1.832603931427002, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.9310803413391113, "logits/rejected": -1.3252551555633545, "logps/chosen": -108.97623443603516, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -114.57369995117188, "loss": 1.3504, "margin_dpo/margin_mean": 10.07722282409668, "margin_dpo/margin_std": 21.492862701416016, "step": 114 }, { "KL/chosen_KL_mean": -13.952949523925781, "KL/mean": -19.18700408935547, "KL/rejected_KL_mean": -24.42105484008789, "KL/std": 19.348716735839844, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.468104362487793, "fcm_dpo/q_t": 0.4901711940765381, "grad_norm": 1.85552978515625, "learning_rate": 4.933595135901732e-07, "logits/chosen": -1.777268409729004, "logits/rejected": -1.3289262056350708, "logps/chosen": -117.13307189941406, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -130.1370086669922, "loss": 1.3491, "margin_dpo/margin_mean": 10.468103408813477, "margin_dpo/margin_std": 22.111629486083984, "step": 115 }, { "KL/chosen_KL_mean": -11.25777816772461, "KL/mean": -15.880006790161133, "KL/rejected_KL_mean": -20.502235412597656, "KL/std": 14.817364692687988, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.244457244873047, "fcm_dpo/q_t": 0.4913122057914734, "grad_norm": 1.6458346843719482, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.8152294158935547, "logits/rejected": -1.403911828994751, "logps/chosen": -92.41615295410156, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -106.80809020996094, "loss": 1.3528, "margin_dpo/margin_mean": 9.244457244873047, "margin_dpo/margin_std": 16.354402542114258, "step": 116 }, { "KL/chosen_KL_mean": -13.989921569824219, "KL/mean": -16.162532806396484, "KL/rejected_KL_mean": -18.33515167236328, "KL/std": 15.213800430297852, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.345232009887695, "fcm_dpo/q_t": 0.4959140419960022, "grad_norm": 1.8440734148025513, "learning_rate": 4.927590601281083e-07, "logits/chosen": -1.5919244289398193, "logits/rejected": -1.0603344440460205, "logps/chosen": -111.07087707519531, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -93.47506713867188, "loss": 1.3716, "margin_dpo/margin_mean": 4.345231533050537, "margin_dpo/margin_std": 20.94216537475586, "step": 117 }, { "KL/chosen_KL_mean": -10.915836334228516, "KL/mean": -15.880392074584961, "KL/rejected_KL_mean": -20.84493637084961, "KL/std": 15.84268569946289, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.929101943969727, "fcm_dpo/q_t": 0.4906686544418335, "grad_norm": 1.641583800315857, "learning_rate": 4.924492340087524e-07, "logits/chosen": -2.0882692337036133, "logits/rejected": -1.7400763034820557, "logps/chosen": -97.45207214355469, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -103.78057861328125, "loss": 1.3504, "margin_dpo/margin_mean": 9.929101943969727, "margin_dpo/margin_std": 17.761028289794922, "step": 118 }, { "KL/chosen_KL_mean": -14.533855438232422, "KL/mean": -20.239587783813477, "KL/rejected_KL_mean": -25.945316314697266, "KL/std": 18.63920783996582, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.411462783813477, "fcm_dpo/q_t": 0.4892829358577728, "grad_norm": 1.947650671005249, "learning_rate": 4.92133019126601e-07, "logits/chosen": -2.0894861221313477, "logits/rejected": -1.7174804210662842, "logps/chosen": -119.14293670654297, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -135.10311889648438, "loss": 1.3458, "margin_dpo/margin_mean": 11.41146183013916, "margin_dpo/margin_std": 24.08114242553711, "step": 119 }, { "KL/chosen_KL_mean": -15.32611083984375, "KL/mean": -21.485897064208984, "KL/rejected_KL_mean": -27.645675659179688, "KL/std": 21.37537384033203, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.319561004638672, "fcm_dpo/q_t": 0.4884437322616577, "grad_norm": 2.0123207569122314, "learning_rate": 4.918104238142103e-07, "logits/chosen": -2.3319880962371826, "logits/rejected": -1.7333265542984009, "logps/chosen": -130.01353454589844, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -138.83154296875, "loss": 1.3432, "margin_dpo/margin_mean": 12.319560050964355, "margin_dpo/margin_std": 27.812061309814453, "step": 120 }, { "KL/chosen_KL_mean": -12.52542495727539, "KL/mean": -19.83320426940918, "KL/rejected_KL_mean": -27.1409912109375, "KL/std": 18.011474609375, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.615571022033691, "fcm_dpo/q_t": 0.48626479506492615, "grad_norm": 2.394714593887329, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.9452611207962036, "logits/rejected": -1.6282463073730469, "logps/chosen": -104.64208984375, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -142.1138916015625, "loss": 1.3335, "margin_dpo/margin_mean": 14.615570068359375, "margin_dpo/margin_std": 20.283851623535156, "step": 121 }, { "KL/chosen_KL_mean": -11.998394012451172, "KL/mean": -20.379356384277344, "KL/rejected_KL_mean": -28.76031494140625, "KL/std": 18.289180755615234, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.761917114257812, "fcm_dpo/q_t": 0.48425790667533875, "grad_norm": 2.3393750190734863, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.796252965927124, "logits/rejected": -1.7952725887298584, "logps/chosen": -96.87596130371094, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -139.84329223632812, "loss": 1.326, "margin_dpo/margin_mean": 16.761917114257812, "margin_dpo/margin_std": 22.488941192626953, "step": 122 }, { "KL/chosen_KL_mean": -14.218936920166016, "KL/mean": -19.51169776916504, "KL/rejected_KL_mean": -24.804462432861328, "KL/std": 16.75397300720215, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.58552360534668, "fcm_dpo/q_t": 0.4900548458099365, "grad_norm": 1.7434991598129272, "learning_rate": 4.908044411417711e-07, "logits/chosen": -2.215064287185669, "logits/rejected": -1.8921101093292236, "logps/chosen": -112.69768524169922, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -118.65863800048828, "loss": 1.3485, "margin_dpo/margin_mean": 10.58552360534668, "margin_dpo/margin_std": 21.293785095214844, "step": 123 }, { "KL/chosen_KL_mean": -15.35445785522461, "KL/mean": -22.458881378173828, "KL/rejected_KL_mean": -29.563308715820312, "KL/std": 21.99361801147461, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.208845138549805, "fcm_dpo/q_t": 0.4866894483566284, "grad_norm": 2.095907688140869, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.9049022197723389, "logits/rejected": -1.6752650737762451, "logps/chosen": -118.11735534667969, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -152.5689239501953, "loss": 1.3367, "margin_dpo/margin_mean": 14.208845138549805, "margin_dpo/margin_std": 29.1323184967041, "step": 124 }, { "KL/chosen_KL_mean": -14.558956146240234, "KL/mean": -18.939876556396484, "KL/rejected_KL_mean": -23.320796966552734, "KL/std": 17.74832534790039, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.76183795928955, "fcm_dpo/q_t": 0.49177074432373047, "grad_norm": 1.7066810131072998, "learning_rate": 4.90102044194588e-07, "logits/chosen": -1.7036805152893066, "logits/rejected": -1.4429402351379395, "logps/chosen": -104.9852294921875, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -112.56980895996094, "loss": 1.3554, "margin_dpo/margin_mean": 8.76183795928955, "margin_dpo/margin_std": 22.675634384155273, "step": 125 }, { "KL/chosen_KL_mean": -14.655536651611328, "KL/mean": -21.364564895629883, "KL/rejected_KL_mean": -28.073596954345703, "KL/std": 21.28351593017578, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.418062210083008, "fcm_dpo/q_t": 0.4874027371406555, "grad_norm": 1.8504302501678467, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.8322618007659912, "logits/rejected": -1.5590441226959229, "logps/chosen": -109.07841491699219, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -131.9551544189453, "loss": 1.3386, "margin_dpo/margin_mean": 13.418062210083008, "margin_dpo/margin_std": 24.87466812133789, "step": 126 }, { "KL/chosen_KL_mean": -14.206920623779297, "KL/mean": -18.448150634765625, "KL/rejected_KL_mean": -22.68938446044922, "KL/std": 17.073810577392578, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.482463836669922, "fcm_dpo/q_t": 0.49203529953956604, "grad_norm": 1.758650302886963, "learning_rate": 4.89374339765481e-07, "logits/chosen": -1.7308814525604248, "logits/rejected": -1.3430821895599365, "logps/chosen": -98.95797729492188, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -106.234619140625, "loss": 1.3563, "margin_dpo/margin_mean": 8.482463836669922, "margin_dpo/margin_std": 21.593029022216797, "step": 127 }, { "KL/chosen_KL_mean": -14.585872650146484, "KL/mean": -19.312314987182617, "KL/rejected_KL_mean": -24.038759231567383, "KL/std": 20.943017959594727, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.452888488769531, "fcm_dpo/q_t": 0.4911378026008606, "grad_norm": 1.8691613674163818, "learning_rate": 4.890010211106795e-07, "logits/chosen": -1.713561773300171, "logits/rejected": -1.1743173599243164, "logps/chosen": -105.20552062988281, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -107.1422348022461, "loss": 1.3538, "margin_dpo/margin_mean": 9.452888488769531, "margin_dpo/margin_std": 27.828866958618164, "step": 128 }, { "KL/chosen_KL_mean": -15.593013763427734, "KL/mean": -21.141332626342773, "KL/rejected_KL_mean": -26.689651489257812, "KL/std": 20.506881713867188, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.09663200378418, "fcm_dpo/q_t": 0.48963189125061035, "grad_norm": 1.7478585243225098, "learning_rate": 4.88621404556699e-07, "logits/chosen": -1.8071861267089844, "logits/rejected": -1.4907528162002563, "logps/chosen": -114.51171875, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -129.97384643554688, "loss": 1.3481, "margin_dpo/margin_mean": 11.09663200378418, "margin_dpo/margin_std": 29.323951721191406, "step": 129 }, { "KL/chosen_KL_mean": -12.362895965576172, "KL/mean": -19.275590896606445, "KL/rejected_KL_mean": -26.188289642333984, "KL/std": 17.823532104492188, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.82539176940918, "fcm_dpo/q_t": 0.4870174527168274, "grad_norm": 1.884419322013855, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.9599928855895996, "logits/rejected": -1.7687820196151733, "logps/chosen": -85.54658508300781, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -114.20983123779297, "loss": 1.3369, "margin_dpo/margin_mean": 13.825392723083496, "margin_dpo/margin_std": 22.98314666748047, "step": 130 }, { "KL/chosen_KL_mean": -11.338382720947266, "KL/mean": -19.285655975341797, "KL/rejected_KL_mean": -27.232929229736328, "KL/std": 20.295347213745117, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.894546508789062, "fcm_dpo/q_t": 0.4850744307041168, "grad_norm": 2.245588541030884, "learning_rate": 4.878433179298909e-07, "logits/chosen": -2.096444606781006, "logits/rejected": -1.952824592590332, "logps/chosen": -85.29483032226562, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -121.68875122070312, "loss": 1.3294, "margin_dpo/margin_mean": 15.894545555114746, "margin_dpo/margin_std": 23.553970336914062, "step": 131 }, { "KL/chosen_KL_mean": -15.291973114013672, "KL/mean": -21.438535690307617, "KL/rejected_KL_mean": -27.58509063720703, "KL/std": 21.42401123046875, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.293122291564941, "fcm_dpo/q_t": 0.4884781837463379, "grad_norm": 1.9617340564727783, "learning_rate": 4.874448683603694e-07, "logits/chosen": -2.2113022804260254, "logits/rejected": -1.792417049407959, "logps/chosen": -107.73146057128906, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -121.71760559082031, "loss": 1.3436, "margin_dpo/margin_mean": 12.293123245239258, "margin_dpo/margin_std": 28.981903076171875, "step": 132 }, { "KL/chosen_KL_mean": -15.099472045898438, "KL/mean": -21.742048263549805, "KL/rejected_KL_mean": -28.384624481201172, "KL/std": 19.578561782836914, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.285152435302734, "fcm_dpo/q_t": 0.4875280261039734, "grad_norm": 1.9026446342468262, "learning_rate": 4.870401618977415e-07, "logits/chosen": -2.384082078933716, "logits/rejected": -1.995592474937439, "logps/chosen": -108.21613311767578, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -130.4795379638672, "loss": 1.3387, "margin_dpo/margin_mean": 13.285152435302734, "margin_dpo/margin_std": 22.695117950439453, "step": 133 }, { "KL/chosen_KL_mean": -14.522712707519531, "KL/mean": -21.771076202392578, "KL/rejected_KL_mean": -29.019439697265625, "KL/std": 18.694076538085938, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.496726036071777, "fcm_dpo/q_t": 0.4863850474357605, "grad_norm": 2.0354928970336914, "learning_rate": 4.866292092063986e-07, "logits/chosen": -2.3649649620056152, "logits/rejected": -2.0470685958862305, "logps/chosen": -98.07283782958984, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -122.90100860595703, "loss": 1.3342, "margin_dpo/margin_mean": 14.496726036071777, "margin_dpo/margin_std": 21.151653289794922, "step": 134 }, { "KL/chosen_KL_mean": -13.12258529663086, "KL/mean": -22.634937286376953, "KL/rejected_KL_mean": -32.14729309082031, "KL/std": 22.838680267333984, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.02469825744629, "fcm_dpo/q_t": 0.48219308257102966, "grad_norm": 2.8875441551208496, "learning_rate": 4.862120211153265e-07, "logits/chosen": -2.1365926265716553, "logits/rejected": -2.1511483192443848, "logps/chosen": -90.70521545410156, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -153.20376586914062, "loss": 1.3192, "margin_dpo/margin_mean": 19.024700164794922, "margin_dpo/margin_std": 28.753055572509766, "step": 135 }, { "KL/chosen_KL_mean": -17.545818328857422, "KL/mean": -24.45079803466797, "KL/rejected_KL_mean": -31.355777740478516, "KL/std": 25.85124969482422, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.809962272644043, "fcm_dpo/q_t": 0.48710912466049194, "grad_norm": 2.133892774581909, "learning_rate": 4.857886086178193e-07, "logits/chosen": -2.3070719242095947, "logits/rejected": -1.9453822374343872, "logps/chosen": -111.71855163574219, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -134.6200714111328, "loss": 1.3385, "margin_dpo/margin_mean": 13.809962272644043, "margin_dpo/margin_std": 31.5003719329834, "step": 136 }, { "KL/chosen_KL_mean": -13.425834655761719, "KL/mean": -26.3138427734375, "KL/rejected_KL_mean": -39.20184326171875, "KL/std": 29.751449584960938, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.776016235351562, "fcm_dpo/q_t": 0.4759446978569031, "grad_norm": 2.641160726547241, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.9102423191070557, "logits/rejected": -1.8042938709259033, "logps/chosen": -91.54457092285156, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -157.12420654296875, "loss": 1.2962, "margin_dpo/margin_mean": 25.776016235351562, "margin_dpo/margin_std": 35.66950988769531, "step": 137 }, { "KL/chosen_KL_mean": -18.014240264892578, "KL/mean": -24.834739685058594, "KL/rejected_KL_mean": -31.65523910522461, "KL/std": 21.567935943603516, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.64100170135498, "fcm_dpo/q_t": 0.48719322681427, "grad_norm": 1.9668843746185303, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.8587028980255127, "logits/rejected": -1.5632383823394775, "logps/chosen": -110.84418487548828, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -130.30364990234375, "loss": 1.3378, "margin_dpo/margin_mean": 13.641002655029297, "margin_dpo/margin_std": 25.048171997070312, "step": 138 }, { "KL/chosen_KL_mean": -13.090530395507812, "KL/mean": -21.042537689208984, "KL/rejected_KL_mean": -28.994544982910156, "KL/std": 19.96964454650879, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.904023170471191, "fcm_dpo/q_t": 0.48506391048431396, "grad_norm": 1.9353021383285522, "learning_rate": 4.844811370781446e-07, "logits/chosen": -2.229693651199341, "logits/rejected": -1.932344913482666, "logps/chosen": -92.09354400634766, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -114.8218765258789, "loss": 1.3297, "margin_dpo/margin_mean": 15.904022216796875, "margin_dpo/margin_std": 25.99072265625, "step": 139 }, { "KL/chosen_KL_mean": -15.905105590820312, "KL/mean": -22.741615295410156, "KL/rejected_KL_mean": -29.578121185302734, "KL/std": 19.976581573486328, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.673019409179688, "fcm_dpo/q_t": 0.48715919256210327, "grad_norm": 1.946937918663025, "learning_rate": 4.840329401637809e-07, "logits/chosen": -2.5170767307281494, "logits/rejected": -2.128420114517212, "logps/chosen": -104.25755310058594, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -120.15289306640625, "loss": 1.3376, "margin_dpo/margin_mean": 13.67302131652832, "margin_dpo/margin_std": 24.29299545288086, "step": 140 }, { "KL/chosen_KL_mean": -21.761993408203125, "KL/mean": -28.51456642150879, "KL/rejected_KL_mean": -35.26713180541992, "KL/std": 29.853912353515625, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.505146026611328, "fcm_dpo/q_t": 0.487299382686615, "grad_norm": 2.110957145690918, "learning_rate": 4.83578576263792e-07, "logits/chosen": -2.121577262878418, "logits/rejected": -1.6628637313842773, "logps/chosen": -128.98214721679688, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -139.9793701171875, "loss": 1.3397, "margin_dpo/margin_mean": 13.505145072937012, "margin_dpo/margin_std": 31.551616668701172, "step": 141 }, { "KL/chosen_KL_mean": -17.6185302734375, "KL/mean": -25.058731079101562, "KL/rejected_KL_mean": -32.49893569946289, "KL/std": 24.494319915771484, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.880409240722656, "fcm_dpo/q_t": 0.4860525131225586, "grad_norm": 1.9830352067947388, "learning_rate": 4.83118057351089e-07, "logits/chosen": -2.0524654388427734, "logits/rejected": -1.8107516765594482, "logps/chosen": -106.34747314453125, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -132.2293243408203, "loss": 1.3336, "margin_dpo/margin_mean": 14.880409240722656, "margin_dpo/margin_std": 26.49492645263672, "step": 142 }, { "KL/chosen_KL_mean": -17.710403442382812, "KL/mean": -22.73883056640625, "KL/rejected_KL_mean": -27.767253875732422, "KL/std": 20.014869689941406, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.056848526000977, "fcm_dpo/q_t": 0.490556001663208, "grad_norm": 2.0982582569122314, "learning_rate": 4.826513955607734e-07, "logits/chosen": -2.423271656036377, "logits/rejected": -2.0048747062683105, "logps/chosen": -103.21498107910156, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -113.57291412353516, "loss": 1.3505, "margin_dpo/margin_mean": 10.05684757232666, "margin_dpo/margin_std": 21.51106071472168, "step": 143 }, { "KL/chosen_KL_mean": -22.250553131103516, "KL/mean": -26.092159271240234, "KL/rejected_KL_mean": -29.933765411376953, "KL/std": 22.336050033569336, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.683212757110596, "fcm_dpo/q_t": 0.49277523159980774, "grad_norm": 2.5046608448028564, "learning_rate": 4.821786031898176e-07, "logits/chosen": -2.268043041229248, "logits/rejected": -1.6632888317108154, "logps/chosen": -126.9376449584961, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -117.65284729003906, "loss": 1.3603, "margin_dpo/margin_mean": 7.683213233947754, "margin_dpo/margin_std": 27.472919464111328, "step": 144 }, { "KL/chosen_KL_mean": -16.57269287109375, "KL/mean": -22.928998947143555, "KL/rejected_KL_mean": -29.28531265258789, "KL/std": 20.22036361694336, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.712616920471191, "fcm_dpo/q_t": 0.4880659878253937, "grad_norm": 2.1981403827667236, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.8920148611068726, "logits/rejected": -1.3593122959136963, "logps/chosen": -105.09258270263672, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -113.05045318603516, "loss": 1.341, "margin_dpo/margin_mean": 12.712617874145508, "margin_dpo/margin_std": 23.377012252807617, "step": 145 }, { "KL/chosen_KL_mean": -18.85649871826172, "KL/mean": -25.00076675415039, "KL/rejected_KL_mean": -31.14502716064453, "KL/std": 21.466808319091797, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.288524627685547, "fcm_dpo/q_t": 0.4884573519229889, "grad_norm": 2.43476939201355, "learning_rate": 4.812146767012779e-07, "logits/chosen": -2.3199334144592285, "logits/rejected": -1.6070849895477295, "logps/chosen": -121.76235961914062, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -120.09439086914062, "loss": 1.3431, "margin_dpo/margin_mean": 12.288524627685547, "margin_dpo/margin_std": 26.741411209106445, "step": 146 }, { "KL/chosen_KL_mean": -17.155933380126953, "KL/mean": -23.658031463623047, "KL/rejected_KL_mean": -30.160133361816406, "KL/std": 19.355243682861328, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.004205703735352, "fcm_dpo/q_t": 0.487796813249588, "grad_norm": 2.2023377418518066, "learning_rate": 4.807235679840536e-07, "logits/chosen": -2.243098735809326, "logits/rejected": -1.669677972793579, "logps/chosen": -106.41812133789062, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -106.66024780273438, "loss": 1.3397, "margin_dpo/margin_mean": 13.004204750061035, "margin_dpo/margin_std": 22.628128051757812, "step": 147 }, { "KL/chosen_KL_mean": -15.518390655517578, "KL/mean": -23.668842315673828, "KL/rejected_KL_mean": -31.819290161132812, "KL/std": 23.872791290283203, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.300901412963867, "fcm_dpo/q_t": 0.4847491979598999, "grad_norm": 2.1001555919647217, "learning_rate": 4.802263794862384e-07, "logits/chosen": -2.077193021774292, "logits/rejected": -1.7084169387817383, "logps/chosen": -107.2812728881836, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -142.20481872558594, "loss": 1.3292, "margin_dpo/margin_mean": 16.300899505615234, "margin_dpo/margin_std": 27.997364044189453, "step": 148 }, { "KL/chosen_KL_mean": -19.361820220947266, "KL/mean": -26.416486740112305, "KL/rejected_KL_mean": -33.47114181518555, "KL/std": 25.459692001342773, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.109317779541016, "fcm_dpo/q_t": 0.48682793974876404, "grad_norm": 2.0347676277160645, "learning_rate": 4.797231243092118e-07, "logits/chosen": -2.229409694671631, "logits/rejected": -1.8569738864898682, "logps/chosen": -112.84584045410156, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -139.11483764648438, "loss": 1.3383, "margin_dpo/margin_mean": 14.109317779541016, "margin_dpo/margin_std": 34.17460632324219, "step": 149 }, { "KL/chosen_KL_mean": -15.81832504272461, "KL/mean": -23.29100227355957, "KL/rejected_KL_mean": -30.763683319091797, "KL/std": 25.18400001525879, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.945354461669922, "fcm_dpo/q_t": 0.48598891496658325, "grad_norm": 2.0218560695648193, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.9714312553405762, "logits/rejected": -1.7712793350219727, "logps/chosen": -91.59701538085938, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -120.79045104980469, "loss": 1.3336, "margin_dpo/margin_mean": 14.945356369018555, "margin_dpo/margin_std": 28.07516098022461, "step": 150 }, { "KL/chosen_KL_mean": -20.208450317382812, "KL/mean": -27.39999771118164, "KL/rejected_KL_mean": -34.59154510498047, "KL/std": 24.995502471923828, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.383101463317871, "fcm_dpo/q_t": 0.4865266680717468, "grad_norm": 2.26743745803833, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.7252001762390137, "logits/rejected": -2.1537907123565674, "logps/chosen": -122.94267272949219, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -140.62936401367188, "loss": 1.3363, "margin_dpo/margin_mean": 14.383099555969238, "margin_dpo/margin_std": 30.62557601928711, "step": 151 }, { "KL/chosen_KL_mean": -17.207748413085938, "KL/mean": -26.498416900634766, "KL/rejected_KL_mean": -35.78907775878906, "KL/std": 24.630258560180664, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.58131980895996, "fcm_dpo/q_t": 0.4825816750526428, "grad_norm": 2.1739070415496826, "learning_rate": 4.78177092112495e-07, "logits/chosen": -2.2305684089660645, "logits/rejected": -1.8285017013549805, "logps/chosen": -114.79849243164062, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -146.2882080078125, "loss": 1.3213, "margin_dpo/margin_mean": 18.581317901611328, "margin_dpo/margin_std": 31.55504608154297, "step": 152 }, { "KL/chosen_KL_mean": -17.82846450805664, "KL/mean": -24.061424255371094, "KL/rejected_KL_mean": -30.294384002685547, "KL/std": 24.049795150756836, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.465923309326172, "fcm_dpo/q_t": 0.48832300305366516, "grad_norm": 2.126957654953003, "learning_rate": 4.776497044244016e-07, "logits/chosen": -2.5173401832580566, "logits/rejected": -2.211559772491455, "logps/chosen": -104.85406494140625, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -121.36292266845703, "loss": 1.3433, "margin_dpo/margin_mean": 12.465921401977539, "margin_dpo/margin_std": 30.838661193847656, "step": 153 }, { "KL/chosen_KL_mean": -17.40404510498047, "KL/mean": -27.790258407592773, "KL/rejected_KL_mean": -38.17646789550781, "KL/std": 25.20508575439453, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.772436141967773, "fcm_dpo/q_t": 0.48053407669067383, "grad_norm": 2.305920362472534, "learning_rate": 4.771163179548808e-07, "logits/chosen": -2.0020737648010254, "logits/rejected": -1.7172210216522217, "logps/chosen": -108.50971221923828, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -149.54885864257812, "loss": 1.3134, "margin_dpo/margin_mean": 20.772436141967773, "margin_dpo/margin_std": 32.578338623046875, "step": 154 }, { "KL/chosen_KL_mean": -19.636581420898438, "KL/mean": -25.864543914794922, "KL/rejected_KL_mean": -32.092506408691406, "KL/std": 24.769248962402344, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.4559326171875, "fcm_dpo/q_t": 0.4882754683494568, "grad_norm": 2.1405575275421143, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -2.380364418029785, "logits/rejected": -1.9519416093826294, "logps/chosen": -114.19755554199219, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -116.86609649658203, "loss": 1.3439, "margin_dpo/margin_mean": 12.4559326171875, "margin_dpo/margin_std": 33.504234313964844, "step": 155 }, { "KL/chosen_KL_mean": -22.403762817382812, "KL/mean": -27.927812576293945, "KL/rejected_KL_mean": -33.45186233520508, "KL/std": 23.984846115112305, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.048100471496582, "fcm_dpo/q_t": 0.48968520760536194, "grad_norm": 2.1949446201324463, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -2.1671767234802246, "logits/rejected": -1.6757879257202148, "logps/chosen": -117.05915069580078, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -117.87120056152344, "loss": 1.3492, "margin_dpo/margin_mean": 11.048097610473633, "margin_dpo/margin_std": 32.47358703613281, "step": 156 }, { "KL/chosen_KL_mean": -20.336463928222656, "KL/mean": -31.510936737060547, "KL/rejected_KL_mean": -42.6854133605957, "KL/std": 28.041786193847656, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.348949432373047, "fcm_dpo/q_t": 0.4790509343147278, "grad_norm": 2.366838216781616, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.60667085647583, "logits/rejected": -2.206681728363037, "logps/chosen": -117.3488540649414, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -154.38674926757812, "loss": 1.3081, "margin_dpo/margin_mean": 22.348947525024414, "margin_dpo/margin_std": 33.39870071411133, "step": 157 }, { "KL/chosen_KL_mean": -19.57443618774414, "KL/mean": -30.24655532836914, "KL/rejected_KL_mean": -40.918678283691406, "KL/std": 30.051069259643555, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.344249725341797, "fcm_dpo/q_t": 0.48006734251976013, "grad_norm": 2.408869981765747, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -2.237668991088867, "logits/rejected": -2.0474178791046143, "logps/chosen": -114.28739929199219, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -161.85601806640625, "loss": 1.3122, "margin_dpo/margin_mean": 21.344249725341797, "margin_dpo/margin_std": 36.51007080078125, "step": 158 }, { "KL/chosen_KL_mean": -21.790023803710938, "KL/mean": -29.488998413085938, "KL/rejected_KL_mean": -37.1879768371582, "KL/std": 28.082319259643555, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.397965431213379, "fcm_dpo/q_t": 0.485576331615448, "grad_norm": 2.3596458435058594, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.730210304260254, "logits/rejected": -2.2260165214538574, "logps/chosen": -130.00827026367188, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -132.02700805664062, "loss": 1.3329, "margin_dpo/margin_mean": 15.397965431213379, "margin_dpo/margin_std": 32.51791763305664, "step": 159 }, { "KL/chosen_KL_mean": -18.31694793701172, "KL/mean": -26.75783920288086, "KL/rejected_KL_mean": -35.198726654052734, "KL/std": 22.432659149169922, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.88176727294922, "fcm_dpo/q_t": 0.48415571451187134, "grad_norm": 2.162680149078369, "learning_rate": 4.737908228387656e-07, "logits/chosen": -2.3376317024230957, "logits/rejected": -1.860319972038269, "logps/chosen": -116.48036193847656, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -137.47970581054688, "loss": 1.3262, "margin_dpo/margin_mean": 16.88176727294922, "margin_dpo/margin_std": 26.067428588867188, "step": 160 }, { "KL/chosen_KL_mean": -18.792457580566406, "KL/mean": -25.691781997680664, "KL/rejected_KL_mean": -32.59110641479492, "KL/std": 21.208423614501953, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.79865550994873, "fcm_dpo/q_t": 0.487063467502594, "grad_norm": 2.119255781173706, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -2.4434123039245605, "logits/rejected": -1.9870522022247314, "logps/chosen": -105.38432312011719, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -121.6075439453125, "loss": 1.3384, "margin_dpo/margin_mean": 13.798654556274414, "margin_dpo/margin_std": 30.162302017211914, "step": 161 }, { "KL/chosen_KL_mean": -18.63315200805664, "KL/mean": -26.898284912109375, "KL/rejected_KL_mean": -35.163421630859375, "KL/std": 26.00421142578125, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.53026580810547, "fcm_dpo/q_t": 0.48451390862464905, "grad_norm": 2.2544848918914795, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.8784273862838745, "logits/rejected": -1.4644958972930908, "logps/chosen": -116.03604125976562, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -131.03265380859375, "loss": 1.3286, "margin_dpo/margin_mean": 16.530263900756836, "margin_dpo/margin_std": 30.91241455078125, "step": 162 }, { "KL/chosen_KL_mean": -16.480274200439453, "KL/mean": -25.69083595275879, "KL/rejected_KL_mean": -34.90140151977539, "KL/std": 25.720428466796875, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.421127319335938, "fcm_dpo/q_t": 0.48276054859161377, "grad_norm": 2.1648406982421875, "learning_rate": 4.720482655449212e-07, "logits/chosen": -2.3249144554138184, "logits/rejected": -1.9463683366775513, "logps/chosen": -98.65567016601562, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -117.76614379882812, "loss": 1.3215, "margin_dpo/margin_mean": 18.421127319335938, "margin_dpo/margin_std": 30.619348526000977, "step": 163 }, { "KL/chosen_KL_mean": -20.903949737548828, "KL/mean": -27.84193229675293, "KL/rejected_KL_mean": -34.77991485595703, "KL/std": 21.145523071289062, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.875960350036621, "fcm_dpo/q_t": 0.48698121309280396, "grad_norm": 2.246858596801758, "learning_rate": 4.714556901942599e-07, "logits/chosen": -2.140094757080078, "logits/rejected": -1.6776025295257568, "logps/chosen": -111.37821960449219, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -121.8080062866211, "loss": 1.3376, "margin_dpo/margin_mean": 13.875959396362305, "margin_dpo/margin_std": 28.024425506591797, "step": 164 }, { "KL/chosen_KL_mean": -15.684490203857422, "KL/mean": -24.439868927001953, "KL/rejected_KL_mean": -33.195247650146484, "KL/std": 21.90814971923828, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.510759353637695, "fcm_dpo/q_t": 0.483608216047287, "grad_norm": 2.048889398574829, "learning_rate": 4.708572792802069e-07, "logits/chosen": -2.378330707550049, "logits/rejected": -1.9174680709838867, "logps/chosen": -102.99771881103516, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -115.09783172607422, "loss": 1.3244, "margin_dpo/margin_mean": 17.510759353637695, "margin_dpo/margin_std": 28.005929946899414, "step": 165 }, { "KL/chosen_KL_mean": -19.772720336914062, "KL/mean": -31.306150436401367, "KL/rejected_KL_mean": -42.83959197998047, "KL/std": 30.460031509399414, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.06686019897461, "fcm_dpo/q_t": 0.47853541374206543, "grad_norm": 2.601170301437378, "learning_rate": 4.702530485714461e-07, "logits/chosen": -2.115992546081543, "logits/rejected": -1.9561371803283691, "logps/chosen": -101.3587875366211, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -146.0047607421875, "loss": 1.3069, "margin_dpo/margin_mean": 23.06686019897461, "margin_dpo/margin_std": 39.302467346191406, "step": 166 }, { "KL/chosen_KL_mean": -19.268898010253906, "KL/mean": -30.340309143066406, "KL/rejected_KL_mean": -41.41171646118164, "KL/std": 26.39439582824707, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.142818450927734, "fcm_dpo/q_t": 0.4792678952217102, "grad_norm": 2.457772970199585, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.4153835773468018, "logits/rejected": -1.274256944656372, "logps/chosen": -102.83056640625, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -140.4647674560547, "loss": 1.3082, "margin_dpo/margin_mean": 22.142818450927734, "margin_dpo/margin_std": 31.4215087890625, "step": 167 }, { "KL/chosen_KL_mean": -17.774131774902344, "KL/mean": -26.893611907958984, "KL/rejected_KL_mean": -36.013092041015625, "KL/std": 23.290193557739258, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.238956451416016, "fcm_dpo/q_t": 0.4829026460647583, "grad_norm": 2.1435091495513916, "learning_rate": 4.690271916109034e-07, "logits/chosen": -2.1773221492767334, "logits/rejected": -1.8361870050430298, "logps/chosen": -97.70536804199219, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -116.1152572631836, "loss": 1.3216, "margin_dpo/margin_mean": 18.238956451416016, "margin_dpo/margin_std": 27.962547302246094, "step": 168 }, { "KL/chosen_KL_mean": -15.760658264160156, "KL/mean": -26.916465759277344, "KL/rejected_KL_mean": -38.072269439697266, "KL/std": 26.823396682739258, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.311616897583008, "fcm_dpo/q_t": 0.4791010916233063, "grad_norm": 2.1101033687591553, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -2.436420202255249, "logits/rejected": -2.2465391159057617, "logps/chosen": -98.66436767578125, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -128.19418334960938, "loss": 1.3074, "margin_dpo/margin_mean": 22.31161880493164, "margin_dpo/margin_std": 29.966922760009766, "step": 169 }, { "KL/chosen_KL_mean": -17.258705139160156, "KL/mean": -25.889793395996094, "KL/rejected_KL_mean": -34.5208854675293, "KL/std": 21.804645538330078, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.26218032836914, "fcm_dpo/q_t": 0.4837963283061981, "grad_norm": 2.3619494438171387, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.9465866088867188, "logits/rejected": -1.763693928718567, "logps/chosen": -92.24067687988281, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -116.68649291992188, "loss": 1.3246, "margin_dpo/margin_mean": 17.26218032836914, "margin_dpo/margin_std": 24.53631591796875, "step": 170 }, { "KL/chosen_KL_mean": -27.477527618408203, "KL/mean": -36.19610595703125, "KL/rejected_KL_mean": -44.9146842956543, "KL/std": 37.320953369140625, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.43715476989746, "fcm_dpo/q_t": 0.48384037613868713, "grad_norm": 2.627336025238037, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -2.3340935707092285, "logits/rejected": -1.758113145828247, "logps/chosen": -146.4560546875, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -160.68399047851562, "loss": 1.3304, "margin_dpo/margin_mean": 17.43715476989746, "margin_dpo/margin_std": 49.729610443115234, "step": 171 }, { "KL/chosen_KL_mean": -26.432022094726562, "KL/mean": -33.17063903808594, "KL/rejected_KL_mean": -39.90924835205078, "KL/std": 25.095430374145508, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.477224349975586, "fcm_dpo/q_t": 0.4873490333557129, "grad_norm": 2.560760736465454, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.9630279541015625, "logits/rejected": -2.1471853256225586, "logps/chosen": -127.5858154296875, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -130.91493225097656, "loss": 1.3382, "margin_dpo/margin_mean": 13.47722339630127, "margin_dpo/margin_std": 23.584453582763672, "step": 172 }, { "KL/chosen_KL_mean": -20.357288360595703, "KL/mean": -27.428783416748047, "KL/rejected_KL_mean": -34.500282287597656, "KL/std": 23.40279769897461, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.14299488067627, "fcm_dpo/q_t": 0.4867244362831116, "grad_norm": 2.182478904724121, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -2.2985358238220215, "logits/rejected": -1.7650365829467773, "logps/chosen": -119.41397094726562, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -118.35101318359375, "loss": 1.3359, "margin_dpo/margin_mean": 14.14299488067627, "margin_dpo/margin_std": 24.41901969909668, "step": 173 }, { "KL/chosen_KL_mean": -20.582590103149414, "KL/mean": -32.914283752441406, "KL/rejected_KL_mean": -45.2459716796875, "KL/std": 33.913856506347656, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.66338539123535, "fcm_dpo/q_t": 0.4770069718360901, "grad_norm": 2.5327768325805664, "learning_rate": 4.652116329460919e-07, "logits/chosen": -2.3354737758636475, "logits/rejected": -2.352814197540283, "logps/chosen": -100.99150085449219, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -153.44769287109375, "loss": 1.3012, "margin_dpo/margin_mean": 24.66338539123535, "margin_dpo/margin_std": 39.778751373291016, "step": 174 }, { "KL/chosen_KL_mean": -24.181076049804688, "KL/mean": -34.961727142333984, "KL/rejected_KL_mean": -45.74237060546875, "KL/std": 29.020261764526367, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.56130027770996, "fcm_dpo/q_t": 0.4797707200050354, "grad_norm": 2.633884906768799, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.7398009300231934, "logits/rejected": -1.4604401588439941, "logps/chosen": -106.89756774902344, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -142.4095001220703, "loss": 1.3104, "margin_dpo/margin_mean": 21.56130027770996, "margin_dpo/margin_std": 31.80907440185547, "step": 175 }, { "KL/chosen_KL_mean": -19.761180877685547, "KL/mean": -32.77143859863281, "KL/rejected_KL_mean": -45.78169631958008, "KL/std": 30.260982513427734, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.0205078125, "fcm_dpo/q_t": 0.4756753444671631, "grad_norm": 2.999523162841797, "learning_rate": 4.638942309888058e-07, "logits/chosen": -2.242743492126465, "logits/rejected": -2.147759199142456, "logps/chosen": -99.39048767089844, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -148.4702911376953, "loss": 1.2955, "margin_dpo/margin_mean": 26.0205078125, "margin_dpo/margin_std": 36.93697738647461, "step": 176 }, { "KL/chosen_KL_mean": -24.08218765258789, "KL/mean": -37.30190658569336, "KL/rejected_KL_mean": -50.52162551879883, "KL/std": 34.79900360107422, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.439434051513672, "fcm_dpo/q_t": 0.4753310978412628, "grad_norm": 2.761540651321411, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -2.201399803161621, "logits/rejected": -1.881829023361206, "logps/chosen": -116.6432876586914, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -153.86001586914062, "loss": 1.2945, "margin_dpo/margin_mean": 26.439434051513672, "margin_dpo/margin_std": 38.399658203125, "step": 177 }, { "KL/chosen_KL_mean": -22.401226043701172, "KL/mean": -38.91693878173828, "KL/rejected_KL_mean": -55.432640075683594, "KL/std": 35.231529235839844, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.03141784667969, "fcm_dpo/q_t": 0.4691542387008667, "grad_norm": 6.399454116821289, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -2.071244239807129, "logits/rejected": -1.9218668937683105, "logps/chosen": -106.66702270507812, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -166.22994995117188, "loss": 1.2709, "margin_dpo/margin_mean": 33.03141784667969, "margin_dpo/margin_std": 38.31219482421875, "step": 178 }, { "KL/chosen_KL_mean": -29.283756256103516, "KL/mean": -37.37403869628906, "KL/rejected_KL_mean": -45.46432113647461, "KL/std": 32.36445617675781, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.180564880371094, "fcm_dpo/q_t": 0.4849242568016052, "grad_norm": 2.698391914367676, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -2.0022692680358887, "logits/rejected": -1.6022377014160156, "logps/chosen": -122.73173522949219, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -139.7197265625, "loss": 1.3325, "margin_dpo/margin_mean": 16.18056297302246, "margin_dpo/margin_std": 41.834495544433594, "step": 179 }, { "KL/chosen_KL_mean": -22.387798309326172, "KL/mean": -36.67138671875, "KL/rejected_KL_mean": -50.95497131347656, "KL/std": 29.8559627532959, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.56717300415039, "fcm_dpo/q_t": 0.47333967685699463, "grad_norm": 3.0783605575561523, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.881057620048523, "logits/rejected": -1.6961443424224854, "logps/chosen": -111.22334289550781, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -154.8619384765625, "loss": 1.2858, "margin_dpo/margin_mean": 28.56717300415039, "margin_dpo/margin_std": 34.096466064453125, "step": 180 }, { "KL/chosen_KL_mean": -21.084606170654297, "KL/mean": -28.68956756591797, "KL/rejected_KL_mean": -36.29452896118164, "KL/std": 27.62065887451172, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.209920883178711, "fcm_dpo/q_t": 0.4857863485813141, "grad_norm": 2.336275100708008, "learning_rate": 4.605024008834863e-07, "logits/chosen": -2.215172052383423, "logits/rejected": -1.7522592544555664, "logps/chosen": -100.31857299804688, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -105.46894836425781, "loss": 1.3337, "margin_dpo/margin_mean": 15.209920883178711, "margin_dpo/margin_std": 32.9734001159668, "step": 181 }, { "KL/chosen_KL_mean": -24.03628921508789, "KL/mean": -37.98310089111328, "KL/rejected_KL_mean": -51.92991638183594, "KL/std": 34.01190185546875, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.89362335205078, "fcm_dpo/q_t": 0.4739624857902527, "grad_norm": 2.8919599056243896, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.5843055248260498, "logits/rejected": -1.430680513381958, "logps/chosen": -100.11846923828125, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -146.89852905273438, "loss": 1.2895, "margin_dpo/margin_mean": 27.89362335205078, "margin_dpo/margin_std": 38.84362030029297, "step": 182 }, { "KL/chosen_KL_mean": -31.61868667602539, "KL/mean": -40.542015075683594, "KL/rejected_KL_mean": -49.46534729003906, "KL/std": 29.431621551513672, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.846654891967773, "fcm_dpo/q_t": 0.4832821488380432, "grad_norm": 2.9559457302093506, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.9353668689727783, "logits/rejected": -1.320439100265503, "logps/chosen": -125.52659606933594, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -133.02597045898438, "loss": 1.324, "margin_dpo/margin_mean": 17.84665298461914, "margin_dpo/margin_std": 31.06645393371582, "step": 183 }, { "KL/chosen_KL_mean": -27.337493896484375, "KL/mean": -34.039710998535156, "KL/rejected_KL_mean": -40.74191665649414, "KL/std": 29.77444839477539, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.404426574707031, "fcm_dpo/q_t": 0.48743748664855957, "grad_norm": 2.4895095825195312, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -2.019181728363037, "logits/rejected": -1.7050652503967285, "logps/chosen": -114.20539855957031, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -125.02647399902344, "loss": 1.3409, "margin_dpo/margin_mean": 13.404426574707031, "margin_dpo/margin_std": 35.08120346069336, "step": 184 }, { "KL/chosen_KL_mean": -28.103076934814453, "KL/mean": -35.055397033691406, "KL/rejected_KL_mean": -42.007713317871094, "KL/std": 28.68169403076172, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.904637336730957, "fcm_dpo/q_t": 0.486990362405777, "grad_norm": 2.6976888179779053, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.5552501678466797, "logits/rejected": -1.9262886047363281, "logps/chosen": -122.12449645996094, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -123.46439361572266, "loss": 1.3386, "margin_dpo/margin_mean": 13.904638290405273, "margin_dpo/margin_std": 33.31956100463867, "step": 185 }, { "KL/chosen_KL_mean": -38.881595611572266, "KL/mean": -46.35780715942383, "KL/rejected_KL_mean": -53.834014892578125, "KL/std": 31.107126235961914, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.952428817749023, "fcm_dpo/q_t": 0.48598945140838623, "grad_norm": 3.3994741439819336, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.627466917037964, "logits/rejected": -1.6476008892059326, "logps/chosen": -153.07327270507812, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -147.1731414794922, "loss": 1.3358, "margin_dpo/margin_mean": 14.952428817749023, "margin_dpo/margin_std": 37.41070556640625, "step": 186 }, { "KL/chosen_KL_mean": -26.893360137939453, "KL/mean": -37.370635986328125, "KL/rejected_KL_mean": -47.847904205322266, "KL/std": 31.571002960205078, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.954551696777344, "fcm_dpo/q_t": 0.4804095923900604, "grad_norm": 2.555356502532959, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -2.3446192741394043, "logits/rejected": -2.1402039527893066, "logps/chosen": -103.76290893554688, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -136.01284790039062, "loss": 1.3136, "margin_dpo/margin_mean": 20.954551696777344, "margin_dpo/margin_std": 36.15287399291992, "step": 187 }, { "KL/chosen_KL_mean": -23.729068756103516, "KL/mean": -35.379783630371094, "KL/rejected_KL_mean": -47.03050231933594, "KL/std": 31.220489501953125, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.301433563232422, "fcm_dpo/q_t": 0.4781527519226074, "grad_norm": 2.724724292755127, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.619107246398926, "logits/rejected": -2.4035162925720215, "logps/chosen": -102.92387390136719, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -143.85940551757812, "loss": 1.3045, "margin_dpo/margin_mean": 23.301433563232422, "margin_dpo/margin_std": 32.73491287231445, "step": 188 }, { "KL/chosen_KL_mean": -29.35335922241211, "KL/mean": -38.44633483886719, "KL/rejected_KL_mean": -47.53931427001953, "KL/std": 29.595035552978516, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.18595314025879, "fcm_dpo/q_t": 0.4830195903778076, "grad_norm": 2.7477781772613525, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -2.0198941230773926, "logits/rejected": -1.3428118228912354, "logps/chosen": -113.24615478515625, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -127.00521087646484, "loss": 1.3236, "margin_dpo/margin_mean": 18.18595314025879, "margin_dpo/margin_std": 35.824249267578125, "step": 189 }, { "KL/chosen_KL_mean": -27.334014892578125, "KL/mean": -40.70348358154297, "KL/rejected_KL_mean": -54.07295227050781, "KL/std": 35.18400955200195, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.738941192626953, "fcm_dpo/q_t": 0.47502341866493225, "grad_norm": 2.8257768154144287, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -2.1897716522216797, "logits/rejected": -1.5973889827728271, "logps/chosen": -130.77548217773438, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -163.28797912597656, "loss": 1.2935, "margin_dpo/margin_mean": 26.738941192626953, "margin_dpo/margin_std": 38.529747009277344, "step": 190 }, { "KL/chosen_KL_mean": -25.75216293334961, "KL/mean": -43.787384033203125, "KL/rejected_KL_mean": -61.82261657714844, "KL/std": 40.069091796875, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.07046127319336, "fcm_dpo/q_t": 0.4664098024368286, "grad_norm": 3.3041231632232666, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -2.080103874206543, "logits/rejected": -1.7944457530975342, "logps/chosen": -125.53767395019531, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -177.82290649414062, "loss": 1.2619, "margin_dpo/margin_mean": 36.07046127319336, "margin_dpo/margin_std": 44.20814514160156, "step": 191 }, { "KL/chosen_KL_mean": -34.37848663330078, "KL/mean": -46.20176696777344, "KL/rejected_KL_mean": -58.02505111694336, "KL/std": 31.59564208984375, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.646574020385742, "fcm_dpo/q_t": 0.4779226779937744, "grad_norm": 2.813488006591797, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.9825165271759033, "logits/rejected": -1.752671718597412, "logps/chosen": -135.2388153076172, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -167.47006225585938, "loss": 1.3045, "margin_dpo/margin_mean": 23.646574020385742, "margin_dpo/margin_std": 38.65959930419922, "step": 192 }, { "KL/chosen_KL_mean": -30.910198211669922, "KL/mean": -38.13055419921875, "KL/rejected_KL_mean": -45.35090637207031, "KL/std": 34.736289978027344, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.44070816040039, "fcm_dpo/q_t": 0.4865609407424927, "grad_norm": 2.7579870223999023, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.4016923904418945, "logits/rejected": -1.80145263671875, "logps/chosen": -123.8485107421875, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -128.4600830078125, "loss": 1.3397, "margin_dpo/margin_mean": 14.44070816040039, "margin_dpo/margin_std": 43.718544006347656, "step": 193 }, { "KL/chosen_KL_mean": -23.63433837890625, "KL/mean": -34.00289535522461, "KL/rejected_KL_mean": -44.371456146240234, "KL/std": 28.016376495361328, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.737125396728516, "fcm_dpo/q_t": 0.4805794954299927, "grad_norm": 2.6150569915771484, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.8486413955688477, "logits/rejected": -2.4697420597076416, "logps/chosen": -87.42591094970703, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -113.09811401367188, "loss": 1.313, "margin_dpo/margin_mean": 20.737125396728516, "margin_dpo/margin_std": 30.418014526367188, "step": 194 }, { "KL/chosen_KL_mean": -27.725528717041016, "KL/mean": -42.08744812011719, "KL/rejected_KL_mean": -56.44936752319336, "KL/std": 32.41444396972656, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.72384262084961, "fcm_dpo/q_t": 0.4731326699256897, "grad_norm": 2.7189524173736572, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.637160301208496, "logits/rejected": -1.5686898231506348, "logps/chosen": -111.61099243164062, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -160.45062255859375, "loss": 1.2858, "margin_dpo/margin_mean": 28.723838806152344, "margin_dpo/margin_std": 36.730064392089844, "step": 195 }, { "KL/chosen_KL_mean": -32.145164489746094, "KL/mean": -44.347816467285156, "KL/rejected_KL_mean": -56.55046463012695, "KL/std": 34.15042495727539, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.405302047729492, "fcm_dpo/q_t": 0.4771324694156647, "grad_norm": 2.7993216514587402, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.632161259651184, "logits/rejected": -1.1745508909225464, "logps/chosen": -121.54652404785156, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -139.89781188964844, "loss": 1.3012, "margin_dpo/margin_mean": 24.405302047729492, "margin_dpo/margin_std": 36.4586067199707, "step": 196 }, { "KL/chosen_KL_mean": -31.207592010498047, "KL/mean": -37.799598693847656, "KL/rejected_KL_mean": -44.391605377197266, "KL/std": 25.758193969726562, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.184011459350586, "fcm_dpo/q_t": 0.487630695104599, "grad_norm": 2.5348827838897705, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.8158825635910034, "logits/rejected": -1.4773838520050049, "logps/chosen": -118.04829406738281, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -127.18450164794922, "loss": 1.3404, "margin_dpo/margin_mean": 13.184011459350586, "margin_dpo/margin_std": 29.58127212524414, "step": 197 }, { "KL/chosen_KL_mean": -34.83221435546875, "KL/mean": -49.10735321044922, "KL/rejected_KL_mean": -63.38250732421875, "KL/std": 35.31700897216797, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.550283432006836, "fcm_dpo/q_t": 0.4733000099658966, "grad_norm": 2.919116735458374, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.9976036548614502, "logits/rejected": -1.873272180557251, "logps/chosen": -119.88046264648438, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -167.4861602783203, "loss": 1.2857, "margin_dpo/margin_mean": 28.55028533935547, "margin_dpo/margin_std": 33.81718444824219, "step": 198 }, { "KL/chosen_KL_mean": -32.66507339477539, "KL/mean": -44.19818878173828, "KL/rejected_KL_mean": -55.7313117980957, "KL/std": 31.641420364379883, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.06623649597168, "fcm_dpo/q_t": 0.47843098640441895, "grad_norm": 2.741696834564209, "learning_rate": 4.471606039587695e-07, "logits/chosen": -2.02994966506958, "logits/rejected": -1.7843990325927734, "logps/chosen": -114.45176696777344, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -146.5220947265625, "loss": 1.3063, "margin_dpo/margin_mean": 23.06623649597168, "margin_dpo/margin_std": 37.673912048339844, "step": 199 }, { "KL/chosen_KL_mean": -31.875057220458984, "KL/mean": -44.491485595703125, "KL/rejected_KL_mean": -57.107913970947266, "KL/std": 32.950565338134766, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.23284912109375, "fcm_dpo/q_t": 0.476433664560318, "grad_norm": 2.890902042388916, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -2.1547908782958984, "logits/rejected": -1.595564842224121, "logps/chosen": -116.14315032958984, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -144.07577514648438, "loss": 1.2997, "margin_dpo/margin_mean": 25.23284912109375, "margin_dpo/margin_std": 41.54100036621094, "step": 200 }, { "KL/chosen_KL_mean": -39.69147872924805, "KL/mean": -51.83430480957031, "KL/rejected_KL_mean": -63.977134704589844, "KL/std": 40.994781494140625, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.285659790039062, "fcm_dpo/q_t": 0.4774027466773987, "grad_norm": 2.8755743503570557, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.7716550827026367, "logits/rejected": -1.4285318851470947, "logps/chosen": -147.46397399902344, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -183.76962280273438, "loss": 1.3044, "margin_dpo/margin_mean": 24.285661697387695, "margin_dpo/margin_std": 44.236717224121094, "step": 201 }, { "KL/chosen_KL_mean": -30.35688018798828, "KL/mean": -42.095855712890625, "KL/rejected_KL_mean": -53.8348388671875, "KL/std": 31.25006103515625, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.477951049804688, "fcm_dpo/q_t": 0.4780980944633484, "grad_norm": 2.7614850997924805, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -2.1085376739501953, "logits/rejected": -2.0068206787109375, "logps/chosen": -106.32933044433594, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -148.29483032226562, "loss": 1.3057, "margin_dpo/margin_mean": 23.477951049804688, "margin_dpo/margin_std": 39.047119140625, "step": 202 }, { "KL/chosen_KL_mean": -31.57367706298828, "KL/mean": -42.32033157348633, "KL/rejected_KL_mean": -53.066986083984375, "KL/std": 30.79452896118164, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.493301391601562, "fcm_dpo/q_t": 0.47990769147872925, "grad_norm": 2.6959047317504883, "learning_rate": 4.439630306414758e-07, "logits/chosen": -2.0698885917663574, "logits/rejected": -1.6133098602294922, "logps/chosen": -126.54083251953125, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -145.95465087890625, "loss": 1.3119, "margin_dpo/margin_mean": 21.493301391601562, "margin_dpo/margin_std": 37.21685791015625, "step": 203 }, { "KL/chosen_KL_mean": -39.63301086425781, "KL/mean": -48.30876159667969, "KL/rejected_KL_mean": -56.98450469970703, "KL/std": 35.06566619873047, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.35149383544922, "fcm_dpo/q_t": 0.48379746079444885, "grad_norm": 3.077220916748047, "learning_rate": 4.431508065452897e-07, "logits/chosen": -2.311122417449951, "logits/rejected": -1.6450445652008057, "logps/chosen": -155.99020385742188, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -150.38209533691406, "loss": 1.3288, "margin_dpo/margin_mean": 17.35149383544922, "margin_dpo/margin_std": 41.838592529296875, "step": 204 }, { "KL/chosen_KL_mean": -35.003414154052734, "KL/mean": -48.88951110839844, "KL/rejected_KL_mean": -62.775604248046875, "KL/std": 36.08696365356445, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.772188186645508, "fcm_dpo/q_t": 0.4739920496940613, "grad_norm": 2.9632060527801514, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.6589226722717285, "logits/rejected": -2.215116024017334, "logps/chosen": -123.86276245117188, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -154.63009643554688, "loss": 1.2883, "margin_dpo/margin_mean": 27.772188186645508, "margin_dpo/margin_std": 32.30433654785156, "step": 205 }, { "KL/chosen_KL_mean": -32.17458724975586, "KL/mean": -45.95695114135742, "KL/rejected_KL_mean": -59.739322662353516, "KL/std": 31.99414825439453, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.56473159790039, "fcm_dpo/q_t": 0.4741447865962982, "grad_norm": 2.7692806720733643, "learning_rate": 4.415111107797445e-07, "logits/chosen": -2.0040812492370605, "logits/rejected": -1.90508234500885, "logps/chosen": -108.72093200683594, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -162.6924591064453, "loss": 1.2883, "margin_dpo/margin_mean": 27.564733505249023, "margin_dpo/margin_std": 29.25942039489746, "step": 206 }, { "KL/chosen_KL_mean": -37.217674255371094, "KL/mean": -51.874244689941406, "KL/rejected_KL_mean": -66.53080749511719, "KL/std": 43.884185791015625, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.31313705444336, "fcm_dpo/q_t": 0.4728306531906128, "grad_norm": 3.1249451637268066, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.228804111480713, "logits/rejected": -1.8011029958724976, "logps/chosen": -123.44932556152344, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -157.18592834472656, "loss": 1.2885, "margin_dpo/margin_mean": 29.31313705444336, "margin_dpo/margin_std": 52.275291442871094, "step": 207 }, { "KL/chosen_KL_mean": -39.750247955322266, "KL/mean": -54.54627227783203, "KL/rejected_KL_mean": -69.34230041503906, "KL/std": 35.587345123291016, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.59204864501953, "fcm_dpo/q_t": 0.4722934365272522, "grad_norm": 3.193610668182373, "learning_rate": 4.398512291636768e-07, "logits/chosen": -2.076310634613037, "logits/rejected": -1.7530120611190796, "logps/chosen": -133.9097900390625, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -170.30462646484375, "loss": 1.2822, "margin_dpo/margin_mean": 29.5920467376709, "margin_dpo/margin_std": 34.12655258178711, "step": 208 }, { "KL/chosen_KL_mean": -35.05459213256836, "KL/mean": -46.087852478027344, "KL/rejected_KL_mean": -57.1211051940918, "KL/std": 33.258460998535156, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.0665225982666, "fcm_dpo/q_t": 0.47938933968544006, "grad_norm": 2.96927809715271, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.518237829208374, "logits/rejected": -2.0496129989624023, "logps/chosen": -119.22515869140625, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -144.74066162109375, "loss": 1.3112, "margin_dpo/margin_mean": 22.0665225982666, "margin_dpo/margin_std": 41.12714385986328, "step": 209 }, { "KL/chosen_KL_mean": -35.84153366088867, "KL/mean": -48.19336700439453, "KL/rejected_KL_mean": -60.54520797729492, "KL/std": 33.683162689208984, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.703672409057617, "fcm_dpo/q_t": 0.47691047191619873, "grad_norm": 2.7228238582611084, "learning_rate": 4.381713366536311e-07, "logits/chosen": -2.2290401458740234, "logits/rejected": -1.8531081676483154, "logps/chosen": -117.01271057128906, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -144.72000122070312, "loss": 1.3007, "margin_dpo/margin_mean": 24.703670501708984, "margin_dpo/margin_std": 38.039039611816406, "step": 210 }, { "KL/chosen_KL_mean": -47.4403190612793, "KL/mean": -57.13762664794922, "KL/rejected_KL_mean": -66.83493041992188, "KL/std": 37.322731018066406, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.394611358642578, "fcm_dpo/q_t": 0.48191946744918823, "grad_norm": 3.415410280227661, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.3795084953308105, "logits/rejected": -1.7288182973861694, "logps/chosen": -156.1530303955078, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -160.39059448242188, "loss": 1.3204, "margin_dpo/margin_mean": 19.394611358642578, "margin_dpo/margin_std": 39.73860168457031, "step": 211 }, { "KL/chosen_KL_mean": -34.13617706298828, "KL/mean": -53.58415222167969, "KL/rejected_KL_mean": -73.0321273803711, "KL/std": 46.402503967285156, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.89594268798828, "fcm_dpo/q_t": 0.46390998363494873, "grad_norm": 3.1416568756103516, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -2.024601936340332, "logits/rejected": -1.6058878898620605, "logps/chosen": -132.4981231689453, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -182.922119140625, "loss": 1.2551, "margin_dpo/margin_mean": 38.89594268798828, "margin_dpo/margin_std": 52.55128479003906, "step": 212 }, { "KL/chosen_KL_mean": -38.22661209106445, "KL/mean": -51.229652404785156, "KL/rejected_KL_mean": -64.23268127441406, "KL/std": 38.76484680175781, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.006061553955078, "fcm_dpo/q_t": 0.4756934642791748, "grad_norm": 3.2746424674987793, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -2.355142593383789, "logits/rejected": -1.5471761226654053, "logps/chosen": -146.28192138671875, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -164.3768310546875, "loss": 1.2982, "margin_dpo/margin_mean": 26.006061553955078, "margin_dpo/margin_std": 46.06051254272461, "step": 213 }, { "KL/chosen_KL_mean": -36.69287872314453, "KL/mean": -49.42802429199219, "KL/rejected_KL_mean": -62.163177490234375, "KL/std": 39.49470520019531, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.470298767089844, "fcm_dpo/q_t": 0.4762842655181885, "grad_norm": 2.8491950035095215, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -2.1699142456054688, "logits/rejected": -2.0498836040496826, "logps/chosen": -114.49761962890625, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -146.43896484375, "loss": 1.2999, "margin_dpo/margin_mean": 25.470298767089844, "margin_dpo/margin_std": 45.42501449584961, "step": 214 }, { "KL/chosen_KL_mean": -33.6776237487793, "KL/mean": -51.376869201660156, "KL/rejected_KL_mean": -69.07611083984375, "KL/std": 40.19752502441406, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.39848327636719, "fcm_dpo/q_t": 0.4670206606388092, "grad_norm": 3.096395254135132, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -2.0989348888397217, "logits/rejected": -1.6502914428710938, "logps/chosen": -118.79151916503906, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -165.9376220703125, "loss": 1.2639, "margin_dpo/margin_mean": 35.39848327636719, "margin_dpo/margin_std": 43.03347396850586, "step": 215 }, { "KL/chosen_KL_mean": -36.65926742553711, "KL/mean": -50.10143280029297, "KL/rejected_KL_mean": -63.543601989746094, "KL/std": 33.143768310546875, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 26.88433074951172, "fcm_dpo/q_t": 0.47486045956611633, "grad_norm": 2.806060791015625, "learning_rate": 4.330133748510036e-07, "logits/chosen": -2.055424928665161, "logits/rejected": -1.6352684497833252, "logps/chosen": -117.25157165527344, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -144.96343994140625, "loss": 1.2931, "margin_dpo/margin_mean": 26.884334564208984, "margin_dpo/margin_std": 38.79054260253906, "step": 216 }, { "KL/chosen_KL_mean": -46.14046859741211, "KL/mean": -57.87847137451172, "KL/rejected_KL_mean": -69.61648559570312, "KL/std": 41.56048583984375, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.476016998291016, "fcm_dpo/q_t": 0.4782150983810425, "grad_norm": 3.3004236221313477, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.348543167114258, "logits/rejected": -2.0554115772247314, "logps/chosen": -139.61305236816406, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -173.10501098632812, "loss": 1.3106, "margin_dpo/margin_mean": 23.476016998291016, "margin_dpo/margin_std": 55.203758239746094, "step": 217 }, { "KL/chosen_KL_mean": -43.794742584228516, "KL/mean": -57.53015899658203, "KL/rejected_KL_mean": -71.26558685302734, "KL/std": 41.57440948486328, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.470844268798828, "fcm_dpo/q_t": 0.47457635402679443, "grad_norm": 3.10245943069458, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.9931758642196655, "logits/rejected": -1.6497151851654053, "logps/chosen": -132.85357666015625, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -165.5723876953125, "loss": 1.2961, "margin_dpo/margin_mean": 27.47084617614746, "margin_dpo/margin_std": 54.40776062011719, "step": 218 }, { "KL/chosen_KL_mean": -42.274776458740234, "KL/mean": -53.472312927246094, "KL/rejected_KL_mean": -64.66986083984375, "KL/std": 46.06688690185547, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.395078659057617, "fcm_dpo/q_t": 0.47912532091140747, "grad_norm": 3.3144888877868652, "learning_rate": 4.303689819449636e-07, "logits/chosen": -2.189531087875366, "logits/rejected": -1.7196552753448486, "logps/chosen": -143.28211975097656, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -157.13780212402344, "loss": 1.3152, "margin_dpo/margin_mean": 22.395078659057617, "margin_dpo/margin_std": 54.590660095214844, "step": 219 }, { "KL/chosen_KL_mean": -40.59363555908203, "KL/mean": -52.10838317871094, "KL/rejected_KL_mean": -63.623130798339844, "KL/std": 34.98394775390625, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.02950668334961, "fcm_dpo/q_t": 0.47845137119293213, "grad_norm": 3.059532642364502, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.7904645204544067, "logits/rejected": -1.458500623703003, "logps/chosen": -136.130859375, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -157.93017578125, "loss": 1.3063, "margin_dpo/margin_mean": 23.02950668334961, "margin_dpo/margin_std": 36.716983795166016, "step": 220 }, { "KL/chosen_KL_mean": -37.13939666748047, "KL/mean": -58.790565490722656, "KL/rejected_KL_mean": -80.44174194335938, "KL/std": 46.09881591796875, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.302345275878906, "fcm_dpo/q_t": 0.4597722291946411, "grad_norm": 3.528750419616699, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.7299917936325073, "logits/rejected": -1.6244182586669922, "logps/chosen": -119.98426055908203, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -191.2535400390625, "loss": 1.2383, "margin_dpo/margin_mean": 43.302345275878906, "margin_dpo/margin_std": 48.66283416748047, "step": 221 }, { "KL/chosen_KL_mean": -47.26339340209961, "KL/mean": -62.1383171081543, "KL/rejected_KL_mean": -77.01325225830078, "KL/std": 45.0401611328125, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.749855041503906, "fcm_dpo/q_t": 0.47232675552368164, "grad_norm": 3.235114812850952, "learning_rate": 4.276818137766118e-07, "logits/chosen": -2.2802953720092773, "logits/rejected": -1.8649625778198242, "logps/chosen": -142.40536499023438, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -183.81765747070312, "loss": 1.2891, "margin_dpo/margin_mean": 29.749858856201172, "margin_dpo/margin_std": 56.94618225097656, "step": 222 }, { "KL/chosen_KL_mean": -44.643272399902344, "KL/mean": -55.63590621948242, "KL/rejected_KL_mean": -66.6285400390625, "KL/std": 37.294769287109375, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 21.985265731811523, "fcm_dpo/q_t": 0.479565292596817, "grad_norm": 3.290008783340454, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.6455376148223877, "logits/rejected": -2.288011074066162, "logps/chosen": -130.2183837890625, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -153.08091735839844, "loss": 1.3127, "margin_dpo/margin_mean": 21.985265731811523, "margin_dpo/margin_std": 45.55609130859375, "step": 223 }, { "KL/chosen_KL_mean": -42.02560043334961, "KL/mean": -56.580780029296875, "KL/rejected_KL_mean": -71.13597106933594, "KL/std": 42.222076416015625, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.110368728637695, "fcm_dpo/q_t": 0.47301870584487915, "grad_norm": 3.2065021991729736, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -2.302281618118286, "logits/rejected": -1.8610870838165283, "logps/chosen": -124.74940490722656, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -153.73135375976562, "loss": 1.2886, "margin_dpo/margin_mean": 29.110370635986328, "margin_dpo/margin_std": 50.40077209472656, "step": 224 }, { "KL/chosen_KL_mean": -48.97197341918945, "KL/mean": -65.55073547363281, "KL/rejected_KL_mean": -82.12950134277344, "KL/std": 44.95241928100586, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.157535552978516, "fcm_dpo/q_t": 0.46949851512908936, "grad_norm": 3.6679956912994385, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.252808094024658, "logits/rejected": -1.7823171615600586, "logps/chosen": -144.649658203125, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -187.22637939453125, "loss": 1.2779, "margin_dpo/margin_mean": 33.157535552978516, "margin_dpo/margin_std": 59.04552459716797, "step": 225 }, { "KL/chosen_KL_mean": -40.82627487182617, "KL/mean": -55.62181091308594, "KL/rejected_KL_mean": -70.4173355102539, "KL/std": 42.42902374267578, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.591053009033203, "fcm_dpo/q_t": 0.4724157452583313, "grad_norm": 3.206827163696289, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.9671905040740967, "logits/rejected": -1.4009177684783936, "logps/chosen": -134.28720092773438, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -157.1190948486328, "loss": 1.2842, "margin_dpo/margin_mean": 29.591053009033203, "margin_dpo/margin_std": 39.901336669921875, "step": 226 }, { "KL/chosen_KL_mean": -50.964054107666016, "KL/mean": -67.93058013916016, "KL/rejected_KL_mean": -84.89710998535156, "KL/std": 45.446380615234375, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 33.933067321777344, "fcm_dpo/q_t": 0.46855658292770386, "grad_norm": 3.5259861946105957, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.980953574180603, "logits/rejected": -1.6434425115585327, "logps/chosen": -143.7783203125, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -189.634033203125, "loss": 1.2731, "margin_dpo/margin_mean": 33.933067321777344, "margin_dpo/margin_std": 54.58409881591797, "step": 227 }, { "KL/chosen_KL_mean": -49.870697021484375, "KL/mean": -61.07789611816406, "KL/rejected_KL_mean": -72.28509521484375, "KL/std": 42.783721923828125, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.414377212524414, "fcm_dpo/q_t": 0.47913190722465515, "grad_norm": 3.349087715148926, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -2.1230201721191406, "logits/rejected": -1.7019457817077637, "logps/chosen": -143.90782165527344, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -168.30661010742188, "loss": 1.3124, "margin_dpo/margin_mean": 22.41437530517578, "margin_dpo/margin_std": 49.53401184082031, "step": 228 }, { "KL/chosen_KL_mean": -47.73883056640625, "KL/mean": -65.20085144042969, "KL/rejected_KL_mean": -82.66287231445312, "KL/std": 48.265403747558594, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.92404556274414, "fcm_dpo/q_t": 0.4678095877170563, "grad_norm": 3.6681969165802, "learning_rate": 4.212490049118951e-07, "logits/chosen": -2.196650505065918, "logits/rejected": -1.5816017389297485, "logps/chosen": -143.31649780273438, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -171.836669921875, "loss": 1.271, "margin_dpo/margin_mean": 34.92404556274414, "margin_dpo/margin_std": 57.89213943481445, "step": 229 }, { "KL/chosen_KL_mean": -39.79048538208008, "KL/mean": -62.80604553222656, "KL/rejected_KL_mean": -85.82159423828125, "KL/std": 45.16272735595703, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 46.03111267089844, "fcm_dpo/q_t": 0.45720237493515015, "grad_norm": 3.849337339401245, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8572266101837158, "logits/rejected": -1.833388090133667, "logps/chosen": -103.50411224365234, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -177.73031616210938, "loss": 1.2283, "margin_dpo/margin_mean": 46.03111267089844, "margin_dpo/margin_std": 47.24932861328125, "step": 230 }, { "KL/chosen_KL_mean": -48.69049072265625, "KL/mean": -62.7978515625, "KL/rejected_KL_mean": -76.90522003173828, "KL/std": 41.06111526489258, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.2147274017334, "fcm_dpo/q_t": 0.47366416454315186, "grad_norm": 3.3135480880737305, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -2.142871379852295, "logits/rejected": -1.5572776794433594, "logps/chosen": -144.1461639404297, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -157.86090087890625, "loss": 1.289, "margin_dpo/margin_mean": 28.214725494384766, "margin_dpo/margin_std": 41.51612854003906, "step": 231 }, { "KL/chosen_KL_mean": -51.760658264160156, "KL/mean": -64.40867614746094, "KL/rejected_KL_mean": -77.05670166015625, "KL/std": 41.65296173095703, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.29604148864746, "fcm_dpo/q_t": 0.4764029383659363, "grad_norm": 3.459150552749634, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.827089786529541, "logits/rejected": -1.351444959640503, "logps/chosen": -148.65792846679688, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -166.82131958007812, "loss": 1.2995, "margin_dpo/margin_mean": 25.296043395996094, "margin_dpo/margin_std": 41.47378158569336, "step": 232 }, { "KL/chosen_KL_mean": -51.44078826904297, "KL/mean": -73.30940246582031, "KL/rejected_KL_mean": -95.1780014038086, "KL/std": 53.29828643798828, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.0037656286731362343, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.73721694946289, "fcm_dpo/q_t": 0.4594994783401489, "grad_norm": 3.6940507888793945, "learning_rate": 4.174733034541245e-07, "logits/chosen": -2.1334328651428223, "logits/rejected": -1.991382122039795, "logps/chosen": -140.49111938476562, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -207.93716430664062, "loss": 1.2406, "margin_dpo/margin_mean": 43.737213134765625, "margin_dpo/margin_std": 58.71673583984375, "step": 233 }, { "KL/chosen_KL_mean": -48.59902572631836, "KL/mean": -71.664306640625, "KL/rejected_KL_mean": -94.7295913696289, "KL/std": 52.20249938964844, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.013825669884681702, "fcm_dpo/delta": 0.08168181777000427, "fcm_dpo/margin": 46.13058090209961, "fcm_dpo/q_t": 0.42717403173446655, "grad_norm": 11.024101257324219, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.5806130170822144, "logits/rejected": -1.5509281158447266, "logps/chosen": -122.91738891601562, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -201.1171875, "loss": 1.0135, "margin_dpo/margin_mean": 46.13058090209961, "margin_dpo/margin_std": 62.62630844116211, "step": 234 }, { "KL/chosen_KL_mean": -60.393985748291016, "KL/mean": -74.62811279296875, "KL/rejected_KL_mean": -88.86224365234375, "KL/std": 45.673213958740234, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.013323214836418629, "fcm_dpo/delta": -0.01074252836406231, "fcm_dpo/margin": 28.468265533447266, "fcm_dpo/q_t": 0.41373100876808167, "grad_norm": 12.34006404876709, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -2.068328619003296, "logits/rejected": -1.534796953201294, "logps/chosen": -158.61102294921875, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -186.1090087890625, "loss": 1.1476, "margin_dpo/margin_mean": 28.468265533447266, "margin_dpo/margin_std": 51.44862747192383, "step": 235 }, { "KL/chosen_KL_mean": -44.05496597290039, "KL/mean": -69.48747253417969, "KL/rejected_KL_mean": -94.91999816894531, "KL/std": 60.93297576904297, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.008107736706733704, "fcm_dpo/delta": -0.0038155466318130493, "fcm_dpo/margin": 50.865028381347656, "fcm_dpo/q_t": 0.408485472202301, "grad_norm": 6.794774532318115, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.8303751945495605, "logits/rejected": -1.8631165027618408, "logps/chosen": -122.8927001953125, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -203.98342895507812, "loss": 1.0929, "margin_dpo/margin_mean": 50.86503219604492, "margin_dpo/margin_std": 68.01996612548828, "step": 236 }, { "KL/chosen_KL_mean": -57.60867691040039, "KL/mean": -72.51579284667969, "KL/rejected_KL_mean": -87.42290496826172, "KL/std": 47.923912048339844, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.014100526459515095, "fcm_dpo/delta": -0.2131621241569519, "fcm_dpo/margin": 29.814228057861328, "fcm_dpo/q_t": 0.4046116769313812, "grad_norm": 17.12820053100586, "learning_rate": 4.136269950853473e-07, "logits/chosen": -2.011935234069824, "logits/rejected": -1.719944953918457, "logps/chosen": -142.81997680664062, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -187.33290100097656, "loss": 1.3285, "margin_dpo/margin_mean": 29.814224243164062, "margin_dpo/margin_std": 63.11576843261719, "step": 237 }, { "KL/chosen_KL_mean": -57.6911735534668, "KL/mean": -75.33876037597656, "KL/rejected_KL_mean": -92.98633575439453, "KL/std": 49.75318145751953, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.295166015625, "fcm_dpo/q_t": 0.49678903818130493, "grad_norm": 0.3823980987071991, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.6957340240478516, "logits/rejected": -1.5723451375961304, "logps/chosen": -136.4224090576172, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -188.4047393798828, "loss": 1.3736, "margin_dpo/margin_mean": 35.295166015625, "margin_dpo/margin_std": 54.56810760498047, "step": 238 }, { "KL/chosen_KL_mean": -55.90031814575195, "KL/mean": -71.54721069335938, "KL/rejected_KL_mean": -87.19410705566406, "KL/std": 42.07535171508789, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.293785095214844, "fcm_dpo/q_t": 0.4971528649330139, "grad_norm": 0.3625796139240265, "learning_rate": 4.116778689174514e-07, "logits/chosen": -2.284397840499878, "logits/rejected": -1.8320980072021484, "logps/chosen": -148.50125122070312, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -187.7117919921875, "loss": 1.375, "margin_dpo/margin_mean": 31.293785095214844, "margin_dpo/margin_std": 50.879852294921875, "step": 239 }, { "KL/chosen_KL_mean": -57.036808013916016, "KL/mean": -69.57403564453125, "KL/rejected_KL_mean": -82.11126708984375, "KL/std": 41.702178955078125, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.074466705322266, "fcm_dpo/q_t": 0.4977186918258667, "grad_norm": 0.3855019211769104, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.582784652709961, "logits/rejected": -2.21639347076416, "logps/chosen": -143.1965789794922, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -162.5669403076172, "loss": 1.3773, "margin_dpo/margin_mean": 25.074466705322266, "margin_dpo/margin_std": 47.29716110229492, "step": 240 }, { "KL/chosen_KL_mean": -54.16182327270508, "KL/mean": -77.92324829101562, "KL/rejected_KL_mean": -101.68467712402344, "KL/std": 48.904537200927734, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 47.52284240722656, "fcm_dpo/q_t": 0.49567657709121704, "grad_norm": 0.4073779582977295, "learning_rate": 4.097117014129903e-07, "logits/chosen": -2.073667526245117, "logits/rejected": -1.3794680833816528, "logps/chosen": -155.207763671875, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -195.73403930664062, "loss": 1.3692, "margin_dpo/margin_mean": 47.52284622192383, "margin_dpo/margin_std": 54.81800079345703, "step": 241 }, { "KL/chosen_KL_mean": -60.4542350769043, "KL/mean": -78.26560974121094, "KL/rejected_KL_mean": -96.07696533203125, "KL/std": 47.5811767578125, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.62272644042969, "fcm_dpo/q_t": 0.49675893783569336, "grad_norm": 0.400572270154953, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.918769359588623, "logits/rejected": -1.2136906385421753, "logps/chosen": -156.12689208984375, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -186.73150634765625, "loss": 1.3735, "margin_dpo/margin_mean": 35.62272644042969, "margin_dpo/margin_std": 56.03547286987305, "step": 242 }, { "KL/chosen_KL_mean": -59.310386657714844, "KL/mean": -75.49266052246094, "KL/rejected_KL_mean": -91.67493438720703, "KL/std": 46.25212097167969, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.36454772949219, "fcm_dpo/q_t": 0.4970554709434509, "grad_norm": 0.3948442041873932, "learning_rate": 4.07728699811968e-07, "logits/chosen": -2.0004444122314453, "logits/rejected": -1.1409938335418701, "logps/chosen": -157.341796875, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -174.86300659179688, "loss": 1.3747, "margin_dpo/margin_mean": 32.36454772949219, "margin_dpo/margin_std": 54.85365295410156, "step": 243 }, { "KL/chosen_KL_mean": -55.742088317871094, "KL/mean": -76.32935333251953, "KL/rejected_KL_mean": -96.91661071777344, "KL/std": 53.07884216308594, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.17452621459961, "fcm_dpo/q_t": 0.4962538778781891, "grad_norm": 0.3851814270019531, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.495643138885498, "logits/rejected": -2.200770378112793, "logps/chosen": -144.63601684570312, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -199.48939514160156, "loss": 1.3714, "margin_dpo/margin_mean": 41.17452621459961, "margin_dpo/margin_std": 46.46287536621094, "step": 244 }, { "KL/chosen_KL_mean": -53.32291030883789, "KL/mean": -69.73786926269531, "KL/rejected_KL_mean": -86.15281677246094, "KL/std": 43.88153076171875, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.82990264892578, "fcm_dpo/q_t": 0.4970131516456604, "grad_norm": 0.3646564483642578, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.67716121673584, "logits/rejected": -1.8381516933441162, "logps/chosen": -157.51690673828125, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -178.80926513671875, "loss": 1.3745, "margin_dpo/margin_mean": 32.82990264892578, "margin_dpo/margin_std": 49.064109802246094, "step": 245 }, { "KL/chosen_KL_mean": -61.264610290527344, "KL/mean": -76.81939697265625, "KL/rejected_KL_mean": -92.37417602539062, "KL/std": 47.39415740966797, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.109575271606445, "fcm_dpo/q_t": 0.4971696734428406, "grad_norm": 0.36137184500694275, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.910382628440857, "logits/rejected": -1.3307101726531982, "logps/chosen": -164.48365783691406, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -183.36639404296875, "loss": 1.3751, "margin_dpo/margin_mean": 31.109573364257812, "margin_dpo/margin_std": 51.7786979675293, "step": 246 }, { "KL/chosen_KL_mean": -55.27613830566406, "KL/mean": -82.02764892578125, "KL/rejected_KL_mean": -108.77915954589844, "KL/std": 55.605167388916016, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.503028869628906, "fcm_dpo/q_t": 0.4951329231262207, "grad_norm": 0.4694412648677826, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.5060763359069824, "logits/rejected": -2.384394645690918, "logps/chosen": -142.2705078125, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -220.1171875, "loss": 1.3671, "margin_dpo/margin_mean": 53.503028869628906, "margin_dpo/margin_std": 61.78107452392578, "step": 247 }, { "KL/chosen_KL_mean": -56.1020393371582, "KL/mean": -75.11825561523438, "KL/rejected_KL_mean": -94.13446807861328, "KL/std": 44.51951599121094, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.03242874145508, "fcm_dpo/q_t": 0.49653974175453186, "grad_norm": 0.3901209533214569, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -2.2209277153015137, "logits/rejected": -2.2333879470825195, "logps/chosen": -130.88755798339844, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -192.41136169433594, "loss": 1.3726, "margin_dpo/margin_mean": 38.03242492675781, "margin_dpo/margin_std": 44.78526306152344, "step": 248 }, { "KL/chosen_KL_mean": -68.63348388671875, "KL/mean": -80.48796081542969, "KL/rejected_KL_mean": -92.34243774414062, "KL/std": 51.08025360107422, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.708961486816406, "fcm_dpo/q_t": 0.4978431463241577, "grad_norm": 0.37152060866355896, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.5973353385925293, "logits/rejected": -2.055696487426758, "logps/chosen": -164.3372802734375, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -179.48892211914062, "loss": 1.3778, "margin_dpo/margin_mean": 23.70896339416504, "margin_dpo/margin_std": 60.57477569580078, "step": 249 }, { "KL/chosen_KL_mean": -62.499691009521484, "KL/mean": -75.36134338378906, "KL/rejected_KL_mean": -88.22297668457031, "KL/std": 43.76087188720703, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.72328758239746, "fcm_dpo/q_t": 0.4976595640182495, "grad_norm": 0.38096582889556885, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.161486864089966, "logits/rejected": -1.1760029792785645, "logps/chosen": -176.55191040039062, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -169.3106689453125, "loss": 1.377, "margin_dpo/margin_mean": 25.72328758239746, "margin_dpo/margin_std": 50.52073669433594, "step": 250 }, { "KL/chosen_KL_mean": -62.56477355957031, "KL/mean": -73.71903991699219, "KL/rejected_KL_mean": -84.87332153320312, "KL/std": 42.35741424560547, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.308544158935547, "fcm_dpo/q_t": 0.4979703724384308, "grad_norm": 0.3943535089492798, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -2.4270381927490234, "logits/rejected": -1.7601121664047241, "logps/chosen": -160.27606201171875, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -168.40072631835938, "loss": 1.3783, "margin_dpo/margin_mean": 22.308544158935547, "margin_dpo/margin_std": 52.485267639160156, "step": 251 }, { "KL/chosen_KL_mean": -60.598060607910156, "KL/mean": -75.49116516113281, "KL/rejected_KL_mean": -90.38426208496094, "KL/std": 43.66533660888672, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.786205291748047, "fcm_dpo/q_t": 0.4972900152206421, "grad_norm": 0.39348453283309937, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.3551197052001953, "logits/rejected": -1.7785565853118896, "logps/chosen": -169.58416748046875, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -181.94851684570312, "loss": 1.3756, "margin_dpo/margin_mean": 29.786205291748047, "margin_dpo/margin_std": 48.90971374511719, "step": 252 }, { "KL/chosen_KL_mean": -64.6201400756836, "KL/mean": -80.17704772949219, "KL/rejected_KL_mean": -95.73396301269531, "KL/std": 45.00267028808594, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.11383056640625, "fcm_dpo/q_t": 0.4971691966056824, "grad_norm": 0.38555580377578735, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.463413953781128, "logits/rejected": -2.2748990058898926, "logps/chosen": -164.83644104003906, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -201.41067504882812, "loss": 1.3751, "margin_dpo/margin_mean": 31.113826751708984, "margin_dpo/margin_std": 45.290313720703125, "step": 253 }, { "KL/chosen_KL_mean": -65.58953857421875, "KL/mean": -76.59654998779297, "KL/rejected_KL_mean": -87.60356903076172, "KL/std": 47.40875244140625, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.014028549194336, "fcm_dpo/q_t": 0.49799707531929016, "grad_norm": 0.3832297921180725, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.3978097438812256, "logits/rejected": -1.9012864828109741, "logps/chosen": -164.32472229003906, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -181.34182739257812, "loss": 1.3784, "margin_dpo/margin_mean": 22.014028549194336, "margin_dpo/margin_std": 48.99956512451172, "step": 254 }, { "KL/chosen_KL_mean": -67.10713195800781, "KL/mean": -87.01411437988281, "KL/rejected_KL_mean": -106.92109680175781, "KL/std": 48.75233459472656, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.81395721435547, "fcm_dpo/q_t": 0.4963776469230652, "grad_norm": 0.418856143951416, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.094536304473877, "logits/rejected": -1.7598210573196411, "logps/chosen": -157.48971557617188, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -203.99734497070312, "loss": 1.3719, "margin_dpo/margin_mean": 39.81395721435547, "margin_dpo/margin_std": 51.875030517578125, "step": 255 }, { "KL/chosen_KL_mean": -61.638729095458984, "KL/mean": -79.88755798339844, "KL/rejected_KL_mean": -98.13639831542969, "KL/std": 50.332942962646484, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.497676849365234, "fcm_dpo/q_t": 0.49667954444885254, "grad_norm": 0.38603174686431885, "learning_rate": 3.944434578520628e-07, "logits/chosen": -2.5607244968414307, "logits/rejected": -2.270542621612549, "logps/chosen": -150.39154052734375, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -196.63023376464844, "loss": 1.3732, "margin_dpo/margin_mean": 36.497676849365234, "margin_dpo/margin_std": 59.016578674316406, "step": 256 }, { "KL/chosen_KL_mean": -64.38579559326172, "KL/mean": -82.75790405273438, "KL/rejected_KL_mean": -101.13002014160156, "KL/std": 52.199615478515625, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.744224548339844, "fcm_dpo/q_t": 0.49665725231170654, "grad_norm": 0.38780486583709717, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.1467926502227783, "logits/rejected": -1.7346665859222412, "logps/chosen": -147.18931579589844, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -186.997802734375, "loss": 1.3731, "margin_dpo/margin_mean": 36.74422073364258, "margin_dpo/margin_std": 60.0453987121582, "step": 257 }, { "KL/chosen_KL_mean": -68.1824951171875, "KL/mean": -86.35082244873047, "KL/rejected_KL_mean": -104.5191650390625, "KL/std": 49.00737762451172, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.33667755126953, "fcm_dpo/q_t": 0.49669405817985535, "grad_norm": 0.4127484858036041, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.549149990081787, "logits/rejected": -2.2171201705932617, "logps/chosen": -158.3702392578125, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -207.98985290527344, "loss": 1.3732, "margin_dpo/margin_mean": 36.33667755126953, "margin_dpo/margin_std": 54.192970275878906, "step": 258 }, { "KL/chosen_KL_mean": -72.45909118652344, "KL/mean": -87.58155059814453, "KL/rejected_KL_mean": -102.70401000976562, "KL/std": 50.721580505371094, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.244903564453125, "fcm_dpo/q_t": 0.49724823236465454, "grad_norm": 0.38939428329467773, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.5042061805725098, "logits/rejected": -1.244484543800354, "logps/chosen": -163.23162841796875, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -197.29217529296875, "loss": 1.3754, "margin_dpo/margin_mean": 30.244901657104492, "margin_dpo/margin_std": 56.13518524169922, "step": 259 }, { "KL/chosen_KL_mean": -69.81595611572266, "KL/mean": -85.67892456054688, "KL/rejected_KL_mean": -101.54188537597656, "KL/std": 48.706207275390625, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.72593879699707, "fcm_dpo/q_t": 0.4971137046813965, "grad_norm": 0.4373262822628021, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -2.005052328109741, "logits/rejected": -2.065290927886963, "logps/chosen": -145.40866088867188, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -202.38742065429688, "loss": 1.3749, "margin_dpo/margin_mean": 31.725940704345703, "margin_dpo/margin_std": 59.580902099609375, "step": 260 }, { "KL/chosen_KL_mean": -64.21780395507812, "KL/mean": -83.2645263671875, "KL/rejected_KL_mean": -102.31124114990234, "KL/std": 50.129737854003906, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.09343719482422, "fcm_dpo/q_t": 0.4965345859527588, "grad_norm": 0.4095863103866577, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.2681331634521484, "logits/rejected": -1.9128162860870361, "logps/chosen": -158.47271728515625, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -195.96823120117188, "loss": 1.3726, "margin_dpo/margin_mean": 38.09343719482422, "margin_dpo/margin_std": 56.10470962524414, "step": 261 }, { "KL/chosen_KL_mean": -66.26140594482422, "KL/mean": -85.22459411621094, "KL/rejected_KL_mean": -104.18777465820312, "KL/std": 49.27330780029297, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 37.92637252807617, "fcm_dpo/q_t": 0.49654948711395264, "grad_norm": 0.4214659631252289, "learning_rate": 3.880912432401264e-07, "logits/chosen": -2.221024513244629, "logits/rejected": -1.7270269393920898, "logps/chosen": -151.52871704101562, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -195.0138702392578, "loss": 1.3726, "margin_dpo/margin_mean": 37.92637634277344, "margin_dpo/margin_std": 49.936885833740234, "step": 262 }, { "KL/chosen_KL_mean": -56.189064025878906, "KL/mean": -83.03018188476562, "KL/rejected_KL_mean": -109.87130737304688, "KL/std": 62.40521240234375, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.68223571777344, "fcm_dpo/q_t": 0.4951166808605194, "grad_norm": 0.42452728748321533, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.3818368911743164, "logits/rejected": -1.8366103172302246, "logps/chosen": -151.94696044921875, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -211.705078125, "loss": 1.367, "margin_dpo/margin_mean": 53.68223571777344, "margin_dpo/margin_std": 69.74070739746094, "step": 263 }, { "KL/chosen_KL_mean": -61.26982116699219, "KL/mean": -83.11402130126953, "KL/rejected_KL_mean": -104.95822143554688, "KL/std": 51.34105682373047, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.68840408325195, "fcm_dpo/q_t": 0.4960252642631531, "grad_norm": 0.43806949257850647, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.208644151687622, "logits/rejected": -1.9389312267303467, "logps/chosen": -151.89248657226562, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -205.2837677001953, "loss": 1.3706, "margin_dpo/margin_mean": 43.68840408325195, "margin_dpo/margin_std": 56.84298324584961, "step": 264 }, { "KL/chosen_KL_mean": -65.95641326904297, "KL/mean": -84.57316589355469, "KL/rejected_KL_mean": -103.18992614746094, "KL/std": 51.370750427246094, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 37.2335205078125, "fcm_dpo/q_t": 0.49661263823509216, "grad_norm": 0.3972235918045044, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.3031697273254395, "logits/rejected": -1.8882498741149902, "logps/chosen": -158.3287353515625, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -197.8175048828125, "loss": 1.3729, "margin_dpo/margin_mean": 37.2335205078125, "margin_dpo/margin_std": 58.81672286987305, "step": 265 }, { "KL/chosen_KL_mean": -77.34361267089844, "KL/mean": -91.66503143310547, "KL/rejected_KL_mean": -105.9864501953125, "KL/std": 56.021217346191406, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.642831802368164, "fcm_dpo/q_t": 0.49739426374435425, "grad_norm": 0.4308432638645172, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.657931327819824, "logits/rejected": -1.8242251873016357, "logps/chosen": -179.54364013671875, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -180.35287475585938, "loss": 1.376, "margin_dpo/margin_mean": 28.642831802368164, "margin_dpo/margin_std": 62.815673828125, "step": 266 }, { "KL/chosen_KL_mean": -64.19775390625, "KL/mean": -84.16668701171875, "KL/rejected_KL_mean": -104.13561248779297, "KL/std": 50.02431106567383, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.9378547668457, "fcm_dpo/q_t": 0.49636685848236084, "grad_norm": 0.4154065251350403, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -2.0333967208862305, "logits/rejected": -1.3918137550354004, "logps/chosen": -168.4837646484375, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -200.122802734375, "loss": 1.3719, "margin_dpo/margin_mean": 39.93785095214844, "margin_dpo/margin_std": 59.95903778076172, "step": 267 }, { "KL/chosen_KL_mean": -63.188209533691406, "KL/mean": -84.78921508789062, "KL/rejected_KL_mean": -106.39022827148438, "KL/std": 51.30052185058594, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.20201873779297, "fcm_dpo/q_t": 0.49606940150260925, "grad_norm": 0.47538384795188904, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.8864874839782715, "logits/rejected": -1.7707188129425049, "logps/chosen": -149.88442993164062, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -215.58206176757812, "loss": 1.3707, "margin_dpo/margin_mean": 43.20201873779297, "margin_dpo/margin_std": 57.410484313964844, "step": 268 }, { "KL/chosen_KL_mean": -73.48660278320312, "KL/mean": -95.80667114257812, "KL/rejected_KL_mean": -118.12675476074219, "KL/std": 57.94461441040039, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.64015579223633, "fcm_dpo/q_t": 0.49593907594680786, "grad_norm": 0.45675161480903625, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -2.173956871032715, "logits/rejected": -1.7567458152770996, "logps/chosen": -178.45840454101562, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -230.60321044921875, "loss": 1.3703, "margin_dpo/margin_mean": 44.640159606933594, "margin_dpo/margin_std": 63.386531829833984, "step": 269 }, { "KL/chosen_KL_mean": -73.05982971191406, "KL/mean": -94.23260498046875, "KL/rejected_KL_mean": -115.40540313720703, "KL/std": 54.21907043457031, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.3455696105957, "fcm_dpo/q_t": 0.4961477816104889, "grad_norm": 0.4385569989681244, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.749652862548828, "logits/rejected": -2.35646390914917, "logps/chosen": -174.1336669921875, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -233.15829467773438, "loss": 1.3711, "margin_dpo/margin_mean": 42.3455696105957, "margin_dpo/margin_std": 67.97145080566406, "step": 270 }, { "KL/chosen_KL_mean": -59.14579391479492, "KL/mean": -83.3553466796875, "KL/rejected_KL_mean": -107.56491088867188, "KL/std": 50.457279205322266, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.41911315917969, "fcm_dpo/q_t": 0.4955947697162628, "grad_norm": 0.4538714587688446, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.827150821685791, "logits/rejected": -1.3909755945205688, "logps/chosen": -155.87039184570312, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -206.08932495117188, "loss": 1.3688, "margin_dpo/margin_mean": 48.41911315917969, "margin_dpo/margin_std": 54.646728515625, "step": 271 }, { "KL/chosen_KL_mean": -51.743919372558594, "KL/mean": -77.3974609375, "KL/rejected_KL_mean": -103.05098724365234, "KL/std": 57.01249694824219, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.30707550048828, "fcm_dpo/q_t": 0.4953323006629944, "grad_norm": 0.46888700127601624, "learning_rate": 3.772161666010912e-07, "logits/chosen": -2.011855363845825, "logits/rejected": -1.9430043697357178, "logps/chosen": -132.72113037109375, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -211.60635375976562, "loss": 1.3678, "margin_dpo/margin_mean": 51.30707550048828, "margin_dpo/margin_std": 61.764644622802734, "step": 272 }, { "KL/chosen_KL_mean": -67.5731201171875, "KL/mean": -91.58149719238281, "KL/rejected_KL_mean": -115.58985900878906, "KL/std": 56.0106315612793, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.016746520996094, "fcm_dpo/q_t": 0.4956318736076355, "grad_norm": 0.46547070145606995, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.9123988151550293, "logits/rejected": -1.5109494924545288, "logps/chosen": -159.7977294921875, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -212.95291137695312, "loss": 1.3691, "margin_dpo/margin_mean": 48.016746520996094, "margin_dpo/margin_std": 66.86575317382812, "step": 273 }, { "KL/chosen_KL_mean": -64.09872436523438, "KL/mean": -83.3919906616211, "KL/rejected_KL_mean": -102.68524169921875, "KL/std": 50.943931579589844, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.58651351928711, "fcm_dpo/q_t": 0.4964892864227295, "grad_norm": 0.4737926125526428, "learning_rate": 3.75e-07, "logits/chosen": -2.2479140758514404, "logits/rejected": -1.875425934791565, "logps/chosen": -146.359619140625, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -186.56224060058594, "loss": 1.3724, "margin_dpo/margin_mean": 38.586509704589844, "margin_dpo/margin_std": 46.091644287109375, "step": 274 }, { "KL/chosen_KL_mean": -58.62002182006836, "KL/mean": -78.39067077636719, "KL/rejected_KL_mean": -98.16131591796875, "KL/std": 53.5073356628418, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.541297912597656, "fcm_dpo/q_t": 0.4964025616645813, "grad_norm": 0.43599364161491394, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.3767666816711426, "logits/rejected": -2.0799734592437744, "logps/chosen": -138.30697631835938, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -196.91229248046875, "loss": 1.3721, "margin_dpo/margin_mean": 39.541297912597656, "margin_dpo/margin_std": 55.3149299621582, "step": 275 }, { "KL/chosen_KL_mean": -65.68757629394531, "KL/mean": -85.82131958007812, "KL/rejected_KL_mean": -105.95507049560547, "KL/std": 53.485870361328125, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 40.26750183105469, "fcm_dpo/q_t": 0.496336966753006, "grad_norm": 0.4480706453323364, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.4238200187683105, "logits/rejected": -1.8556790351867676, "logps/chosen": -152.22727966308594, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -183.80902099609375, "loss": 1.3719, "margin_dpo/margin_mean": 40.26750183105469, "margin_dpo/margin_std": 69.35868835449219, "step": 276 }, { "KL/chosen_KL_mean": -67.07992553710938, "KL/mean": -84.27149963378906, "KL/rejected_KL_mean": -101.46308898925781, "KL/std": 47.57292175292969, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.383155822753906, "fcm_dpo/q_t": 0.4968717098236084, "grad_norm": 0.431079626083374, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.4258360862731934, "logits/rejected": -2.0311741828918457, "logps/chosen": -151.32403564453125, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -200.29730224609375, "loss": 1.3739, "margin_dpo/margin_mean": 34.383155822753906, "margin_dpo/margin_std": 45.89862060546875, "step": 277 }, { "KL/chosen_KL_mean": -61.50092315673828, "KL/mean": -83.33283996582031, "KL/rejected_KL_mean": -105.16474914550781, "KL/std": 52.6975212097168, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.663818359375, "fcm_dpo/q_t": 0.4960274398326874, "grad_norm": 0.4308766722679138, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.225346803665161, "logits/rejected": -2.0065505504608154, "logps/chosen": -143.93280029296875, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -203.02166748046875, "loss": 1.3706, "margin_dpo/margin_mean": 43.663822174072266, "margin_dpo/margin_std": 56.605224609375, "step": 278 }, { "KL/chosen_KL_mean": -66.3231430053711, "KL/mean": -92.26366424560547, "KL/rejected_KL_mean": -118.20419311523438, "KL/std": 62.291648864746094, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.88105010986328, "fcm_dpo/q_t": 0.4952806234359741, "grad_norm": 0.5649306178092957, "learning_rate": 3.6940245560867e-07, "logits/chosen": -2.139366388320923, "logits/rejected": -1.9556715488433838, "logps/chosen": -151.49114990234375, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -212.33084106445312, "loss": 1.3677, "margin_dpo/margin_mean": 51.88105010986328, "margin_dpo/margin_std": 68.84480285644531, "step": 279 }, { "KL/chosen_KL_mean": -60.25834655761719, "KL/mean": -87.26072692871094, "KL/rejected_KL_mean": -114.26310729980469, "KL/std": 53.28746032714844, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.00474548339844, "fcm_dpo/q_t": 0.4950866103172302, "grad_norm": 0.5035830140113831, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.961017370223999, "logits/rejected": -1.783928394317627, "logps/chosen": -146.11476135253906, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -218.38169860839844, "loss": 1.3668, "margin_dpo/margin_mean": 54.00474166870117, "margin_dpo/margin_std": 55.35174560546875, "step": 280 }, { "KL/chosen_KL_mean": -64.06562805175781, "KL/mean": -89.23621368408203, "KL/rejected_KL_mean": -114.40678405761719, "KL/std": 58.6932487487793, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.341156005859375, "fcm_dpo/q_t": 0.49542033672332764, "grad_norm": 0.4447258412837982, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.296375036239624, "logits/rejected": -1.5671701431274414, "logps/chosen": -150.616455078125, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -203.03546142578125, "loss": 1.3682, "margin_dpo/margin_mean": 50.341156005859375, "margin_dpo/margin_std": 64.49791717529297, "step": 281 }, { "KL/chosen_KL_mean": -64.27935791015625, "KL/mean": -95.81797790527344, "KL/rejected_KL_mean": -127.35659790039062, "KL/std": 66.81671142578125, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.077247619628906, "fcm_dpo/q_t": 0.494262158870697, "grad_norm": 0.46972402930259705, "learning_rate": 3.660059364023408e-07, "logits/chosen": -2.0709939002990723, "logits/rejected": -1.706782579421997, "logps/chosen": -169.38446044921875, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -230.2099609375, "loss": 1.3636, "margin_dpo/margin_mean": 63.077247619628906, "margin_dpo/margin_std": 71.7579345703125, "step": 282 }, { "KL/chosen_KL_mean": -78.5650634765625, "KL/mean": -99.89405059814453, "KL/rejected_KL_mean": -121.2230453491211, "KL/std": 61.614227294921875, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.657981872558594, "fcm_dpo/q_t": 0.4961192011833191, "grad_norm": 0.5413599014282227, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.6243228912353516, "logits/rejected": -2.312777519226074, "logps/chosen": -173.61766052246094, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -224.76759338378906, "loss": 1.371, "margin_dpo/margin_mean": 42.657981872558594, "margin_dpo/margin_std": 67.44954681396484, "step": 283 }, { "KL/chosen_KL_mean": -63.72255325317383, "KL/mean": -93.76842498779297, "KL/rejected_KL_mean": -123.81429290771484, "KL/std": 64.97592163085938, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.09173583984375, "fcm_dpo/q_t": 0.4945337176322937, "grad_norm": 0.5273388624191284, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -2.0984437465667725, "logits/rejected": -1.7558673620224, "logps/chosen": -151.3890380859375, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -222.56533813476562, "loss": 1.3647, "margin_dpo/margin_mean": 60.09173583984375, "margin_dpo/margin_std": 68.29450988769531, "step": 284 }, { "KL/chosen_KL_mean": -74.01020050048828, "KL/mean": -94.277099609375, "KL/rejected_KL_mean": -114.54399871826172, "KL/std": 49.18408966064453, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 40.53379821777344, "fcm_dpo/q_t": 0.4963122010231018, "grad_norm": 0.46756792068481445, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.814209222793579, "logits/rejected": -2.5483243465423584, "logps/chosen": -168.4432373046875, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -218.61593627929688, "loss": 1.3717, "margin_dpo/margin_mean": 40.53379821777344, "margin_dpo/margin_std": 54.813697814941406, "step": 285 }, { "KL/chosen_KL_mean": -82.05018615722656, "KL/mean": -108.37709045410156, "KL/rejected_KL_mean": -134.70401000976562, "KL/std": 65.91907501220703, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.65380859375, "fcm_dpo/q_t": 0.4952099323272705, "grad_norm": 0.501090943813324, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.3067121505737305, "logits/rejected": -1.859205961227417, "logps/chosen": -185.7705841064453, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -230.96176147460938, "loss": 1.3674, "margin_dpo/margin_mean": 52.65380859375, "margin_dpo/margin_std": 68.73916625976562, "step": 286 }, { "KL/chosen_KL_mean": -87.45057678222656, "KL/mean": -103.10128784179688, "KL/rejected_KL_mean": -118.75199890136719, "KL/std": 52.15273666381836, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.30141830444336, "fcm_dpo/q_t": 0.4971526265144348, "grad_norm": 0.4614596962928772, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.641435146331787, "logits/rejected": -2.053424835205078, "logps/chosen": -181.3404541015625, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -202.08566284179688, "loss": 1.3751, "margin_dpo/margin_mean": 31.30141830444336, "margin_dpo/margin_std": 67.46745300292969, "step": 287 }, { "KL/chosen_KL_mean": -82.4139404296875, "KL/mean": -110.38633728027344, "KL/rejected_KL_mean": -138.35874938964844, "KL/std": 62.62823486328125, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.94480895996094, "fcm_dpo/q_t": 0.4949108064174652, "grad_norm": 0.511562168598175, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -2.175168514251709, "logits/rejected": -1.7686786651611328, "logps/chosen": -170.5699462890625, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -231.6407012939453, "loss": 1.3662, "margin_dpo/margin_mean": 55.94480895996094, "margin_dpo/margin_std": 70.8388671875, "step": 288 }, { "KL/chosen_KL_mean": -79.27298736572266, "KL/mean": -109.95760345458984, "KL/rejected_KL_mean": -140.6422119140625, "KL/std": 66.03662109375, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.36921310424805, "fcm_dpo/q_t": 0.4944177269935608, "grad_norm": 0.5269072651863098, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.3235690593719482, "logits/rejected": -2.1279563903808594, "logps/chosen": -154.66592407226562, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -233.79649353027344, "loss": 1.3643, "margin_dpo/margin_mean": 61.36921691894531, "margin_dpo/margin_std": 77.78003692626953, "step": 289 }, { "KL/chosen_KL_mean": -92.54629516601562, "KL/mean": -123.64572143554688, "KL/rejected_KL_mean": -154.74513244628906, "KL/std": 76.39749145507812, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 62.19883728027344, "fcm_dpo/q_t": 0.494343101978302, "grad_norm": 0.6138589382171631, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.5879628658294678, "logits/rejected": -1.6669304370880127, "logps/chosen": -180.58827209472656, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -277.957275390625, "loss": 1.364, "margin_dpo/margin_mean": 62.19883728027344, "margin_dpo/margin_std": 86.45903015136719, "step": 290 }, { "KL/chosen_KL_mean": -89.29346466064453, "KL/mean": -112.14572143554688, "KL/rejected_KL_mean": -134.99798583984375, "KL/std": 67.00045013427734, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.70451354980469, "fcm_dpo/q_t": 0.49584224820137024, "grad_norm": 0.5047085881233215, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.759077548980713, "logits/rejected": -2.314127206802368, "logps/chosen": -183.38870239257812, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -231.0480499267578, "loss": 1.3699, "margin_dpo/margin_mean": 45.70451354980469, "margin_dpo/margin_std": 76.01837158203125, "step": 291 }, { "KL/chosen_KL_mean": -86.78099060058594, "KL/mean": -114.64785766601562, "KL/rejected_KL_mean": -142.5147247314453, "KL/std": 65.37959289550781, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.733734130859375, "fcm_dpo/q_t": 0.494929701089859, "grad_norm": 0.5114029049873352, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.3668980598449707, "logits/rejected": -1.9508452415466309, "logps/chosen": -175.03140258789062, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -238.932373046875, "loss": 1.3662, "margin_dpo/margin_mean": 55.733734130859375, "margin_dpo/margin_std": 62.76366424560547, "step": 292 }, { "KL/chosen_KL_mean": -87.58837890625, "KL/mean": -108.8157958984375, "KL/rejected_KL_mean": -130.043212890625, "KL/std": 71.17256927490234, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.45484161376953, "fcm_dpo/q_t": 0.496138334274292, "grad_norm": 0.5141710042953491, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.819126605987549, "logits/rejected": -2.2716569900512695, "logps/chosen": -174.96493530273438, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -215.7990264892578, "loss": 1.3711, "margin_dpo/margin_mean": 42.45484161376953, "margin_dpo/margin_std": 75.37640380859375, "step": 293 }, { "KL/chosen_KL_mean": -86.51016235351562, "KL/mean": -104.98387908935547, "KL/rejected_KL_mean": -123.45760345458984, "KL/std": 60.51080322265625, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 36.94744110107422, "fcm_dpo/q_t": 0.49663883447647095, "grad_norm": 0.5028398036956787, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.9443440437316895, "logits/rejected": -1.8030061721801758, "logps/chosen": -160.0181121826172, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -211.54638671875, "loss": 1.3731, "margin_dpo/margin_mean": 36.94744873046875, "margin_dpo/margin_std": 69.68022155761719, "step": 294 }, { "KL/chosen_KL_mean": -85.63993835449219, "KL/mean": -112.86396789550781, "KL/rejected_KL_mean": -140.08799743652344, "KL/std": 70.26445007324219, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.44804382324219, "fcm_dpo/q_t": 0.4950474202632904, "grad_norm": 0.5187733769416809, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.9427385330200195, "logits/rejected": -1.7889024019241333, "logps/chosen": -163.40542602539062, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -234.33526611328125, "loss": 1.3668, "margin_dpo/margin_mean": 54.44804382324219, "margin_dpo/margin_std": 80.09774017333984, "step": 295 }, { "KL/chosen_KL_mean": -103.17127990722656, "KL/mean": -128.64019775390625, "KL/rejected_KL_mean": -154.10911560058594, "KL/std": 65.76932525634766, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.93783950805664, "fcm_dpo/q_t": 0.4953664541244507, "grad_norm": 0.5457687377929688, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.8183341026306152, "logits/rejected": -2.339557409286499, "logps/chosen": -201.02769470214844, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -254.92543029785156, "loss": 1.368, "margin_dpo/margin_mean": 50.93783950805664, "margin_dpo/margin_std": 77.70854187011719, "step": 296 }, { "KL/chosen_KL_mean": -103.27163696289062, "KL/mean": -127.75170135498047, "KL/rejected_KL_mean": -152.2317657470703, "KL/std": 67.92790222167969, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.96013641357422, "fcm_dpo/q_t": 0.49554625153541565, "grad_norm": 0.5602902770042419, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2856303453445435, "logits/rejected": -0.9939242601394653, "logps/chosen": -191.83746337890625, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -253.788330078125, "loss": 1.3687, "margin_dpo/margin_mean": 48.96013641357422, "margin_dpo/margin_std": 70.83642578125, "step": 297 }, { "KL/chosen_KL_mean": -93.95957946777344, "KL/mean": -123.56320190429688, "KL/rejected_KL_mean": -153.16683959960938, "KL/std": 75.34748077392578, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.207252502441406, "fcm_dpo/q_t": 0.49461495876312256, "grad_norm": 0.6004077792167664, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.2587051391601562, "logits/rejected": -2.2941136360168457, "logps/chosen": -188.84002685546875, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -275.47784423828125, "loss": 1.3651, "margin_dpo/margin_mean": 59.207252502441406, "margin_dpo/margin_std": 80.2012939453125, "step": 298 }, { "KL/chosen_KL_mean": -94.76151275634766, "KL/mean": -124.74179077148438, "KL/rejected_KL_mean": -154.72207641601562, "KL/std": 65.50479888916016, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.96056365966797, "fcm_dpo/q_t": 0.4945456385612488, "grad_norm": 0.57039475440979, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.1956722736358643, "logits/rejected": -0.9969459176063538, "logps/chosen": -175.16986083984375, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -244.25924682617188, "loss": 1.3648, "margin_dpo/margin_mean": 59.96056365966797, "margin_dpo/margin_std": 73.24144744873047, "step": 299 }, { "KL/chosen_KL_mean": -104.41131591796875, "KL/mean": -133.02633666992188, "KL/rejected_KL_mean": -161.64138793945312, "KL/std": 73.70620727539062, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.23007583618164, "fcm_dpo/q_t": 0.49479418992996216, "grad_norm": 0.6704532504081726, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.7621468305587769, "logits/rejected": -1.5680410861968994, "logps/chosen": -192.57022094726562, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -262.58056640625, "loss": 1.3658, "margin_dpo/margin_mean": 57.230072021484375, "margin_dpo/margin_std": 82.76898193359375, "step": 300 }, { "KL/chosen_KL_mean": -96.98553466796875, "KL/mean": -130.77798461914062, "KL/rejected_KL_mean": -164.5704345703125, "KL/std": 71.33174133300781, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.58489227294922, "fcm_dpo/q_t": 0.493852436542511, "grad_norm": 0.594471275806427, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -2.0409111976623535, "logits/rejected": -1.6474981307983398, "logps/chosen": -182.49044799804688, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -263.4872741699219, "loss": 1.3621, "margin_dpo/margin_mean": 67.58489227294922, "margin_dpo/margin_std": 83.41411590576172, "step": 301 }, { "KL/chosen_KL_mean": -96.13658905029297, "KL/mean": -123.20683288574219, "KL/rejected_KL_mean": -150.277099609375, "KL/std": 72.52355194091797, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.1404914855957, "fcm_dpo/q_t": 0.4950747489929199, "grad_norm": 0.5614818930625916, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3619815111160278, "logits/rejected": -1.2101081609725952, "logps/chosen": -175.58700561523438, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -253.33619689941406, "loss": 1.3669, "margin_dpo/margin_mean": 54.14049530029297, "margin_dpo/margin_std": 77.02583312988281, "step": 302 }, { "KL/chosen_KL_mean": -95.25529479980469, "KL/mean": -125.26386260986328, "KL/rejected_KL_mean": -155.27243041992188, "KL/std": 72.72525024414062, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.017127990722656, "fcm_dpo/q_t": 0.494540274143219, "grad_norm": 0.541359007358551, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.9448398351669312, "logits/rejected": -1.4812731742858887, "logps/chosen": -190.8535919189453, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -247.33700561523438, "loss": 1.3648, "margin_dpo/margin_mean": 60.01713180541992, "margin_dpo/margin_std": 78.48544311523438, "step": 303 }, { "KL/chosen_KL_mean": -99.51309204101562, "KL/mean": -129.34442138671875, "KL/rejected_KL_mean": -159.17572021484375, "KL/std": 72.62190246582031, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.66264343261719, "fcm_dpo/q_t": 0.49457240104675293, "grad_norm": 0.5903537273406982, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.2780168056488037, "logits/rejected": -1.8753902912139893, "logps/chosen": -198.96624755859375, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -257.25799560546875, "loss": 1.3649, "margin_dpo/margin_mean": 59.66264343261719, "margin_dpo/margin_std": 73.67073059082031, "step": 304 }, { "KL/chosen_KL_mean": -105.79988098144531, "KL/mean": -133.5484619140625, "KL/rejected_KL_mean": -161.29705810546875, "KL/std": 75.13532257080078, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.497188568115234, "fcm_dpo/q_t": 0.4949517250061035, "grad_norm": 0.5181828737258911, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.070046901702881, "logits/rejected": -1.5920658111572266, "logps/chosen": -209.0632781982422, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -259.12603759765625, "loss": 1.3665, "margin_dpo/margin_mean": 55.49718475341797, "margin_dpo/margin_std": 86.9449462890625, "step": 305 }, { "KL/chosen_KL_mean": -95.47161865234375, "KL/mean": -128.48162841796875, "KL/rejected_KL_mean": -161.49163818359375, "KL/std": 66.19195556640625, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.02001190185547, "fcm_dpo/q_t": 0.49399399757385254, "grad_norm": 0.6592018604278564, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.3579845428466797, "logits/rejected": -1.8126777410507202, "logps/chosen": -195.77545166015625, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -253.72250366210938, "loss": 1.3625, "margin_dpo/margin_mean": 66.02001190185547, "margin_dpo/margin_std": 63.879425048828125, "step": 306 }, { "KL/chosen_KL_mean": -97.0665283203125, "KL/mean": -125.2652587890625, "KL/rejected_KL_mean": -153.46395874023438, "KL/std": 72.29367065429688, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.39742660522461, "fcm_dpo/q_t": 0.49487006664276123, "grad_norm": 0.5368165373802185, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.233731746673584, "logits/rejected": -1.889369249343872, "logps/chosen": -193.71629333496094, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -246.81256103515625, "loss": 1.3661, "margin_dpo/margin_mean": 56.397430419921875, "margin_dpo/margin_std": 90.20610046386719, "step": 307 }, { "KL/chosen_KL_mean": -101.65605163574219, "KL/mean": -131.77162170410156, "KL/rejected_KL_mean": -161.88717651367188, "KL/std": 74.72151184082031, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.23111343383789, "fcm_dpo/q_t": 0.49452096223831177, "grad_norm": 0.6848556995391846, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7518043518066406, "logits/rejected": -1.4484599828720093, "logps/chosen": -187.3807373046875, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -248.31036376953125, "loss": 1.3647, "margin_dpo/margin_mean": 60.231117248535156, "margin_dpo/margin_std": 79.66980743408203, "step": 308 }, { "KL/chosen_KL_mean": -113.58496856689453, "KL/mean": -138.85247802734375, "KL/rejected_KL_mean": -164.11996459960938, "KL/std": 83.8274917602539, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.53499221801758, "fcm_dpo/q_t": 0.49540311098098755, "grad_norm": 0.6120714545249939, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.8171327114105225, "logits/rejected": -1.3903783559799194, "logps/chosen": -196.73641967773438, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -250.70599365234375, "loss": 1.3682, "margin_dpo/margin_mean": 50.534996032714844, "margin_dpo/margin_std": 87.61152648925781, "step": 309 }, { "KL/chosen_KL_mean": -104.87617492675781, "KL/mean": -132.5391845703125, "KL/rejected_KL_mean": -160.20220947265625, "KL/std": 71.02485656738281, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.32602310180664, "fcm_dpo/q_t": 0.49496710300445557, "grad_norm": 0.6255085468292236, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.4935412406921387, "logits/rejected": -2.0935585498809814, "logps/chosen": -194.61416625976562, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -247.27304077148438, "loss": 1.3665, "margin_dpo/margin_mean": 55.326026916503906, "margin_dpo/margin_std": 76.76844024658203, "step": 310 }, { "KL/chosen_KL_mean": -123.14608764648438, "KL/mean": -154.6707305908203, "KL/rejected_KL_mean": -186.1953582763672, "KL/std": 73.74455261230469, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.04926300048828, "fcm_dpo/q_t": 0.4942648708820343, "grad_norm": 0.6044071316719055, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.7669131755828857, "logits/rejected": -1.0473551750183105, "logps/chosen": -211.6725311279297, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -268.46142578125, "loss": 1.3637, "margin_dpo/margin_mean": 63.04926300048828, "margin_dpo/margin_std": 80.36540222167969, "step": 311 }, { "KL/chosen_KL_mean": -115.89427185058594, "KL/mean": -145.19268798828125, "KL/rejected_KL_mean": -174.4910888671875, "KL/std": 82.65101623535156, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.59680938720703, "fcm_dpo/q_t": 0.494669646024704, "grad_norm": 0.7197696566581726, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.6603741645812988, "logits/rejected": -1.1405774354934692, "logps/chosen": -196.2949676513672, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -256.938232421875, "loss": 1.3653, "margin_dpo/margin_mean": 58.59680938720703, "margin_dpo/margin_std": 85.7784194946289, "step": 312 }, { "KL/chosen_KL_mean": -114.14349365234375, "KL/mean": -141.85757446289062, "KL/rejected_KL_mean": -169.57168579101562, "KL/std": 82.99652862548828, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.428199768066406, "fcm_dpo/q_t": 0.49495813250541687, "grad_norm": 0.6220037937164307, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.523918867111206, "logits/rejected": -2.1386287212371826, "logps/chosen": -198.285400390625, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -262.99853515625, "loss": 1.3665, "margin_dpo/margin_mean": 55.428199768066406, "margin_dpo/margin_std": 84.48899841308594, "step": 313 }, { "KL/chosen_KL_mean": -121.7081298828125, "KL/mean": -150.68707275390625, "KL/rejected_KL_mean": -179.66600036621094, "KL/std": 89.51388549804688, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.9578857421875, "fcm_dpo/q_t": 0.49472856521606445, "grad_norm": 0.6284196972846985, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -2.0430755615234375, "logits/rejected": -1.843724012374878, "logps/chosen": -210.51785278320312, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -283.8249206542969, "loss": 1.3656, "margin_dpo/margin_mean": 57.957881927490234, "margin_dpo/margin_std": 94.6346435546875, "step": 314 }, { "KL/chosen_KL_mean": -110.37505340576172, "KL/mean": -140.90536499023438, "KL/rejected_KL_mean": -171.4356689453125, "KL/std": 86.46977233886719, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.060604095458984, "fcm_dpo/q_t": 0.4944458603858948, "grad_norm": 0.6105300784111023, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.5778260231018066, "logits/rejected": -1.4280219078063965, "logps/chosen": -191.16876220703125, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -263.57275390625, "loss": 1.3645, "margin_dpo/margin_mean": 61.060604095458984, "margin_dpo/margin_std": 94.01798248291016, "step": 315 }, { "KL/chosen_KL_mean": -120.70732116699219, "KL/mean": -156.82171630859375, "KL/rejected_KL_mean": -192.93612670898438, "KL/std": 93.74090576171875, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.22879028320312, "fcm_dpo/q_t": 0.49343132972717285, "grad_norm": 0.7058082818984985, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.7478408813476562, "logits/rejected": -1.7572659254074097, "logps/chosen": -204.89157104492188, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -298.626953125, "loss": 1.3605, "margin_dpo/margin_mean": 72.22879028320312, "margin_dpo/margin_std": 104.33418273925781, "step": 316 }, { "KL/chosen_KL_mean": -126.76493072509766, "KL/mean": -168.20443725585938, "KL/rejected_KL_mean": -209.64395141601562, "KL/std": 99.00001525878906, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.8790054321289, "fcm_dpo/q_t": 0.49246281385421753, "grad_norm": 0.7400802969932556, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.8679721355438232, "logits/rejected": -1.4022815227508545, "logps/chosen": -214.638427734375, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -317.9091491699219, "loss": 1.3567, "margin_dpo/margin_mean": 82.8790054321289, "margin_dpo/margin_std": 103.33180236816406, "step": 317 }, { "KL/chosen_KL_mean": -132.30374145507812, "KL/mean": -163.41293334960938, "KL/rejected_KL_mean": -194.5221405029297, "KL/std": 78.21662902832031, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 62.21839141845703, "fcm_dpo/q_t": 0.49434077739715576, "grad_norm": 0.8151686787605286, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.488161563873291, "logits/rejected": -1.7811187505722046, "logps/chosen": -230.50927734375, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -282.658447265625, "loss": 1.3641, "margin_dpo/margin_mean": 62.21839904785156, "margin_dpo/margin_std": 92.67713928222656, "step": 318 }, { "KL/chosen_KL_mean": -108.31716918945312, "KL/mean": -149.04617309570312, "KL/rejected_KL_mean": -189.7751922607422, "KL/std": 91.43504333496094, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.45802307128906, "fcm_dpo/q_t": 0.49259161949157715, "grad_norm": 0.7332903742790222, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.734549641609192, "logits/rejected": -1.706794023513794, "logps/chosen": -190.820556640625, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -308.4258728027344, "loss": 1.3572, "margin_dpo/margin_mean": 81.45802307128906, "margin_dpo/margin_std": 102.8504638671875, "step": 319 }, { "KL/chosen_KL_mean": -105.89178466796875, "KL/mean": -141.0335235595703, "KL/rejected_KL_mean": -176.17526245117188, "KL/std": 86.29802703857422, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.28347778320312, "fcm_dpo/q_t": 0.49360716342926025, "grad_norm": 0.6260523796081543, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.138866901397705, "logits/rejected": -1.9578328132629395, "logps/chosen": -193.02902221679688, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -279.3993835449219, "loss": 1.3611, "margin_dpo/margin_mean": 70.28347778320312, "margin_dpo/margin_std": 88.19691467285156, "step": 320 }, { "KL/chosen_KL_mean": -129.58059692382812, "KL/mean": -165.00338745117188, "KL/rejected_KL_mean": -200.4261474609375, "KL/std": 89.25773620605469, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.84557342529297, "fcm_dpo/q_t": 0.49355560541152954, "grad_norm": 0.7722080945968628, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -2.1803717613220215, "logits/rejected": -1.8023591041564941, "logps/chosen": -222.6852569580078, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -297.894287109375, "loss": 1.3609, "margin_dpo/margin_mean": 70.84556579589844, "margin_dpo/margin_std": 84.67974090576172, "step": 321 }, { "KL/chosen_KL_mean": -120.62291717529297, "KL/mean": -163.04608154296875, "KL/rejected_KL_mean": -205.46925354003906, "KL/std": 86.5671615600586, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.84634399414062, "fcm_dpo/q_t": 0.49228277802467346, "grad_norm": 0.9280350208282471, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.8844470977783203, "logits/rejected": -1.5391461849212646, "logps/chosen": -212.81582641601562, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -306.36248779296875, "loss": 1.356, "margin_dpo/margin_mean": 84.84634399414062, "margin_dpo/margin_std": 92.27838134765625, "step": 322 }, { "KL/chosen_KL_mean": -140.92686462402344, "KL/mean": -171.42723083496094, "KL/rejected_KL_mean": -201.92758178710938, "KL/std": 87.6961441040039, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.000732421875, "fcm_dpo/q_t": 0.4944518208503723, "grad_norm": 0.7313146591186523, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0472300052642822, "logits/rejected": -1.671252965927124, "logps/chosen": -221.198974609375, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -282.2309265136719, "loss": 1.3645, "margin_dpo/margin_mean": 61.000732421875, "margin_dpo/margin_std": 95.22064208984375, "step": 323 }, { "KL/chosen_KL_mean": -148.60018920898438, "KL/mean": -180.0680694580078, "KL/rejected_KL_mean": -211.53591918945312, "KL/std": 97.90856170654297, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 62.93574523925781, "fcm_dpo/q_t": 0.4942760169506073, "grad_norm": 0.757982075214386, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.547990083694458, "logits/rejected": -0.9075179100036621, "logps/chosen": -244.3295440673828, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -305.7051696777344, "loss": 1.3639, "margin_dpo/margin_mean": 62.93573760986328, "margin_dpo/margin_std": 99.79803466796875, "step": 324 }, { "KL/chosen_KL_mean": -133.6934356689453, "KL/mean": -166.76138305664062, "KL/rejected_KL_mean": -199.82931518554688, "KL/std": 84.0896987915039, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.13590240478516, "fcm_dpo/q_t": 0.49398449063301086, "grad_norm": 0.8935292959213257, "learning_rate": 3.147047612756302e-07, "logits/chosen": -2.005892753601074, "logits/rejected": -1.284029245376587, "logps/chosen": -232.40029907226562, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -276.2374267578125, "loss": 1.3626, "margin_dpo/margin_mean": 66.13589477539062, "margin_dpo/margin_std": 89.38803100585938, "step": 325 }, { "KL/chosen_KL_mean": -145.3929443359375, "KL/mean": -177.97201538085938, "KL/rejected_KL_mean": -210.55108642578125, "KL/std": 83.67611694335938, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.15811157226562, "fcm_dpo/q_t": 0.49407297372817993, "grad_norm": 0.6680540442466736, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.540010690689087, "logits/rejected": -1.8570656776428223, "logps/chosen": -250.05426025390625, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -297.37646484375, "loss": 1.363, "margin_dpo/margin_mean": 65.15811920166016, "margin_dpo/margin_std": 89.78076934814453, "step": 326 }, { "KL/chosen_KL_mean": -142.866943359375, "KL/mean": -178.5404815673828, "KL/rejected_KL_mean": -214.21405029296875, "KL/std": 78.52259063720703, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.34710693359375, "fcm_dpo/q_t": 0.493510901927948, "grad_norm": 0.6842023134231567, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -2.1012120246887207, "logits/rejected": -1.5639615058898926, "logps/chosen": -250.30718994140625, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -331.26727294921875, "loss": 1.3608, "margin_dpo/margin_mean": 71.34710693359375, "margin_dpo/margin_std": 92.82408905029297, "step": 327 }, { "KL/chosen_KL_mean": -140.35289001464844, "KL/mean": -185.05911254882812, "KL/rejected_KL_mean": -229.76535034179688, "KL/std": 94.36602783203125, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.41246032714844, "fcm_dpo/q_t": 0.4918683171272278, "grad_norm": 0.7003490924835205, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.8437340259552002, "logits/rejected": -1.5540169477462769, "logps/chosen": -226.96359252929688, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -327.167236328125, "loss": 1.3544, "margin_dpo/margin_mean": 89.41246032714844, "margin_dpo/margin_std": 101.45838928222656, "step": 328 }, { "KL/chosen_KL_mean": -151.4568328857422, "KL/mean": -185.93223571777344, "KL/rejected_KL_mean": -220.4076385498047, "KL/std": 94.54548645019531, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.9508056640625, "fcm_dpo/q_t": 0.49372899532318115, "grad_norm": 0.8190925717353821, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.541459321975708, "logits/rejected": -1.044100284576416, "logps/chosen": -252.06991577148438, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -325.361328125, "loss": 1.3617, "margin_dpo/margin_mean": 68.9508056640625, "margin_dpo/margin_std": 104.92478942871094, "step": 329 }, { "KL/chosen_KL_mean": -122.18350982666016, "KL/mean": -160.2194061279297, "KL/rejected_KL_mean": -198.25531005859375, "KL/std": 86.59149932861328, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.07177734375, "fcm_dpo/q_t": 0.4930799603462219, "grad_norm": 0.6898196339607239, "learning_rate": 3.084861204504122e-07, "logits/chosen": -2.02060604095459, "logits/rejected": -1.9541997909545898, "logps/chosen": -199.14193725585938, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -291.07025146484375, "loss": 1.359, "margin_dpo/margin_mean": 76.07177734375, "margin_dpo/margin_std": 81.2776870727539, "step": 330 }, { "KL/chosen_KL_mean": -133.2657928466797, "KL/mean": -174.69540405273438, "KL/rejected_KL_mean": -216.12501525878906, "KL/std": 81.15861511230469, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.85923767089844, "fcm_dpo/q_t": 0.49246352910995483, "grad_norm": 0.8207454681396484, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.894834041595459, "logits/rejected": -1.6030985116958618, "logps/chosen": -216.32521057128906, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -310.25616455078125, "loss": 1.3566, "margin_dpo/margin_mean": 82.85923767089844, "margin_dpo/margin_std": 89.53767395019531, "step": 331 }, { "KL/chosen_KL_mean": -139.2109375, "KL/mean": -167.0198974609375, "KL/rejected_KL_mean": -194.82882690429688, "KL/std": 91.18476867675781, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.61789321899414, "fcm_dpo/q_t": 0.49494093656539917, "grad_norm": 0.7257262468338013, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.184760570526123, "logits/rejected": -1.7605326175689697, "logps/chosen": -221.141845703125, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -277.67474365234375, "loss": 1.3664, "margin_dpo/margin_mean": 55.61789321899414, "margin_dpo/margin_std": 91.4469985961914, "step": 332 }, { "KL/chosen_KL_mean": -162.44064331054688, "KL/mean": -201.37088012695312, "KL/rejected_KL_mean": -240.30111694335938, "KL/std": 101.54875183105469, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.86046600341797, "fcm_dpo/q_t": 0.492919385433197, "grad_norm": 0.6999876499176025, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.6394717693328857, "logits/rejected": -1.3062536716461182, "logps/chosen": -250.97323608398438, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -342.76715087890625, "loss": 1.3586, "margin_dpo/margin_mean": 77.86046600341797, "margin_dpo/margin_std": 115.288330078125, "step": 333 }, { "KL/chosen_KL_mean": -150.29730224609375, "KL/mean": -182.28341674804688, "KL/rejected_KL_mean": -214.26956176757812, "KL/std": 97.10531616210938, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.972259521484375, "fcm_dpo/q_t": 0.49418145418167114, "grad_norm": 0.7139880657196045, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.6103546619415283, "logits/rejected": -1.3189562559127808, "logps/chosen": -248.05508422851562, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -314.275390625, "loss": 1.3636, "margin_dpo/margin_mean": 63.972259521484375, "margin_dpo/margin_std": 110.68311309814453, "step": 334 }, { "KL/chosen_KL_mean": -146.12216186523438, "KL/mean": -177.39920043945312, "KL/rejected_KL_mean": -208.67626953125, "KL/std": 95.54989624023438, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 62.55409622192383, "fcm_dpo/q_t": 0.4943101406097412, "grad_norm": 0.7912931442260742, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.413041830062866, "logits/rejected": -1.9847434759140015, "logps/chosen": -242.49819946289062, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -303.60028076171875, "loss": 1.364, "margin_dpo/margin_mean": 62.55409622192383, "margin_dpo/margin_std": 103.68998718261719, "step": 335 }, { "KL/chosen_KL_mean": -149.3192596435547, "KL/mean": -188.77877807617188, "KL/rejected_KL_mean": -228.23831176757812, "KL/std": 104.25688171386719, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.9190673828125, "fcm_dpo/q_t": 0.4928240180015564, "grad_norm": 0.9397751092910767, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.5487431287765503, "logits/rejected": -1.41886568069458, "logps/chosen": -237.84878540039062, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -337.7940979003906, "loss": 1.3582, "margin_dpo/margin_mean": 78.9190673828125, "margin_dpo/margin_std": 115.75193786621094, "step": 336 }, { "KL/chosen_KL_mean": -148.67715454101562, "KL/mean": -183.18775939941406, "KL/rejected_KL_mean": -217.6983642578125, "KL/std": 99.23714447021484, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.0212173461914, "fcm_dpo/q_t": 0.4937232732772827, "grad_norm": 1.222922444343567, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.5389790534973145, "logits/rejected": -2.2782750129699707, "logps/chosen": -245.98953247070312, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -327.8297119140625, "loss": 1.3618, "margin_dpo/margin_mean": 69.0212173461914, "margin_dpo/margin_std": 119.0255355834961, "step": 337 }, { "KL/chosen_KL_mean": -147.28213500976562, "KL/mean": -182.8008575439453, "KL/rejected_KL_mean": -218.319580078125, "KL/std": 96.6544189453125, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.03744506835938, "fcm_dpo/q_t": 0.4935382604598999, "grad_norm": 0.8707118034362793, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.9207526445388794, "logits/rejected": -1.7913457155227661, "logps/chosen": -230.91224670410156, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -312.0119323730469, "loss": 1.3609, "margin_dpo/margin_mean": 71.03744506835938, "margin_dpo/margin_std": 92.04019165039062, "step": 338 }, { "KL/chosen_KL_mean": -167.58238220214844, "KL/mean": -203.53329467773438, "KL/rejected_KL_mean": -239.48422241210938, "KL/std": 99.42959594726562, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.9018325805664, "fcm_dpo/q_t": 0.4934612810611725, "grad_norm": 0.8349969983100891, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.941159725189209, "logits/rejected": -2.5226404666900635, "logps/chosen": -267.4342041015625, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -348.55859375, "loss": 1.3608, "margin_dpo/margin_mean": 71.90184020996094, "margin_dpo/margin_std": 112.38922882080078, "step": 339 }, { "KL/chosen_KL_mean": -179.77813720703125, "KL/mean": -211.97381591796875, "KL/rejected_KL_mean": -244.1694793701172, "KL/std": 94.88607788085938, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.3913345336914, "fcm_dpo/q_t": 0.49414363503456116, "grad_norm": 0.7166052460670471, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.5899147987365723, "logits/rejected": -1.1042330265045166, "logps/chosen": -264.7521057128906, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -322.9698181152344, "loss": 1.3634, "margin_dpo/margin_mean": 64.3913345336914, "margin_dpo/margin_std": 109.90341186523438, "step": 340 }, { "KL/chosen_KL_mean": -181.21702575683594, "KL/mean": -214.32345581054688, "KL/rejected_KL_mean": -247.4298858642578, "KL/std": 100.5274658203125, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.2128677368164, "fcm_dpo/q_t": 0.4939785897731781, "grad_norm": 1.2967149019241333, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.4057916402816772, "logits/rejected": -1.1847453117370605, "logps/chosen": -266.48101806640625, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -353.5158386230469, "loss": 1.3628, "margin_dpo/margin_mean": 66.21287536621094, "margin_dpo/margin_std": 118.70549011230469, "step": 341 }, { "KL/chosen_KL_mean": -154.27752685546875, "KL/mean": -200.57716369628906, "KL/rejected_KL_mean": -246.8767852783203, "KL/std": 95.897705078125, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.59925842285156, "fcm_dpo/q_t": 0.49157750606536865, "grad_norm": 0.810247540473938, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.6059767007827759, "logits/rejected": -0.8535606265068054, "logps/chosen": -255.31613159179688, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -345.7752990722656, "loss": 1.3532, "margin_dpo/margin_mean": 92.59925842285156, "margin_dpo/margin_std": 101.7650146484375, "step": 342 }, { "KL/chosen_KL_mean": -160.68618774414062, "KL/mean": -203.82254028320312, "KL/rejected_KL_mean": -246.9588623046875, "KL/std": 102.24070739746094, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.2726821899414, "fcm_dpo/q_t": 0.49215346574783325, "grad_norm": 0.8888170719146729, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.182816982269287, "logits/rejected": -2.103832721710205, "logps/chosen": -244.43356323242188, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -363.5379638671875, "loss": 1.3555, "margin_dpo/margin_mean": 86.2726821899414, "margin_dpo/margin_std": 102.39237976074219, "step": 343 }, { "KL/chosen_KL_mean": -159.1710662841797, "KL/mean": -206.58258056640625, "KL/rejected_KL_mean": -253.99407958984375, "KL/std": 110.3947525024414, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.82304382324219, "fcm_dpo/q_t": 0.49137866497039795, "grad_norm": 1.00442373752594, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.6581742763519287, "logits/rejected": -1.423604965209961, "logps/chosen": -237.42503356933594, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -349.42279052734375, "loss": 1.3526, "margin_dpo/margin_mean": 94.82304382324219, "margin_dpo/margin_std": 118.95082092285156, "step": 344 }, { "KL/chosen_KL_mean": -172.69143676757812, "KL/mean": -218.20645141601562, "KL/rejected_KL_mean": -263.721435546875, "KL/std": 110.43985748291016, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.03003692626953, "fcm_dpo/q_t": 0.49172243475914, "grad_norm": 0.9692233800888062, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.6049047708511353, "logits/rejected": -1.3242138624191284, "logps/chosen": -270.77093505859375, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -367.7562255859375, "loss": 1.3539, "margin_dpo/margin_mean": 91.03003692626953, "margin_dpo/margin_std": 120.76396179199219, "step": 345 }, { "KL/chosen_KL_mean": -192.4693145751953, "KL/mean": -226.9765625, "KL/rejected_KL_mean": -261.48382568359375, "KL/std": 100.59674835205078, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.01448059082031, "fcm_dpo/q_t": 0.49372348189353943, "grad_norm": 0.7467343211174011, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.9147953987121582, "logits/rejected": -1.6191020011901855, "logps/chosen": -273.1528015136719, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -347.37640380859375, "loss": 1.3617, "margin_dpo/margin_mean": 69.01448059082031, "margin_dpo/margin_std": 111.23995971679688, "step": 346 }, { "KL/chosen_KL_mean": -186.00994873046875, "KL/mean": -219.91299438476562, "KL/rejected_KL_mean": -253.81607055664062, "KL/std": 106.09712219238281, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.80613708496094, "fcm_dpo/q_t": 0.49383389949798584, "grad_norm": 0.7186636924743652, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.8870346546173096, "logits/rejected": -1.5816588401794434, "logps/chosen": -279.6893310546875, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -351.35858154296875, "loss": 1.3622, "margin_dpo/margin_mean": 67.80613708496094, "margin_dpo/margin_std": 119.1959228515625, "step": 347 }, { "KL/chosen_KL_mean": -167.26681518554688, "KL/mean": -202.7482452392578, "KL/rejected_KL_mean": -238.22967529296875, "KL/std": 98.35514831542969, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.96286010742188, "fcm_dpo/q_t": 0.4935460686683655, "grad_norm": 0.7997406721115112, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.0083837509155273, "logits/rejected": -1.7451856136322021, "logps/chosen": -257.5190734863281, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -333.99993896484375, "loss": 1.361, "margin_dpo/margin_mean": 70.96286010742188, "margin_dpo/margin_std": 99.32627868652344, "step": 348 }, { "KL/chosen_KL_mean": -184.60604858398438, "KL/mean": -215.485595703125, "KL/rejected_KL_mean": -246.36512756347656, "KL/std": 109.29093933105469, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.759063720703125, "fcm_dpo/q_t": 0.4943845868110657, "grad_norm": 0.915273904800415, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.230034828186035, "logits/rejected": -2.023677349090576, "logps/chosen": -276.66412353515625, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -350.33038330078125, "loss": 1.3645, "margin_dpo/margin_mean": 61.759063720703125, "margin_dpo/margin_std": 129.05435180664062, "step": 349 }, { "KL/chosen_KL_mean": -188.12440490722656, "KL/mean": -217.86611938476562, "KL/rejected_KL_mean": -247.6078338623047, "KL/std": 97.84568786621094, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.48340606689453, "fcm_dpo/q_t": 0.4945898652076721, "grad_norm": 0.769838809967041, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.908294916152954, "logits/rejected": -1.6903955936431885, "logps/chosen": -275.1785888671875, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -343.32977294921875, "loss": 1.3652, "margin_dpo/margin_mean": 59.48340606689453, "margin_dpo/margin_std": 114.35772705078125, "step": 350 }, { "KL/chosen_KL_mean": -184.76634216308594, "KL/mean": -228.49021911621094, "KL/rejected_KL_mean": -272.214111328125, "KL/std": 111.65129852294922, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.44773864746094, "fcm_dpo/q_t": 0.4920479655265808, "grad_norm": 0.9345192313194275, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.4993529319763184, "logits/rejected": -1.2291303873062134, "logps/chosen": -273.93536376953125, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -380.5121154785156, "loss": 1.3552, "margin_dpo/margin_mean": 87.44773864746094, "margin_dpo/margin_std": 124.15311431884766, "step": 351 }, { "KL/chosen_KL_mean": -177.83920288085938, "KL/mean": -213.4480438232422, "KL/rejected_KL_mean": -249.056884765625, "KL/std": 106.60469055175781, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.21765899658203, "fcm_dpo/q_t": 0.49352508783340454, "grad_norm": 0.9877608418464661, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.7789299488067627, "logits/rejected": -1.2107391357421875, "logps/chosen": -274.65069580078125, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -333.4617004394531, "loss": 1.3611, "margin_dpo/margin_mean": 71.21766662597656, "margin_dpo/margin_std": 125.68152618408203, "step": 352 }, { "KL/chosen_KL_mean": -193.91583251953125, "KL/mean": -218.3944854736328, "KL/rejected_KL_mean": -242.87313842773438, "KL/std": 94.61516571044922, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.957313537597656, "fcm_dpo/q_t": 0.49554720520973206, "grad_norm": 0.8663977384567261, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.2101526260375977, "logits/rejected": -1.820171594619751, "logps/chosen": -279.2783203125, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -330.2404479980469, "loss": 1.369, "margin_dpo/margin_mean": 48.957313537597656, "margin_dpo/margin_std": 108.5419921875, "step": 353 }, { "KL/chosen_KL_mean": -169.31204223632812, "KL/mean": -208.84152221679688, "KL/rejected_KL_mean": -248.3710174560547, "KL/std": 98.509521484375, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.05896759033203, "fcm_dpo/q_t": 0.49281030893325806, "grad_norm": 0.9006252288818359, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8994472026824951, "logits/rejected": -1.4418854713439941, "logps/chosen": -265.11322021484375, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -343.2371520996094, "loss": 1.3581, "margin_dpo/margin_mean": 79.05896759033203, "margin_dpo/margin_std": 101.05210876464844, "step": 354 }, { "KL/chosen_KL_mean": -167.02377319335938, "KL/mean": -205.9095458984375, "KL/rejected_KL_mean": -244.79531860351562, "KL/std": 97.71288299560547, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.77151489257812, "fcm_dpo/q_t": 0.49292677640914917, "grad_norm": 0.8648291230201721, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4507951736450195, "logits/rejected": -1.9625849723815918, "logps/chosen": -253.20933532714844, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -333.25689697265625, "loss": 1.3586, "margin_dpo/margin_mean": 77.77151489257812, "margin_dpo/margin_std": 103.86575317382812, "step": 355 }, { "KL/chosen_KL_mean": -204.48880004882812, "KL/mean": -227.94973754882812, "KL/rejected_KL_mean": -251.4106903076172, "KL/std": 99.31417846679688, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 46.92190933227539, "fcm_dpo/q_t": 0.49573349952697754, "grad_norm": 0.8260735869407654, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.6962158679962158, "logits/rejected": -1.2159392833709717, "logps/chosen": -295.306884765625, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -333.53802490234375, "loss": 1.3698, "margin_dpo/margin_mean": 46.921913146972656, "margin_dpo/margin_std": 117.75009155273438, "step": 356 }, { "KL/chosen_KL_mean": -172.89608764648438, "KL/mean": -217.02703857421875, "KL/rejected_KL_mean": -261.15802001953125, "KL/std": 104.23750305175781, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.26194763183594, "fcm_dpo/q_t": 0.49197280406951904, "grad_norm": 1.0038914680480957, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.7554752826690674, "logits/rejected": -1.3898425102233887, "logps/chosen": -260.40081787109375, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -353.40740966796875, "loss": 1.3548, "margin_dpo/margin_mean": 88.2619400024414, "margin_dpo/margin_std": 100.63441467285156, "step": 357 }, { "KL/chosen_KL_mean": -178.49664306640625, "KL/mean": -215.6006317138672, "KL/rejected_KL_mean": -252.70462036132812, "KL/std": 101.59156799316406, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.20796203613281, "fcm_dpo/q_t": 0.4932519197463989, "grad_norm": 1.0003002882003784, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.6032872200012207, "logits/rejected": -1.1514511108398438, "logps/chosen": -273.5521240234375, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -349.18072509765625, "loss": 1.3599, "margin_dpo/margin_mean": 74.20795440673828, "margin_dpo/margin_std": 108.20745849609375, "step": 358 }, { "KL/chosen_KL_mean": -199.01348876953125, "KL/mean": -232.92236328125, "KL/rejected_KL_mean": -266.83123779296875, "KL/std": 102.5389404296875, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.81775665283203, "fcm_dpo/q_t": 0.493831992149353, "grad_norm": 0.905453622341156, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4211071729660034, "logits/rejected": -0.8205487728118896, "logps/chosen": -286.3868408203125, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -355.6031188964844, "loss": 1.3622, "margin_dpo/margin_mean": 67.8177490234375, "margin_dpo/margin_std": 115.3590087890625, "step": 359 }, { "KL/chosen_KL_mean": -184.0244140625, "KL/mean": -222.63845825195312, "KL/rejected_KL_mean": -261.2525329589844, "KL/std": 92.87191772460938, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.22813415527344, "fcm_dpo/q_t": 0.49297547340393066, "grad_norm": 1.0166014432907104, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.5261640548706055, "logits/rejected": -1.1875019073486328, "logps/chosen": -259.24786376953125, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -337.3465270996094, "loss": 1.3587, "margin_dpo/margin_mean": 77.22813415527344, "margin_dpo/margin_std": 100.05503845214844, "step": 360 }, { "KL/chosen_KL_mean": -181.20518493652344, "KL/mean": -222.42855834960938, "KL/rejected_KL_mean": -263.6519470214844, "KL/std": 110.0662841796875, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.44676208496094, "fcm_dpo/q_t": 0.4925026297569275, "grad_norm": 0.815161406993866, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.528104543685913, "logits/rejected": -1.2955803871154785, "logps/chosen": -263.15850830078125, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -360.3311462402344, "loss": 1.3569, "margin_dpo/margin_mean": 82.44676208496094, "margin_dpo/margin_std": 110.7667236328125, "step": 361 }, { "KL/chosen_KL_mean": -187.51629638671875, "KL/mean": -228.94691467285156, "KL/rejected_KL_mean": -270.3775329589844, "KL/std": 113.00533294677734, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.86121368408203, "fcm_dpo/q_t": 0.4924650490283966, "grad_norm": 0.9034834504127502, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.9891437292099, "logits/rejected": -1.8070474863052368, "logps/chosen": -276.85601806640625, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -380.8795166015625, "loss": 1.3569, "margin_dpo/margin_mean": 82.86121368408203, "margin_dpo/margin_std": 121.80889892578125, "step": 362 }, { "KL/chosen_KL_mean": -171.81039428710938, "KL/mean": -225.1159210205078, "KL/rejected_KL_mean": -278.42144775390625, "KL/std": 109.88695526123047, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.611083984375, "fcm_dpo/q_t": 0.4903060793876648, "grad_norm": 0.9357336759567261, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -2.048159599304199, "logits/rejected": -1.6483769416809082, "logps/chosen": -252.58538818359375, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -357.8541259765625, "loss": 1.3483, "margin_dpo/margin_mean": 106.611083984375, "margin_dpo/margin_std": 114.93002319335938, "step": 363 }, { "KL/chosen_KL_mean": -165.46412658691406, "KL/mean": -208.24130249023438, "KL/rejected_KL_mean": -251.01846313476562, "KL/std": 101.43978881835938, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.5543212890625, "fcm_dpo/q_t": 0.49221867322921753, "grad_norm": 1.0311967134475708, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.2013206481933594, "logits/rejected": -2.0865535736083984, "logps/chosen": -241.99374389648438, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -348.5802001953125, "loss": 1.3557, "margin_dpo/margin_mean": 85.5543212890625, "margin_dpo/margin_std": 99.4864273071289, "step": 364 }, { "KL/chosen_KL_mean": -177.32022094726562, "KL/mean": -219.65313720703125, "KL/rejected_KL_mean": -261.986083984375, "KL/std": 108.89871215820312, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.66582489013672, "fcm_dpo/q_t": 0.492303729057312, "grad_norm": 1.0799362659454346, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7997865676879883, "logits/rejected": -1.7093557119369507, "logps/chosen": -257.8651123046875, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -361.3092346191406, "loss": 1.3563, "margin_dpo/margin_mean": 84.66582489013672, "margin_dpo/margin_std": 125.13111114501953, "step": 365 }, { "KL/chosen_KL_mean": -177.12368774414062, "KL/mean": -213.56826782226562, "KL/rejected_KL_mean": -250.01284790039062, "KL/std": 113.86600494384766, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.8891372680664, "fcm_dpo/q_t": 0.4933742880821228, "grad_norm": 0.8691852688789368, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -2.0426485538482666, "logits/rejected": -2.1078741550445557, "logps/chosen": -249.3617706298828, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -341.6846923828125, "loss": 1.3605, "margin_dpo/margin_mean": 72.88914489746094, "margin_dpo/margin_std": 125.07225036621094, "step": 366 }, { "KL/chosen_KL_mean": -166.27252197265625, "KL/mean": -208.83001708984375, "KL/rejected_KL_mean": -251.38751220703125, "KL/std": 98.39833068847656, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.11499786376953, "fcm_dpo/q_t": 0.49225935339927673, "grad_norm": 0.9662667512893677, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.854824185371399, "logits/rejected": -1.4920262098312378, "logps/chosen": -251.69451904296875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -334.9053039550781, "loss": 1.3559, "margin_dpo/margin_mean": 85.11500549316406, "margin_dpo/margin_std": 102.79495239257812, "step": 367 }, { "KL/chosen_KL_mean": -143.25274658203125, "KL/mean": -196.10162353515625, "KL/rejected_KL_mean": -248.95046997070312, "KL/std": 109.72242736816406, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.69770812988281, "fcm_dpo/q_t": 0.4903867840766907, "grad_norm": 0.9113496541976929, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.7766039371490479, "logits/rejected": -1.7688448429107666, "logps/chosen": -217.0938720703125, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -353.645751953125, "loss": 1.3486, "margin_dpo/margin_mean": 105.69770812988281, "margin_dpo/margin_std": 101.72529602050781, "step": 368 }, { "KL/chosen_KL_mean": -159.89561462402344, "KL/mean": -205.93450927734375, "KL/rejected_KL_mean": -251.97341918945312, "KL/std": 104.65704345703125, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.07780456542969, "fcm_dpo/q_t": 0.4916269779205322, "grad_norm": 0.8797938823699951, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.8370492458343506, "logits/rejected": -1.496963381767273, "logps/chosen": -258.3575744628906, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -351.130126953125, "loss": 1.3535, "margin_dpo/margin_mean": 92.07780456542969, "margin_dpo/margin_std": 106.89232635498047, "step": 369 }, { "KL/chosen_KL_mean": -188.60708618164062, "KL/mean": -218.0814666748047, "KL/rejected_KL_mean": -247.55587768554688, "KL/std": 101.82417297363281, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.948760986328125, "fcm_dpo/q_t": 0.49463951587677, "grad_norm": 0.8821626305580139, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -2.096390724182129, "logits/rejected": -1.9175591468811035, "logps/chosen": -285.3828125, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -350.29998779296875, "loss": 1.3655, "margin_dpo/margin_mean": 58.948760986328125, "margin_dpo/margin_std": 127.63455200195312, "step": 370 }, { "KL/chosen_KL_mean": -178.73919677734375, "KL/mean": -212.51214599609375, "KL/rejected_KL_mean": -246.28512573242188, "KL/std": 97.90868377685547, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.5459213256836, "fcm_dpo/q_t": 0.49385666847229004, "grad_norm": 1.0349375009536743, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -2.0471489429473877, "logits/rejected": -1.6975362300872803, "logps/chosen": -281.7294006347656, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -351.6357421875, "loss": 1.3622, "margin_dpo/margin_mean": 67.54592895507812, "margin_dpo/margin_std": 104.92131042480469, "step": 371 }, { "KL/chosen_KL_mean": -159.487548828125, "KL/mean": -208.59080505371094, "KL/rejected_KL_mean": -257.6940612792969, "KL/std": 114.43161010742188, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.20652770996094, "fcm_dpo/q_t": 0.49107176065444946, "grad_norm": 1.0204379558563232, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.9200867414474487, "logits/rejected": -1.2818241119384766, "logps/chosen": -250.51377868652344, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -343.3042907714844, "loss": 1.3514, "margin_dpo/margin_mean": 98.20652770996094, "margin_dpo/margin_std": 126.39222717285156, "step": 372 }, { "KL/chosen_KL_mean": -186.43389892578125, "KL/mean": -216.46633911132812, "KL/rejected_KL_mean": -246.498779296875, "KL/std": 113.82173156738281, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.06488800048828, "fcm_dpo/q_t": 0.4945371448993683, "grad_norm": 1.135353684425354, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.2397830486297607, "logits/rejected": -1.7184360027313232, "logps/chosen": -286.5069580078125, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -337.6151123046875, "loss": 1.365, "margin_dpo/margin_mean": 60.06488800048828, "margin_dpo/margin_std": 114.65261840820312, "step": 373 }, { "KL/chosen_KL_mean": -157.58544921875, "KL/mean": -199.19424438476562, "KL/rejected_KL_mean": -240.80303955078125, "KL/std": 105.68367004394531, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.21757507324219, "fcm_dpo/q_t": 0.4924326539039612, "grad_norm": 0.9221986532211304, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.3380837440490723, "logits/rejected": -1.9697742462158203, "logps/chosen": -255.59779357910156, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -341.5552978515625, "loss": 1.3566, "margin_dpo/margin_mean": 83.21757507324219, "margin_dpo/margin_std": 109.4050064086914, "step": 374 }, { "KL/chosen_KL_mean": -149.45709228515625, "KL/mean": -199.14419555664062, "KL/rejected_KL_mean": -248.83128356933594, "KL/std": 94.68896484375, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.37420654296875, "fcm_dpo/q_t": 0.4909617006778717, "grad_norm": 1.0185810327529907, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.5717110633850098, "logits/rejected": -1.3357819318771362, "logps/chosen": -233.85337829589844, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -344.5501403808594, "loss": 1.3508, "margin_dpo/margin_mean": 99.37419891357422, "margin_dpo/margin_std": 106.06827545166016, "step": 375 }, { "KL/chosen_KL_mean": -147.46893310546875, "KL/mean": -204.2501983642578, "KL/rejected_KL_mean": -261.031494140625, "KL/std": 121.10807800292969, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.56254577636719, "fcm_dpo/q_t": 0.489674836397171, "grad_norm": 0.8469047546386719, "learning_rate": 2.5e-07, "logits/chosen": -1.7598469257354736, "logits/rejected": -1.4238927364349365, "logps/chosen": -243.20663452148438, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -373.0047607421875, "loss": 1.346, "margin_dpo/margin_mean": 113.56254577636719, "margin_dpo/margin_std": 130.0355224609375, "step": 376 }, { "KL/chosen_KL_mean": -162.67572021484375, "KL/mean": -218.12339782714844, "KL/rejected_KL_mean": -273.5710754394531, "KL/std": 118.41307830810547, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.89534759521484, "fcm_dpo/q_t": 0.489915132522583, "grad_norm": 1.115671157836914, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.3885087966918945, "logits/rejected": -1.2946076393127441, "logps/chosen": -241.60267639160156, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -379.4700012207031, "loss": 1.3468, "margin_dpo/margin_mean": 110.89534759521484, "margin_dpo/margin_std": 110.572998046875, "step": 377 }, { "KL/chosen_KL_mean": -149.98641967773438, "KL/mean": -194.03610229492188, "KL/rejected_KL_mean": -238.08575439453125, "KL/std": 111.81681823730469, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.09934997558594, "fcm_dpo/q_t": 0.4919881224632263, "grad_norm": 0.9591884613037109, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.1435415744781494, "logits/rejected": -1.891158103942871, "logps/chosen": -225.35882568359375, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -331.78985595703125, "loss": 1.355, "margin_dpo/margin_mean": 88.0993423461914, "margin_dpo/margin_std": 122.89102172851562, "step": 378 }, { "KL/chosen_KL_mean": -178.26634216308594, "KL/mean": -217.96075439453125, "KL/rejected_KL_mean": -257.6551818847656, "KL/std": 112.94235229492188, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.38883972167969, "fcm_dpo/q_t": 0.49278220534324646, "grad_norm": 0.8373602032661438, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.928919792175293, "logits/rejected": -1.9124579429626465, "logps/chosen": -267.12200927734375, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -379.03631591796875, "loss": 1.3582, "margin_dpo/margin_mean": 79.38883972167969, "margin_dpo/margin_std": 131.4962921142578, "step": 379 }, { "KL/chosen_KL_mean": -184.84556579589844, "KL/mean": -240.80767822265625, "KL/rejected_KL_mean": -296.769775390625, "KL/std": 124.38056945800781, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.9241943359375, "fcm_dpo/q_t": 0.48982590436935425, "grad_norm": 1.0556137561798096, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.4230661392211914, "logits/rejected": -1.4161427021026611, "logps/chosen": -264.67877197265625, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -407.5145263671875, "loss": 1.3467, "margin_dpo/margin_mean": 111.92418670654297, "margin_dpo/margin_std": 143.48963928222656, "step": 380 }, { "KL/chosen_KL_mean": -176.75955200195312, "KL/mean": -213.9329833984375, "KL/rejected_KL_mean": -251.10641479492188, "KL/std": 116.61373138427734, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.34683990478516, "fcm_dpo/q_t": 0.4932399392127991, "grad_norm": 0.9770979285240173, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.9034305810928345, "logits/rejected": -1.389075756072998, "logps/chosen": -263.3263854980469, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -335.5231628417969, "loss": 1.3601, "margin_dpo/margin_mean": 74.34684753417969, "margin_dpo/margin_std": 138.21649169921875, "step": 381 }, { "KL/chosen_KL_mean": -169.05430603027344, "KL/mean": -214.29786682128906, "KL/rejected_KL_mean": -259.54144287109375, "KL/std": 107.33595275878906, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.48712921142578, "fcm_dpo/q_t": 0.49177175760269165, "grad_norm": 1.6794415712356567, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.8790152072906494, "logits/rejected": -1.6445090770721436, "logps/chosen": -273.51611328125, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -374.1436767578125, "loss": 1.3541, "margin_dpo/margin_mean": 90.48712921142578, "margin_dpo/margin_std": 122.25260162353516, "step": 382 }, { "KL/chosen_KL_mean": -187.34671020507812, "KL/mean": -228.21807861328125, "KL/rejected_KL_mean": -269.0894470214844, "KL/std": 120.78411102294922, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.74274444580078, "fcm_dpo/q_t": 0.4925670027732849, "grad_norm": 1.1881667375564575, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.8493716716766357, "logits/rejected": -1.7301526069641113, "logps/chosen": -269.4627685546875, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -367.8666076660156, "loss": 1.3573, "margin_dpo/margin_mean": 81.74274444580078, "margin_dpo/margin_std": 125.46829223632812, "step": 383 }, { "KL/chosen_KL_mean": -197.28875732421875, "KL/mean": -239.13865661621094, "KL/rejected_KL_mean": -280.98858642578125, "KL/std": 120.67817687988281, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.69982147216797, "fcm_dpo/q_t": 0.49238863587379456, "grad_norm": 1.602678656578064, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.8336693048477173, "logits/rejected": -1.6251020431518555, "logps/chosen": -277.8739929199219, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -372.43646240234375, "loss": 1.3567, "margin_dpo/margin_mean": 83.69982147216797, "margin_dpo/margin_std": 133.8952178955078, "step": 384 }, { "KL/chosen_KL_mean": -166.47813415527344, "KL/mean": -215.008056640625, "KL/rejected_KL_mean": -263.53802490234375, "KL/std": 101.20652770996094, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.05984497070312, "fcm_dpo/q_t": 0.4911741018295288, "grad_norm": 1.4249235391616821, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.2267589569091797, "logits/rejected": -1.632882833480835, "logps/chosen": -259.2383117675781, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -347.99798583984375, "loss": 1.3518, "margin_dpo/margin_mean": 97.05984497070312, "margin_dpo/margin_std": 125.04959869384766, "step": 385 }, { "KL/chosen_KL_mean": -178.54898071289062, "KL/mean": -226.53125, "KL/rejected_KL_mean": -274.5135498046875, "KL/std": 104.94932556152344, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.96458435058594, "fcm_dpo/q_t": 0.4912724494934082, "grad_norm": 0.9700907468795776, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.6265153884887695, "logits/rejected": -1.3574599027633667, "logps/chosen": -263.9079284667969, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -371.50421142578125, "loss": 1.3522, "margin_dpo/margin_mean": 95.96458435058594, "margin_dpo/margin_std": 120.54728698730469, "step": 386 }, { "KL/chosen_KL_mean": -205.60906982421875, "KL/mean": -244.04669189453125, "KL/rejected_KL_mean": -282.48431396484375, "KL/std": 114.22113037109375, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.87522888183594, "fcm_dpo/q_t": 0.49300870299339294, "grad_norm": 1.2254066467285156, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.459016799926758, "logits/rejected": -1.7884490489959717, "logps/chosen": -308.2972106933594, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -381.9128723144531, "loss": 1.3589, "margin_dpo/margin_mean": 76.87522888183594, "margin_dpo/margin_std": 106.61904907226562, "step": 387 }, { "KL/chosen_KL_mean": -198.00827026367188, "KL/mean": -249.09182739257812, "KL/rejected_KL_mean": -300.1754150390625, "KL/std": 121.22393798828125, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.16712188720703, "fcm_dpo/q_t": 0.49070942401885986, "grad_norm": 0.9585520029067993, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.677984356880188, "logits/rejected": -1.601196050643921, "logps/chosen": -281.593505859375, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -410.2757568359375, "loss": 1.35, "margin_dpo/margin_mean": 102.1671142578125, "margin_dpo/margin_std": 127.39134216308594, "step": 388 }, { "KL/chosen_KL_mean": -215.88612365722656, "KL/mean": -253.49069213867188, "KL/rejected_KL_mean": -291.0952453613281, "KL/std": 122.9730224609375, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.2091293334961, "fcm_dpo/q_t": 0.4931609630584717, "grad_norm": 1.0877854824066162, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.233241081237793, "logits/rejected": -1.7751067876815796, "logps/chosen": -305.7083740234375, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -375.94903564453125, "loss": 1.3597, "margin_dpo/margin_mean": 75.20912170410156, "margin_dpo/margin_std": 136.73983764648438, "step": 389 }, { "KL/chosen_KL_mean": -203.630859375, "KL/mean": -249.55914306640625, "KL/rejected_KL_mean": -295.4873962402344, "KL/std": 122.56779479980469, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.8565444946289, "fcm_dpo/q_t": 0.49165207147598267, "grad_norm": 0.8791910409927368, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.5720950365066528, "logits/rejected": -1.3773651123046875, "logps/chosen": -281.1632995605469, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -388.0713195800781, "loss": 1.3539, "margin_dpo/margin_mean": 91.85655212402344, "margin_dpo/margin_std": 145.87574768066406, "step": 390 }, { "KL/chosen_KL_mean": -200.3042449951172, "KL/mean": -234.88409423828125, "KL/rejected_KL_mean": -269.4639587402344, "KL/std": 116.33213806152344, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.15969848632812, "fcm_dpo/q_t": 0.4937098026275635, "grad_norm": 1.2944446802139282, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.8807950019836426, "logits/rejected": -1.4598252773284912, "logps/chosen": -288.53717041015625, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -362.47149658203125, "loss": 1.3618, "margin_dpo/margin_mean": 69.15969848632812, "margin_dpo/margin_std": 118.82960510253906, "step": 391 }, { "KL/chosen_KL_mean": -187.063232421875, "KL/mean": -247.23727416992188, "KL/rejected_KL_mean": -307.41131591796875, "KL/std": 134.86997985839844, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.34811401367188, "fcm_dpo/q_t": 0.48905888199806213, "grad_norm": 1.028505802154541, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.5383713245391846, "logits/rejected": -1.4393025636672974, "logps/chosen": -274.8076477050781, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -415.9820556640625, "loss": 1.3437, "margin_dpo/margin_mean": 120.34811401367188, "margin_dpo/margin_std": 145.052001953125, "step": 392 }, { "KL/chosen_KL_mean": -177.40011596679688, "KL/mean": -221.87188720703125, "KL/rejected_KL_mean": -266.3436584472656, "KL/std": 118.01399230957031, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.94352722167969, "fcm_dpo/q_t": 0.491911917924881, "grad_norm": 0.8497743606567383, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.7141625881195068, "logits/rejected": -1.3971697092056274, "logps/chosen": -253.96392822265625, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -346.06982421875, "loss": 1.3547, "margin_dpo/margin_mean": 88.94353485107422, "margin_dpo/margin_std": 120.73057556152344, "step": 393 }, { "KL/chosen_KL_mean": -189.3004913330078, "KL/mean": -240.79592895507812, "KL/rejected_KL_mean": -292.2913818359375, "KL/std": 115.5928955078125, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.99090576171875, "fcm_dpo/q_t": 0.49063509702682495, "grad_norm": 1.3508741855621338, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6443471908569336, "logits/rejected": -1.434368371963501, "logps/chosen": -273.9991455078125, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -389.7491455078125, "loss": 1.3496, "margin_dpo/margin_mean": 102.99090576171875, "margin_dpo/margin_std": 120.57392883300781, "step": 394 }, { "KL/chosen_KL_mean": -195.06524658203125, "KL/mean": -236.36114501953125, "KL/rejected_KL_mean": -277.65704345703125, "KL/std": 118.40348815917969, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.59181213378906, "fcm_dpo/q_t": 0.49249279499053955, "grad_norm": 1.0467205047607422, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5405683517456055, "logits/rejected": -2.145899772644043, "logps/chosen": -283.952880859375, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -368.70257568359375, "loss": 1.3573, "margin_dpo/margin_mean": 82.59181213378906, "margin_dpo/margin_std": 159.75503540039062, "step": 395 }, { "KL/chosen_KL_mean": -211.0719451904297, "KL/mean": -244.45932006835938, "KL/rejected_KL_mean": -277.84674072265625, "KL/std": 105.89325714111328, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.77479553222656, "fcm_dpo/q_t": 0.4939284324645996, "grad_norm": 1.0817525386810303, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8913463354110718, "logits/rejected": -1.4163239002227783, "logps/chosen": -313.54327392578125, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -371.0815124511719, "loss": 1.3627, "margin_dpo/margin_mean": 66.77479553222656, "margin_dpo/margin_std": 125.77687072753906, "step": 396 }, { "KL/chosen_KL_mean": -191.84962463378906, "KL/mean": -258.509033203125, "KL/rejected_KL_mean": -325.16845703125, "KL/std": 124.88023376464844, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.31883239746094, "fcm_dpo/q_t": 0.4878804683685303, "grad_norm": 1.051343321800232, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.2475037574768066, "logits/rejected": -1.6700012683868408, "logps/chosen": -292.89404296875, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -434.01959228515625, "loss": 1.3391, "margin_dpo/margin_mean": 133.31881713867188, "margin_dpo/margin_std": 148.19677734375, "step": 397 }, { "KL/chosen_KL_mean": -212.04051208496094, "KL/mean": -256.9162902832031, "KL/rejected_KL_mean": -301.79205322265625, "KL/std": 124.27288818359375, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.75151062011719, "fcm_dpo/q_t": 0.4918413460254669, "grad_norm": 0.8932135105133057, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.0038039684295654, "logits/rejected": -1.7471816539764404, "logps/chosen": -307.0897216796875, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -415.79766845703125, "loss": 1.3547, "margin_dpo/margin_mean": 89.75151824951172, "margin_dpo/margin_std": 157.50247192382812, "step": 398 }, { "KL/chosen_KL_mean": -176.28807067871094, "KL/mean": -234.19964599609375, "KL/rejected_KL_mean": -292.1111755371094, "KL/std": 111.22946166992188, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.82308959960938, "fcm_dpo/q_t": 0.4894695580005646, "grad_norm": 0.9833790063858032, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.0446536540985107, "logits/rejected": -1.2256288528442383, "logps/chosen": -272.9266357421875, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -375.64764404296875, "loss": 1.3452, "margin_dpo/margin_mean": 115.82308959960938, "margin_dpo/margin_std": 134.67637634277344, "step": 399 }, { "KL/chosen_KL_mean": -193.1552734375, "KL/mean": -252.4359130859375, "KL/rejected_KL_mean": -311.7165832519531, "KL/std": 121.08788299560547, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.56129455566406, "fcm_dpo/q_t": 0.4892216920852661, "grad_norm": 1.3723735809326172, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.1169614791870117, "logits/rejected": -1.6685991287231445, "logps/chosen": -298.45843505859375, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -420.37213134765625, "loss": 1.3444, "margin_dpo/margin_mean": 118.5613021850586, "margin_dpo/margin_std": 150.3964080810547, "step": 400 }, { "KL/chosen_KL_mean": -207.37384033203125, "KL/mean": -242.18992614746094, "KL/rejected_KL_mean": -277.0059814453125, "KL/std": 128.53662109375, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.6321792602539, "fcm_dpo/q_t": 0.4936734735965729, "grad_norm": 1.2752323150634766, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.438228726387024, "logits/rejected": -1.1983909606933594, "logps/chosen": -291.66302490234375, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -367.9490966796875, "loss": 1.3619, "margin_dpo/margin_mean": 69.6321792602539, "margin_dpo/margin_std": 157.20834350585938, "step": 401 }, { "KL/chosen_KL_mean": -182.39230346679688, "KL/mean": -253.97817993164062, "KL/rejected_KL_mean": -325.5640869140625, "KL/std": 134.62110900878906, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 143.17178344726562, "fcm_dpo/q_t": 0.4869849383831024, "grad_norm": 1.2138631343841553, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.964569330215454, "logits/rejected": -1.5195369720458984, "logps/chosen": -265.98541259765625, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -413.37432861328125, "loss": 1.3355, "margin_dpo/margin_mean": 143.17178344726562, "margin_dpo/margin_std": 140.76400756835938, "step": 402 }, { "KL/chosen_KL_mean": -184.8344268798828, "KL/mean": -239.397705078125, "KL/rejected_KL_mean": -293.96099853515625, "KL/std": 126.30580139160156, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.1265640258789, "fcm_dpo/q_t": 0.4900781512260437, "grad_norm": 1.237794041633606, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.329070568084717, "logits/rejected": -1.9343056678771973, "logps/chosen": -258.5875244140625, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -377.8811340332031, "loss": 1.3476, "margin_dpo/margin_mean": 109.1265640258789, "margin_dpo/margin_std": 135.99649047851562, "step": 403 }, { "KL/chosen_KL_mean": -219.57293701171875, "KL/mean": -260.28546142578125, "KL/rejected_KL_mean": -300.9979248046875, "KL/std": 118.1670150756836, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.42499542236328, "fcm_dpo/q_t": 0.4925977885723114, "grad_norm": 2.1119422912597656, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.8524386882781982, "logits/rejected": -1.5204455852508545, "logps/chosen": -299.2491149902344, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -385.2784423828125, "loss": 1.3576, "margin_dpo/margin_mean": 81.42499542236328, "margin_dpo/margin_std": 145.00726318359375, "step": 404 }, { "KL/chosen_KL_mean": -223.13638305664062, "KL/mean": -263.7894287109375, "KL/rejected_KL_mean": -304.4424743652344, "KL/std": 116.52339172363281, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.30609893798828, "fcm_dpo/q_t": 0.49260827898979187, "grad_norm": 1.1063090562820435, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.601543664932251, "logits/rejected": -1.1352825164794922, "logps/chosen": -319.59832763671875, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -397.31317138671875, "loss": 1.3575, "margin_dpo/margin_mean": 81.30609130859375, "margin_dpo/margin_std": 134.81668090820312, "step": 405 }, { "KL/chosen_KL_mean": -206.9559326171875, "KL/mean": -248.4728546142578, "KL/rejected_KL_mean": -289.98974609375, "KL/std": 119.68988800048828, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.03382110595703, "fcm_dpo/q_t": 0.4924490749835968, "grad_norm": 1.4097647666931152, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.8554019927978516, "logits/rejected": -1.7015700340270996, "logps/chosen": -285.08990478515625, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -388.27337646484375, "loss": 1.3569, "margin_dpo/margin_mean": 83.03382110595703, "margin_dpo/margin_std": 130.78445434570312, "step": 406 }, { "KL/chosen_KL_mean": -205.68972778320312, "KL/mean": -241.38262939453125, "KL/rejected_KL_mean": -277.07550048828125, "KL/std": 114.71207427978516, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.38578796386719, "fcm_dpo/q_t": 0.4935082793235779, "grad_norm": 0.985300600528717, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.0839614868164062, "logits/rejected": -1.564816951751709, "logps/chosen": -289.70257568359375, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -359.8565673828125, "loss": 1.361, "margin_dpo/margin_mean": 71.38578796386719, "margin_dpo/margin_std": 121.4251480102539, "step": 407 }, { "KL/chosen_KL_mean": -218.18521118164062, "KL/mean": -247.17364501953125, "KL/rejected_KL_mean": -276.1620788574219, "KL/std": 117.0627212524414, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.97690200805664, "fcm_dpo/q_t": 0.4947285056114197, "grad_norm": 1.1877293586730957, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.271998882293701, "logits/rejected": -1.6563382148742676, "logps/chosen": -322.64697265625, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -372.5342712402344, "loss": 1.366, "margin_dpo/margin_mean": 57.976898193359375, "margin_dpo/margin_std": 145.94131469726562, "step": 408 }, { "KL/chosen_KL_mean": -216.83087158203125, "KL/mean": -253.64871215820312, "KL/rejected_KL_mean": -290.46661376953125, "KL/std": 117.92543029785156, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.63573455810547, "fcm_dpo/q_t": 0.4933035671710968, "grad_norm": 1.0551276206970215, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.626796007156372, "logits/rejected": -1.197108507156372, "logps/chosen": -314.4991760253906, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -380.512451171875, "loss": 1.3603, "margin_dpo/margin_mean": 73.63572692871094, "margin_dpo/margin_std": 135.65740966796875, "step": 409 }, { "KL/chosen_KL_mean": -206.41868591308594, "KL/mean": -252.99142456054688, "KL/rejected_KL_mean": -299.56414794921875, "KL/std": 119.91096496582031, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.14546203613281, "fcm_dpo/q_t": 0.49152863025665283, "grad_norm": 1.2341649532318115, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.7841274738311768, "logits/rejected": -1.6734273433685303, "logps/chosen": -282.887939453125, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -388.2048034667969, "loss": 1.3532, "margin_dpo/margin_mean": 93.14546966552734, "margin_dpo/margin_std": 120.74440002441406, "step": 410 }, { "KL/chosen_KL_mean": -222.02255249023438, "KL/mean": -264.6455078125, "KL/rejected_KL_mean": -307.26849365234375, "KL/std": 120.98237609863281, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.24595642089844, "fcm_dpo/q_t": 0.4922531247138977, "grad_norm": 1.181662917137146, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.1155920028686523, "logits/rejected": -1.708309531211853, "logps/chosen": -309.1888427734375, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -394.364501953125, "loss": 1.3562, "margin_dpo/margin_mean": 85.24595642089844, "margin_dpo/margin_std": 143.66213989257812, "step": 411 }, { "KL/chosen_KL_mean": -211.48501586914062, "KL/mean": -256.6719665527344, "KL/rejected_KL_mean": -301.8589172363281, "KL/std": 119.15702056884766, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.37390899658203, "fcm_dpo/q_t": 0.49178546667099, "grad_norm": 1.2977638244628906, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.4247288703918457, "logits/rejected": -2.2487239837646484, "logps/chosen": -290.432373046875, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -407.9644775390625, "loss": 1.3545, "margin_dpo/margin_mean": 90.37390899658203, "margin_dpo/margin_std": 154.7036590576172, "step": 412 }, { "KL/chosen_KL_mean": -227.95477294921875, "KL/mean": -275.67822265625, "KL/rejected_KL_mean": -323.401611328125, "KL/std": 132.4185791015625, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.44683837890625, "fcm_dpo/q_t": 0.4913187026977539, "grad_norm": 1.1291053295135498, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.3407810926437378, "logits/rejected": -1.0945894718170166, "logps/chosen": -323.64947509765625, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -431.36248779296875, "loss": 1.3528, "margin_dpo/margin_mean": 95.44683074951172, "margin_dpo/margin_std": 167.28082275390625, "step": 413 }, { "KL/chosen_KL_mean": -215.22140502929688, "KL/mean": -257.13763427734375, "KL/rejected_KL_mean": -299.0538330078125, "KL/std": 120.03974914550781, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.83241271972656, "fcm_dpo/q_t": 0.4923799932003021, "grad_norm": 2.2677652835845947, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.650420069694519, "logits/rejected": -1.3068745136260986, "logps/chosen": -303.4980773925781, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -391.92388916015625, "loss": 1.3569, "margin_dpo/margin_mean": 83.83241271972656, "margin_dpo/margin_std": 161.726318359375, "step": 414 }, { "KL/chosen_KL_mean": -237.62579345703125, "KL/mean": -270.13653564453125, "KL/rejected_KL_mean": -302.64727783203125, "KL/std": 112.86087036132812, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.021484375, "fcm_dpo/q_t": 0.49408912658691406, "grad_norm": 1.125251054763794, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.5502837896347046, "logits/rejected": -1.196222186088562, "logps/chosen": -346.54168701171875, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -410.11865234375, "loss": 1.3635, "margin_dpo/margin_mean": 65.021484375, "margin_dpo/margin_std": 149.48646545410156, "step": 415 }, { "KL/chosen_KL_mean": -225.70355224609375, "KL/mean": -280.2988586425781, "KL/rejected_KL_mean": -334.8941650390625, "KL/std": 130.92701721191406, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.19062805175781, "fcm_dpo/q_t": 0.4900730848312378, "grad_norm": 0.9622937440872192, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8542230129241943, "logits/rejected": -1.3697822093963623, "logps/chosen": -319.1024169921875, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -432.567138671875, "loss": 1.3476, "margin_dpo/margin_mean": 109.19062042236328, "margin_dpo/margin_std": 135.30125427246094, "step": 416 }, { "KL/chosen_KL_mean": -206.27529907226562, "KL/mean": -250.29833984375, "KL/rejected_KL_mean": -294.32135009765625, "KL/std": 111.29560852050781, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.04609680175781, "fcm_dpo/q_t": 0.4919931888580322, "grad_norm": 1.1508241891860962, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.001636505126953, "logits/rejected": -1.0756847858428955, "logps/chosen": -289.8106384277344, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -368.7632141113281, "loss": 1.355, "margin_dpo/margin_mean": 88.04609680175781, "margin_dpo/margin_std": 115.64031982421875, "step": 417 }, { "KL/chosen_KL_mean": -199.2671356201172, "KL/mean": -264.85205078125, "KL/rejected_KL_mean": -330.43701171875, "KL/std": 129.3358917236328, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.16986083984375, "fcm_dpo/q_t": 0.4880755841732025, "grad_norm": 1.5258660316467285, "learning_rate": 1.965167291983757e-07, "logits/chosen": -1.9789018630981445, "logits/rejected": -1.3162562847137451, "logps/chosen": -307.4886474609375, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -442.3016357421875, "loss": 1.3398, "margin_dpo/margin_mean": 131.16986083984375, "margin_dpo/margin_std": 129.9786376953125, "step": 418 }, { "KL/chosen_KL_mean": -212.3191375732422, "KL/mean": -264.7310485839844, "KL/rejected_KL_mean": -317.1429443359375, "KL/std": 115.19749450683594, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.82377624511719, "fcm_dpo/q_t": 0.4904698431491852, "grad_norm": 1.2451626062393188, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.6446306705474854, "logits/rejected": -1.2191293239593506, "logps/chosen": -309.50244140625, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -415.3282470703125, "loss": 1.3492, "margin_dpo/margin_mean": 104.82377624511719, "margin_dpo/margin_std": 137.223388671875, "step": 419 }, { "KL/chosen_KL_mean": -214.3306884765625, "KL/mean": -257.2616882324219, "KL/rejected_KL_mean": -300.19268798828125, "KL/std": 118.12338256835938, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.86197662353516, "fcm_dpo/q_t": 0.4921945333480835, "grad_norm": 1.4017235040664673, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.2489523887634277, "logits/rejected": -1.174484372138977, "logps/chosen": -328.63916015625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -375.876220703125, "loss": 1.356, "margin_dpo/margin_mean": 85.86197662353516, "margin_dpo/margin_std": 145.96072387695312, "step": 420 }, { "KL/chosen_KL_mean": -198.78170776367188, "KL/mean": -252.2711639404297, "KL/rejected_KL_mean": -305.7606201171875, "KL/std": 122.08566284179688, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.97888946533203, "fcm_dpo/q_t": 0.4902721643447876, "grad_norm": 0.9198900461196899, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -2.0408246517181396, "logits/rejected": -1.7042707204818726, "logps/chosen": -284.66156005859375, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -402.09710693359375, "loss": 1.3484, "margin_dpo/margin_mean": 106.97889709472656, "margin_dpo/margin_std": 140.58285522460938, "step": 421 }, { "KL/chosen_KL_mean": -196.875732421875, "KL/mean": -250.972412109375, "KL/rejected_KL_mean": -305.069091796875, "KL/std": 113.023681640625, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.19334411621094, "fcm_dpo/q_t": 0.4901624917984009, "grad_norm": 1.1005600690841675, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.4773216247558594, "logits/rejected": -2.1418914794921875, "logps/chosen": -297.3563232421875, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -399.477294921875, "loss": 1.3479, "margin_dpo/margin_mean": 108.19334411621094, "margin_dpo/margin_std": 133.833251953125, "step": 422 }, { "KL/chosen_KL_mean": -176.04852294921875, "KL/mean": -234.06109619140625, "KL/rejected_KL_mean": -292.07366943359375, "KL/std": 137.47471618652344, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.02516174316406, "fcm_dpo/q_t": 0.48945456743240356, "grad_norm": 1.563063621520996, "learning_rate": 1.902669377503756e-07, "logits/chosen": -2.1429243087768555, "logits/rejected": -2.0337021350860596, "logps/chosen": -254.4984588623047, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -384.1202087402344, "loss": 1.3454, "margin_dpo/margin_mean": 116.02516174316406, "margin_dpo/margin_std": 161.16860961914062, "step": 423 }, { "KL/chosen_KL_mean": -211.32186889648438, "KL/mean": -255.22006225585938, "KL/rejected_KL_mean": -299.1182861328125, "KL/std": 127.58609008789062, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.79641723632812, "fcm_dpo/q_t": 0.49201729893684387, "grad_norm": 1.3755840063095093, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7744395732879639, "logits/rejected": -1.2133219242095947, "logps/chosen": -298.9642333984375, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -371.48394775390625, "loss": 1.3553, "margin_dpo/margin_mean": 87.79641723632812, "margin_dpo/margin_std": 140.54977416992188, "step": 424 }, { "KL/chosen_KL_mean": -198.8984832763672, "KL/mean": -238.583984375, "KL/rejected_KL_mean": -278.26947021484375, "KL/std": 130.1495819091797, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.37100219726562, "fcm_dpo/q_t": 0.49278712272644043, "grad_norm": 1.2784063816070557, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.6140975952148438, "logits/rejected": -1.5705825090408325, "logps/chosen": -277.41827392578125, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -381.01812744140625, "loss": 1.3584, "margin_dpo/margin_mean": 79.3709945678711, "margin_dpo/margin_std": 156.01913452148438, "step": 425 }, { "KL/chosen_KL_mean": -192.32785034179688, "KL/mean": -238.9544677734375, "KL/rejected_KL_mean": -285.5810852050781, "KL/std": 116.59796142578125, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.25321960449219, "fcm_dpo/q_t": 0.4915212392807007, "grad_norm": 1.057815670967102, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.8050282001495361, "logits/rejected": -1.0586919784545898, "logps/chosen": -300.83367919921875, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -373.8811340332031, "loss": 1.3533, "margin_dpo/margin_mean": 93.25321960449219, "margin_dpo/margin_std": 143.46258544921875, "step": 426 }, { "KL/chosen_KL_mean": -157.63092041015625, "KL/mean": -197.07656860351562, "KL/rejected_KL_mean": -236.52220153808594, "KL/std": 102.88623046875, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.89128112792969, "fcm_dpo/q_t": 0.49282559752464294, "grad_norm": 1.605826497077942, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.5074687004089355, "logits/rejected": -1.910810947418213, "logps/chosen": -256.7513732910156, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -322.246826171875, "loss": 1.3582, "margin_dpo/margin_mean": 78.89128112792969, "margin_dpo/margin_std": 114.67684173583984, "step": 427 }, { "KL/chosen_KL_mean": -225.1114501953125, "KL/mean": -266.06781005859375, "KL/rejected_KL_mean": -307.02410888671875, "KL/std": 122.73931884765625, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.9126968383789, "fcm_dpo/q_t": 0.49255335330963135, "grad_norm": 1.3326665163040161, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.823284387588501, "logits/rejected": -1.5749711990356445, "logps/chosen": -331.0806884765625, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -416.1263122558594, "loss": 1.3574, "margin_dpo/margin_mean": 81.91268920898438, "margin_dpo/margin_std": 146.6439208984375, "step": 428 }, { "KL/chosen_KL_mean": -214.685791015625, "KL/mean": -264.3059387207031, "KL/rejected_KL_mean": -313.92608642578125, "KL/std": 131.53396606445312, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.24031066894531, "fcm_dpo/q_t": 0.4909781217575073, "grad_norm": 1.1373257637023926, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9559352397918701, "logits/rejected": -1.2601736783981323, "logps/chosen": -328.23065185546875, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -412.1681213378906, "loss": 1.3513, "margin_dpo/margin_mean": 99.24031066894531, "margin_dpo/margin_std": 149.8839111328125, "step": 429 }, { "KL/chosen_KL_mean": -193.497802734375, "KL/mean": -235.42178344726562, "KL/rejected_KL_mean": -277.34576416015625, "KL/std": 124.89234924316406, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.84793090820312, "fcm_dpo/q_t": 0.49238109588623047, "grad_norm": 1.1782426834106445, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4254720211029053, "logits/rejected": -1.409998893737793, "logps/chosen": -284.8171691894531, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -387.45538330078125, "loss": 1.3568, "margin_dpo/margin_mean": 83.84793090820312, "margin_dpo/margin_std": 158.88136291503906, "step": 430 }, { "KL/chosen_KL_mean": -193.17691040039062, "KL/mean": -242.40145874023438, "KL/rejected_KL_mean": -291.62603759765625, "KL/std": 133.22412109375, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.44915008544922, "fcm_dpo/q_t": 0.49104905128479004, "grad_norm": 1.209434151649475, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8824633359909058, "logits/rejected": -1.5226737260818481, "logps/chosen": -286.3581237792969, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -389.75830078125, "loss": 1.3515, "margin_dpo/margin_mean": 98.44914245605469, "margin_dpo/margin_std": 149.8339080810547, "step": 431 }, { "KL/chosen_KL_mean": -191.02978515625, "KL/mean": -243.3031463623047, "KL/rejected_KL_mean": -295.5765075683594, "KL/std": 130.2408447265625, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.54672241210938, "fcm_dpo/q_t": 0.4904947280883789, "grad_norm": 1.2171269655227661, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.5353196859359741, "logits/rejected": -1.0105890035629272, "logps/chosen": -295.4645690917969, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -400.66607666015625, "loss": 1.3492, "margin_dpo/margin_mean": 104.54672241210938, "margin_dpo/margin_std": 132.76707458496094, "step": 432 }, { "KL/chosen_KL_mean": -209.67279052734375, "KL/mean": -254.8739013671875, "KL/rejected_KL_mean": -300.07501220703125, "KL/std": 121.84899139404297, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.40221405029297, "fcm_dpo/q_t": 0.4917805790901184, "grad_norm": 1.1343106031417847, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.658353328704834, "logits/rejected": -1.3694684505462646, "logps/chosen": -299.5160217285156, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -401.8084716796875, "loss": 1.3543, "margin_dpo/margin_mean": 90.40220642089844, "margin_dpo/margin_std": 135.516845703125, "step": 433 }, { "KL/chosen_KL_mean": -185.01266479492188, "KL/mean": -232.9998779296875, "KL/rejected_KL_mean": -280.987060546875, "KL/std": 139.98255920410156, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.97442626953125, "fcm_dpo/q_t": 0.49128466844558716, "grad_norm": 1.612027883529663, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.456101417541504, "logits/rejected": -1.7854692935943604, "logps/chosen": -282.7052001953125, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -365.078369140625, "loss": 1.3527, "margin_dpo/margin_mean": 95.97442626953125, "margin_dpo/margin_std": 172.64657592773438, "step": 434 }, { "KL/chosen_KL_mean": -183.20713806152344, "KL/mean": -234.2652130126953, "KL/rejected_KL_mean": -285.32330322265625, "KL/std": 125.69647216796875, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.11614990234375, "fcm_dpo/q_t": 0.490719735622406, "grad_norm": 1.1337751150131226, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.416731834411621, "logits/rejected": -1.9927961826324463, "logps/chosen": -269.3790588378906, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -379.07501220703125, "loss": 1.3502, "margin_dpo/margin_mean": 102.11614990234375, "margin_dpo/margin_std": 149.9251708984375, "step": 435 }, { "KL/chosen_KL_mean": -163.34854125976562, "KL/mean": -214.0811309814453, "KL/rejected_KL_mean": -264.813720703125, "KL/std": 107.28119659423828, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.46515655517578, "fcm_dpo/q_t": 0.4907728135585785, "grad_norm": 1.6816062927246094, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.6090291738510132, "logits/rejected": -0.778298556804657, "logps/chosen": -269.1356506347656, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -353.43841552734375, "loss": 1.3502, "margin_dpo/margin_mean": 101.46515655517578, "margin_dpo/margin_std": 114.98629760742188, "step": 436 }, { "KL/chosen_KL_mean": -175.71194458007812, "KL/mean": -217.8163299560547, "KL/rejected_KL_mean": -259.92071533203125, "KL/std": 110.5042724609375, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.20880126953125, "fcm_dpo/q_t": 0.49234145879745483, "grad_norm": 1.594247579574585, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.7516825199127197, "logits/rejected": -1.0866731405258179, "logps/chosen": -271.77398681640625, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -348.93292236328125, "loss": 1.3563, "margin_dpo/margin_mean": 84.20880126953125, "margin_dpo/margin_std": 116.1947021484375, "step": 437 }, { "KL/chosen_KL_mean": -187.36749267578125, "KL/mean": -229.4556884765625, "KL/rejected_KL_mean": -271.5438232421875, "KL/std": 115.34988403320312, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.17634582519531, "fcm_dpo/q_t": 0.49234485626220703, "grad_norm": 1.1680872440338135, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1620349884033203, "logits/rejected": -1.400120735168457, "logps/chosen": -289.2228698730469, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -360.99151611328125, "loss": 1.3565, "margin_dpo/margin_mean": 84.17634582519531, "margin_dpo/margin_std": 128.7117156982422, "step": 438 }, { "KL/chosen_KL_mean": -195.17208862304688, "KL/mean": -239.2177276611328, "KL/rejected_KL_mean": -283.26336669921875, "KL/std": 127.03926086425781, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.09130096435547, "fcm_dpo/q_t": 0.49199140071868896, "grad_norm": 1.439738154411316, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.4751160144805908, "logits/rejected": -1.2606828212738037, "logps/chosen": -276.927734375, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -373.8497314453125, "loss": 1.3551, "margin_dpo/margin_mean": 88.09129333496094, "margin_dpo/margin_std": 130.72140502929688, "step": 439 }, { "KL/chosen_KL_mean": -185.93008422851562, "KL/mean": -239.72247314453125, "KL/rejected_KL_mean": -293.5148620605469, "KL/std": 130.09527587890625, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.58479309082031, "fcm_dpo/q_t": 0.49022069573402405, "grad_norm": 0.9338685870170593, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.2722907066345215, "logits/rejected": -1.782204031944275, "logps/chosen": -291.5711669921875, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -396.9158630371094, "loss": 1.3482, "margin_dpo/margin_mean": 107.58479309082031, "margin_dpo/margin_std": 144.60543823242188, "step": 440 }, { "KL/chosen_KL_mean": -228.3004150390625, "KL/mean": -270.8326416015625, "KL/rejected_KL_mean": -313.3648986816406, "KL/std": 128.08969116210938, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.06449127197266, "fcm_dpo/q_t": 0.4922717213630676, "grad_norm": 1.0376414060592651, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.6420317888259888, "logits/rejected": -1.3245062828063965, "logps/chosen": -319.8299560546875, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -416.9842834472656, "loss": 1.3564, "margin_dpo/margin_mean": 85.06449890136719, "margin_dpo/margin_std": 159.05276489257812, "step": 441 }, { "KL/chosen_KL_mean": -220.42779541015625, "KL/mean": -274.090576171875, "KL/rejected_KL_mean": -327.7532958984375, "KL/std": 120.3879165649414, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.32551574707031, "fcm_dpo/q_t": 0.49024271965026855, "grad_norm": 1.7691465616226196, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.3618794679641724, "logits/rejected": -0.8833746910095215, "logps/chosen": -305.20538330078125, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -411.57745361328125, "loss": 1.3483, "margin_dpo/margin_mean": 107.32551574707031, "margin_dpo/margin_std": 142.24887084960938, "step": 442 }, { "KL/chosen_KL_mean": -217.49900817871094, "KL/mean": -249.42796325683594, "KL/rejected_KL_mean": -281.35693359375, "KL/std": 125.46501159667969, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.85790252685547, "fcm_dpo/q_t": 0.4941936433315277, "grad_norm": 1.2781754732131958, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.866733193397522, "logits/rejected": -1.3077478408813477, "logps/chosen": -320.1482849121094, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -374.3949890136719, "loss": 1.3638, "margin_dpo/margin_mean": 63.85790252685547, "margin_dpo/margin_std": 129.806640625, "step": 443 }, { "KL/chosen_KL_mean": -225.17474365234375, "KL/mean": -269.8009033203125, "KL/rejected_KL_mean": -314.4271240234375, "KL/std": 126.92528533935547, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.25239562988281, "fcm_dpo/q_t": 0.491885781288147, "grad_norm": 1.5561860799789429, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5354111194610596, "logits/rejected": -1.3780322074890137, "logps/chosen": -313.0944519042969, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -417.7505798339844, "loss": 1.3548, "margin_dpo/margin_mean": 89.25239562988281, "margin_dpo/margin_std": 147.5731964111328, "step": 444 }, { "KL/chosen_KL_mean": -207.03897094726562, "KL/mean": -254.80068969726562, "KL/rejected_KL_mean": -302.56243896484375, "KL/std": 121.69742584228516, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.52345275878906, "fcm_dpo/q_t": 0.49131426215171814, "grad_norm": 1.1319053173065186, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.8694086074829102, "logits/rejected": -1.470777988433838, "logps/chosen": -308.4398193359375, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -401.600341796875, "loss": 1.3524, "margin_dpo/margin_mean": 95.52345275878906, "margin_dpo/margin_std": 134.99661254882812, "step": 445 }, { "KL/chosen_KL_mean": -201.62742614746094, "KL/mean": -240.94940185546875, "KL/rejected_KL_mean": -280.2713623046875, "KL/std": 121.82521057128906, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.64392852783203, "fcm_dpo/q_t": 0.4928485155105591, "grad_norm": 1.2530763149261475, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.8012892007827759, "logits/rejected": -1.6881077289581299, "logps/chosen": -289.0497741699219, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -386.97210693359375, "loss": 1.3585, "margin_dpo/margin_mean": 78.64392852783203, "margin_dpo/margin_std": 132.45956420898438, "step": 446 }, { "KL/chosen_KL_mean": -192.95877075195312, "KL/mean": -253.91043090820312, "KL/rejected_KL_mean": -314.86212158203125, "KL/std": 134.91900634765625, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.90333557128906, "fcm_dpo/q_t": 0.48892122507095337, "grad_norm": 1.3521136045455933, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.153425455093384, "logits/rejected": -2.1216909885406494, "logps/chosen": -276.57403564453125, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -419.7745361328125, "loss": 1.3433, "margin_dpo/margin_mean": 121.9033432006836, "margin_dpo/margin_std": 161.2356719970703, "step": 447 }, { "KL/chosen_KL_mean": -204.72186279296875, "KL/mean": -263.73077392578125, "KL/rejected_KL_mean": -322.7396545410156, "KL/std": 136.4146728515625, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.01779174804688, "fcm_dpo/q_t": 0.4892716407775879, "grad_norm": 1.489800214767456, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.9205386638641357, "logits/rejected": -1.669029951095581, "logps/chosen": -297.2976379394531, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -430.429443359375, "loss": 1.3446, "margin_dpo/margin_mean": 118.01779174804688, "margin_dpo/margin_std": 155.318603515625, "step": 448 }, { "KL/chosen_KL_mean": -207.05532836914062, "KL/mean": -266.0755615234375, "KL/rejected_KL_mean": -325.0958251953125, "KL/std": 128.35470581054688, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.04047393798828, "fcm_dpo/q_t": 0.4892694056034088, "grad_norm": 1.5926631689071655, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4137320518493652, "logits/rejected": -0.9777640104293823, "logps/chosen": -309.45428466796875, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -420.24468994140625, "loss": 1.3446, "margin_dpo/margin_mean": 118.04048156738281, "margin_dpo/margin_std": 155.619873046875, "step": 449 }, { "KL/chosen_KL_mean": -187.0884552001953, "KL/mean": -246.85662841796875, "KL/rejected_KL_mean": -306.62481689453125, "KL/std": 133.586669921875, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.53633880615234, "fcm_dpo/q_t": 0.4891315996646881, "grad_norm": 1.2129921913146973, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3072023391723633, "logits/rejected": -2.1238694190979004, "logps/chosen": -274.081298828125, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -415.1568603515625, "loss": 1.3438, "margin_dpo/margin_mean": 119.53633117675781, "margin_dpo/margin_std": 131.26165771484375, "step": 450 }, { "KL/chosen_KL_mean": -202.91717529296875, "KL/mean": -240.45196533203125, "KL/rejected_KL_mean": -277.9867248535156, "KL/std": 125.43379211425781, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.06954956054688, "fcm_dpo/q_t": 0.49317482113838196, "grad_norm": 2.319932460784912, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.3842310905456543, "logits/rejected": -1.896211862564087, "logps/chosen": -289.72845458984375, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -357.8423156738281, "loss": 1.36, "margin_dpo/margin_mean": 75.06954956054688, "margin_dpo/margin_std": 157.05776977539062, "step": 451 }, { "KL/chosen_KL_mean": -178.70372009277344, "KL/mean": -228.15872192382812, "KL/rejected_KL_mean": -277.6137390136719, "KL/std": 119.38697814941406, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.91000366210938, "fcm_dpo/q_t": 0.49100637435913086, "grad_norm": 1.084477186203003, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.830923080444336, "logits/rejected": -1.8063544034957886, "logps/chosen": -258.0833740234375, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -381.3291320800781, "loss": 1.3512, "margin_dpo/margin_mean": 98.91000366210938, "margin_dpo/margin_std": 127.22932434082031, "step": 452 }, { "KL/chosen_KL_mean": -220.1751251220703, "KL/mean": -276.32537841796875, "KL/rejected_KL_mean": -332.4757080078125, "KL/std": 118.25814819335938, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.30059051513672, "fcm_dpo/q_t": 0.48979121446609497, "grad_norm": 1.080251693725586, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.8142940998077393, "logits/rejected": -1.3601171970367432, "logps/chosen": -307.87030029296875, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -422.533935546875, "loss": 1.3465, "margin_dpo/margin_mean": 112.30059814453125, "margin_dpo/margin_std": 143.5760955810547, "step": 453 }, { "KL/chosen_KL_mean": -223.78147888183594, "KL/mean": -270.38629150390625, "KL/rejected_KL_mean": -316.9910888671875, "KL/std": 123.91271209716797, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.2096176147461, "fcm_dpo/q_t": 0.49152636528015137, "grad_norm": 1.2518094778060913, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.9763963222503662, "logits/rejected": -1.5450630187988281, "logps/chosen": -313.34771728515625, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -409.91217041015625, "loss": 1.3536, "margin_dpo/margin_mean": 93.20960998535156, "margin_dpo/margin_std": 166.69815063476562, "step": 454 }, { "KL/chosen_KL_mean": -213.89828491210938, "KL/mean": -269.12469482421875, "KL/rejected_KL_mean": -324.35107421875, "KL/std": 141.9357452392578, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.45278930664062, "fcm_dpo/q_t": 0.48996320366859436, "grad_norm": 1.393744707107544, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8411903381347656, "logits/rejected": -1.8268795013427734, "logps/chosen": -291.52825927734375, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -443.32904052734375, "loss": 1.3474, "margin_dpo/margin_mean": 110.45278930664062, "margin_dpo/margin_std": 163.38267517089844, "step": 455 }, { "KL/chosen_KL_mean": -215.51504516601562, "KL/mean": -270.21142578125, "KL/rejected_KL_mean": -324.9078369140625, "KL/std": 130.93084716796875, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.3927993774414, "fcm_dpo/q_t": 0.4900534152984619, "grad_norm": 1.4738166332244873, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.064818859100342, "logits/rejected": -1.9441179037094116, "logps/chosen": -305.13189697265625, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -434.4675598144531, "loss": 1.3474, "margin_dpo/margin_mean": 109.39279174804688, "margin_dpo/margin_std": 124.47409057617188, "step": 456 }, { "KL/chosen_KL_mean": -180.71231079101562, "KL/mean": -242.7852020263672, "KL/rejected_KL_mean": -304.8581237792969, "KL/std": 125.56108093261719, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.14579010009766, "fcm_dpo/q_t": 0.48871180415153503, "grad_norm": 1.2241476774215698, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -1.9472465515136719, "logits/rejected": -1.635411024093628, "logps/chosen": -268.03399658203125, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -395.6247253417969, "loss": 1.3422, "margin_dpo/margin_mean": 124.14579010009766, "margin_dpo/margin_std": 117.72716522216797, "step": 457 }, { "KL/chosen_KL_mean": -213.75442504882812, "KL/mean": -254.4448699951172, "KL/rejected_KL_mean": -295.13531494140625, "KL/std": 125.61604309082031, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.3808822631836, "fcm_dpo/q_t": 0.49260395765304565, "grad_norm": 1.7903943061828613, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.6780837774276733, "logits/rejected": -1.4624168872833252, "logps/chosen": -307.2748718261719, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -398.5043029785156, "loss": 1.3578, "margin_dpo/margin_mean": 81.3808822631836, "margin_dpo/margin_std": 162.7209930419922, "step": 458 }, { "KL/chosen_KL_mean": -192.52938842773438, "KL/mean": -250.52796936035156, "KL/rejected_KL_mean": -308.52655029296875, "KL/std": 124.76286315917969, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.99714660644531, "fcm_dpo/q_t": 0.48945507407188416, "grad_norm": 1.1488556861877441, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.7645294666290283, "logits/rejected": -1.552501916885376, "logps/chosen": -268.21759033203125, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -400.697021484375, "loss": 1.3452, "margin_dpo/margin_mean": 115.99714660644531, "margin_dpo/margin_std": 137.31597900390625, "step": 459 }, { "KL/chosen_KL_mean": -215.01849365234375, "KL/mean": -262.3026123046875, "KL/rejected_KL_mean": -309.58673095703125, "KL/std": 124.95025634765625, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.56825256347656, "fcm_dpo/q_t": 0.49140214920043945, "grad_norm": 1.5790084600448608, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.1015806198120117, "logits/rejected": -2.158827066421509, "logps/chosen": -296.1363525390625, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -419.89910888671875, "loss": 1.3529, "margin_dpo/margin_mean": 94.5682601928711, "margin_dpo/margin_std": 145.4624481201172, "step": 460 }, { "KL/chosen_KL_mean": -230.23422241210938, "KL/mean": -276.3617248535156, "KL/rejected_KL_mean": -322.4892272949219, "KL/std": 113.78703308105469, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.2550048828125, "fcm_dpo/q_t": 0.49161192774772644, "grad_norm": 1.0411123037338257, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.6853283643722534, "logits/rejected": -1.510180950164795, "logps/chosen": -311.8177490234375, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -416.3663330078125, "loss": 1.3536, "margin_dpo/margin_mean": 92.2550048828125, "margin_dpo/margin_std": 131.12799072265625, "step": 461 }, { "KL/chosen_KL_mean": -189.92825317382812, "KL/mean": -241.18862915039062, "KL/rejected_KL_mean": -292.44903564453125, "KL/std": 121.39903259277344, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.52076721191406, "fcm_dpo/q_t": 0.49067896604537964, "grad_norm": 1.1100726127624512, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.185206174850464, "logits/rejected": -1.7075309753417969, "logps/chosen": -284.1268005371094, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -378.08062744140625, "loss": 1.3499, "margin_dpo/margin_mean": 102.52076721191406, "margin_dpo/margin_std": 130.77490234375, "step": 462 }, { "KL/chosen_KL_mean": -183.3933868408203, "KL/mean": -251.04864501953125, "KL/rejected_KL_mean": -318.7038879394531, "KL/std": 132.03543090820312, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.31051635742188, "fcm_dpo/q_t": 0.48769956827163696, "grad_norm": 1.3634824752807617, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.7432148456573486, "logits/rejected": -1.3445067405700684, "logps/chosen": -269.318115234375, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -415.605712890625, "loss": 1.3384, "margin_dpo/margin_mean": 135.31051635742188, "margin_dpo/margin_std": 140.26654052734375, "step": 463 }, { "KL/chosen_KL_mean": -223.70034790039062, "KL/mean": -283.45928955078125, "KL/rejected_KL_mean": -343.2182312011719, "KL/std": 130.31866455078125, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.51788330078125, "fcm_dpo/q_t": 0.48913758993148804, "grad_norm": 1.7988463640213013, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.6609601974487305, "logits/rejected": -1.7540407180786133, "logps/chosen": -303.3895263671875, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -450.51055908203125, "loss": 1.3441, "margin_dpo/margin_mean": 119.51789093017578, "margin_dpo/margin_std": 157.24026489257812, "step": 464 }, { "KL/chosen_KL_mean": -211.84432983398438, "KL/mean": -273.32684326171875, "KL/rejected_KL_mean": -334.8093566894531, "KL/std": 144.781982421875, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.96501159667969, "fcm_dpo/q_t": 0.48882049322128296, "grad_norm": 1.4276649951934814, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6545532941818237, "logits/rejected": -1.389270305633545, "logps/chosen": -303.7045593261719, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -453.51934814453125, "loss": 1.3427, "margin_dpo/margin_mean": 122.96501159667969, "margin_dpo/margin_std": 136.1651153564453, "step": 465 }, { "KL/chosen_KL_mean": -187.78369140625, "KL/mean": -248.41531372070312, "KL/rejected_KL_mean": -309.04693603515625, "KL/std": 133.1949005126953, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.26324462890625, "fcm_dpo/q_t": 0.4889755845069885, "grad_norm": 0.986527681350708, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.487891435623169, "logits/rejected": -1.1897718906402588, "logps/chosen": -272.485107421875, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -405.0977783203125, "loss": 1.3434, "margin_dpo/margin_mean": 121.26324462890625, "margin_dpo/margin_std": 148.54696655273438, "step": 466 }, { "KL/chosen_KL_mean": -250.74281311035156, "KL/mean": -298.21612548828125, "KL/rejected_KL_mean": -345.689453125, "KL/std": 148.59051513671875, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.94664001464844, "fcm_dpo/q_t": 0.49136897921562195, "grad_norm": 1.342187523841858, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.2689499855041504, "logits/rejected": -1.7093749046325684, "logps/chosen": -360.0411376953125, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -454.5331115722656, "loss": 1.3531, "margin_dpo/margin_mean": 94.94664001464844, "margin_dpo/margin_std": 176.24453735351562, "step": 467 }, { "KL/chosen_KL_mean": -186.73731994628906, "KL/mean": -237.9687042236328, "KL/rejected_KL_mean": -289.2000732421875, "KL/std": 141.23046875, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.4627456665039, "fcm_dpo/q_t": 0.4906880855560303, "grad_norm": 1.3160651922225952, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.8962513208389282, "logits/rejected": -1.6688036918640137, "logps/chosen": -285.0589599609375, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -395.88055419921875, "loss": 1.3503, "margin_dpo/margin_mean": 102.4627456665039, "margin_dpo/margin_std": 170.70376586914062, "step": 468 }, { "KL/chosen_KL_mean": -227.30914306640625, "KL/mean": -286.134765625, "KL/rejected_KL_mean": -344.96038818359375, "KL/std": 136.4553985595703, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.6512222290039, "fcm_dpo/q_t": 0.4893065094947815, "grad_norm": 1.4665937423706055, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.2916266918182373, "logits/rejected": -1.8988697528839111, "logps/chosen": -324.0733642578125, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -454.55535888671875, "loss": 1.3447, "margin_dpo/margin_mean": 117.6512222290039, "margin_dpo/margin_std": 149.55702209472656, "step": 469 }, { "KL/chosen_KL_mean": -202.37921142578125, "KL/mean": -261.4927978515625, "KL/rejected_KL_mean": -320.6064147949219, "KL/std": 126.10218811035156, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.22721862792969, "fcm_dpo/q_t": 0.4892507791519165, "grad_norm": 1.4553412199020386, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.6545372009277344, "logits/rejected": -1.261196255683899, "logps/chosen": -285.8758544921875, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -409.0921936035156, "loss": 1.3443, "margin_dpo/margin_mean": 118.22721862792969, "margin_dpo/margin_std": 126.57218933105469, "step": 470 }, { "KL/chosen_KL_mean": -233.83831787109375, "KL/mean": -284.53125, "KL/rejected_KL_mean": -335.22418212890625, "KL/std": 129.54754638671875, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.3858413696289, "fcm_dpo/q_t": 0.4907827377319336, "grad_norm": 1.0913476943969727, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.4510018825531006, "logits/rejected": -1.0054879188537598, "logps/chosen": -322.3126220703125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -425.70587158203125, "loss": 1.3503, "margin_dpo/margin_mean": 101.3858413696289, "margin_dpo/margin_std": 132.80740356445312, "step": 471 }, { "KL/chosen_KL_mean": -228.4903106689453, "KL/mean": -290.489990234375, "KL/rejected_KL_mean": -352.4896240234375, "KL/std": 134.1337127685547, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.99931335449219, "fcm_dpo/q_t": 0.48873215913772583, "grad_norm": 1.3819903135299683, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.123788356781006, "logits/rejected": -1.4337353706359863, "logps/chosen": -353.7239990234375, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -473.5431213378906, "loss": 1.3426, "margin_dpo/margin_mean": 123.99931335449219, "margin_dpo/margin_std": 162.02560424804688, "step": 472 }, { "KL/chosen_KL_mean": -263.16424560546875, "KL/mean": -294.146240234375, "KL/rejected_KL_mean": -325.1282653808594, "KL/std": 139.3316650390625, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.9639892578125, "fcm_dpo/q_t": 0.4943707585334778, "grad_norm": 1.0603415966033936, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.3978759050369263, "logits/rejected": -0.886802613735199, "logps/chosen": -358.775634765625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -413.2794189453125, "loss": 1.3648, "margin_dpo/margin_mean": 61.963985443115234, "margin_dpo/margin_std": 168.30084228515625, "step": 473 }, { "KL/chosen_KL_mean": -236.97222900390625, "KL/mean": -275.8983154296875, "KL/rejected_KL_mean": -314.8243713378906, "KL/std": 133.86373901367188, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.85213470458984, "fcm_dpo/q_t": 0.4929227828979492, "grad_norm": 1.8815666437149048, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.391800880432129, "logits/rejected": -1.215080738067627, "logps/chosen": -318.45196533203125, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -411.2900085449219, "loss": 1.3592, "margin_dpo/margin_mean": 77.85213470458984, "margin_dpo/margin_std": 171.57681274414062, "step": 474 }, { "KL/chosen_KL_mean": -177.35501098632812, "KL/mean": -241.58963012695312, "KL/rejected_KL_mean": -305.82427978515625, "KL/std": 133.80117797851562, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.46925354003906, "fcm_dpo/q_t": 0.48832717537879944, "grad_norm": 1.3093132972717285, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.0719263553619385, "logits/rejected": -2.0382347106933594, "logps/chosen": -251.55099487304688, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -400.5166931152344, "loss": 1.341, "margin_dpo/margin_mean": 128.46926879882812, "margin_dpo/margin_std": 165.99461364746094, "step": 475 }, { "KL/chosen_KL_mean": -212.21922302246094, "KL/mean": -261.82122802734375, "KL/rejected_KL_mean": -311.4232177734375, "KL/std": 134.0870361328125, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.2040023803711, "fcm_dpo/q_t": 0.4909840226173401, "grad_norm": 1.2890536785125732, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.823391079902649, "logits/rejected": -1.4781482219696045, "logps/chosen": -284.1903076171875, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -391.68548583984375, "loss": 1.3513, "margin_dpo/margin_mean": 99.20399475097656, "margin_dpo/margin_std": 149.17369079589844, "step": 476 }, { "KL/chosen_KL_mean": -212.4242706298828, "KL/mean": -264.4588928222656, "KL/rejected_KL_mean": -316.4935607910156, "KL/std": 113.25377655029297, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.06926727294922, "fcm_dpo/q_t": 0.4905369281768799, "grad_norm": 1.2752678394317627, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -1.8781952857971191, "logits/rejected": -0.9043726325035095, "logps/chosen": -317.4298095703125, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -398.3719787597656, "loss": 1.3494, "margin_dpo/margin_mean": 104.06926727294922, "margin_dpo/margin_std": 140.34413146972656, "step": 477 }, { "KL/chosen_KL_mean": -201.14138793945312, "KL/mean": -259.1689147949219, "KL/rejected_KL_mean": -317.19647216796875, "KL/std": 127.32273864746094, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.0550537109375, "fcm_dpo/q_t": 0.4894506335258484, "grad_norm": 1.5693960189819336, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.539062738418579, "logits/rejected": -1.3643951416015625, "logps/chosen": -277.9295959472656, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -407.6363830566406, "loss": 1.3453, "margin_dpo/margin_mean": 116.05506134033203, "margin_dpo/margin_std": 153.4639892578125, "step": 478 }, { "KL/chosen_KL_mean": -201.07809448242188, "KL/mean": -258.6812744140625, "KL/rejected_KL_mean": -316.2844543457031, "KL/std": 134.64480590820312, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.20635986328125, "fcm_dpo/q_t": 0.4895276427268982, "grad_norm": 2.3810460567474365, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.2901099920272827, "logits/rejected": -1.1524243354797363, "logps/chosen": -282.4404296875, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -417.3756103515625, "loss": 1.3456, "margin_dpo/margin_mean": 115.20635986328125, "margin_dpo/margin_std": 156.03213500976562, "step": 479 }, { "KL/chosen_KL_mean": -241.4727783203125, "KL/mean": -283.8801574707031, "KL/rejected_KL_mean": -326.28753662109375, "KL/std": 125.17633819580078, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.81478881835938, "fcm_dpo/q_t": 0.49228817224502563, "grad_norm": 1.4294754266738892, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.6384773254394531, "logits/rejected": -0.9873009920120239, "logps/chosen": -338.2466735839844, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -412.6922912597656, "loss": 1.3563, "margin_dpo/margin_mean": 84.81478881835938, "margin_dpo/margin_std": 135.29940795898438, "step": 480 }, { "KL/chosen_KL_mean": -227.48672485351562, "KL/mean": -278.081787109375, "KL/rejected_KL_mean": -328.6768798828125, "KL/std": 129.54893493652344, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.19014739990234, "fcm_dpo/q_t": 0.49080073833465576, "grad_norm": 1.4760825634002686, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.7632849216461182, "logits/rejected": -1.3502529859542847, "logps/chosen": -319.1568908691406, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -427.37176513671875, "loss": 1.3505, "margin_dpo/margin_mean": 101.19013977050781, "margin_dpo/margin_std": 141.78890991210938, "step": 481 }, { "KL/chosen_KL_mean": -225.3126983642578, "KL/mean": -274.40313720703125, "KL/rejected_KL_mean": -323.49359130859375, "KL/std": 136.76844787597656, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.18087768554688, "fcm_dpo/q_t": 0.4910729229450226, "grad_norm": 1.4333518743515015, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.4765095710754395, "logits/rejected": -1.1385408639907837, "logps/chosen": -323.83282470703125, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -418.32305908203125, "loss": 1.3517, "margin_dpo/margin_mean": 98.18087768554688, "margin_dpo/margin_std": 154.32546997070312, "step": 482 }, { "KL/chosen_KL_mean": -237.8358154296875, "KL/mean": -285.8502197265625, "KL/rejected_KL_mean": -333.86468505859375, "KL/std": 131.451904296875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.02888488769531, "fcm_dpo/q_t": 0.4912678301334381, "grad_norm": 1.187623143196106, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.8724706172943115, "logits/rejected": -1.2341727018356323, "logps/chosen": -344.95440673828125, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -434.97967529296875, "loss": 1.3523, "margin_dpo/margin_mean": 96.02888488769531, "margin_dpo/margin_std": 138.7087860107422, "step": 483 }, { "KL/chosen_KL_mean": -205.47267150878906, "KL/mean": -267.48492431640625, "KL/rejected_KL_mean": -329.49713134765625, "KL/std": 134.38156127929688, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.02447509765625, "fcm_dpo/q_t": 0.4887256920337677, "grad_norm": 1.2202513217926025, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.4089269638061523, "logits/rejected": -1.9681293964385986, "logps/chosen": -292.45257568359375, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -420.2208251953125, "loss": 1.3424, "margin_dpo/margin_mean": 124.02447509765625, "margin_dpo/margin_std": 150.463623046875, "step": 484 }, { "KL/chosen_KL_mean": -200.2897186279297, "KL/mean": -265.67156982421875, "KL/rejected_KL_mean": -331.053466796875, "KL/std": 150.381591796875, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.76373291015625, "fcm_dpo/q_t": 0.4881165027618408, "grad_norm": 1.3072179555892944, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.6861554384231567, "logits/rejected": -1.3097057342529297, "logps/chosen": -294.34844970703125, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -439.616455078125, "loss": 1.3402, "margin_dpo/margin_mean": 130.76373291015625, "margin_dpo/margin_std": 169.04812622070312, "step": 485 }, { "KL/chosen_KL_mean": -230.041259765625, "KL/mean": -288.87335205078125, "KL/rejected_KL_mean": -347.7054443359375, "KL/std": 142.84767150878906, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.66419982910156, "fcm_dpo/q_t": 0.48930272459983826, "grad_norm": 1.441987156867981, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.7319356203079224, "logits/rejected": -1.041558861732483, "logps/chosen": -323.78717041015625, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -445.776123046875, "loss": 1.3448, "margin_dpo/margin_mean": 117.66419219970703, "margin_dpo/margin_std": 162.48834228515625, "step": 486 }, { "KL/chosen_KL_mean": -218.41683959960938, "KL/mean": -272.8494873046875, "KL/rejected_KL_mean": -327.2821044921875, "KL/std": 125.84562683105469, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.86524963378906, "fcm_dpo/q_t": 0.4901004433631897, "grad_norm": 1.118082880973816, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -1.8526530265808105, "logits/rejected": -1.53386390209198, "logps/chosen": -306.4504089355469, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -425.75421142578125, "loss": 1.3477, "margin_dpo/margin_mean": 108.86524963378906, "margin_dpo/margin_std": 138.0690155029297, "step": 487 }, { "KL/chosen_KL_mean": -185.95751953125, "KL/mean": -243.49520874023438, "KL/rejected_KL_mean": -301.03289794921875, "KL/std": 138.6363525390625, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.07540130615234, "fcm_dpo/q_t": 0.48953771591186523, "grad_norm": 1.2510218620300293, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.4665780067443848, "logits/rejected": -1.2076163291931152, "logps/chosen": -270.7471618652344, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -391.3063659667969, "loss": 1.3455, "margin_dpo/margin_mean": 115.07539367675781, "margin_dpo/margin_std": 137.2046356201172, "step": 488 }, { "KL/chosen_KL_mean": -231.2078857421875, "KL/mean": -277.13140869140625, "KL/rejected_KL_mean": -323.054931640625, "KL/std": 126.50961303710938, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.84703826904297, "fcm_dpo/q_t": 0.4916495382785797, "grad_norm": 1.3026013374328613, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.4385499954223633, "logits/rejected": -0.959219753742218, "logps/chosen": -321.67718505859375, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -409.4525451660156, "loss": 1.3538, "margin_dpo/margin_mean": 91.8470458984375, "margin_dpo/margin_std": 140.81517028808594, "step": 489 }, { "KL/chosen_KL_mean": -218.31240844726562, "KL/mean": -274.1784362792969, "KL/rejected_KL_mean": -330.04449462890625, "KL/std": 144.55630493164062, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.73208618164062, "fcm_dpo/q_t": 0.48984888195991516, "grad_norm": 1.3615212440490723, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.0914506912231445, "logits/rejected": -2.203991413116455, "logps/chosen": -303.6325378417969, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -446.038330078125, "loss": 1.3473, "margin_dpo/margin_mean": 111.73208618164062, "margin_dpo/margin_std": 193.05340576171875, "step": 490 }, { "KL/chosen_KL_mean": -224.49839782714844, "KL/mean": -263.5049133300781, "KL/rejected_KL_mean": -302.51141357421875, "KL/std": 138.72564697265625, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.01304626464844, "fcm_dpo/q_t": 0.4929063618183136, "grad_norm": 1.3926090002059937, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.6915675401687622, "logits/rejected": -1.226719856262207, "logps/chosen": -310.64190673828125, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -383.19091796875, "loss": 1.3588, "margin_dpo/margin_mean": 78.01304626464844, "margin_dpo/margin_std": 143.27713012695312, "step": 491 }, { "KL/chosen_KL_mean": -211.14300537109375, "KL/mean": -246.52919006347656, "KL/rejected_KL_mean": -281.91534423828125, "KL/std": 150.15316772460938, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.77232360839844, "fcm_dpo/q_t": 0.49356818199157715, "grad_norm": 1.5438599586486816, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.752314805984497, "logits/rejected": -1.3796977996826172, "logps/chosen": -292.2506103515625, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -362.6673583984375, "loss": 1.3617, "margin_dpo/margin_mean": 70.77232360839844, "margin_dpo/margin_std": 176.40798950195312, "step": 492 }, { "KL/chosen_KL_mean": -214.25796508789062, "KL/mean": -253.70657348632812, "KL/rejected_KL_mean": -293.1551513671875, "KL/std": 123.95536804199219, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.89718627929688, "fcm_dpo/q_t": 0.4928267002105713, "grad_norm": 1.532279133796692, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.4282188415527344, "logits/rejected": -1.7629042863845825, "logps/chosen": -326.46533203125, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -386.7623596191406, "loss": 1.3585, "margin_dpo/margin_mean": 78.89718627929688, "margin_dpo/margin_std": 142.81576538085938, "step": 493 }, { "KL/chosen_KL_mean": -196.0376434326172, "KL/mean": -260.84197998046875, "KL/rejected_KL_mean": -325.64630126953125, "KL/std": 130.868896484375, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.608642578125, "fcm_dpo/q_t": 0.4882183074951172, "grad_norm": 1.532448649406433, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.520850419998169, "logits/rejected": -1.468914270401001, "logps/chosen": -269.15252685546875, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -417.8092956542969, "loss": 1.3405, "margin_dpo/margin_mean": 129.608642578125, "margin_dpo/margin_std": 153.21302795410156, "step": 494 }, { "KL/chosen_KL_mean": -213.83392333984375, "KL/mean": -263.9111328125, "KL/rejected_KL_mean": -313.9884033203125, "KL/std": 127.50297546386719, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.15448760986328, "fcm_dpo/q_t": 0.49089735746383667, "grad_norm": 1.5817948579788208, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.2524399757385254, "logits/rejected": -1.792403221130371, "logps/chosen": -313.354248046875, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -411.9192810058594, "loss": 1.3511, "margin_dpo/margin_mean": 100.15448760986328, "margin_dpo/margin_std": 164.18215942382812, "step": 495 }, { "KL/chosen_KL_mean": -241.04049682617188, "KL/mean": -284.5957946777344, "KL/rejected_KL_mean": -328.1510925292969, "KL/std": 125.81867980957031, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.1106185913086, "fcm_dpo/q_t": 0.4920784831047058, "grad_norm": 1.6127132177352905, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -1.912793517112732, "logits/rejected": -1.2630382776260376, "logps/chosen": -348.8972473144531, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -420.9216613769531, "loss": 1.3555, "margin_dpo/margin_mean": 87.1106185913086, "margin_dpo/margin_std": 141.43544006347656, "step": 496 }, { "KL/chosen_KL_mean": -244.12612915039062, "KL/mean": -272.3963623046875, "KL/rejected_KL_mean": -300.6666259765625, "KL/std": 132.38937377929688, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.54047393798828, "fcm_dpo/q_t": 0.49486127495765686, "grad_norm": 1.080565094947815, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.7551138401031494, "logits/rejected": -1.3312263488769531, "logps/chosen": -336.2093505859375, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -382.461669921875, "loss": 1.3666, "margin_dpo/margin_mean": 56.54046630859375, "margin_dpo/margin_std": 147.32797241210938, "step": 497 }, { "KL/chosen_KL_mean": -211.84930419921875, "KL/mean": -257.46435546875, "KL/rejected_KL_mean": -303.0794677734375, "KL/std": 140.48878479003906, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.23013305664062, "fcm_dpo/q_t": 0.491704523563385, "grad_norm": 1.4889013767242432, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.6594362258911133, "logits/rejected": -1.1586934328079224, "logps/chosen": -310.0436706542969, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -393.76690673828125, "loss": 1.3544, "margin_dpo/margin_mean": 91.2301254272461, "margin_dpo/margin_std": 178.34352111816406, "step": 498 }, { "KL/chosen_KL_mean": -212.55532836914062, "KL/mean": -268.9793701171875, "KL/rejected_KL_mean": -325.4033508300781, "KL/std": 144.71109008789062, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.84805297851562, "fcm_dpo/q_t": 0.48974278569221497, "grad_norm": 1.183454155921936, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.142685890197754, "logits/rejected": -2.098834753036499, "logps/chosen": -293.59521484375, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -432.3250732421875, "loss": 1.3466, "margin_dpo/margin_mean": 112.84805297851562, "margin_dpo/margin_std": 170.67840576171875, "step": 499 }, { "KL/chosen_KL_mean": -234.53134155273438, "KL/mean": -284.47039794921875, "KL/rejected_KL_mean": -334.40948486328125, "KL/std": 138.0274658203125, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.87814331054688, "fcm_dpo/q_t": 0.49092262983322144, "grad_norm": 1.3471401929855347, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.66791832447052, "logits/rejected": -1.5001310110092163, "logps/chosen": -323.7794189453125, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -434.8197021484375, "loss": 1.3512, "margin_dpo/margin_mean": 99.87813568115234, "margin_dpo/margin_std": 166.26011657714844, "step": 500 }, { "KL/chosen_KL_mean": -215.7557373046875, "KL/mean": -264.0111083984375, "KL/rejected_KL_mean": -312.2665100097656, "KL/std": 150.98638916015625, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.51077270507812, "fcm_dpo/q_t": 0.49123066663742065, "grad_norm": 1.2567087411880493, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.604017734527588, "logits/rejected": -1.194427490234375, "logps/chosen": -306.3526611328125, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -397.56378173828125, "loss": 1.3526, "margin_dpo/margin_mean": 96.51077270507812, "margin_dpo/margin_std": 180.25949096679688, "step": 501 }, { "KL/chosen_KL_mean": -215.19261169433594, "KL/mean": -255.19873046875, "KL/rejected_KL_mean": -295.204833984375, "KL/std": 116.01655578613281, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.01224517822266, "fcm_dpo/q_t": 0.49272382259368896, "grad_norm": 1.3566211462020874, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.8724292516708374, "logits/rejected": -1.1818046569824219, "logps/chosen": -318.663330078125, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -384.2490234375, "loss": 1.358, "margin_dpo/margin_mean": 80.01225280761719, "margin_dpo/margin_std": 136.80911254882812, "step": 502 }, { "KL/chosen_KL_mean": -210.64071655273438, "KL/mean": -259.3628845214844, "KL/rejected_KL_mean": -308.0850830078125, "KL/std": 124.54035949707031, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.44436645507812, "fcm_dpo/q_t": 0.4911414384841919, "grad_norm": 1.591650128364563, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.6072678565979004, "logits/rejected": -1.1237056255340576, "logps/chosen": -304.3706970214844, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -395.69403076171875, "loss": 1.3519, "margin_dpo/margin_mean": 97.44436645507812, "margin_dpo/margin_std": 148.94207763671875, "step": 503 }, { "KL/chosen_KL_mean": -222.60354614257812, "KL/mean": -287.56524658203125, "KL/rejected_KL_mean": -352.5269775390625, "KL/std": 140.22885131835938, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.92343139648438, "fcm_dpo/q_t": 0.4881902039051056, "grad_norm": 1.615248441696167, "learning_rate": 9.730107739932805e-08, "logits/chosen": -1.7978065013885498, "logits/rejected": -1.4776687622070312, "logps/chosen": -314.8954162597656, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -461.56463623046875, "loss": 1.3404, "margin_dpo/margin_mean": 129.92343139648438, "margin_dpo/margin_std": 161.79417419433594, "step": 504 }, { "KL/chosen_KL_mean": -249.1166229248047, "KL/mean": -291.00592041015625, "KL/rejected_KL_mean": -332.89520263671875, "KL/std": 138.8397979736328, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.77857971191406, "fcm_dpo/q_t": 0.49238139390945435, "grad_norm": 1.3969202041625977, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.38433837890625, "logits/rejected": -1.8676379919052124, "logps/chosen": -355.3935546875, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -427.101318359375, "loss": 1.3568, "margin_dpo/margin_mean": 83.77857971191406, "margin_dpo/margin_std": 145.863525390625, "step": 505 }, { "KL/chosen_KL_mean": -223.79164123535156, "KL/mean": -275.4461669921875, "KL/rejected_KL_mean": -327.1007080078125, "KL/std": 122.47463989257812, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.30907440185547, "fcm_dpo/q_t": 0.4906066656112671, "grad_norm": 1.6615079641342163, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.7085609436035156, "logits/rejected": -1.35092031955719, "logps/chosen": -305.23248291015625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -412.76513671875, "loss": 1.3497, "margin_dpo/margin_mean": 103.30908203125, "margin_dpo/margin_std": 136.96994018554688, "step": 506 }, { "KL/chosen_KL_mean": -238.9376220703125, "KL/mean": -288.0364990234375, "KL/rejected_KL_mean": -337.13543701171875, "KL/std": 139.08627319335938, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.19783020019531, "fcm_dpo/q_t": 0.49107491970062256, "grad_norm": 1.4654449224472046, "learning_rate": 9.427092687124691e-08, "logits/chosen": -1.874941110610962, "logits/rejected": -1.4364969730377197, "logps/chosen": -338.36456298828125, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -439.38836669921875, "loss": 1.3518, "margin_dpo/margin_mean": 98.19783020019531, "margin_dpo/margin_std": 160.1425323486328, "step": 507 }, { "KL/chosen_KL_mean": -236.0667266845703, "KL/mean": -279.02166748046875, "KL/rejected_KL_mean": -321.97662353515625, "KL/std": 141.71432495117188, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.90990447998047, "fcm_dpo/q_t": 0.49219217896461487, "grad_norm": 1.0850905179977417, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.8919941186904907, "logits/rejected": -1.4204915761947632, "logps/chosen": -333.98236083984375, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -425.77447509765625, "loss": 1.3563, "margin_dpo/margin_mean": 85.909912109375, "margin_dpo/margin_std": 176.93125915527344, "step": 508 }, { "KL/chosen_KL_mean": -217.5289306640625, "KL/mean": -254.47073364257812, "KL/rejected_KL_mean": -291.4125061035156, "KL/std": 117.330078125, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.88357543945312, "fcm_dpo/q_t": 0.493280291557312, "grad_norm": 1.728564739227295, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.038396120071411, "logits/rejected": -1.413188099861145, "logps/chosen": -318.2587890625, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -378.35589599609375, "loss": 1.3605, "margin_dpo/margin_mean": 73.88357543945312, "margin_dpo/margin_std": 160.56405639648438, "step": 509 }, { "KL/chosen_KL_mean": -252.77386474609375, "KL/mean": -301.6528625488281, "KL/rejected_KL_mean": -350.5318908691406, "KL/std": 129.88673400878906, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.75801849365234, "fcm_dpo/q_t": 0.4911108613014221, "grad_norm": 1.2370424270629883, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.5077199935913086, "logits/rejected": -1.355818748474121, "logps/chosen": -341.9432373046875, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -458.4512939453125, "loss": 1.3516, "margin_dpo/margin_mean": 97.75801849365234, "margin_dpo/margin_std": 130.74880981445312, "step": 510 }, { "KL/chosen_KL_mean": -221.97689819335938, "KL/mean": -279.25830078125, "KL/rejected_KL_mean": -336.5396728515625, "KL/std": 137.59400939941406, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.56280517578125, "fcm_dpo/q_t": 0.48958826065063477, "grad_norm": 1.7217494249343872, "learning_rate": 9.028829858700973e-08, "logits/chosen": -1.9741308689117432, "logits/rejected": -1.4161567687988281, "logps/chosen": -311.7266845703125, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -434.2709655761719, "loss": 1.3459, "margin_dpo/margin_mean": 114.56280517578125, "margin_dpo/margin_std": 161.41232299804688, "step": 511 }, { "KL/chosen_KL_mean": -218.8033447265625, "KL/mean": -271.3383483886719, "KL/rejected_KL_mean": -323.8733825683594, "KL/std": 138.19630432128906, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.07003021240234, "fcm_dpo/q_t": 0.49044710397720337, "grad_norm": 1.6084272861480713, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.876635193824768, "logits/rejected": -1.461219072341919, "logps/chosen": -304.5384826660156, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -411.4982604980469, "loss": 1.3492, "margin_dpo/margin_mean": 105.07002258300781, "margin_dpo/margin_std": 151.35934448242188, "step": 512 }, { "KL/chosen_KL_mean": -230.9978790283203, "KL/mean": -273.217041015625, "KL/rejected_KL_mean": -315.43621826171875, "KL/std": 129.878662109375, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.43832397460938, "fcm_dpo/q_t": 0.49232247471809387, "grad_norm": 1.3969497680664062, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.072983980178833, "logits/rejected": -1.3799635171890259, "logps/chosen": -327.1883850097656, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -398.35394287109375, "loss": 1.3566, "margin_dpo/margin_mean": 84.43832397460938, "margin_dpo/margin_std": 154.8363037109375, "step": 513 }, { "KL/chosen_KL_mean": -233.09539794921875, "KL/mean": -273.984619140625, "KL/rejected_KL_mean": -314.87384033203125, "KL/std": 132.27398681640625, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.7784423828125, "fcm_dpo/q_t": 0.49256592988967896, "grad_norm": 1.912717342376709, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.5975807905197144, "logits/rejected": -1.036444902420044, "logps/chosen": -330.9190368652344, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -400.24365234375, "loss": 1.3576, "margin_dpo/margin_mean": 81.7784423828125, "margin_dpo/margin_std": 155.91966247558594, "step": 514 }, { "KL/chosen_KL_mean": -203.45083618164062, "KL/mean": -255.56295776367188, "KL/rejected_KL_mean": -307.6750793457031, "KL/std": 133.6326904296875, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.2242431640625, "fcm_dpo/q_t": 0.4905243515968323, "grad_norm": 1.3206520080566406, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.6961365938186646, "logits/rejected": -1.4786815643310547, "logps/chosen": -285.3843994140625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -401.25164794921875, "loss": 1.3495, "margin_dpo/margin_mean": 104.2242431640625, "margin_dpo/margin_std": 154.39849853515625, "step": 515 }, { "KL/chosen_KL_mean": -229.89297485351562, "KL/mean": -276.09930419921875, "KL/rejected_KL_mean": -322.30560302734375, "KL/std": 162.87176513671875, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.4126205444336, "fcm_dpo/q_t": 0.49160605669021606, "grad_norm": 1.5753997564315796, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.286804676055908, "logits/rejected": -2.180054187774658, "logps/chosen": -326.9846496582031, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -440.86224365234375, "loss": 1.3543, "margin_dpo/margin_mean": 92.41262817382812, "margin_dpo/margin_std": 202.94235229492188, "step": 516 }, { "KL/chosen_KL_mean": -241.20608520507812, "KL/mean": -298.5009460449219, "KL/rejected_KL_mean": -355.79583740234375, "KL/std": 137.04371643066406, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.58978271484375, "fcm_dpo/q_t": 0.48958346247673035, "grad_norm": 1.0855655670166016, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.824014663696289, "logits/rejected": -1.4196107387542725, "logps/chosen": -334.39263916015625, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -451.3886413574219, "loss": 1.3459, "margin_dpo/margin_mean": 114.58977508544922, "margin_dpo/margin_std": 153.72708129882812, "step": 517 }, { "KL/chosen_KL_mean": -238.85641479492188, "KL/mean": -278.89642333984375, "KL/rejected_KL_mean": -318.9364013671875, "KL/std": 128.30589294433594, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.07994079589844, "fcm_dpo/q_t": 0.4927215576171875, "grad_norm": 1.6169328689575195, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.861922025680542, "logits/rejected": -1.558349370956421, "logps/chosen": -325.8778076171875, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -410.18096923828125, "loss": 1.3582, "margin_dpo/margin_mean": 80.07994842529297, "margin_dpo/margin_std": 160.91848754882812, "step": 518 }, { "KL/chosen_KL_mean": -228.51361083984375, "KL/mean": -275.32244873046875, "KL/rejected_KL_mean": -322.1312561035156, "KL/std": 121.50465393066406, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.61763000488281, "fcm_dpo/q_t": 0.49149006605148315, "grad_norm": 1.2509772777557373, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.8296997547149658, "logits/rejected": -1.4754791259765625, "logps/chosen": -329.201171875, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -431.16168212890625, "loss": 1.3531, "margin_dpo/margin_mean": 93.61763000488281, "margin_dpo/margin_std": 135.3308563232422, "step": 519 }, { "KL/chosen_KL_mean": -220.57669067382812, "KL/mean": -270.0523681640625, "KL/rejected_KL_mean": -319.528076171875, "KL/std": 145.37222290039062, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.95137786865234, "fcm_dpo/q_t": 0.49100276827812195, "grad_norm": 1.3034907579421997, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.3850593566894531, "logits/rejected": -0.972121000289917, "logps/chosen": -319.35430908203125, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -426.4737854003906, "loss": 1.3514, "margin_dpo/margin_mean": 98.95137023925781, "margin_dpo/margin_std": 147.31546020507812, "step": 520 }, { "KL/chosen_KL_mean": -209.36990356445312, "KL/mean": -264.2769470214844, "KL/rejected_KL_mean": -319.18402099609375, "KL/std": 120.14476776123047, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.81409454345703, "fcm_dpo/q_t": 0.49001505970954895, "grad_norm": 1.5551319122314453, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9185186624526978, "logits/rejected": -1.5932320356369019, "logps/chosen": -295.72100830078125, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -409.57720947265625, "loss": 1.3473, "margin_dpo/margin_mean": 109.81410217285156, "margin_dpo/margin_std": 129.33724975585938, "step": 521 }, { "KL/chosen_KL_mean": -212.08590698242188, "KL/mean": -262.1612243652344, "KL/rejected_KL_mean": -312.236572265625, "KL/std": 145.10726928710938, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.15064239501953, "fcm_dpo/q_t": 0.4908941686153412, "grad_norm": 1.1942366361618042, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.5835715532302856, "logits/rejected": -1.2258533239364624, "logps/chosen": -288.09393310546875, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -386.6778259277344, "loss": 1.351, "margin_dpo/margin_mean": 100.15065002441406, "margin_dpo/margin_std": 154.26805114746094, "step": 522 }, { "KL/chosen_KL_mean": -246.61451721191406, "KL/mean": -281.2411193847656, "KL/rejected_KL_mean": -315.86773681640625, "KL/std": 144.89715576171875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.2531967163086, "fcm_dpo/q_t": 0.4937039613723755, "grad_norm": 1.6137934923171997, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.8659441471099854, "logits/rejected": -1.2018120288848877, "logps/chosen": -343.11077880859375, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -406.88299560546875, "loss": 1.3623, "margin_dpo/margin_mean": 69.25318908691406, "margin_dpo/margin_std": 178.01451110839844, "step": 523 }, { "KL/chosen_KL_mean": -236.80909729003906, "KL/mean": -281.243896484375, "KL/rejected_KL_mean": -325.6787414550781, "KL/std": 135.7559814453125, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.86965942382812, "fcm_dpo/q_t": 0.49192163348197937, "grad_norm": 1.5842705965042114, "learning_rate": 7.781828926091535e-08, "logits/chosen": -1.8518600463867188, "logits/rejected": -1.1303952932357788, "logps/chosen": -340.214599609375, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -413.5888671875, "loss": 1.3551, "margin_dpo/margin_mean": 88.86965942382812, "margin_dpo/margin_std": 159.5693359375, "step": 524 }, { "KL/chosen_KL_mean": -205.56283569335938, "KL/mean": -270.6584777832031, "KL/rejected_KL_mean": -335.754150390625, "KL/std": 142.8565673828125, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.19131469726562, "fcm_dpo/q_t": 0.48816946148872375, "grad_norm": 1.3365238904953003, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.4250292778015137, "logits/rejected": -1.342210292816162, "logps/chosen": -289.7693176269531, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -442.0157470703125, "loss": 1.3405, "margin_dpo/margin_mean": 130.19131469726562, "margin_dpo/margin_std": 172.88055419921875, "step": 525 }, { "KL/chosen_KL_mean": -223.3436279296875, "KL/mean": -273.8338623046875, "KL/rejected_KL_mean": -324.32415771484375, "KL/std": 122.38346862792969, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.98051452636719, "fcm_dpo/q_t": 0.49081942439079285, "grad_norm": 1.4286041259765625, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7077335119247437, "logits/rejected": -1.0902087688446045, "logps/chosen": -326.19775390625, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -417.9755859375, "loss": 1.3506, "margin_dpo/margin_mean": 100.98051452636719, "margin_dpo/margin_std": 148.9225616455078, "step": 526 }, { "KL/chosen_KL_mean": -216.984130859375, "KL/mean": -269.2605285644531, "KL/rejected_KL_mean": -321.53692626953125, "KL/std": 127.36724853515625, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.5528335571289, "fcm_dpo/q_t": 0.49049264192581177, "grad_norm": 1.359908103942871, "learning_rate": 7.504749238082414e-08, "logits/chosen": -2.891925811767578, "logits/rejected": -2.1005899906158447, "logps/chosen": -313.74957275390625, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -406.93212890625, "loss": 1.3491, "margin_dpo/margin_mean": 104.5528335571289, "margin_dpo/margin_std": 117.47905731201172, "step": 527 }, { "KL/chosen_KL_mean": -207.22940063476562, "KL/mean": -262.7825622558594, "KL/rejected_KL_mean": -318.335693359375, "KL/std": 134.67123413085938, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.10631561279297, "fcm_dpo/q_t": 0.48990124464035034, "grad_norm": 1.3166943788528442, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.1763148307800293, "logits/rejected": -1.6734592914581299, "logps/chosen": -301.3943176269531, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -418.7080078125, "loss": 1.3472, "margin_dpo/margin_mean": 111.10631561279297, "margin_dpo/margin_std": 163.90121459960938, "step": 528 }, { "KL/chosen_KL_mean": -229.60946655273438, "KL/mean": -284.8818359375, "KL/rejected_KL_mean": -340.15423583984375, "KL/std": 138.5322265625, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.54476928710938, "fcm_dpo/q_t": 0.4899531602859497, "grad_norm": 1.8147159814834595, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.808337926864624, "logits/rejected": -1.6539735794067383, "logps/chosen": -310.3862609863281, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -437.34539794921875, "loss": 1.3474, "margin_dpo/margin_mean": 110.54476928710938, "margin_dpo/margin_std": 165.53543090820312, "step": 529 }, { "KL/chosen_KL_mean": -218.55160522460938, "KL/mean": -266.0648193359375, "KL/rejected_KL_mean": -313.578125, "KL/std": 132.4784698486328, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.0264892578125, "fcm_dpo/q_t": 0.4913633465766907, "grad_norm": 1.308121919631958, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.7055540084838867, "logits/rejected": -1.423275113105774, "logps/chosen": -298.32733154296875, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -405.63201904296875, "loss": 1.3527, "margin_dpo/margin_mean": 95.02649688720703, "margin_dpo/margin_std": 143.35870361328125, "step": 530 }, { "KL/chosen_KL_mean": -200.96441650390625, "KL/mean": -263.37188720703125, "KL/rejected_KL_mean": -325.779296875, "KL/std": 139.5245819091797, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.81488037109375, "fcm_dpo/q_t": 0.4886533319950104, "grad_norm": 1.273372769355774, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.8546128273010254, "logits/rejected": -1.2819756269454956, "logps/chosen": -280.28363037109375, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -403.88763427734375, "loss": 1.3421, "margin_dpo/margin_mean": 124.81488037109375, "margin_dpo/margin_std": 142.94557189941406, "step": 531 }, { "KL/chosen_KL_mean": -230.81863403320312, "KL/mean": -293.7120361328125, "KL/rejected_KL_mean": -356.60546875, "KL/std": 136.46438598632812, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.78684997558594, "fcm_dpo/q_t": 0.4885653257369995, "grad_norm": 1.3982964754104614, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.8524370193481445, "logits/rejected": -1.2592790126800537, "logps/chosen": -332.4555358886719, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -449.95086669921875, "loss": 1.3419, "margin_dpo/margin_mean": 125.78684997558594, "margin_dpo/margin_std": 159.63821411132812, "step": 532 }, { "KL/chosen_KL_mean": -216.20458984375, "KL/mean": -253.63616943359375, "KL/rejected_KL_mean": -291.0677490234375, "KL/std": 136.33724975585938, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.86318969726562, "fcm_dpo/q_t": 0.49319761991500854, "grad_norm": 1.337302565574646, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.4079771041870117, "logits/rejected": -1.8055026531219482, "logps/chosen": -306.95123291015625, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -373.9562683105469, "loss": 1.3601, "margin_dpo/margin_mean": 74.86318969726562, "margin_dpo/margin_std": 156.84097290039062, "step": 533 }, { "KL/chosen_KL_mean": -187.625244140625, "KL/mean": -244.61813354492188, "KL/rejected_KL_mean": -301.61102294921875, "KL/std": 127.88544464111328, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.98580169677734, "fcm_dpo/q_t": 0.48963606357574463, "grad_norm": 1.1319295167922974, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.9218432903289795, "logits/rejected": -1.5368304252624512, "logps/chosen": -274.2560729980469, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -386.4830017089844, "loss": 1.3458, "margin_dpo/margin_mean": 113.98580932617188, "margin_dpo/margin_std": 135.1536865234375, "step": 534 }, { "KL/chosen_KL_mean": -196.69805908203125, "KL/mean": -257.7487487792969, "KL/rejected_KL_mean": -318.7994384765625, "KL/std": 127.67987823486328, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.10136413574219, "fcm_dpo/q_t": 0.4889012277126312, "grad_norm": 1.2520966529846191, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.7372514009475708, "logits/rejected": -1.5765430927276611, "logps/chosen": -287.04345703125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -419.577392578125, "loss": 1.343, "margin_dpo/margin_mean": 122.10136413574219, "margin_dpo/margin_std": 142.51438903808594, "step": 535 }, { "KL/chosen_KL_mean": -186.85440063476562, "KL/mean": -249.2871551513672, "KL/rejected_KL_mean": -311.71990966796875, "KL/std": 130.90774536132812, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.865478515625, "fcm_dpo/q_t": 0.4886496663093567, "grad_norm": 1.7963075637817383, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.7049306631088257, "logits/rejected": -1.4675121307373047, "logps/chosen": -265.81396484375, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -404.80767822265625, "loss": 1.3421, "margin_dpo/margin_mean": 124.865478515625, "margin_dpo/margin_std": 149.51405334472656, "step": 536 }, { "KL/chosen_KL_mean": -180.0478973388672, "KL/mean": -237.11093139648438, "KL/rejected_KL_mean": -294.1739501953125, "KL/std": 137.26785278320312, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.12606048583984, "fcm_dpo/q_t": 0.489624559879303, "grad_norm": 1.10858952999115, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.5227255821228027, "logits/rejected": -0.8840628862380981, "logps/chosen": -266.7572937011719, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -373.0816345214844, "loss": 1.346, "margin_dpo/margin_mean": 114.12606048583984, "margin_dpo/margin_std": 147.21047973632812, "step": 537 }, { "KL/chosen_KL_mean": -244.043212890625, "KL/mean": -291.51654052734375, "KL/rejected_KL_mean": -338.9898681640625, "KL/std": 123.71240234375, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.94664001464844, "fcm_dpo/q_t": 0.49136781692504883, "grad_norm": 1.0933558940887451, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.9200063943862915, "logits/rejected": -1.4036803245544434, "logps/chosen": -342.4853515625, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -430.0845031738281, "loss": 1.3527, "margin_dpo/margin_mean": 94.94664764404297, "margin_dpo/margin_std": 137.60369873046875, "step": 538 }, { "KL/chosen_KL_mean": -206.58184814453125, "KL/mean": -260.77801513671875, "KL/rejected_KL_mean": -314.9742126464844, "KL/std": 131.7769775390625, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.39237976074219, "fcm_dpo/q_t": 0.4901456832885742, "grad_norm": 1.1839953660964966, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.5132765769958496, "logits/rejected": -2.053689479827881, "logps/chosen": -297.51397705078125, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -412.419677734375, "loss": 1.3479, "margin_dpo/margin_mean": 108.39237976074219, "margin_dpo/margin_std": 142.0021209716797, "step": 539 }, { "KL/chosen_KL_mean": -238.70672607421875, "KL/mean": -303.78143310546875, "KL/rejected_KL_mean": -368.8560791015625, "KL/std": 161.0806884765625, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.14938354492188, "fcm_dpo/q_t": 0.48818129301071167, "grad_norm": 1.387000322341919, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9841508865356445, "logits/rejected": -1.7754669189453125, "logps/chosen": -346.5766296386719, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -490.2496032714844, "loss": 1.3409, "margin_dpo/margin_mean": 130.1493682861328, "margin_dpo/margin_std": 205.59262084960938, "step": 540 }, { "KL/chosen_KL_mean": -250.32455444335938, "KL/mean": -292.8800964355469, "KL/rejected_KL_mean": -335.43560791015625, "KL/std": 124.82654571533203, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.11106872558594, "fcm_dpo/q_t": 0.49226367473602295, "grad_norm": 1.878108024597168, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.4527981281280518, "logits/rejected": -2.0403223037719727, "logps/chosen": -356.89202880859375, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -444.501953125, "loss": 1.3563, "margin_dpo/margin_mean": 85.11106872558594, "margin_dpo/margin_std": 136.0174102783203, "step": 541 }, { "KL/chosen_KL_mean": -201.30450439453125, "KL/mean": -267.1488952636719, "KL/rejected_KL_mean": -332.9932861328125, "KL/std": 138.34011840820312, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.68878173828125, "fcm_dpo/q_t": 0.4880279302597046, "grad_norm": 1.5025057792663574, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.2199223041534424, "logits/rejected": -2.1057114601135254, "logps/chosen": -288.6737976074219, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -435.81365966796875, "loss": 1.3397, "margin_dpo/margin_mean": 131.68878173828125, "margin_dpo/margin_std": 149.2720947265625, "step": 542 }, { "KL/chosen_KL_mean": -245.6800537109375, "KL/mean": -280.73101806640625, "KL/rejected_KL_mean": -315.781982421875, "KL/std": 152.51522827148438, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.10189056396484, "fcm_dpo/q_t": 0.493630051612854, "grad_norm": 1.4123289585113525, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.4957821369171143, "logits/rejected": -1.5793166160583496, "logps/chosen": -332.8326416015625, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -427.3136901855469, "loss": 1.3622, "margin_dpo/margin_mean": 70.10189056396484, "margin_dpo/margin_std": 193.31002807617188, "step": 543 }, { "KL/chosen_KL_mean": -252.8023681640625, "KL/mean": -299.6568603515625, "KL/rejected_KL_mean": -346.51141357421875, "KL/std": 148.5951385498047, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.70903015136719, "fcm_dpo/q_t": 0.49149441719055176, "grad_norm": 1.3667747974395752, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.6579030752182007, "logits/rejected": -1.585377812385559, "logps/chosen": -338.94927978515625, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -452.90533447265625, "loss": 1.3536, "margin_dpo/margin_mean": 93.70903015136719, "margin_dpo/margin_std": 182.00241088867188, "step": 544 }, { "KL/chosen_KL_mean": -227.00839233398438, "KL/mean": -281.1431884765625, "KL/rejected_KL_mean": -335.2779541015625, "KL/std": 142.0379638671875, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.26956176757812, "fcm_dpo/q_t": 0.4901590347290039, "grad_norm": 1.7261921167373657, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.745758295059204, "logits/rejected": -1.3517372608184814, "logps/chosen": -323.4326171875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -436.01202392578125, "loss": 1.3482, "margin_dpo/margin_mean": 108.26956176757812, "margin_dpo/margin_std": 164.57266235351562, "step": 545 }, { "KL/chosen_KL_mean": -241.48052978515625, "KL/mean": -283.5916442871094, "KL/rejected_KL_mean": -325.7027587890625, "KL/std": 135.96165466308594, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.22225952148438, "fcm_dpo/q_t": 0.4923417866230011, "grad_norm": 1.319090723991394, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.394700527191162, "logits/rejected": -1.0598866939544678, "logps/chosen": -327.50164794921875, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -413.0754089355469, "loss": 1.3567, "margin_dpo/margin_mean": 84.2222671508789, "margin_dpo/margin_std": 154.5341796875, "step": 546 }, { "KL/chosen_KL_mean": -230.82200622558594, "KL/mean": -285.8389892578125, "KL/rejected_KL_mean": -340.8559875488281, "KL/std": 138.82980346679688, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.03398132324219, "fcm_dpo/q_t": 0.48999589681625366, "grad_norm": 1.1235092878341675, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.7568588256835938, "logits/rejected": -1.303643822669983, "logps/chosen": -331.4730224609375, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -438.7677307128906, "loss": 1.3472, "margin_dpo/margin_mean": 110.03398132324219, "margin_dpo/margin_std": 130.71463012695312, "step": 547 }, { "KL/chosen_KL_mean": -224.74990844726562, "KL/mean": -268.9020690917969, "KL/rejected_KL_mean": -313.05419921875, "KL/std": 136.29962158203125, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.30427551269531, "fcm_dpo/q_t": 0.49197477102279663, "grad_norm": 1.8005226850509644, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.204517364501953, "logits/rejected": -1.8633087873458862, "logps/chosen": -320.780029296875, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -411.4674072265625, "loss": 1.3554, "margin_dpo/margin_mean": 88.30427551269531, "margin_dpo/margin_std": 170.8684539794922, "step": 548 }, { "KL/chosen_KL_mean": -212.9886474609375, "KL/mean": -252.43612670898438, "KL/rejected_KL_mean": -291.88360595703125, "KL/std": 128.372802734375, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.89495086669922, "fcm_dpo/q_t": 0.4928264319896698, "grad_norm": 1.3965439796447754, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.7226332426071167, "logits/rejected": -1.3340373039245605, "logps/chosen": -296.43963623046875, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -373.26385498046875, "loss": 1.3588, "margin_dpo/margin_mean": 78.89494323730469, "margin_dpo/margin_std": 166.16514587402344, "step": 549 }, { "KL/chosen_KL_mean": -232.6486053466797, "KL/mean": -275.18096923828125, "KL/rejected_KL_mean": -317.71331787109375, "KL/std": 126.42316436767578, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.06472778320312, "fcm_dpo/q_t": 0.4922679364681244, "grad_norm": 1.680233359336853, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.7040235996246338, "logits/rejected": -1.52479887008667, "logps/chosen": -322.2429504394531, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -419.02880859375, "loss": 1.3563, "margin_dpo/margin_mean": 85.06472778320312, "margin_dpo/margin_std": 147.43011474609375, "step": 550 }, { "KL/chosen_KL_mean": -205.40823364257812, "KL/mean": -270.95013427734375, "KL/rejected_KL_mean": -336.4920654296875, "KL/std": 144.10508728027344, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.08383178710938, "fcm_dpo/q_t": 0.48808830976486206, "grad_norm": 1.2473042011260986, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.6766180992126465, "logits/rejected": -1.5569493770599365, "logps/chosen": -300.4903869628906, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -454.2113037109375, "loss": 1.34, "margin_dpo/margin_mean": 131.08383178710938, "margin_dpo/margin_std": 160.82327270507812, "step": 551 }, { "KL/chosen_KL_mean": -215.60186767578125, "KL/mean": -293.6191711425781, "KL/rejected_KL_mean": -371.636474609375, "KL/std": 160.28924560546875, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.0346221923828, "fcm_dpo/q_t": 0.485821396112442, "grad_norm": 2.3583102226257324, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.104945421218872, "logits/rejected": -1.7772026062011719, "logps/chosen": -317.01104736328125, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -482.47491455078125, "loss": 1.3314, "margin_dpo/margin_mean": 156.0346221923828, "margin_dpo/margin_std": 175.00418090820312, "step": 552 }, { "KL/chosen_KL_mean": -222.27947998046875, "KL/mean": -268.271484375, "KL/rejected_KL_mean": -314.26348876953125, "KL/std": 145.79220581054688, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.98401641845703, "fcm_dpo/q_t": 0.4916377663612366, "grad_norm": 1.4818267822265625, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.591939091682434, "logits/rejected": -1.2843761444091797, "logps/chosen": -320.24334716796875, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -417.8406982421875, "loss": 1.3539, "margin_dpo/margin_mean": 91.9840087890625, "margin_dpo/margin_std": 157.61248779296875, "step": 553 }, { "KL/chosen_KL_mean": -229.1710205078125, "KL/mean": -282.5264892578125, "KL/rejected_KL_mean": -335.88201904296875, "KL/std": 140.5336151123047, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.71099853515625, "fcm_dpo/q_t": 0.49030134081840515, "grad_norm": 0.9630252122879028, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.914034128189087, "logits/rejected": -1.1705381870269775, "logps/chosen": -332.85784912109375, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -428.8741455078125, "loss": 1.349, "margin_dpo/margin_mean": 106.71099853515625, "margin_dpo/margin_std": 182.19970703125, "step": 554 }, { "KL/chosen_KL_mean": -235.69052124023438, "KL/mean": -285.06689453125, "KL/rejected_KL_mean": -334.4432373046875, "KL/std": 148.77706909179688, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.7527084350586, "fcm_dpo/q_t": 0.4910257160663605, "grad_norm": 1.341017484664917, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.72976553440094, "logits/rejected": -1.6568920612335205, "logps/chosen": -321.8941650390625, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -448.99041748046875, "loss": 1.3517, "margin_dpo/margin_mean": 98.7527084350586, "margin_dpo/margin_std": 173.00955200195312, "step": 555 }, { "KL/chosen_KL_mean": -213.22293090820312, "KL/mean": -268.15869140625, "KL/rejected_KL_mean": -323.094482421875, "KL/std": 125.10711669921875, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.8715591430664, "fcm_dpo/q_t": 0.49001091718673706, "grad_norm": 1.488216519355774, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.5739505290985107, "logits/rejected": -1.3266757726669312, "logps/chosen": -301.62640380859375, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -429.73358154296875, "loss": 1.3474, "margin_dpo/margin_mean": 109.87155151367188, "margin_dpo/margin_std": 142.60140991210938, "step": 556 }, { "KL/chosen_KL_mean": -237.88278198242188, "KL/mean": -297.8853759765625, "KL/rejected_KL_mean": -357.8879089355469, "KL/std": 159.34915161132812, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.0051498413086, "fcm_dpo/q_t": 0.48909592628479004, "grad_norm": 1.2532081604003906, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9798691272735596, "logits/rejected": -1.825594186782837, "logps/chosen": -336.718017578125, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -473.53680419921875, "loss": 1.3443, "margin_dpo/margin_mean": 120.00515747070312, "margin_dpo/margin_std": 185.64053344726562, "step": 557 }, { "KL/chosen_KL_mean": -235.6900634765625, "KL/mean": -296.75396728515625, "KL/rejected_KL_mean": -357.81781005859375, "KL/std": 142.95631408691406, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.12774658203125, "fcm_dpo/q_t": 0.48889821767807007, "grad_norm": 1.6884140968322754, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3168187141418457, "logits/rejected": -1.7428547143936157, "logps/chosen": -341.701904296875, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -466.2091369628906, "loss": 1.343, "margin_dpo/margin_mean": 122.12774658203125, "margin_dpo/margin_std": 146.31329345703125, "step": 558 }, { "KL/chosen_KL_mean": -251.79202270507812, "KL/mean": -298.00634765625, "KL/rejected_KL_mean": -344.220703125, "KL/std": 143.93344116210938, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.42866516113281, "fcm_dpo/q_t": 0.4915962815284729, "grad_norm": 1.5290822982788086, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.321533679962158, "logits/rejected": -1.876450777053833, "logps/chosen": -352.2167663574219, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -444.6333923339844, "loss": 1.3539, "margin_dpo/margin_mean": 92.42867279052734, "margin_dpo/margin_std": 169.63247680664062, "step": 559 }, { "KL/chosen_KL_mean": -227.11923217773438, "KL/mean": -275.7327880859375, "KL/rejected_KL_mean": -324.3463134765625, "KL/std": 133.1451416015625, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.22708129882812, "fcm_dpo/q_t": 0.49116212129592896, "grad_norm": 1.513019323348999, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.4115982055664062, "logits/rejected": -1.0054408311843872, "logps/chosen": -314.7547607421875, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -411.72613525390625, "loss": 1.3521, "margin_dpo/margin_mean": 97.2270736694336, "margin_dpo/margin_std": 158.92120361328125, "step": 560 }, { "KL/chosen_KL_mean": -235.67169189453125, "KL/mean": -290.56341552734375, "KL/rejected_KL_mean": -345.4552001953125, "KL/std": 131.8497314453125, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.78350067138672, "fcm_dpo/q_t": 0.4900193214416504, "grad_norm": 1.3038712739944458, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.7599000930786133, "logits/rejected": -1.7529546022415161, "logps/chosen": -310.5531005859375, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -451.5031433105469, "loss": 1.3474, "margin_dpo/margin_mean": 109.78350830078125, "margin_dpo/margin_std": 135.86599731445312, "step": 561 }, { "KL/chosen_KL_mean": -215.78262329101562, "KL/mean": -262.787841796875, "KL/rejected_KL_mean": -309.7930908203125, "KL/std": 133.54171752929688, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.01046752929688, "fcm_dpo/q_t": 0.49145689606666565, "grad_norm": 1.3660366535186768, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.17975115776062, "logits/rejected": -1.757237195968628, "logps/chosen": -316.4458312988281, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -412.033935546875, "loss": 1.3534, "margin_dpo/margin_mean": 94.01046752929688, "margin_dpo/margin_std": 171.0333251953125, "step": 562 }, { "KL/chosen_KL_mean": -230.93568420410156, "KL/mean": -267.5196228027344, "KL/rejected_KL_mean": -304.1035461425781, "KL/std": 133.96408081054688, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.16787719726562, "fcm_dpo/q_t": 0.4933473467826843, "grad_norm": 1.5234521627426147, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.8962435722351074, "logits/rejected": -1.6049884557724, "logps/chosen": -327.19488525390625, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -405.30859375, "loss": 1.3605, "margin_dpo/margin_mean": 73.16787719726562, "margin_dpo/margin_std": 137.6486358642578, "step": 563 }, { "KL/chosen_KL_mean": -227.21707153320312, "KL/mean": -270.55810546875, "KL/rejected_KL_mean": -313.899169921875, "KL/std": 127.83708953857422, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.68209838867188, "fcm_dpo/q_t": 0.4921172857284546, "grad_norm": 1.3515064716339111, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.4696030616760254, "logits/rejected": -1.9251725673675537, "logps/chosen": -340.177490234375, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -413.4272155761719, "loss": 1.3556, "margin_dpo/margin_mean": 86.68209838867188, "margin_dpo/margin_std": 134.59152221679688, "step": 564 }, { "KL/chosen_KL_mean": -231.71719360351562, "KL/mean": -279.9974060058594, "KL/rejected_KL_mean": -328.27764892578125, "KL/std": 147.35995483398438, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.56043243408203, "fcm_dpo/q_t": 0.4912245273590088, "grad_norm": 1.4101845026016235, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.498381495475769, "logits/rejected": -1.0147336721420288, "logps/chosen": -327.082763671875, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -420.4949951171875, "loss": 1.3524, "margin_dpo/margin_mean": 96.5604248046875, "margin_dpo/margin_std": 172.33413696289062, "step": 565 }, { "KL/chosen_KL_mean": -240.8166046142578, "KL/mean": -290.54376220703125, "KL/rejected_KL_mean": -340.2708740234375, "KL/std": 149.3006134033203, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.45426940917969, "fcm_dpo/q_t": 0.4909588098526001, "grad_norm": 1.3209724426269531, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9818880558013916, "logits/rejected": -1.5844101905822754, "logps/chosen": -339.789794921875, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -451.091796875, "loss": 1.3512, "margin_dpo/margin_mean": 99.45426940917969, "margin_dpo/margin_std": 151.80812072753906, "step": 566 }, { "KL/chosen_KL_mean": -225.69778442382812, "KL/mean": -266.2763366699219, "KL/rejected_KL_mean": -306.85491943359375, "KL/std": 127.26952362060547, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.15716552734375, "fcm_dpo/q_t": 0.4926193058490753, "grad_norm": 1.1284000873565674, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.176835536956787, "logits/rejected": -1.520682454109192, "logps/chosen": -318.4013671875, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -398.0792236328125, "loss": 1.3577, "margin_dpo/margin_mean": 81.15716552734375, "margin_dpo/margin_std": 139.14242553710938, "step": 567 }, { "KL/chosen_KL_mean": -210.0182647705078, "KL/mean": -273.23114013671875, "KL/rejected_KL_mean": -336.4439697265625, "KL/std": 137.57666015625, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.42573547363281, "fcm_dpo/q_t": 0.4885084629058838, "grad_norm": 1.3686751127243042, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3413128852844238, "logits/rejected": -1.2455995082855225, "logps/chosen": -283.1505126953125, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -431.3966369628906, "loss": 1.3416, "margin_dpo/margin_mean": 126.42573547363281, "margin_dpo/margin_std": 152.33377075195312, "step": 568 }, { "KL/chosen_KL_mean": -224.77645874023438, "KL/mean": -276.4371643066406, "KL/rejected_KL_mean": -328.097900390625, "KL/std": 143.65467834472656, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.3214340209961, "fcm_dpo/q_t": 0.4906090497970581, "grad_norm": 1.3294892311096191, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.8929035663604736, "logits/rejected": -1.5218733549118042, "logps/chosen": -313.9935302734375, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -414.5089416503906, "loss": 1.3502, "margin_dpo/margin_mean": 103.3214340209961, "margin_dpo/margin_std": 180.6341552734375, "step": 569 }, { "KL/chosen_KL_mean": -205.11883544921875, "KL/mean": -268.8258056640625, "KL/rejected_KL_mean": -332.53277587890625, "KL/std": 134.943115234375, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.41390991210938, "fcm_dpo/q_t": 0.48841553926467896, "grad_norm": 1.4354383945465088, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.225943088531494, "logits/rejected": -1.5794090032577515, "logps/chosen": -299.881591796875, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -421.37969970703125, "loss": 1.3411, "margin_dpo/margin_mean": 127.41390991210938, "margin_dpo/margin_std": 143.6767578125, "step": 570 }, { "KL/chosen_KL_mean": -208.236328125, "KL/mean": -272.5758972167969, "KL/rejected_KL_mean": -336.9154968261719, "KL/std": 132.4944610595703, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.67916870117188, "fcm_dpo/q_t": 0.4883033037185669, "grad_norm": 1.6984244585037231, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.461467981338501, "logits/rejected": -1.3020431995391846, "logps/chosen": -289.0576171875, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -433.9307861328125, "loss": 1.3409, "margin_dpo/margin_mean": 128.67916870117188, "margin_dpo/margin_std": 158.65196228027344, "step": 571 }, { "KL/chosen_KL_mean": -214.38385009765625, "KL/mean": -277.10943603515625, "KL/rejected_KL_mean": -339.83502197265625, "KL/std": 136.91493225097656, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.45114135742188, "fcm_dpo/q_t": 0.48859745264053345, "grad_norm": 1.3011760711669922, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.5517966747283936, "logits/rejected": -1.3889210224151611, "logps/chosen": -291.7637939453125, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -430.4010009765625, "loss": 1.3419, "margin_dpo/margin_mean": 125.45115661621094, "margin_dpo/margin_std": 151.60354614257812, "step": 572 }, { "KL/chosen_KL_mean": -209.3701171875, "KL/mean": -263.08123779296875, "KL/rejected_KL_mean": -316.7923889160156, "KL/std": 135.92234802246094, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.4222640991211, "fcm_dpo/q_t": 0.4902353584766388, "grad_norm": 1.479299545288086, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1896510124206543, "logits/rejected": -1.9723482131958008, "logps/chosen": -293.0762023925781, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -407.4898681640625, "loss": 1.3484, "margin_dpo/margin_mean": 107.42225646972656, "margin_dpo/margin_std": 158.64999389648438, "step": 573 }, { "KL/chosen_KL_mean": -228.9401092529297, "KL/mean": -287.7364196777344, "KL/rejected_KL_mean": -346.53271484375, "KL/std": 138.7572021484375, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.59262084960938, "fcm_dpo/q_t": 0.48930954933166504, "grad_norm": 1.3537989854812622, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.2493650913238525, "logits/rejected": -1.9631407260894775, "logps/chosen": -314.069580078125, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -445.5867614746094, "loss": 1.3447, "margin_dpo/margin_mean": 117.59261322021484, "margin_dpo/margin_std": 149.71913146972656, "step": 574 }, { "KL/chosen_KL_mean": -246.0197296142578, "KL/mean": -292.6722717285156, "KL/rejected_KL_mean": -339.32476806640625, "KL/std": 143.23031616210938, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.30506134033203, "fcm_dpo/q_t": 0.49152034521102905, "grad_norm": 1.692467212677002, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.2987326383590698, "logits/rejected": -0.8373672962188721, "logps/chosen": -351.79058837890625, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -440.27423095703125, "loss": 1.3537, "margin_dpo/margin_mean": 93.30506134033203, "margin_dpo/margin_std": 178.9981689453125, "step": 575 }, { "KL/chosen_KL_mean": -224.7119140625, "KL/mean": -282.2330627441406, "KL/rejected_KL_mean": -339.75421142578125, "KL/std": 133.9973907470703, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.04229736328125, "fcm_dpo/q_t": 0.4895413815975189, "grad_norm": 1.6518588066101074, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.8489116430282593, "logits/rejected": -1.8132587671279907, "logps/chosen": -305.91644287109375, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -445.16363525390625, "loss": 1.3457, "margin_dpo/margin_mean": 115.04229736328125, "margin_dpo/margin_std": 155.98648071289062, "step": 576 }, { "KL/chosen_KL_mean": -208.5750274658203, "KL/mean": -269.1076354980469, "KL/rejected_KL_mean": -329.6402587890625, "KL/std": 136.67034912109375, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.06523132324219, "fcm_dpo/q_t": 0.48899713158607483, "grad_norm": 1.7903372049331665, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.7740964889526367, "logits/rejected": -1.1027222871780396, "logps/chosen": -310.03924560546875, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -418.96661376953125, "loss": 1.3436, "margin_dpo/margin_mean": 121.06523132324219, "margin_dpo/margin_std": 163.35018920898438, "step": 577 }, { "KL/chosen_KL_mean": -246.45130920410156, "KL/mean": -290.68927001953125, "KL/rejected_KL_mean": -334.9272155761719, "KL/std": 129.21597290039062, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.47590637207031, "fcm_dpo/q_t": 0.4919559955596924, "grad_norm": 1.7569173574447632, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.214076042175293, "logits/rejected": -0.983075737953186, "logps/chosen": -323.74371337890625, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -423.1742248535156, "loss": 1.3551, "margin_dpo/margin_mean": 88.47589874267578, "margin_dpo/margin_std": 149.4856719970703, "step": 578 }, { "KL/chosen_KL_mean": -224.828369140625, "KL/mean": -270.6837463378906, "KL/rejected_KL_mean": -316.53912353515625, "KL/std": 136.4702911376953, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.71076965332031, "fcm_dpo/q_t": 0.49166518449783325, "grad_norm": 1.9986103773117065, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.5586861371994019, "logits/rejected": -1.3251712322235107, "logps/chosen": -312.1714172363281, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -418.66015625, "loss": 1.354, "margin_dpo/margin_mean": 91.71076965332031, "margin_dpo/margin_std": 146.7569580078125, "step": 579 }, { "KL/chosen_KL_mean": -212.24114990234375, "KL/mean": -272.56341552734375, "KL/rejected_KL_mean": -332.8857421875, "KL/std": 143.4878387451172, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.64458465576172, "fcm_dpo/q_t": 0.48904091119766235, "grad_norm": 1.3253934383392334, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.2448136806488037, "logits/rejected": -0.9304983615875244, "logps/chosen": -301.169921875, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -421.5823669433594, "loss": 1.344, "margin_dpo/margin_mean": 120.64458465576172, "margin_dpo/margin_std": 180.18927001953125, "step": 580 }, { "KL/chosen_KL_mean": -229.17425537109375, "KL/mean": -278.528076171875, "KL/rejected_KL_mean": -327.8819274902344, "KL/std": 130.377197265625, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.70765686035156, "fcm_dpo/q_t": 0.49102693796157837, "grad_norm": 1.2537689208984375, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.8122143745422363, "logits/rejected": -1.4828100204467773, "logps/chosen": -321.3974914550781, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -427.2713623046875, "loss": 1.3515, "margin_dpo/margin_mean": 98.70765686035156, "margin_dpo/margin_std": 157.49752807617188, "step": 581 }, { "KL/chosen_KL_mean": -200.88623046875, "KL/mean": -254.44369506835938, "KL/rejected_KL_mean": -308.00115966796875, "KL/std": 149.11911010742188, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.11492156982422, "fcm_dpo/q_t": 0.49027180671691895, "grad_norm": 1.0486972332000732, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.046031951904297, "logits/rejected": -2.0475945472717285, "logps/chosen": -288.22186279296875, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -423.15557861328125, "loss": 1.3488, "margin_dpo/margin_mean": 107.11491394042969, "margin_dpo/margin_std": 177.82369995117188, "step": 582 }, { "KL/chosen_KL_mean": -239.0076904296875, "KL/mean": -295.9010009765625, "KL/rejected_KL_mean": -352.7943115234375, "KL/std": 146.55023193359375, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.78657531738281, "fcm_dpo/q_t": 0.4896610379219055, "grad_norm": 1.5454033613204956, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5560364723205566, "logits/rejected": -1.3311630487442017, "logps/chosen": -322.7247619628906, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -447.37896728515625, "loss": 1.3465, "margin_dpo/margin_mean": 113.78659057617188, "margin_dpo/margin_std": 183.21823120117188, "step": 583 }, { "KL/chosen_KL_mean": -214.94927978515625, "KL/mean": -276.2021484375, "KL/rejected_KL_mean": -337.45501708984375, "KL/std": 141.59811401367188, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.50572204589844, "fcm_dpo/q_t": 0.4888641834259033, "grad_norm": 1.0804193019866943, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9775047302246094, "logits/rejected": -1.5636026859283447, "logps/chosen": -309.2088623046875, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -441.93768310546875, "loss": 1.3431, "margin_dpo/margin_mean": 122.50572204589844, "margin_dpo/margin_std": 162.48976135253906, "step": 584 }, { "KL/chosen_KL_mean": -228.69781494140625, "KL/mean": -294.15673828125, "KL/rejected_KL_mean": -359.6156005859375, "KL/std": 160.39144897460938, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.91781616210938, "fcm_dpo/q_t": 0.48810717463493347, "grad_norm": 1.351098656654358, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.8273212909698486, "logits/rejected": -1.5677375793457031, "logps/chosen": -314.3345642089844, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -449.9010925292969, "loss": 1.3405, "margin_dpo/margin_mean": 130.91783142089844, "margin_dpo/margin_std": 196.95590209960938, "step": 585 }, { "KL/chosen_KL_mean": -245.8614044189453, "KL/mean": -289.77337646484375, "KL/rejected_KL_mean": -333.6854248046875, "KL/std": 137.66607666015625, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.82398986816406, "fcm_dpo/q_t": 0.4920145273208618, "grad_norm": 2.0301010608673096, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.7595069408416748, "logits/rejected": -1.5209012031555176, "logps/chosen": -329.54254150390625, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -432.0445556640625, "loss": 1.3552, "margin_dpo/margin_mean": 87.82398986816406, "margin_dpo/margin_std": 138.7886199951172, "step": 586 }, { "KL/chosen_KL_mean": -201.19610595703125, "KL/mean": -244.39300537109375, "KL/rejected_KL_mean": -287.5898742675781, "KL/std": 136.7624969482422, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.39378356933594, "fcm_dpo/q_t": 0.49214422702789307, "grad_norm": 1.7666727304458618, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.320841073989868, "logits/rejected": -2.0735151767730713, "logps/chosen": -278.77362060546875, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -376.91021728515625, "loss": 1.3558, "margin_dpo/margin_mean": 86.39378356933594, "margin_dpo/margin_std": 145.81398010253906, "step": 587 }, { "KL/chosen_KL_mean": -230.41065979003906, "KL/mean": -280.56256103515625, "KL/rejected_KL_mean": -330.7144775390625, "KL/std": 150.8777313232422, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.30379486083984, "fcm_dpo/q_t": 0.4908895492553711, "grad_norm": 1.928987979888916, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.6582820415496826, "logits/rejected": -1.6475504636764526, "logps/chosen": -314.21490478515625, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -443.83392333984375, "loss": 1.3513, "margin_dpo/margin_mean": 100.30379486083984, "margin_dpo/margin_std": 184.2015380859375, "step": 588 }, { "KL/chosen_KL_mean": -241.93655395507812, "KL/mean": -300.8431091308594, "KL/rejected_KL_mean": -359.7496337890625, "KL/std": 135.36849975585938, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.81309509277344, "fcm_dpo/q_t": 0.48928824067115784, "grad_norm": 1.2383195161819458, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.582207202911377, "logits/rejected": -1.2300772666931152, "logps/chosen": -338.4015808105469, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -475.786865234375, "loss": 1.3445, "margin_dpo/margin_mean": 117.81309509277344, "margin_dpo/margin_std": 135.1048126220703, "step": 589 }, { "KL/chosen_KL_mean": -230.77854919433594, "KL/mean": -285.69720458984375, "KL/rejected_KL_mean": -340.6158447265625, "KL/std": 125.02320861816406, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.83733367919922, "fcm_dpo/q_t": 0.4900161623954773, "grad_norm": 2.5948238372802734, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.899955153465271, "logits/rejected": -1.8126505613327026, "logps/chosen": -319.2064208984375, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -450.81463623046875, "loss": 1.3474, "margin_dpo/margin_mean": 109.83733367919922, "margin_dpo/margin_std": 143.81607055664062, "step": 590 }, { "KL/chosen_KL_mean": -205.45977783203125, "KL/mean": -270.99822998046875, "KL/rejected_KL_mean": -336.5367126464844, "KL/std": 130.99505615234375, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.07691955566406, "fcm_dpo/q_t": 0.4880834221839905, "grad_norm": 1.1645410060882568, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.8659775257110596, "logits/rejected": -1.2887282371520996, "logps/chosen": -293.3094482421875, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -426.11590576171875, "loss": 1.3398, "margin_dpo/margin_mean": 131.07691955566406, "margin_dpo/margin_std": 137.01629638671875, "step": 591 }, { "KL/chosen_KL_mean": -247.90548706054688, "KL/mean": -294.3731689453125, "KL/rejected_KL_mean": -340.84088134765625, "KL/std": 131.29327392578125, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.93539428710938, "fcm_dpo/q_t": 0.491550087928772, "grad_norm": 1.7956303358078003, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7327682971954346, "logits/rejected": -1.4217722415924072, "logps/chosen": -322.8812255859375, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -422.39349365234375, "loss": 1.3537, "margin_dpo/margin_mean": 92.93538665771484, "margin_dpo/margin_std": 173.05859375, "step": 592 }, { "KL/chosen_KL_mean": -220.73265075683594, "KL/mean": -256.926513671875, "KL/rejected_KL_mean": -293.1203918457031, "KL/std": 131.6584014892578, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.38774871826172, "fcm_dpo/q_t": 0.49341827630996704, "grad_norm": 1.746701955795288, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.6231648921966553, "logits/rejected": -0.32334697246551514, "logps/chosen": -292.9716796875, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -363.2169189453125, "loss": 1.3609, "margin_dpo/margin_mean": 72.38774871826172, "margin_dpo/margin_std": 152.6986083984375, "step": 593 }, { "KL/chosen_KL_mean": -210.3105010986328, "KL/mean": -267.4587707519531, "KL/rejected_KL_mean": -324.6070556640625, "KL/std": 130.5922393798828, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.29653930664062, "fcm_dpo/q_t": 0.48961102962493896, "grad_norm": 1.6515247821807861, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9128252267837524, "logits/rejected": -1.4221749305725098, "logps/chosen": -300.74822998046875, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -416.8746337890625, "loss": 1.3459, "margin_dpo/margin_mean": 114.29654693603516, "margin_dpo/margin_std": 145.2410888671875, "step": 594 }, { "KL/chosen_KL_mean": -218.18704223632812, "KL/mean": -271.2037048339844, "KL/rejected_KL_mean": -324.2203369140625, "KL/std": 133.42660522460938, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.03327941894531, "fcm_dpo/q_t": 0.49036315083503723, "grad_norm": 1.4991977214813232, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.7916855812072754, "logits/rejected": -1.4304553270339966, "logps/chosen": -299.70184326171875, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -410.74078369140625, "loss": 1.3489, "margin_dpo/margin_mean": 106.03327941894531, "margin_dpo/margin_std": 160.24530029296875, "step": 595 }, { "KL/chosen_KL_mean": -228.92234802246094, "KL/mean": -272.08319091796875, "KL/rejected_KL_mean": -315.2440185546875, "KL/std": 135.37033081054688, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.3216781616211, "fcm_dpo/q_t": 0.49215370416641235, "grad_norm": 1.3997477293014526, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.4744374752044678, "logits/rejected": -0.9049133062362671, "logps/chosen": -317.1304016113281, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -401.3377990722656, "loss": 1.356, "margin_dpo/margin_mean": 86.32167053222656, "margin_dpo/margin_std": 163.84317016601562, "step": 596 }, { "KL/chosen_KL_mean": -216.90113830566406, "KL/mean": -280.4420166015625, "KL/rejected_KL_mean": -343.9828796386719, "KL/std": 144.5499267578125, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.08175659179688, "fcm_dpo/q_t": 0.4884474575519562, "grad_norm": 1.331176996231079, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.3097362518310547, "logits/rejected": -1.216961145401001, "logps/chosen": -300.4911193847656, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -450.92730712890625, "loss": 1.3413, "margin_dpo/margin_mean": 127.08175659179688, "margin_dpo/margin_std": 146.40628051757812, "step": 597 }, { "KL/chosen_KL_mean": -213.69586181640625, "KL/mean": -266.47564697265625, "KL/rejected_KL_mean": -319.25543212890625, "KL/std": 135.9591064453125, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.5595703125, "fcm_dpo/q_t": 0.4904016852378845, "grad_norm": 1.515350341796875, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.881174087524414, "logits/rejected": -1.5876262187957764, "logps/chosen": -303.87847900390625, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -414.05401611328125, "loss": 1.3491, "margin_dpo/margin_mean": 105.5595703125, "margin_dpo/margin_std": 150.4593048095703, "step": 598 }, { "KL/chosen_KL_mean": -210.4183807373047, "KL/mean": -258.1647033691406, "KL/rejected_KL_mean": -305.9110107421875, "KL/std": 138.53240966796875, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.49263763427734, "fcm_dpo/q_t": 0.4913164973258972, "grad_norm": 1.4090205430984497, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.6098897457122803, "logits/rejected": -1.508292555809021, "logps/chosen": -288.2606506347656, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -395.108642578125, "loss": 1.3526, "margin_dpo/margin_mean": 95.49263000488281, "margin_dpo/margin_std": 145.65313720703125, "step": 599 }, { "KL/chosen_KL_mean": -242.9329833984375, "KL/mean": -292.9346618652344, "KL/rejected_KL_mean": -342.93634033203125, "KL/std": 134.55630493164062, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.00334167480469, "fcm_dpo/q_t": 0.49091002345085144, "grad_norm": 1.1847206354141235, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.162911891937256, "logits/rejected": -1.8301777839660645, "logps/chosen": -335.12078857421875, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -435.890625, "loss": 1.351, "margin_dpo/margin_mean": 100.00334167480469, "margin_dpo/margin_std": 154.40420532226562, "step": 600 }, { "KL/chosen_KL_mean": -244.49351501464844, "KL/mean": -298.8528747558594, "KL/rejected_KL_mean": -353.21221923828125, "KL/std": 135.27601623535156, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.71871948242188, "fcm_dpo/q_t": 0.4901161789894104, "grad_norm": 1.4203492403030396, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.6374378204345703, "logits/rejected": -1.3311494588851929, "logps/chosen": -333.70965576171875, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -453.3827819824219, "loss": 1.3477, "margin_dpo/margin_mean": 108.71871948242188, "margin_dpo/margin_std": 135.02020263671875, "step": 601 }, { "KL/chosen_KL_mean": -195.9742889404297, "KL/mean": -262.2476806640625, "KL/rejected_KL_mean": -328.52105712890625, "KL/std": 141.33265686035156, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.5467529296875, "fcm_dpo/q_t": 0.48795539140701294, "grad_norm": 1.450485110282898, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.9536635875701904, "logits/rejected": -1.6854503154754639, "logps/chosen": -276.0318908691406, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -421.71856689453125, "loss": 1.3396, "margin_dpo/margin_mean": 132.5467529296875, "margin_dpo/margin_std": 166.76577758789062, "step": 602 }, { "KL/chosen_KL_mean": -207.1727294921875, "KL/mean": -264.6861877441406, "KL/rejected_KL_mean": -322.19964599609375, "KL/std": 128.03976440429688, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.02689361572266, "fcm_dpo/q_t": 0.48954468965530396, "grad_norm": 1.404901385307312, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.137207508087158, "logits/rejected": -1.601407527923584, "logps/chosen": -309.4822998046875, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -430.26849365234375, "loss": 1.3457, "margin_dpo/margin_mean": 115.02689361572266, "margin_dpo/margin_std": 160.87945556640625, "step": 603 }, { "KL/chosen_KL_mean": -212.21835327148438, "KL/mean": -273.8084716796875, "KL/rejected_KL_mean": -335.39862060546875, "KL/std": 139.64578247070312, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.18025970458984, "fcm_dpo/q_t": 0.4888021945953369, "grad_norm": 1.764539122581482, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.615762710571289, "logits/rejected": -1.4765417575836182, "logps/chosen": -290.3924560546875, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -434.89471435546875, "loss": 1.3428, "margin_dpo/margin_mean": 123.18025970458984, "margin_dpo/margin_std": 154.06007385253906, "step": 604 }, { "KL/chosen_KL_mean": -234.2135009765625, "KL/mean": -286.96966552734375, "KL/rejected_KL_mean": -339.7258605957031, "KL/std": 123.01370239257812, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.51237487792969, "fcm_dpo/q_t": 0.4904071092605591, "grad_norm": 1.693343997001648, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.7024314403533936, "logits/rejected": -1.23195481300354, "logps/chosen": -328.98681640625, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -447.03076171875, "loss": 1.3489, "margin_dpo/margin_mean": 105.51237487792969, "margin_dpo/margin_std": 134.88214111328125, "step": 605 }, { "KL/chosen_KL_mean": -226.82058715820312, "KL/mean": -279.2626953125, "KL/rejected_KL_mean": -331.70477294921875, "KL/std": 140.89700317382812, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.88418579101562, "fcm_dpo/q_t": 0.49046531319618225, "grad_norm": 1.5250120162963867, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.7871960401535034, "logits/rejected": -1.4706482887268066, "logps/chosen": -314.573974609375, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -435.9469909667969, "loss": 1.3493, "margin_dpo/margin_mean": 104.8841781616211, "margin_dpo/margin_std": 153.72372436523438, "step": 606 }, { "KL/chosen_KL_mean": -213.12637329101562, "KL/mean": -270.724609375, "KL/rejected_KL_mean": -328.3228759765625, "KL/std": 143.23129272460938, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.19647216796875, "fcm_dpo/q_t": 0.48953238129615784, "grad_norm": 1.7933545112609863, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.5177137851715088, "logits/rejected": -1.0120415687561035, "logps/chosen": -301.4554138183594, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -415.0909729003906, "loss": 1.346, "margin_dpo/margin_mean": 115.19647216796875, "margin_dpo/margin_std": 190.28561401367188, "step": 607 }, { "KL/chosen_KL_mean": -205.70248413085938, "KL/mean": -265.0831298828125, "KL/rejected_KL_mean": -324.46380615234375, "KL/std": 140.42660522460938, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.76131439208984, "fcm_dpo/q_t": 0.48920297622680664, "grad_norm": 1.0633465051651, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.973042607307434, "logits/rejected": -1.7091739177703857, "logps/chosen": -291.47186279296875, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -424.69659423828125, "loss": 1.3443, "margin_dpo/margin_mean": 118.76130676269531, "margin_dpo/margin_std": 148.90289306640625, "step": 608 }, { "KL/chosen_KL_mean": -228.84652709960938, "KL/mean": -280.2555847167969, "KL/rejected_KL_mean": -331.6646423339844, "KL/std": 141.60870361328125, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.81808471679688, "fcm_dpo/q_t": 0.4906551241874695, "grad_norm": 1.3710808753967285, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.0283102989196777, "logits/rejected": -1.628585934638977, "logps/chosen": -321.8131103515625, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -427.58282470703125, "loss": 1.3501, "margin_dpo/margin_mean": 102.81808471679688, "margin_dpo/margin_std": 158.6423797607422, "step": 609 }, { "KL/chosen_KL_mean": -208.54515075683594, "KL/mean": -268.61456298828125, "KL/rejected_KL_mean": -328.6839904785156, "KL/std": 139.17343139648438, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.13886260986328, "fcm_dpo/q_t": 0.48907679319381714, "grad_norm": 1.3466321229934692, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.791795253753662, "logits/rejected": -1.2901978492736816, "logps/chosen": -303.24542236328125, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -418.37139892578125, "loss": 1.3438, "margin_dpo/margin_mean": 120.13885498046875, "margin_dpo/margin_std": 151.05416870117188, "step": 610 }, { "KL/chosen_KL_mean": -219.42237854003906, "KL/mean": -280.22869873046875, "KL/rejected_KL_mean": -341.0350341796875, "KL/std": 139.6595916748047, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.61260986328125, "fcm_dpo/q_t": 0.488945871591568, "grad_norm": 1.773148536682129, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.0366454124450684, "logits/rejected": -1.7239127159118652, "logps/chosen": -306.2057189941406, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -430.8780822753906, "loss": 1.3433, "margin_dpo/margin_mean": 121.61260986328125, "margin_dpo/margin_std": 151.7537384033203, "step": 611 }, { "KL/chosen_KL_mean": -245.00921630859375, "KL/mean": -303.1357421875, "KL/rejected_KL_mean": -361.26226806640625, "KL/std": 138.34400939941406, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.2530517578125, "fcm_dpo/q_t": 0.4894331395626068, "grad_norm": 1.3645261526107788, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.9875160455703735, "logits/rejected": -1.187069296836853, "logps/chosen": -346.02935791015625, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -455.0452880859375, "loss": 1.3453, "margin_dpo/margin_mean": 116.2530517578125, "margin_dpo/margin_std": 159.74888610839844, "step": 612 }, { "KL/chosen_KL_mean": -195.66128540039062, "KL/mean": -267.02557373046875, "KL/rejected_KL_mean": -338.38983154296875, "KL/std": 135.2791748046875, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 142.7285614013672, "fcm_dpo/q_t": 0.4870239496231079, "grad_norm": 1.3145759105682373, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.7016212940216064, "logits/rejected": -1.5170788764953613, "logps/chosen": -284.64990234375, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -448.3853454589844, "loss": 1.3356, "margin_dpo/margin_mean": 142.72854614257812, "margin_dpo/margin_std": 128.56886291503906, "step": 613 }, { "KL/chosen_KL_mean": -232.323974609375, "KL/mean": -292.01220703125, "KL/rejected_KL_mean": -351.700439453125, "KL/std": 130.44183349609375, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.3764419555664, "fcm_dpo/q_t": 0.4891471266746521, "grad_norm": 1.5397840738296509, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.340468406677246, "logits/rejected": -1.352636694908142, "logps/chosen": -312.5240478515625, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -461.56280517578125, "loss": 1.3443, "margin_dpo/margin_mean": 119.37643432617188, "margin_dpo/margin_std": 168.7695770263672, "step": 614 }, { "KL/chosen_KL_mean": -226.08694458007812, "KL/mean": -268.90911865234375, "KL/rejected_KL_mean": -311.7312927246094, "KL/std": 132.42791748046875, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.64435577392578, "fcm_dpo/q_t": 0.4922129809856415, "grad_norm": 1.2678147554397583, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.5571417808532715, "logits/rejected": -1.1913896799087524, "logps/chosen": -326.522216796875, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -412.911376953125, "loss": 1.356, "margin_dpo/margin_mean": 85.64435577392578, "margin_dpo/margin_std": 136.1597900390625, "step": 615 }, { "KL/chosen_KL_mean": -234.41159057617188, "KL/mean": -289.3996276855469, "KL/rejected_KL_mean": -344.38763427734375, "KL/std": 129.77932739257812, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.97603607177734, "fcm_dpo/q_t": 0.490004301071167, "grad_norm": 1.4584693908691406, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.6534991264343262, "logits/rejected": -1.174759864807129, "logps/chosen": -326.904541015625, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -426.44830322265625, "loss": 1.3476, "margin_dpo/margin_mean": 109.97604370117188, "margin_dpo/margin_std": 166.7244873046875, "step": 616 }, { "KL/chosen_KL_mean": -238.14019775390625, "KL/mean": -279.19696044921875, "KL/rejected_KL_mean": -320.2537536621094, "KL/std": 123.48951721191406, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.11357116699219, "fcm_dpo/q_t": 0.4925338625907898, "grad_norm": 1.4027494192123413, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.5286438465118408, "logits/rejected": -0.9164419174194336, "logps/chosen": -329.2371826171875, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -390.663818359375, "loss": 1.3573, "margin_dpo/margin_mean": 82.11357879638672, "margin_dpo/margin_std": 140.49459838867188, "step": 617 }, { "KL/chosen_KL_mean": -240.85191345214844, "KL/mean": -295.05023193359375, "KL/rejected_KL_mean": -349.24847412109375, "KL/std": 144.12353515625, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.39657592773438, "fcm_dpo/q_t": 0.4901438355445862, "grad_norm": 1.2898865938186646, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.901164174079895, "logits/rejected": -1.5423259735107422, "logps/chosen": -342.87249755859375, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -449.0496826171875, "loss": 1.3481, "margin_dpo/margin_mean": 108.39657592773438, "margin_dpo/margin_std": 161.1641845703125, "step": 618 }, { "KL/chosen_KL_mean": -233.73138427734375, "KL/mean": -282.6120910644531, "KL/rejected_KL_mean": -331.4927978515625, "KL/std": 142.014892578125, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.76140594482422, "fcm_dpo/q_t": 0.49111735820770264, "grad_norm": 1.2545284032821655, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.247955560684204, "logits/rejected": -0.8983598351478577, "logps/chosen": -323.4727783203125, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -431.3941650390625, "loss": 1.352, "margin_dpo/margin_mean": 97.76140594482422, "margin_dpo/margin_std": 166.3153076171875, "step": 619 }, { "KL/chosen_KL_mean": -227.59776306152344, "KL/mean": -277.9017639160156, "KL/rejected_KL_mean": -328.20574951171875, "KL/std": 148.2431640625, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.60797119140625, "fcm_dpo/q_t": 0.49085533618927, "grad_norm": 1.2179439067840576, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.096388816833496, "logits/rejected": -1.5978420972824097, "logps/chosen": -312.7220764160156, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -413.6182861328125, "loss": 1.351, "margin_dpo/margin_mean": 100.60797119140625, "margin_dpo/margin_std": 164.88360595703125, "step": 620 }, { "KL/chosen_KL_mean": -216.21820068359375, "KL/mean": -273.700439453125, "KL/rejected_KL_mean": -331.18267822265625, "KL/std": 145.95465087890625, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.96449279785156, "fcm_dpo/q_t": 0.4895522892475128, "grad_norm": 1.5833624601364136, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.6870248317718506, "logits/rejected": -1.425366997718811, "logps/chosen": -305.46661376953125, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -426.6473388671875, "loss": 1.3461, "margin_dpo/margin_mean": 114.96449279785156, "margin_dpo/margin_std": 178.24256896972656, "step": 621 }, { "KL/chosen_KL_mean": -229.35679626464844, "KL/mean": -282.6197509765625, "KL/rejected_KL_mean": -335.8827209472656, "KL/std": 129.4134063720703, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.52593231201172, "fcm_dpo/q_t": 0.4903165102005005, "grad_norm": 1.752244234085083, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.1364340782165527, "logits/rejected": -0.4978974461555481, "logps/chosen": -329.15093994140625, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -426.7109375, "loss": 1.3486, "margin_dpo/margin_mean": 106.52593994140625, "margin_dpo/margin_std": 147.96026611328125, "step": 622 }, { "KL/chosen_KL_mean": -234.94317626953125, "KL/mean": -268.534912109375, "KL/rejected_KL_mean": -302.12664794921875, "KL/std": 146.97402954101562, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.1834716796875, "fcm_dpo/q_t": 0.4938926100730896, "grad_norm": 1.4696574211120605, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.156374931335449, "logits/rejected": -1.7532413005828857, "logps/chosen": -325.39874267578125, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -393.44940185546875, "loss": 1.3632, "margin_dpo/margin_mean": 67.1834716796875, "margin_dpo/margin_std": 184.85931396484375, "step": 623 }, { "KL/chosen_KL_mean": -248.8751678466797, "KL/mean": -310.89654541015625, "KL/rejected_KL_mean": -372.91790771484375, "KL/std": 140.95399475097656, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.04273223876953, "fcm_dpo/q_t": 0.488725483417511, "grad_norm": 1.6702913045883179, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.3571369647979736, "logits/rejected": -1.5065135955810547, "logps/chosen": -358.7503967285156, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -477.69110107421875, "loss": 1.3426, "margin_dpo/margin_mean": 124.04273223876953, "margin_dpo/margin_std": 165.765869140625, "step": 624 }, { "KL/chosen_KL_mean": -249.03677368164062, "KL/mean": -295.9581298828125, "KL/rejected_KL_mean": -342.8794860839844, "KL/std": 134.85549926757812, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.84271240234375, "fcm_dpo/q_t": 0.49146977066993713, "grad_norm": 1.5781930685043335, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.5395351648330688, "logits/rejected": -1.1825315952301025, "logps/chosen": -336.2049560546875, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -434.740966796875, "loss": 1.3532, "margin_dpo/margin_mean": 93.84271240234375, "margin_dpo/margin_std": 150.32525634765625, "step": 625 }, { "KL/chosen_KL_mean": -239.57720947265625, "KL/mean": -282.11505126953125, "KL/rejected_KL_mean": -324.65289306640625, "KL/std": 149.40951538085938, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.07569122314453, "fcm_dpo/q_t": 0.4922667145729065, "grad_norm": 1.4581763744354248, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.4075006246566772, "logits/rejected": -1.0036838054656982, "logps/chosen": -343.8014221191406, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -428.83038330078125, "loss": 1.3565, "margin_dpo/margin_mean": 85.07568359375, "margin_dpo/margin_std": 169.05987548828125, "step": 626 }, { "KL/chosen_KL_mean": -199.0731201171875, "KL/mean": -265.67071533203125, "KL/rejected_KL_mean": -332.26837158203125, "KL/std": 155.18414306640625, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.19522094726562, "fcm_dpo/q_t": 0.48790037631988525, "grad_norm": 1.1659889221191406, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.4735091924667358, "logits/rejected": -1.6355674266815186, "logps/chosen": -274.00775146484375, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -444.84124755859375, "loss": 1.3396, "margin_dpo/margin_mean": 133.19522094726562, "margin_dpo/margin_std": 186.50167846679688, "step": 627 }, { "KL/chosen_KL_mean": -230.17123413085938, "KL/mean": -291.81317138671875, "KL/rejected_KL_mean": -353.4551086425781, "KL/std": 148.59918212890625, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.28387451171875, "fcm_dpo/q_t": 0.48879700899124146, "grad_norm": 1.66887366771698, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.8854752779006958, "logits/rejected": -1.6521716117858887, "logps/chosen": -323.8623046875, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -460.799072265625, "loss": 1.3429, "margin_dpo/margin_mean": 123.28387451171875, "margin_dpo/margin_std": 171.766845703125, "step": 628 }, { "KL/chosen_KL_mean": -244.23976135253906, "KL/mean": -298.1212158203125, "KL/rejected_KL_mean": -352.0027160644531, "KL/std": 142.1669921875, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.76295471191406, "fcm_dpo/q_t": 0.4902014136314392, "grad_norm": 1.4489983320236206, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.8941720724105835, "logits/rejected": -1.4594390392303467, "logps/chosen": -347.47015380859375, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -449.171142578125, "loss": 1.3483, "margin_dpo/margin_mean": 107.76295471191406, "margin_dpo/margin_std": 152.7852783203125, "step": 629 }, { "KL/chosen_KL_mean": -225.06729125976562, "KL/mean": -249.10043334960938, "KL/rejected_KL_mean": -273.13360595703125, "KL/std": 131.01629638671875, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.066314697265625, "fcm_dpo/q_t": 0.4956273138523102, "grad_norm": 1.4192891120910645, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.0261151790618896, "logits/rejected": -1.566742181777954, "logps/chosen": -318.96484375, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -362.50799560546875, "loss": 1.3696, "margin_dpo/margin_mean": 48.066314697265625, "margin_dpo/margin_std": 150.2725372314453, "step": 630 }, { "KL/chosen_KL_mean": -205.35214233398438, "KL/mean": -262.2412109375, "KL/rejected_KL_mean": -319.13031005859375, "KL/std": 131.1405029296875, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.7781982421875, "fcm_dpo/q_t": 0.48965686559677124, "grad_norm": 1.4673633575439453, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.3659682273864746, "logits/rejected": -1.9587020874023438, "logps/chosen": -293.1229553222656, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -407.812744140625, "loss": 1.3461, "margin_dpo/margin_mean": 113.7781982421875, "margin_dpo/margin_std": 155.79501342773438, "step": 631 }, { "KL/chosen_KL_mean": -209.44259643554688, "KL/mean": -264.0088806152344, "KL/rejected_KL_mean": -318.5751953125, "KL/std": 134.52273559570312, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.132568359375, "fcm_dpo/q_t": 0.4900820851325989, "grad_norm": 1.8181419372558594, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.8664073944091797, "logits/rejected": -1.447324514389038, "logps/chosen": -298.0691223144531, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -410.0260925292969, "loss": 1.3481, "margin_dpo/margin_mean": 109.13257598876953, "margin_dpo/margin_std": 180.57809448242188, "step": 632 }, { "KL/chosen_KL_mean": -253.2836456298828, "KL/mean": -285.93341064453125, "KL/rejected_KL_mean": -318.5831604003906, "KL/std": 137.2220916748047, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.29949951171875, "fcm_dpo/q_t": 0.49406859278678894, "grad_norm": 1.2742242813110352, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.0824685096740723, "logits/rejected": -1.9164131879806519, "logps/chosen": -334.6580810546875, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -417.2088623046875, "loss": 1.3639, "margin_dpo/margin_mean": 65.29949951171875, "margin_dpo/margin_std": 187.76548767089844, "step": 633 }, { "KL/chosen_KL_mean": -233.90390014648438, "KL/mean": -281.5899658203125, "KL/rejected_KL_mean": -329.2760314941406, "KL/std": 122.26199340820312, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.37210845947266, "fcm_dpo/q_t": 0.49132853746414185, "grad_norm": 1.466221809387207, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.2663811445236206, "logits/rejected": -1.1162302494049072, "logps/chosen": -330.0267333984375, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -442.12384033203125, "loss": 1.3525, "margin_dpo/margin_mean": 95.37210845947266, "margin_dpo/margin_std": 128.84921264648438, "step": 634 }, { "KL/chosen_KL_mean": -255.53207397460938, "KL/mean": -281.13580322265625, "KL/rejected_KL_mean": -306.73956298828125, "KL/std": 145.5662841796875, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.207435607910156, "fcm_dpo/q_t": 0.49534744024276733, "grad_norm": 1.425372838973999, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.6933493614196777, "logits/rejected": -1.1568870544433594, "logps/chosen": -354.2161865234375, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -396.63873291015625, "loss": 1.3688, "margin_dpo/margin_mean": 51.20743179321289, "margin_dpo/margin_std": 177.87088012695312, "step": 635 }, { "KL/chosen_KL_mean": -212.37069702148438, "KL/mean": -272.450439453125, "KL/rejected_KL_mean": -332.5301513671875, "KL/std": 135.81869506835938, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.15943908691406, "fcm_dpo/q_t": 0.489075243473053, "grad_norm": 1.2453855276107788, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.6245635747909546, "logits/rejected": -1.1739835739135742, "logps/chosen": -302.78863525390625, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -420.2370300292969, "loss": 1.3436, "margin_dpo/margin_mean": 120.15943908691406, "margin_dpo/margin_std": 133.84805297851562, "step": 636 }, { "KL/chosen_KL_mean": -215.70919799804688, "KL/mean": -281.42108154296875, "KL/rejected_KL_mean": -347.1329345703125, "KL/std": 152.3313446044922, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.4237060546875, "fcm_dpo/q_t": 0.48805201053619385, "grad_norm": 1.2991691827774048, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.7452445030212402, "logits/rejected": -1.6465566158294678, "logps/chosen": -298.158935546875, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -451.1615295410156, "loss": 1.3397, "margin_dpo/margin_mean": 131.4237060546875, "margin_dpo/margin_std": 140.70486450195312, "step": 637 }, { "KL/chosen_KL_mean": -216.25918579101562, "KL/mean": -268.8521728515625, "KL/rejected_KL_mean": -321.4451904296875, "KL/std": 138.57391357421875, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.18601989746094, "fcm_dpo/q_t": 0.4904358983039856, "grad_norm": 1.2507277727127075, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.375159740447998, "logits/rejected": -0.7276169061660767, "logps/chosen": -308.18414306640625, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -407.73223876953125, "loss": 1.3492, "margin_dpo/margin_mean": 105.18601989746094, "margin_dpo/margin_std": 155.5662841796875, "step": 638 }, { "KL/chosen_KL_mean": -276.25811767578125, "KL/mean": -296.6956787109375, "KL/rejected_KL_mean": -317.1332092285156, "KL/std": 125.15509033203125, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 40.87510299682617, "fcm_dpo/q_t": 0.4962824285030365, "grad_norm": 1.3309545516967773, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.2663021087646484, "logits/rejected": -0.5249555706977844, "logps/chosen": -380.78564453125, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -398.6135559082031, "loss": 1.3724, "margin_dpo/margin_mean": 40.87510681152344, "margin_dpo/margin_std": 161.4024200439453, "step": 639 }, { "KL/chosen_KL_mean": -229.2627410888672, "KL/mean": -289.701171875, "KL/rejected_KL_mean": -350.1396484375, "KL/std": 146.80062866210938, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.87687683105469, "fcm_dpo/q_t": 0.48901376128196716, "grad_norm": 1.6588349342346191, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.7149131298065186, "logits/rejected": -1.2239015102386475, "logps/chosen": -327.14801025390625, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -462.84466552734375, "loss": 1.3437, "margin_dpo/margin_mean": 120.87687683105469, "margin_dpo/margin_std": 158.21798706054688, "step": 640 }, { "KL/chosen_KL_mean": -216.69053649902344, "KL/mean": -272.09417724609375, "KL/rejected_KL_mean": -327.49786376953125, "KL/std": 162.19119262695312, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.80733489990234, "fcm_dpo/q_t": 0.48994481563568115, "grad_norm": 1.2550052404403687, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.201937675476074, "logits/rejected": -1.6700432300567627, "logps/chosen": -313.13623046875, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -422.633544921875, "loss": 1.3476, "margin_dpo/margin_mean": 110.80732727050781, "margin_dpo/margin_std": 192.61529541015625, "step": 641 }, { "KL/chosen_KL_mean": -243.242431640625, "KL/mean": -295.277099609375, "KL/rejected_KL_mean": -347.3117980957031, "KL/std": 144.02980041503906, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.0693588256836, "fcm_dpo/q_t": 0.4905507564544678, "grad_norm": 1.3675718307495117, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.520108699798584, "logits/rejected": -1.2000648975372314, "logps/chosen": -344.0022888183594, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -462.0194396972656, "loss": 1.3501, "margin_dpo/margin_mean": 104.06935119628906, "margin_dpo/margin_std": 192.39376831054688, "step": 642 }, { "KL/chosen_KL_mean": -225.69921875, "KL/mean": -275.3413391113281, "KL/rejected_KL_mean": -324.98345947265625, "KL/std": 136.58155822753906, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.28419494628906, "fcm_dpo/q_t": 0.4909749925136566, "grad_norm": 1.526860237121582, "learning_rate": 4.993270631642038e-09, "logits/chosen": -1.9938781261444092, "logits/rejected": -1.6580390930175781, "logps/chosen": -310.44287109375, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -419.3018798828125, "loss": 1.3514, "margin_dpo/margin_mean": 99.28419494628906, "margin_dpo/margin_std": 159.60052490234375, "step": 643 }, { "KL/chosen_KL_mean": -233.57669067382812, "KL/mean": -271.7669372558594, "KL/rejected_KL_mean": -309.95721435546875, "KL/std": 137.5450439453125, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.38053131103516, "fcm_dpo/q_t": 0.4930558502674103, "grad_norm": 1.692925214767456, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.8165006637573242, "logits/rejected": -1.5859838724136353, "logps/chosen": -318.8994140625, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -404.5658264160156, "loss": 1.3595, "margin_dpo/margin_mean": 76.38053131103516, "margin_dpo/margin_std": 155.11380004882812, "step": 644 }, { "KL/chosen_KL_mean": -245.29324340820312, "KL/mean": -270.58026123046875, "KL/rejected_KL_mean": -295.86724853515625, "KL/std": 140.84878540039062, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.574005126953125, "fcm_dpo/q_t": 0.495402455329895, "grad_norm": 1.3473420143127441, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.4048900604248047, "logits/rejected": -1.2763041257858276, "logps/chosen": -327.88348388671875, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -398.87103271484375, "loss": 1.3689, "margin_dpo/margin_mean": 50.57400894165039, "margin_dpo/margin_std": 165.8172607421875, "step": 645 }, { "KL/chosen_KL_mean": -219.01678466796875, "KL/mean": -270.1850280761719, "KL/rejected_KL_mean": -321.3533020019531, "KL/std": 125.06463623046875, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.33649444580078, "fcm_dpo/q_t": 0.490694135427475, "grad_norm": 1.0723785161972046, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.5940406322479248, "logits/rejected": -1.0231525897979736, "logps/chosen": -310.20880126953125, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -405.1316223144531, "loss": 1.3499, "margin_dpo/margin_mean": 102.33650207519531, "margin_dpo/margin_std": 122.92570495605469, "step": 646 }, { "KL/chosen_KL_mean": -225.8936309814453, "KL/mean": -273.8428955078125, "KL/rejected_KL_mean": -321.79217529296875, "KL/std": 144.59640502929688, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.89855194091797, "fcm_dpo/q_t": 0.49128299951553345, "grad_norm": 1.2274832725524902, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.1849069595336914, "logits/rejected": -1.0571948289871216, "logps/chosen": -314.73809814453125, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -421.2904968261719, "loss": 1.3526, "margin_dpo/margin_mean": 95.89854431152344, "margin_dpo/margin_std": 163.76416015625, "step": 647 }, { "KL/chosen_KL_mean": -217.9874267578125, "KL/mean": -285.9239501953125, "KL/rejected_KL_mean": -353.8604736328125, "KL/std": 152.80508422851562, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.87306213378906, "fcm_dpo/q_t": 0.4876495897769928, "grad_norm": 1.33875572681427, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.8676542043685913, "logits/rejected": -1.8008723258972168, "logps/chosen": -305.83551025390625, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -458.5305480957031, "loss": 1.3384, "margin_dpo/margin_mean": 135.87306213378906, "margin_dpo/margin_std": 167.4234619140625, "step": 648 }, { "KL/chosen_KL_mean": -207.22348022460938, "KL/mean": -258.79547119140625, "KL/rejected_KL_mean": -310.36749267578125, "KL/std": 134.2249755859375, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.14404296875, "fcm_dpo/q_t": 0.4906225800514221, "grad_norm": 2.1754472255706787, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.1229329109191895, "logits/rejected": -1.7200000286102295, "logps/chosen": -296.9159851074219, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -399.0740966796875, "loss": 1.3497, "margin_dpo/margin_mean": 103.14403533935547, "margin_dpo/margin_std": 136.93667602539062, "step": 649 }, { "KL/chosen_KL_mean": -243.7003173828125, "KL/mean": -298.59246826171875, "KL/rejected_KL_mean": -353.48455810546875, "KL/std": 155.85052490234375, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.78427124023438, "fcm_dpo/q_t": 0.4900275468826294, "grad_norm": 1.4109262228012085, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.1995060443878174, "logits/rejected": -0.9953488111495972, "logps/chosen": -333.63092041015625, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -456.0973815917969, "loss": 1.3481, "margin_dpo/margin_mean": 109.78427124023438, "margin_dpo/margin_std": 202.22552490234375, "step": 650 }, { "KL/chosen_KL_mean": -194.81790161132812, "KL/mean": -266.25775146484375, "KL/rejected_KL_mean": -337.6976318359375, "KL/std": 147.70590209960938, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 142.8797149658203, "fcm_dpo/q_t": 0.48701250553131104, "grad_norm": 1.992139220237732, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.053314447402954, "logits/rejected": -1.8810372352600098, "logps/chosen": -274.0052490234375, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -443.6309814453125, "loss": 1.3357, "margin_dpo/margin_mean": 142.8797149658203, "margin_dpo/margin_std": 147.1202392578125, "step": 651 }, { "KL/chosen_KL_mean": -213.09654235839844, "KL/mean": -264.321533203125, "KL/rejected_KL_mean": -315.5465087890625, "KL/std": 129.14602661132812, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.4499740600586, "fcm_dpo/q_t": 0.4906855523586273, "grad_norm": 1.1549783945083618, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.502163887023926, "logits/rejected": -2.255155086517334, "logps/chosen": -314.88677978515625, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -431.87109375, "loss": 1.3499, "margin_dpo/margin_mean": 102.44998168945312, "margin_dpo/margin_std": 130.84469604492188, "step": 652 }, { "KL/chosen_KL_mean": -240.38258361816406, "KL/mean": -294.7884521484375, "KL/rejected_KL_mean": -349.19427490234375, "KL/std": 145.90829467773438, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.81169128417969, "fcm_dpo/q_t": 0.4901064932346344, "grad_norm": 1.3277246952056885, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.125239372253418, "logits/rejected": -1.8768677711486816, "logps/chosen": -333.7361755371094, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -458.3175048828125, "loss": 1.3479, "margin_dpo/margin_mean": 108.81169128417969, "margin_dpo/margin_std": 152.5767822265625, "step": 653 }, { "KL/chosen_KL_mean": -229.85910034179688, "KL/mean": -277.35223388671875, "KL/rejected_KL_mean": -324.84539794921875, "KL/std": 137.8497772216797, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.98631286621094, "fcm_dpo/q_t": 0.491370290517807, "grad_norm": 1.5333549976348877, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.18053936958313, "logits/rejected": -2.014281988143921, "logps/chosen": -318.9746398925781, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -429.7653503417969, "loss": 1.353, "margin_dpo/margin_mean": 94.9863052368164, "margin_dpo/margin_std": 167.09979248046875, "step": 654 }, { "KL/chosen_KL_mean": -215.47061157226562, "KL/mean": -271.3269958496094, "KL/rejected_KL_mean": -327.1833801269531, "KL/std": 152.38134765625, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.71275329589844, "fcm_dpo/q_t": 0.4898463487625122, "grad_norm": 1.58479905128479, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.8100078105926514, "logits/rejected": -1.829290747642517, "logps/chosen": -296.5067138671875, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -429.9857177734375, "loss": 1.347, "margin_dpo/margin_mean": 111.71275329589844, "margin_dpo/margin_std": 167.95777893066406, "step": 655 }, { "KL/chosen_KL_mean": -228.39154052734375, "KL/mean": -295.823486328125, "KL/rejected_KL_mean": -363.25543212890625, "KL/std": 152.21609497070312, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.86386108398438, "fcm_dpo/q_t": 0.48774343729019165, "grad_norm": 1.5740361213684082, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.3829023838043213, "logits/rejected": -1.4543681144714355, "logps/chosen": -313.7168884277344, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -481.5940856933594, "loss": 1.3387, "margin_dpo/margin_mean": 134.86387634277344, "margin_dpo/margin_std": 159.04019165039062, "step": 656 }, { "KL/chosen_KL_mean": -238.7667999267578, "KL/mean": -296.2598876953125, "KL/rejected_KL_mean": -353.7529296875, "KL/std": 146.40243530273438, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.98617553710938, "fcm_dpo/q_t": 0.48955363035202026, "grad_norm": 1.4652671813964844, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.8721195459365845, "logits/rejected": -1.7055325508117676, "logps/chosen": -318.96453857421875, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -456.3342590332031, "loss": 1.346, "margin_dpo/margin_mean": 114.9861831665039, "margin_dpo/margin_std": 178.43234252929688, "step": 657 }, { "KL/chosen_KL_mean": -244.04071044921875, "KL/mean": -285.47021484375, "KL/rejected_KL_mean": -326.899658203125, "KL/std": 129.0428009033203, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.85896301269531, "fcm_dpo/q_t": 0.4924665689468384, "grad_norm": 1.3818362951278687, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.9651684761047363, "logits/rejected": -1.530510663986206, "logps/chosen": -338.67523193359375, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -428.5312805175781, "loss": 1.357, "margin_dpo/margin_mean": 82.85896301269531, "margin_dpo/margin_std": 137.40762329101562, "step": 658 }, { "KL/chosen_KL_mean": -222.9498291015625, "KL/mean": -275.7210693359375, "KL/rejected_KL_mean": -328.4923095703125, "KL/std": 134.9240264892578, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.54249572753906, "fcm_dpo/q_t": 0.49040722846984863, "grad_norm": 1.192875623703003, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.13419771194458, "logits/rejected": -1.2147327661514282, "logps/chosen": -335.50567626953125, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -422.4139404296875, "loss": 1.3492, "margin_dpo/margin_mean": 105.54249572753906, "margin_dpo/margin_std": 166.92892456054688, "step": 659 }, { "KL/chosen_KL_mean": -224.22293090820312, "KL/mean": -282.5438232421875, "KL/rejected_KL_mean": -340.86474609375, "KL/std": 131.55426025390625, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.64180755615234, "fcm_dpo/q_t": 0.48939651250839233, "grad_norm": 1.4155142307281494, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.7241791486740112, "logits/rejected": -1.1189115047454834, "logps/chosen": -317.600341796875, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -422.2595520019531, "loss": 1.345, "margin_dpo/margin_mean": 116.64181518554688, "margin_dpo/margin_std": 141.79949951171875, "step": 660 }, { "KL/chosen_KL_mean": -219.68325805664062, "KL/mean": -285.6512451171875, "KL/rejected_KL_mean": -351.61920166015625, "KL/std": 164.92117309570312, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.93594360351562, "fcm_dpo/q_t": 0.4880141615867615, "grad_norm": 1.2806204557418823, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.731147289276123, "logits/rejected": -1.2433744668960571, "logps/chosen": -307.5384216308594, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -444.02252197265625, "loss": 1.3401, "margin_dpo/margin_mean": 131.93594360351562, "margin_dpo/margin_std": 185.71823120117188, "step": 661 }, { "KL/chosen_KL_mean": -226.00527954101562, "KL/mean": -280.9569396972656, "KL/rejected_KL_mean": -335.90863037109375, "KL/std": 129.31370544433594, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.90336608886719, "fcm_dpo/q_t": 0.4900086224079132, "grad_norm": 1.4917068481445312, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.696298360824585, "logits/rejected": -1.1456780433654785, "logps/chosen": -328.7850646972656, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -431.1339416503906, "loss": 1.3475, "margin_dpo/margin_mean": 109.90336608886719, "margin_dpo/margin_std": 156.3925018310547, "step": 662 }, { "KL/chosen_KL_mean": -213.5937042236328, "KL/mean": -275.109130859375, "KL/rejected_KL_mean": -336.6246032714844, "KL/std": 148.34017944335938, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.03091430664062, "fcm_dpo/q_t": 0.4888218939304352, "grad_norm": 1.9462416172027588, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.084744453430176, "logits/rejected": -1.8936957120895386, "logps/chosen": -309.9403076171875, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -457.15106201171875, "loss": 1.3431, "margin_dpo/margin_mean": 123.0309066772461, "margin_dpo/margin_std": 177.39727783203125, "step": 663 }, { "KL/chosen_KL_mean": -243.3436279296875, "KL/mean": -296.7142333984375, "KL/rejected_KL_mean": -350.0848083496094, "KL/std": 150.9837646484375, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.74120330810547, "fcm_dpo/q_t": 0.49030470848083496, "grad_norm": 1.5346823930740356, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.5729939937591553, "logits/rejected": -1.4265449047088623, "logps/chosen": -335.18603515625, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -461.9215087890625, "loss": 1.3488, "margin_dpo/margin_mean": 106.7412109375, "margin_dpo/margin_std": 172.2030792236328, "step": 664 }, { "KL/chosen_KL_mean": -206.29421997070312, "KL/mean": -258.61627197265625, "KL/rejected_KL_mean": -310.9383544921875, "KL/std": 131.9756622314453, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.6441650390625, "fcm_dpo/q_t": 0.4904859960079193, "grad_norm": 1.192274570465088, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.2719807624816895, "logits/rejected": -1.8646246194839478, "logps/chosen": -294.48040771484375, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -402.8503723144531, "loss": 1.3492, "margin_dpo/margin_mean": 104.6441650390625, "margin_dpo/margin_std": 135.92141723632812, "step": 665 }, { "KL/chosen_KL_mean": -229.60818481445312, "KL/mean": -283.57183837890625, "KL/rejected_KL_mean": -337.53546142578125, "KL/std": 134.85519409179688, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.92725372314453, "fcm_dpo/q_t": 0.4901903569698334, "grad_norm": 1.3281044960021973, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.524195671081543, "logits/rejected": -1.1464462280273438, "logps/chosen": -330.582763671875, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -438.7853698730469, "loss": 1.3482, "margin_dpo/margin_mean": 107.92725372314453, "margin_dpo/margin_std": 154.51214599609375, "step": 666 }, { "KL/chosen_KL_mean": -254.58877563476562, "KL/mean": -305.22967529296875, "KL/rejected_KL_mean": -355.8705749511719, "KL/std": 136.72254943847656, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.28181457519531, "fcm_dpo/q_t": 0.4907909333705902, "grad_norm": 1.144588828086853, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.3386204242706299, "logits/rejected": -0.7127670049667358, "logps/chosen": -352.1599426269531, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -449.455322265625, "loss": 1.3504, "margin_dpo/margin_mean": 101.28181457519531, "margin_dpo/margin_std": 140.80117797851562, "step": 667 }, { "KL/chosen_KL_mean": -225.80313110351562, "KL/mean": -290.3489990234375, "KL/rejected_KL_mean": -354.8948669433594, "KL/std": 144.80484008789062, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.09173583984375, "fcm_dpo/q_t": 0.4882687032222748, "grad_norm": 1.273056983947754, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.0590052604675293, "logits/rejected": -1.5532677173614502, "logps/chosen": -328.378173828125, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -463.7125549316406, "loss": 1.3408, "margin_dpo/margin_mean": 129.09173583984375, "margin_dpo/margin_std": 169.97396850585938, "step": 668 }, { "KL/chosen_KL_mean": -201.46636962890625, "KL/mean": -262.97955322265625, "KL/rejected_KL_mean": -324.4927978515625, "KL/std": 140.24380493164062, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.02641296386719, "fcm_dpo/q_t": 0.48881715536117554, "grad_norm": 1.6806139945983887, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.6980552673339844, "logits/rejected": -1.3369419574737549, "logps/chosen": -290.96112060546875, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -407.01226806640625, "loss": 1.3428, "margin_dpo/margin_mean": 123.02641296386719, "margin_dpo/margin_std": 152.68316650390625, "step": 669 }, { "KL/chosen_KL_mean": -214.75491333007812, "KL/mean": -281.1017761230469, "KL/rejected_KL_mean": -347.4486389160156, "KL/std": 152.75840759277344, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.69371032714844, "fcm_dpo/q_t": 0.48793697357177734, "grad_norm": 1.3497326374053955, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.6722712516784668, "logits/rejected": -1.1674315929412842, "logps/chosen": -310.20953369140625, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -448.9815673828125, "loss": 1.3394, "margin_dpo/margin_mean": 132.69371032714844, "margin_dpo/margin_std": 158.087646484375, "step": 670 }, { "KL/chosen_KL_mean": -242.2982635498047, "KL/mean": -288.9319763183594, "KL/rejected_KL_mean": -335.5657043457031, "KL/std": 139.82489013671875, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.26744842529297, "fcm_dpo/q_t": 0.49152153730392456, "grad_norm": 1.3667782545089722, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6273458003997803, "logits/rejected": -1.4211280345916748, "logps/chosen": -324.42138671875, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -425.785400390625, "loss": 1.3536, "margin_dpo/margin_mean": 93.26744842529297, "margin_dpo/margin_std": 169.07919311523438, "step": 671 }, { "KL/chosen_KL_mean": -248.28729248046875, "KL/mean": -287.35345458984375, "KL/rejected_KL_mean": -326.4196472167969, "KL/std": 129.1230926513672, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.13236999511719, "fcm_dpo/q_t": 0.492897629737854, "grad_norm": 2.0968644618988037, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.9617589712142944, "logits/rejected": -1.6762751340866089, "logps/chosen": -338.3492126464844, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -426.8728942871094, "loss": 1.3589, "margin_dpo/margin_mean": 78.13236999511719, "margin_dpo/margin_std": 148.447998046875, "step": 672 }, { "KL/chosen_KL_mean": -222.6824951171875, "KL/mean": -266.8341369628906, "KL/rejected_KL_mean": -310.98577880859375, "KL/std": 132.83645629882812, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.30327606201172, "fcm_dpo/q_t": 0.49197280406951904, "grad_norm": 1.3028767108917236, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.8254225254058838, "logits/rejected": -1.4891750812530518, "logps/chosen": -301.9456481933594, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -402.33502197265625, "loss": 1.3551, "margin_dpo/margin_mean": 88.30327606201172, "margin_dpo/margin_std": 140.77066040039062, "step": 673 }, { "KL/chosen_KL_mean": -194.15835571289062, "KL/mean": -262.90081787109375, "KL/rejected_KL_mean": -331.6432800292969, "KL/std": 132.58355712890625, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.48492431640625, "fcm_dpo/q_t": 0.48750340938568115, "grad_norm": 1.5625756978988647, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.4738422632217407, "logits/rejected": -1.0830705165863037, "logps/chosen": -269.6166687011719, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -407.846923828125, "loss": 1.3376, "margin_dpo/margin_mean": 137.48492431640625, "margin_dpo/margin_std": 144.63613891601562, "step": 674 }, { "KL/chosen_KL_mean": -225.7226104736328, "KL/mean": -282.5733642578125, "KL/rejected_KL_mean": -339.42413330078125, "KL/std": 141.37728881835938, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.70149230957031, "fcm_dpo/q_t": 0.4896652400493622, "grad_norm": 1.2629005908966064, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.3649225234985352, "logits/rejected": -1.026859998703003, "logps/chosen": -305.63214111328125, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -420.6423645019531, "loss": 1.3464, "margin_dpo/margin_mean": 113.70149993896484, "margin_dpo/margin_std": 179.7775421142578, "step": 675 }, { "KL/chosen_KL_mean": -236.42254638671875, "KL/mean": -293.7616271972656, "KL/rejected_KL_mean": -351.1007080078125, "KL/std": 137.4601287841797, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.67816162109375, "fcm_dpo/q_t": 0.4895763397216797, "grad_norm": 1.1398276090621948, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.860346794128418, "logits/rejected": -1.2837889194488525, "logps/chosen": -334.5936584472656, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -446.1253662109375, "loss": 1.3459, "margin_dpo/margin_mean": 114.67816162109375, "margin_dpo/margin_std": 166.82305908203125, "step": 676 }, { "KL/chosen_KL_mean": -226.10263061523438, "KL/mean": -275.0416564941406, "KL/rejected_KL_mean": -323.98065185546875, "KL/std": 136.26846313476562, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.8780746459961, "fcm_dpo/q_t": 0.4911019802093506, "grad_norm": 1.2947571277618408, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.5773019790649414, "logits/rejected": -0.9424566626548767, "logps/chosen": -317.4819030761719, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -406.85845947265625, "loss": 1.3517, "margin_dpo/margin_mean": 97.8780746459961, "margin_dpo/margin_std": 144.3620147705078, "step": 677 }, { "KL/chosen_KL_mean": -200.00686645507812, "KL/mean": -268.3798828125, "KL/rejected_KL_mean": -336.7529602050781, "KL/std": 146.05279541015625, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 136.74609375, "fcm_dpo/q_t": 0.4875701665878296, "grad_norm": 1.2188920974731445, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.5671298503875732, "logits/rejected": -1.2588754892349243, "logps/chosen": -275.6527099609375, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -423.71905517578125, "loss": 1.3379, "margin_dpo/margin_mean": 136.74610900878906, "margin_dpo/margin_std": 148.8485107421875, "step": 678 }, { "KL/chosen_KL_mean": -234.0555419921875, "KL/mean": -287.52423095703125, "KL/rejected_KL_mean": -340.99285888671875, "KL/std": 119.00814056396484, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.93734741210938, "fcm_dpo/q_t": 0.49027904868125916, "grad_norm": 1.1385034322738647, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.251692295074463, "logits/rejected": -1.770925760269165, "logps/chosen": -314.8289794921875, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -423.87139892578125, "loss": 1.3484, "margin_dpo/margin_mean": 106.93734741210938, "margin_dpo/margin_std": 141.4192657470703, "step": 679 }, { "KL/chosen_KL_mean": -242.19873046875, "KL/mean": -304.9227294921875, "KL/rejected_KL_mean": -367.646728515625, "KL/std": 151.58428955078125, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.447998046875, "fcm_dpo/q_t": 0.48859965801239014, "grad_norm": 1.4815016984939575, "learning_rate": 1.31753782067201e-11, "logits/chosen": -2.033252477645874, "logits/rejected": -1.6960684061050415, "logps/chosen": -349.88165283203125, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -483.7416076660156, "loss": 1.3421, "margin_dpo/margin_mean": 125.44800567626953, "margin_dpo/margin_std": 168.80014038085938, "step": 680 }, { "KL/chosen_KL_mean": -218.32070922851562, "KL/mean": -272.51141357421875, "KL/rejected_KL_mean": -326.7021484375, "KL/std": 139.86032104492188, "epoch": 1.0, "fcm_dpo/beta": 0.0003639644419308752, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.38145446777344, "fcm_dpo/q_t": 0.49015066027641296, "grad_norm": 1.3287807703018188, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.373037576675415, "logits/rejected": -2.086273431777954, "logps/chosen": -311.331787109375, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -421.5243225097656, "loss": 1.3482, "margin_dpo/margin_mean": 108.38145446777344, "margin_dpo/margin_std": 165.59652709960938, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.346229954613113, "train_runtime": 1970.8718, "train_samples_per_second": 22.121, "train_steps_per_second": 0.346 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }