{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.821475982666016, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.757366180419922, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.05213165283203125, "KL/mean": 0.02123851329088211, "KL/rejected_KL_mean": -0.009647369384765625, "KL/std": 0.30469638109207153, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.061782434582710266, "fcm_dpo/q_t": 0.4984560012817383, "grad_norm": 25.964136123657227, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42700856924057007, "logits/rejected": 0.6502017974853516, "logps/chosen": -93.75885009765625, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.23915100097656, "loss": 1.3807, "margin_dpo/margin_mean": 0.06178255379199982, "margin_dpo/margin_std": 0.4693487286567688, "step": 3 }, { "KL/chosen_KL_mean": 0.008319854736328125, "KL/mean": -0.0022910013794898987, "KL/rejected_KL_mean": -0.012905120849609375, "KL/std": 0.33126771450042725, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.021229371428489685, "fcm_dpo/q_t": 0.4994674623012543, "grad_norm": 32.80839920043945, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6482337117195129, "logits/rejected": 0.7797278761863708, "logps/chosen": -87.28414916992188, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.837158203125, "loss": 1.3847, "margin_dpo/margin_mean": 0.021229222416877747, "margin_dpo/margin_std": 0.4484151601791382, "step": 4 }, { "KL/chosen_KL_mean": -0.039531707763671875, "KL/mean": -0.027082368731498718, "KL/rejected_KL_mean": -0.014636993408203125, "KL/std": 0.2979029715061188, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02489754557609558, "fcm_dpo/q_t": 0.5006207227706909, "grad_norm": 35.387393951416016, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4465378522872925, "logits/rejected": 0.6580582857131958, "logps/chosen": -89.37628936767578, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.76246643066406, "loss": 1.3892, "margin_dpo/margin_mean": -0.02489730715751648, "margin_dpo/margin_std": 0.4088785648345947, "step": 5 }, { "KL/chosen_KL_mean": 0.003940582275390625, "KL/mean": -0.003724142909049988, "KL/rejected_KL_mean": -0.01139068603515625, "KL/std": 0.292846143245697, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0153256356716156, "fcm_dpo/q_t": 0.49961599707603455, "grad_norm": 36.03197479248047, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.7300682067871094, "logits/rejected": 1.0666272640228271, "logps/chosen": -97.31753540039062, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.89483642578125, "loss": 1.3852, "margin_dpo/margin_mean": 0.015324920415878296, "margin_dpo/margin_std": 0.4150603711605072, "step": 6 }, { "KL/chosen_KL_mean": 0.00576019287109375, "KL/mean": -0.027363106608390808, "KL/rejected_KL_mean": -0.06049346923828125, "KL/std": 0.26753008365631104, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06624773144721985, "fcm_dpo/q_t": 0.4983446002006531, "grad_norm": 37.939456939697266, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4963763952255249, "logits/rejected": 0.5440800189971924, "logps/chosen": -86.64277648925781, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.67667388916016, "loss": 1.3801, "margin_dpo/margin_mean": 0.06624764204025269, "margin_dpo/margin_std": 0.39998042583465576, "step": 7 }, { "KL/chosen_KL_mean": -0.030071258544921875, "KL/mean": 0.015311740338802338, "KL/rejected_KL_mean": 0.060699462890625, "KL/std": 0.3527182936668396, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09076635539531708, "fcm_dpo/q_t": 0.5022671818733215, "grad_norm": 32.85660171508789, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4456387460231781, "logits/rejected": 0.7620406150817871, "logps/chosen": -89.95375061035156, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.16734313964844, "loss": 1.396, "margin_dpo/margin_mean": -0.090766042470932, "margin_dpo/margin_std": 0.4774355888366699, "step": 8 }, { "KL/chosen_KL_mean": 0.02001953125, "KL/mean": -0.004378870129585266, "KL/rejected_KL_mean": -0.028774261474609375, "KL/std": 0.3540073037147522, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.048785120248794556, "fcm_dpo/q_t": 0.4987806975841522, "grad_norm": 36.07113265991211, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7219189405441284, "logits/rejected": 0.905205488204956, "logps/chosen": -103.83711242675781, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.34809875488281, "loss": 1.3819, "margin_dpo/margin_mean": 0.04878535866737366, "margin_dpo/margin_std": 0.41844016313552856, "step": 9 }, { "KL/chosen_KL_mean": 0.051403045654296875, "KL/mean": 0.08129139989614487, "KL/rejected_KL_mean": 0.111175537109375, "KL/std": 0.29876604676246643, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.059779539704322815, "fcm_dpo/q_t": 0.5014941692352295, "grad_norm": 33.32407760620117, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5896826982498169, "logits/rejected": 0.8054566383361816, "logps/chosen": -76.15296173095703, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.04092407226562, "loss": 1.3927, "margin_dpo/margin_mean": -0.05977940559387207, "margin_dpo/margin_std": 0.42056939005851746, "step": 10 }, { "KL/chosen_KL_mean": 0.07837677001953125, "KL/mean": 0.035002999007701874, "KL/rejected_KL_mean": -0.008373260498046875, "KL/std": 0.3122630715370178, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08674506843090057, "fcm_dpo/q_t": 0.49783316254615784, "grad_norm": 37.058773040771484, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.2816029489040375, "logits/rejected": 0.49706321954727173, "logps/chosen": -82.28811645507812, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.27299499511719, "loss": 1.3782, "margin_dpo/margin_mean": 0.08674581348896027, "margin_dpo/margin_std": 0.4525449872016907, "step": 11 }, { "KL/chosen_KL_mean": 0.000164031982421875, "KL/mean": -0.013757966458797455, "KL/rejected_KL_mean": -0.027675628662109375, "KL/std": 0.3647559881210327, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02783392369747162, "fcm_dpo/q_t": 0.49930480122566223, "grad_norm": 40.19915771484375, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.4683685898780823, "logits/rejected": 0.599022388458252, "logps/chosen": -99.10533142089844, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.29908752441406, "loss": 1.3842, "margin_dpo/margin_mean": 0.0278339684009552, "margin_dpo/margin_std": 0.5427765846252441, "step": 12 }, { "KL/chosen_KL_mean": 0.02599334716796875, "KL/mean": 0.01120806485414505, "KL/rejected_KL_mean": -0.003574371337890625, "KL/std": 0.3564586043357849, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.029575034976005554, "fcm_dpo/q_t": 0.4992622137069702, "grad_norm": 41.62953186035156, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.4014189541339874, "logits/rejected": 0.6501343250274658, "logps/chosen": -90.53373718261719, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.69467163085938, "loss": 1.384, "margin_dpo/margin_mean": 0.02957439422607422, "margin_dpo/margin_std": 0.5223793983459473, "step": 13 }, { "KL/chosen_KL_mean": -0.035495758056640625, "KL/mean": -0.007676973938941956, "KL/rejected_KL_mean": 0.0201416015625, "KL/std": 0.29156261682510376, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05564120411872864, "fcm_dpo/q_t": 0.5013890862464905, "grad_norm": 35.99645233154297, "learning_rate": 9.420289855072464e-08, "logits/chosen": 1.0424894094467163, "logits/rejected": 1.1364803314208984, "logps/chosen": -99.8626708984375, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.92185974121094, "loss": 1.3923, "margin_dpo/margin_mean": -0.05564120411872864, "margin_dpo/margin_std": 0.42240291833877563, "step": 14 }, { "KL/chosen_KL_mean": 0.00502777099609375, "KL/mean": 0.007549792528152466, "KL/rejected_KL_mean": 0.01007080078125, "KL/std": 0.2964215874671936, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.00504070520401001, "fcm_dpo/q_t": 0.5001264214515686, "grad_norm": 31.949726104736328, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.393198698759079, "logits/rejected": 0.5600039958953857, "logps/chosen": -78.90493774414062, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.05227661132812, "loss": 1.3872, "margin_dpo/margin_mean": -0.00504070520401001, "margin_dpo/margin_std": 0.3776414692401886, "step": 15 }, { "KL/chosen_KL_mean": 0.01799774169921875, "KL/mean": -0.015829920768737793, "KL/rejected_KL_mean": -0.0496673583984375, "KL/std": 0.3318990468978882, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06766515970230103, "fcm_dpo/q_t": 0.49830901622772217, "grad_norm": 34.50741958618164, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.500816285610199, "logits/rejected": 0.7521071434020996, "logps/chosen": -97.40528106689453, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.64912414550781, "loss": 1.3801, "margin_dpo/margin_mean": 0.06766489148139954, "margin_dpo/margin_std": 0.4656125605106354, "step": 16 }, { "KL/chosen_KL_mean": 0.036670684814453125, "KL/mean": 0.01893872767686844, "KL/rejected_KL_mean": 0.001201629638671875, "KL/std": 0.33163076639175415, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03547142446041107, "fcm_dpo/q_t": 0.499111533164978, "grad_norm": 35.608882904052734, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.6628004312515259, "logits/rejected": 0.9156403541564941, "logps/chosen": -104.32763671875, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.46652221679688, "loss": 1.3834, "margin_dpo/margin_mean": 0.03547009825706482, "margin_dpo/margin_std": 0.5154414176940918, "step": 17 }, { "KL/chosen_KL_mean": 0.019748687744140625, "KL/mean": 0.0162547305226326, "KL/rejected_KL_mean": 0.01276397705078125, "KL/std": 0.328279584646225, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.006982162594795227, "fcm_dpo/q_t": 0.49982550740242004, "grad_norm": 41.72677993774414, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.4237491488456726, "logits/rejected": 0.7731245756149292, "logps/chosen": -87.07220458984375, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.83795166015625, "loss": 1.3862, "margin_dpo/margin_mean": 0.006982743740081787, "margin_dpo/margin_std": 0.5070095658302307, "step": 18 }, { "KL/chosen_KL_mean": 0.02716064453125, "KL/mean": 0.031385183334350586, "KL/rejected_KL_mean": 0.03560638427734375, "KL/std": 0.2878996729850769, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.008450210094451904, "fcm_dpo/q_t": 0.5002115964889526, "grad_norm": 31.514612197875977, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6067254543304443, "logits/rejected": 0.8830981254577637, "logps/chosen": -105.84638214111328, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.89462280273438, "loss": 1.3875, "margin_dpo/margin_mean": -0.008449554443359375, "margin_dpo/margin_std": 0.3756786584854126, "step": 19 }, { "KL/chosen_KL_mean": -0.016845703125, "KL/mean": -0.024066537618637085, "KL/rejected_KL_mean": -0.031291961669921875, "KL/std": 0.3088562786579132, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014448165893554688, "fcm_dpo/q_t": 0.49963873624801636, "grad_norm": 32.52676010131836, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.22031639516353607, "logits/rejected": 0.4947779178619385, "logps/chosen": -90.77496337890625, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.94361114501953, "loss": 1.3853, "margin_dpo/margin_mean": 0.014448463916778564, "margin_dpo/margin_std": 0.42588090896606445, "step": 20 }, { "KL/chosen_KL_mean": 0.08238983154296875, "KL/mean": 0.025366492569446564, "KL/rejected_KL_mean": -0.031658172607421875, "KL/std": 0.2928202152252197, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11404521763324738, "fcm_dpo/q_t": 0.49715161323547363, "grad_norm": 31.399518966674805, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5744367837905884, "logits/rejected": 0.7540575265884399, "logps/chosen": -80.25106811523438, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.96539306640625, "loss": 1.3754, "margin_dpo/margin_mean": 0.1140454113483429, "margin_dpo/margin_std": 0.44449806213378906, "step": 21 }, { "KL/chosen_KL_mean": 0.036102294921875, "KL/mean": 0.02506348490715027, "KL/rejected_KL_mean": 0.014026641845703125, "KL/std": 0.3325667381286621, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.022068381309509277, "fcm_dpo/q_t": 0.4994485080242157, "grad_norm": 42.915897369384766, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.779334545135498, "logits/rejected": 0.9065211415290833, "logps/chosen": -95.35919952392578, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.45948791503906, "loss": 1.3846, "margin_dpo/margin_mean": 0.02206823229789734, "margin_dpo/margin_std": 0.4258459210395813, "step": 22 }, { "KL/chosen_KL_mean": -0.018903732299804688, "KL/mean": 0.022981911897659302, "KL/rejected_KL_mean": 0.06487655639648438, "KL/std": 0.333385169506073, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08377939462661743, "fcm_dpo/q_t": 0.5020897388458252, "grad_norm": 31.454723358154297, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.6354077458381653, "logits/rejected": 0.8085711002349854, "logps/chosen": -90.65641784667969, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.52938079833984, "loss": 1.3953, "margin_dpo/margin_mean": -0.08377933502197266, "margin_dpo/margin_std": 0.4790075421333313, "step": 23 }, { "KL/chosen_KL_mean": 0.04962348937988281, "KL/mean": 0.012520663440227509, "KL/rejected_KL_mean": -0.024585723876953125, "KL/std": 0.3016190528869629, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07420895993709564, "fcm_dpo/q_t": 0.49814629554748535, "grad_norm": 44.449913024902344, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.4520428478717804, "logits/rejected": 0.4859141409397125, "logps/chosen": -69.86766815185547, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.63449096679688, "loss": 1.3793, "margin_dpo/margin_mean": 0.07420960068702698, "margin_dpo/margin_std": 0.3892441391944885, "step": 24 }, { "KL/chosen_KL_mean": -0.01080322265625, "KL/mean": -0.04771599918603897, "KL/rejected_KL_mean": -0.08463668823242188, "KL/std": 0.2791096866130829, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07382743060588837, "fcm_dpo/q_t": 0.4981558620929718, "grad_norm": 37.017086029052734, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.44776245951652527, "logits/rejected": 0.613357424736023, "logps/chosen": -80.83628845214844, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.041748046875, "loss": 1.3793, "margin_dpo/margin_mean": 0.07382696866989136, "margin_dpo/margin_std": 0.3829525113105774, "step": 25 }, { "KL/chosen_KL_mean": -0.021968841552734375, "KL/mean": -0.013098865747451782, "KL/rejected_KL_mean": -0.004222869873046875, "KL/std": 0.3084653317928314, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.017745554447174072, "fcm_dpo/q_t": 0.5004435777664185, "grad_norm": 41.04769515991211, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.20780444145202637, "logits/rejected": 0.41232481598854065, "logps/chosen": -88.92314147949219, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.83241271972656, "loss": 1.3886, "margin_dpo/margin_mean": -0.017745256423950195, "margin_dpo/margin_std": 0.4410586357116699, "step": 26 }, { "KL/chosen_KL_mean": -0.010578155517578125, "KL/mean": -0.09648218005895615, "KL/rejected_KL_mean": -0.18238449096679688, "KL/std": 0.34528160095214844, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17180438339710236, "fcm_dpo/q_t": 0.49570852518081665, "grad_norm": 44.21283721923828, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.23756228387355804, "logits/rejected": 0.4268535375595093, "logps/chosen": -77.6065902709961, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.12089538574219, "loss": 1.3698, "margin_dpo/margin_mean": 0.1718047857284546, "margin_dpo/margin_std": 0.4879463016986847, "step": 27 }, { "KL/chosen_KL_mean": 0.038936614990234375, "KL/mean": -0.008876144886016846, "KL/rejected_KL_mean": -0.0566864013671875, "KL/std": 0.32462751865386963, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09562146663665771, "fcm_dpo/q_t": 0.49761098623275757, "grad_norm": 36.368953704833984, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5566167831420898, "logits/rejected": 0.7736653685569763, "logps/chosen": -102.18963623046875, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.01618194580078, "loss": 1.3772, "margin_dpo/margin_mean": 0.09562188386917114, "margin_dpo/margin_std": 0.4157450199127197, "step": 28 }, { "KL/chosen_KL_mean": 0.016559600830078125, "KL/mean": -0.022749975323677063, "KL/rejected_KL_mean": -0.0620574951171875, "KL/std": 0.34623080492019653, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07862827181816101, "fcm_dpo/q_t": 0.4980344772338867, "grad_norm": 42.554988861083984, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.31471747159957886, "logits/rejected": 0.5718276500701904, "logps/chosen": -88.6304931640625, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.02217102050781, "loss": 1.3789, "margin_dpo/margin_mean": 0.07862880825996399, "margin_dpo/margin_std": 0.41622698307037354, "step": 29 }, { "KL/chosen_KL_mean": -0.024204254150390625, "KL/mean": -0.060240238904953, "KL/rejected_KL_mean": -0.09627914428710938, "KL/std": 0.3244808316230774, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07207274436950684, "fcm_dpo/q_t": 0.4982002079486847, "grad_norm": 39.0716438293457, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.6168072819709778, "logits/rejected": 0.7326011657714844, "logps/chosen": -88.41259002685547, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.41517639160156, "loss": 1.3795, "margin_dpo/margin_mean": 0.07207253575325012, "margin_dpo/margin_std": 0.38867026567459106, "step": 30 }, { "KL/chosen_KL_mean": -0.01702117919921875, "KL/mean": -0.06890757381916046, "KL/rejected_KL_mean": -0.12079238891601562, "KL/std": 0.32098305225372314, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10376748442649841, "fcm_dpo/q_t": 0.49740755558013916, "grad_norm": 31.35635757446289, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.6492427587509155, "logits/rejected": 0.8862093687057495, "logps/chosen": -101.14268493652344, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.97921752929688, "loss": 1.3764, "margin_dpo/margin_mean": 0.10376802086830139, "margin_dpo/margin_std": 0.4157770872116089, "step": 31 }, { "KL/chosen_KL_mean": -0.028469085693359375, "KL/mean": -0.0712999776005745, "KL/rejected_KL_mean": -0.11413192749023438, "KL/std": 0.3352075219154358, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08566714823246002, "fcm_dpo/q_t": 0.49785852432250977, "grad_norm": 35.396522521972656, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.4871651530265808, "logits/rejected": 0.6312373876571655, "logps/chosen": -96.64549255371094, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.3385009765625, "loss": 1.3784, "margin_dpo/margin_mean": 0.08566747605800629, "margin_dpo/margin_std": 0.49307262897491455, "step": 32 }, { "KL/chosen_KL_mean": 0.0281219482421875, "KL/mean": -0.05220947414636612, "KL/rejected_KL_mean": -0.13254165649414062, "KL/std": 0.33622103929519653, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16066186130046844, "fcm_dpo/q_t": 0.49598586559295654, "grad_norm": 33.56657028198242, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6170715689659119, "logits/rejected": 0.7986421585083008, "logps/chosen": -81.49288940429688, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.93849182128906, "loss": 1.3708, "margin_dpo/margin_mean": 0.16066250205039978, "margin_dpo/margin_std": 0.44578516483306885, "step": 33 }, { "KL/chosen_KL_mean": 0.059764862060546875, "KL/mean": -0.06535378843545914, "KL/rejected_KL_mean": -0.19047164916992188, "KL/std": 0.4213768243789673, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2502385973930359, "fcm_dpo/q_t": 0.4937604069709778, "grad_norm": 40.56326675415039, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.638230562210083, "logits/rejected": 0.8112214803695679, "logps/chosen": -77.14227294921875, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.90924072265625, "loss": 1.3624, "margin_dpo/margin_mean": 0.25023889541625977, "margin_dpo/margin_std": 0.6150074005126953, "step": 34 }, { "KL/chosen_KL_mean": 0.053348541259765625, "KL/mean": -0.06372416019439697, "KL/rejected_KL_mean": -0.18079757690429688, "KL/std": 0.3433344066143036, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23414570093154907, "fcm_dpo/q_t": 0.4941514730453491, "grad_norm": 42.68899154663086, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.3354959487915039, "logits/rejected": 0.49060869216918945, "logps/chosen": -77.51699829101562, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.36935424804688, "loss": 1.3636, "margin_dpo/margin_mean": 0.2341463565826416, "margin_dpo/margin_std": 0.4703635275363922, "step": 35 }, { "KL/chosen_KL_mean": -0.00391387939453125, "KL/mean": -0.08189772814512253, "KL/rejected_KL_mean": -0.15988922119140625, "KL/std": 0.35309967398643494, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1559712439775467, "fcm_dpo/q_t": 0.4961051642894745, "grad_norm": 32.6065673828125, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.45249712467193604, "logits/rejected": 0.6444553136825562, "logps/chosen": -83.29216003417969, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.91730499267578, "loss": 1.3713, "margin_dpo/margin_mean": 0.1559714525938034, "margin_dpo/margin_std": 0.4470587968826294, "step": 36 }, { "KL/chosen_KL_mean": -0.029575347900390625, "KL/mean": -0.14033129811286926, "KL/rejected_KL_mean": -0.2510871887207031, "KL/std": 0.43875962495803833, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22150462865829468, "fcm_dpo/q_t": 0.4944715201854706, "grad_norm": 36.02985763549805, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.6072508692741394, "logits/rejected": 0.76311194896698, "logps/chosen": -94.80066680908203, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -85.96835327148438, "loss": 1.3652, "margin_dpo/margin_mean": 0.22150462865829468, "margin_dpo/margin_std": 0.6122845411300659, "step": 37 }, { "KL/chosen_KL_mean": 0.041164398193359375, "KL/mean": -0.07342269271612167, "KL/rejected_KL_mean": -0.18800735473632812, "KL/std": 0.4407653510570526, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22917841374874115, "fcm_dpo/q_t": 0.49428001046180725, "grad_norm": 36.94073486328125, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.41596081852912903, "logits/rejected": 0.5534640550613403, "logps/chosen": -75.93733978271484, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.29202270507812, "loss": 1.3644, "margin_dpo/margin_mean": 0.2291794717311859, "margin_dpo/margin_std": 0.5785521864891052, "step": 38 }, { "KL/chosen_KL_mean": 0.0107421875, "KL/mean": -0.14146539568901062, "KL/rejected_KL_mean": -0.2936744689941406, "KL/std": 0.39912474155426025, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.30441612005233765, "fcm_dpo/q_t": 0.49239736795425415, "grad_norm": 36.210731506347656, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.4538946747779846, "logits/rejected": 0.6406049132347107, "logps/chosen": -81.17503356933594, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.48959350585938, "loss": 1.3568, "margin_dpo/margin_mean": 0.3044161796569824, "margin_dpo/margin_std": 0.5159787535667419, "step": 39 }, { "KL/chosen_KL_mean": 0.06087493896484375, "KL/mean": -0.05862073600292206, "KL/rejected_KL_mean": -0.1781158447265625, "KL/std": 0.33346205949783325, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23898854851722717, "fcm_dpo/q_t": 0.4940286874771118, "grad_norm": 29.057310104370117, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.7142466306686401, "logits/rejected": 0.9179277420043945, "logps/chosen": -83.27169799804688, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.43402862548828, "loss": 1.363, "margin_dpo/margin_mean": 0.23898768424987793, "margin_dpo/margin_std": 0.4170268774032593, "step": 40 }, { "KL/chosen_KL_mean": -0.038600921630859375, "KL/mean": -0.17350825667381287, "KL/rejected_KL_mean": -0.3084144592285156, "KL/std": 0.4457316994667053, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2698156237602234, "fcm_dpo/q_t": 0.49326491355895996, "grad_norm": 33.422122955322266, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.4682924747467041, "logits/rejected": 0.6467391848564148, "logps/chosen": -93.18726348876953, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.38761901855469, "loss": 1.3605, "margin_dpo/margin_mean": 0.2698156237602234, "margin_dpo/margin_std": 0.6239008903503418, "step": 41 }, { "KL/chosen_KL_mean": 0.0256805419921875, "KL/mean": -0.1792878955602646, "KL/rejected_KL_mean": -0.3842620849609375, "KL/std": 0.5033321380615234, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40993908047676086, "fcm_dpo/q_t": 0.48976224660873413, "grad_norm": 40.78578186035156, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.29466933012008667, "logits/rejected": 0.43526387214660645, "logps/chosen": -90.67594146728516, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.24890899658203, "loss": 1.3466, "margin_dpo/margin_mean": 0.40993955731391907, "margin_dpo/margin_std": 0.5887066125869751, "step": 42 }, { "KL/chosen_KL_mean": -0.014873504638671875, "KL/mean": -0.23074953258037567, "KL/rejected_KL_mean": -0.44663238525390625, "KL/std": 0.4737606942653656, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4317546784877777, "fcm_dpo/q_t": 0.4892197847366333, "grad_norm": 38.24826431274414, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.7082161903381348, "logits/rejected": 0.9047064781188965, "logps/chosen": -89.65890502929688, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.39848327636719, "loss": 1.3444, "margin_dpo/margin_mean": 0.431755006313324, "margin_dpo/margin_std": 0.5760371685028076, "step": 43 }, { "KL/chosen_KL_mean": -0.0157623291015625, "KL/mean": -0.24162331223487854, "KL/rejected_KL_mean": -0.4674835205078125, "KL/std": 0.5125470757484436, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.45172548294067383, "fcm_dpo/q_t": 0.4887210428714752, "grad_norm": 34.277626037597656, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.9362626075744629, "logits/rejected": 1.0547525882720947, "logps/chosen": -81.89408111572266, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.88911437988281, "loss": 1.3424, "margin_dpo/margin_mean": 0.451725572347641, "margin_dpo/margin_std": 0.5561680793762207, "step": 44 }, { "KL/chosen_KL_mean": -0.10937118530273438, "KL/mean": -0.21997100114822388, "KL/rejected_KL_mean": -0.33056640625, "KL/std": 0.42387402057647705, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22119711339473724, "fcm_dpo/q_t": 0.49447542428970337, "grad_norm": 24.384540557861328, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.5753566026687622, "logits/rejected": 0.6857389211654663, "logps/chosen": -77.45396423339844, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.57830810546875, "loss": 1.3651, "margin_dpo/margin_mean": 0.22119756042957306, "margin_dpo/margin_std": 0.5594868659973145, "step": 45 }, { "KL/chosen_KL_mean": 0.0009002685546875, "KL/mean": -0.2501962184906006, "KL/rejected_KL_mean": -0.5012931823730469, "KL/std": 0.5349624156951904, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5021981596946716, "fcm_dpo/q_t": 0.48745983839035034, "grad_norm": 33.23622512817383, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.5993818640708923, "logits/rejected": 0.8469666242599487, "logps/chosen": -90.33992004394531, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.17781066894531, "loss": 1.3376, "margin_dpo/margin_mean": 0.5021975636482239, "margin_dpo/margin_std": 0.5783181190490723, "step": 46 }, { "KL/chosen_KL_mean": -0.07114028930664062, "KL/mean": -0.238023042678833, "KL/rejected_KL_mean": -0.4049072265625, "KL/std": 0.49713367223739624, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3337647318840027, "fcm_dpo/q_t": 0.4916689395904541, "grad_norm": 34.18733215332031, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.5437924861907959, "logits/rejected": 0.7313907146453857, "logps/chosen": -104.48243713378906, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.65980529785156, "loss": 1.3544, "margin_dpo/margin_mean": 0.33376431465148926, "margin_dpo/margin_std": 0.6865592002868652, "step": 47 }, { "KL/chosen_KL_mean": 0.0165557861328125, "KL/mean": -0.2888548672199249, "KL/rejected_KL_mean": -0.5942649841308594, "KL/std": 0.6951754093170166, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.610822319984436, "fcm_dpo/q_t": 0.48476821184158325, "grad_norm": 38.61543655395508, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.200358048081398, "logits/rejected": 0.39991044998168945, "logps/chosen": -91.0837173461914, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.67484283447266, "loss": 1.328, "margin_dpo/margin_mean": 0.610822856426239, "margin_dpo/margin_std": 0.868547797203064, "step": 48 }, { "KL/chosen_KL_mean": 0.062389373779296875, "KL/mean": -0.3189862072467804, "KL/rejected_KL_mean": -0.7003593444824219, "KL/std": 0.8289512395858765, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7627428770065308, "fcm_dpo/q_t": 0.48101240396499634, "grad_norm": 39.60076904296875, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.14819221198558807, "logits/rejected": 0.6239709854125977, "logps/chosen": -92.94129180908203, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.44935607910156, "loss": 1.314, "margin_dpo/margin_mean": 0.7627429962158203, "margin_dpo/margin_std": 1.0111594200134277, "step": 49 }, { "KL/chosen_KL_mean": -0.05675506591796875, "KL/mean": -0.37378230690956116, "KL/rejected_KL_mean": -0.6908035278320312, "KL/std": 0.7271406650543213, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6340419054031372, "fcm_dpo/q_t": 0.4841960072517395, "grad_norm": 33.3209114074707, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.42532938718795776, "logits/rejected": 0.5910958647727966, "logps/chosen": -94.68356323242188, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.26516723632812, "loss": 1.3257, "margin_dpo/margin_mean": 0.6340416073799133, "margin_dpo/margin_std": 0.843346357345581, "step": 50 }, { "KL/chosen_KL_mean": -0.043796539306640625, "KL/mean": -0.4066208004951477, "KL/rejected_KL_mean": -0.7694435119628906, "KL/std": 0.8937298059463501, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7256404161453247, "fcm_dpo/q_t": 0.48196056485176086, "grad_norm": 29.72462272644043, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.544784665107727, "logits/rejected": 0.6821225881576538, "logps/chosen": -87.55107116699219, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.2417984008789, "loss": 1.3182, "margin_dpo/margin_mean": 0.725640058517456, "margin_dpo/margin_std": 1.12073814868927, "step": 51 }, { "KL/chosen_KL_mean": 0.00920867919921875, "KL/mean": -0.4253120422363281, "KL/rejected_KL_mean": -0.8598442077636719, "KL/std": 0.8275116682052612, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8690494298934937, "fcm_dpo/q_t": 0.4783685803413391, "grad_norm": 32.13676834106445, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.3826545774936676, "logits/rejected": 0.5919617414474487, "logps/chosen": -90.62105560302734, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.89891052246094, "loss": 1.3036, "margin_dpo/margin_mean": 0.8690494298934937, "margin_dpo/margin_std": 0.956383466720581, "step": 52 }, { "KL/chosen_KL_mean": -0.011692047119140625, "KL/mean": -0.5706908106803894, "KL/rejected_KL_mean": -1.1296920776367188, "KL/std": 1.117661714553833, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1180002689361572, "fcm_dpo/q_t": 0.4723230004310608, "grad_norm": 39.28775405883789, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.11750403046607971, "logits/rejected": -0.010614126920700073, "logps/chosen": -81.59477233886719, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.79122161865234, "loss": 1.2826, "margin_dpo/margin_mean": 1.11799955368042, "margin_dpo/margin_std": 1.3521897792816162, "step": 53 }, { "KL/chosen_KL_mean": -0.0873565673828125, "KL/mean": -0.6340787410736084, "KL/rejected_KL_mean": -1.1808013916015625, "KL/std": 1.3180814981460571, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0934383869171143, "fcm_dpo/q_t": 0.4728393256664276, "grad_norm": 39.939125061035156, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.3601967990398407, "logits/rejected": 0.5388259291648865, "logps/chosen": -88.99752807617188, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.29835510253906, "loss": 1.2842, "margin_dpo/margin_mean": 1.0934383869171143, "margin_dpo/margin_std": 1.3030112981796265, "step": 54 }, { "KL/chosen_KL_mean": 0.11523818969726562, "KL/mean": -0.4584028124809265, "KL/rejected_KL_mean": -1.0320396423339844, "KL/std": 1.2030868530273438, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1472759246826172, "fcm_dpo/q_t": 0.4716120660305023, "grad_norm": 35.2214469909668, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.10238924622535706, "logits/rejected": 0.4109187424182892, "logps/chosen": -92.34068298339844, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.00297546386719, "loss": 1.281, "margin_dpo/margin_mean": 1.147275686264038, "margin_dpo/margin_std": 1.530487298965454, "step": 55 }, { "KL/chosen_KL_mean": -0.033321380615234375, "KL/mean": -0.6018686294555664, "KL/rejected_KL_mean": -1.1704139709472656, "KL/std": 1.1662425994873047, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1370903253555298, "fcm_dpo/q_t": 0.47190243005752563, "grad_norm": 32.97835922241211, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.2760552763938904, "logits/rejected": 0.6161391735076904, "logps/chosen": -87.26997375488281, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.81595611572266, "loss": 1.2824, "margin_dpo/margin_mean": 1.1370902061462402, "margin_dpo/margin_std": 1.6295883655548096, "step": 56 }, { "KL/chosen_KL_mean": -0.10442733764648438, "KL/mean": -0.631227970123291, "KL/rejected_KL_mean": -1.1580352783203125, "KL/std": 1.2112032175064087, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0536186695098877, "fcm_dpo/q_t": 0.47391778230667114, "grad_norm": 33.80556106567383, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.3832647502422333, "logits/rejected": 0.602504551410675, "logps/chosen": -98.25517272949219, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.21087646484375, "loss": 1.2898, "margin_dpo/margin_mean": 1.0536195039749146, "margin_dpo/margin_std": 1.5559934377670288, "step": 57 }, { "KL/chosen_KL_mean": 0.2710914611816406, "KL/mean": -0.5070213079452515, "KL/rejected_KL_mean": -1.285125732421875, "KL/std": 1.432263731956482, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.556209683418274, "fcm_dpo/q_t": 0.46176934242248535, "grad_norm": 37.75045394897461, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.39911893010139465, "logits/rejected": 0.7484442591667175, "logps/chosen": -99.30987548828125, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.50740051269531, "loss": 1.2466, "margin_dpo/margin_mean": 1.5562095642089844, "margin_dpo/margin_std": 1.9012987613677979, "step": 58 }, { "KL/chosen_KL_mean": 0.047893524169921875, "KL/mean": -0.5771614909172058, "KL/rejected_KL_mean": -1.2022132873535156, "KL/std": 1.1531610488891602, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2501047849655151, "fcm_dpo/q_t": 0.46901440620422363, "grad_norm": 34.36309051513672, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.06413343548774719, "logits/rejected": 0.2503090500831604, "logps/chosen": -89.75443267822266, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.47888946533203, "loss": 1.2701, "margin_dpo/margin_mean": 1.2501044273376465, "margin_dpo/margin_std": 1.3920749425888062, "step": 59 }, { "KL/chosen_KL_mean": 0.0740203857421875, "KL/mean": -0.5358163118362427, "KL/rejected_KL_mean": -1.1456527709960938, "KL/std": 1.2732292413711548, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2196729183197021, "fcm_dpo/q_t": 0.46974730491638184, "grad_norm": 28.586130142211914, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.3476541340351105, "logits/rejected": 0.6150113940238953, "logps/chosen": -95.08169555664062, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.5180892944336, "loss": 1.2729, "margin_dpo/margin_mean": 1.2196730375289917, "margin_dpo/margin_std": 1.3969335556030273, "step": 60 }, { "KL/chosen_KL_mean": -0.23726654052734375, "KL/mean": -0.783356785774231, "KL/rejected_KL_mean": -1.3294448852539062, "KL/std": 1.4031500816345215, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0921752452850342, "fcm_dpo/q_t": 0.4731889069080353, "grad_norm": 28.377683639526367, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.22575867176055908, "logits/rejected": 0.36646145582199097, "logps/chosen": -85.96957397460938, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.29520416259766, "loss": 1.2888, "margin_dpo/margin_mean": 1.0921757221221924, "margin_dpo/margin_std": 1.828963041305542, "step": 61 }, { "KL/chosen_KL_mean": -0.11860275268554688, "KL/mean": -0.7271952629089355, "KL/rejected_KL_mean": -1.3357887268066406, "KL/std": 1.477699637413025, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2171878814697266, "fcm_dpo/q_t": 0.4699515104293823, "grad_norm": 28.266677856445312, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.1340886354446411, "logits/rejected": 0.32614439725875854, "logps/chosen": -81.75398254394531, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.37409973144531, "loss": 1.2786, "margin_dpo/margin_mean": 1.2171870470046997, "margin_dpo/margin_std": 1.9535574913024902, "step": 62 }, { "KL/chosen_KL_mean": -0.02603912353515625, "KL/mean": -0.7968390583992004, "KL/rejected_KL_mean": -1.5676422119140625, "KL/std": 1.3708903789520264, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5416043996810913, "fcm_dpo/q_t": 0.4618118703365326, "grad_norm": 35.06855773925781, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.25060391426086426, "logits/rejected": 0.45625168085098267, "logps/chosen": -103.65010070800781, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.97067260742188, "loss": 1.2443, "margin_dpo/margin_mean": 1.5416042804718018, "margin_dpo/margin_std": 1.5575233697891235, "step": 63 }, { "KL/chosen_KL_mean": -0.2126007080078125, "KL/mean": -1.0183178186416626, "KL/rejected_KL_mean": -1.8240432739257812, "KL/std": 1.8184094429016113, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6114435195922852, "fcm_dpo/q_t": 0.4605445861816406, "grad_norm": 35.0161247253418, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.5890347957611084, "logits/rejected": 0.8106449842453003, "logps/chosen": -87.21418762207031, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.40951538085938, "loss": 1.2435, "margin_dpo/margin_mean": 1.611443281173706, "margin_dpo/margin_std": 2.0928924083709717, "step": 64 }, { "KL/chosen_KL_mean": -0.0146636962890625, "KL/mean": -0.9369107484817505, "KL/rejected_KL_mean": -1.8591651916503906, "KL/std": 1.6740304231643677, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8444950580596924, "fcm_dpo/q_t": 0.454690158367157, "grad_norm": 38.105918884277344, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.10184075683355331, "logits/rejected": 0.2475431263446808, "logps/chosen": -91.23657989501953, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.1947021484375, "loss": 1.2211, "margin_dpo/margin_mean": 1.8444955348968506, "margin_dpo/margin_std": 2.074845790863037, "step": 65 }, { "KL/chosen_KL_mean": -0.31584930419921875, "KL/mean": -0.9498295783996582, "KL/rejected_KL_mean": -1.5838050842285156, "KL/std": 1.5764840841293335, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2679508924484253, "fcm_dpo/q_t": 0.4688006043434143, "grad_norm": 25.590856552124023, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.017530532553792, "logits/rejected": 0.2128189504146576, "logps/chosen": -84.1000747680664, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.33590698242188, "loss": 1.2728, "margin_dpo/margin_mean": 1.2679510116577148, "margin_dpo/margin_std": 1.8610610961914062, "step": 66 }, { "KL/chosen_KL_mean": -0.8628463745117188, "KL/mean": -1.425905466079712, "KL/rejected_KL_mean": -1.9889602661132812, "KL/std": 2.02144718170166, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1261157989501953, "fcm_dpo/q_t": 0.47281843423843384, "grad_norm": 24.35548210144043, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.34441930055618286, "logits/rejected": 0.6219497323036194, "logps/chosen": -88.53581237792969, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.07571411132812, "loss": 1.2956, "margin_dpo/margin_mean": 1.1261157989501953, "margin_dpo/margin_std": 2.7534079551696777, "step": 67 }, { "KL/chosen_KL_mean": -0.2273712158203125, "KL/mean": -1.290507197380066, "KL/rejected_KL_mean": -2.3536376953125, "KL/std": 2.215397357940674, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.12975220382213593, "fcm_dpo/delta": 0.16678282618522644, "fcm_dpo/margin": 2.1262645721435547, "fcm_dpo/q_t": 0.4442985951900482, "grad_norm": 33.7536735534668, "learning_rate": 4.855072463768116e-07, "logits/chosen": 0.01098564825952053, "logits/rejected": 0.31069740653038025, "logps/chosen": -97.30621337890625, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.47236633300781, "loss": 1.1612, "margin_dpo/margin_mean": 2.1262640953063965, "margin_dpo/margin_std": 2.741765022277832, "step": 68 }, { "KL/chosen_KL_mean": -0.27097320556640625, "KL/mean": -1.3898770809173584, "KL/rejected_KL_mean": -2.508777618408203, "KL/std": 2.4313292503356934, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.15801313519477844, "fcm_dpo/delta": 0.0674431174993515, "fcm_dpo/margin": 2.2378058433532715, "fcm_dpo/q_t": 0.42351439595222473, "grad_norm": 40.93582534790039, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.0931001752614975, "logits/rejected": 0.205903559923172, "logps/chosen": -85.99068450927734, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.98898315429688, "loss": 1.1141, "margin_dpo/margin_mean": 2.237806558609009, "margin_dpo/margin_std": 2.9925875663757324, "step": 69 }, { "KL/chosen_KL_mean": -0.6336822509765625, "KL/mean": -1.9103761911392212, "KL/rejected_KL_mean": -3.1870689392089844, "KL/std": 2.7556700706481934, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.15868568420410156, "fcm_dpo/delta": -0.008746134117245674, "fcm_dpo/margin": 2.5533857345581055, "fcm_dpo/q_t": 0.40846139192581177, "grad_norm": 38.4747200012207, "learning_rate": 5e-07, "logits/chosen": 0.2843042016029358, "logits/rejected": 0.4623540937900543, "logps/chosen": -95.64363098144531, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.39979553222656, "loss": 1.0849, "margin_dpo/margin_mean": 2.5533852577209473, "margin_dpo/margin_std": 3.287351608276367, "step": 70 }, { "KL/chosen_KL_mean": 0.007877349853515625, "KL/mean": -1.8587327003479004, "KL/rejected_KL_mean": -3.7253456115722656, "KL/std": 3.5019378662109375, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.12101645767688751, "fcm_dpo/delta": -0.1268956959247589, "fcm_dpo/margin": 3.733224868774414, "fcm_dpo/q_t": 0.383242130279541, "grad_norm": 38.21384811401367, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.2877994775772095, "logits/rejected": 0.06802543252706528, "logps/chosen": -90.96947479248047, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.31637573242188, "loss": 1.0356, "margin_dpo/margin_mean": 3.733224630355835, "margin_dpo/margin_std": 3.965754508972168, "step": 71 }, { "KL/chosen_KL_mean": -0.5295333862304688, "KL/mean": -1.9942858219146729, "KL/rejected_KL_mean": -3.4590377807617188, "KL/std": 2.906968593597412, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.12798023223876953, "fcm_dpo/delta": 0.04027131199836731, "fcm_dpo/margin": 2.929507255554199, "fcm_dpo/q_t": 0.41516679525375366, "grad_norm": 34.411537170410156, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.18199267983436584, "logits/rejected": 0.142164945602417, "logps/chosen": -98.42105865478516, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.6507568359375, "loss": 1.0881, "margin_dpo/margin_mean": 2.929506301879883, "margin_dpo/margin_std": 3.279536724090576, "step": 72 }, { "KL/chosen_KL_mean": -0.7138824462890625, "KL/mean": -2.2803893089294434, "KL/rejected_KL_mean": -3.846893310546875, "KL/std": 3.7853920459747314, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.1300668716430664, "fcm_dpo/delta": -0.012645280919969082, "fcm_dpo/margin": 3.1330132484436035, "fcm_dpo/q_t": 0.41098320484161377, "grad_norm": 32.395938873291016, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.2871820330619812, "logits/rejected": 0.02070710062980652, "logps/chosen": -96.48290252685547, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.77932739257812, "loss": 1.1051, "margin_dpo/margin_mean": 3.1330130100250244, "margin_dpo/margin_std": 4.819911003112793, "step": 73 }, { "KL/chosen_KL_mean": -0.9393234252929688, "KL/mean": -2.7251346111297607, "KL/rejected_KL_mean": -4.5109405517578125, "KL/std": 3.8043198585510254, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.11784079670906067, "fcm_dpo/delta": -0.040358271449804306, "fcm_dpo/margin": 3.5716192722320557, "fcm_dpo/q_t": 0.4008851945400238, "grad_norm": 33.6810188293457, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.2261185348033905, "logits/rejected": -0.08399015665054321, "logps/chosen": -79.74771881103516, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.15771484375, "loss": 1.0719, "margin_dpo/margin_mean": 3.571619987487793, "margin_dpo/margin_std": 4.459634780883789, "step": 74 }, { "KL/chosen_KL_mean": -1.0582923889160156, "KL/mean": -3.3414793014526367, "KL/rejected_KL_mean": -5.624664306640625, "KL/std": 4.560938835144043, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.10258522629737854, "fcm_dpo/delta": -0.12704604864120483, "fcm_dpo/margin": 4.566375732421875, "fcm_dpo/q_t": 0.384893536567688, "grad_norm": 32.54334259033203, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.3859766721725464, "logits/rejected": -0.1824646294116974, "logps/chosen": -79.34014892578125, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.02777862548828, "loss": 1.0443, "margin_dpo/margin_mean": 4.566375732421875, "margin_dpo/margin_std": 5.388866901397705, "step": 75 }, { "KL/chosen_KL_mean": -1.4658546447753906, "KL/mean": -2.9989418983459473, "KL/rejected_KL_mean": -4.532035827636719, "KL/std": 3.8019421100616455, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.1080910712480545, "fcm_dpo/delta": 0.0995396226644516, "fcm_dpo/margin": 3.0661802291870117, "fcm_dpo/q_t": 0.4300139844417572, "grad_norm": 24.346710205078125, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.3781723976135254, "logits/rejected": 0.02148185297846794, "logps/chosen": -89.3429946899414, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.25171661376953, "loss": 1.1362, "margin_dpo/margin_mean": 3.0661802291870117, "margin_dpo/margin_std": 4.518192291259766, "step": 76 }, { "KL/chosen_KL_mean": -1.5833091735839844, "KL/mean": -3.2625255584716797, "KL/rejected_KL_mean": -4.941745758056641, "KL/std": 4.407753944396973, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.11446798592805862, "fcm_dpo/delta": 0.024975329637527466, "fcm_dpo/margin": 3.3584322929382324, "fcm_dpo/q_t": 0.4146745502948761, "grad_norm": 30.046106338500977, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.28037023544311523, "logits/rejected": 0.07915257662534714, "logps/chosen": -101.28366088867188, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.14750671386719, "loss": 1.1142, "margin_dpo/margin_mean": 3.3584322929382324, "margin_dpo/margin_std": 4.9894208908081055, "step": 77 }, { "KL/chosen_KL_mean": -0.6632652282714844, "KL/mean": -2.478687286376953, "KL/rejected_KL_mean": -4.294105529785156, "KL/std": 3.666003704071045, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.11342023313045502, "fcm_dpo/delta": -0.020740443840622902, "fcm_dpo/margin": 3.6308388710021973, "fcm_dpo/q_t": 0.40385574102401733, "grad_norm": 26.898517608642578, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.003077227622270584, "logits/rejected": 0.22786158323287964, "logps/chosen": -90.95997619628906, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.43183135986328, "loss": 1.0717, "margin_dpo/margin_mean": 3.6308398246765137, "margin_dpo/margin_std": 4.37663459777832, "step": 78 }, { "KL/chosen_KL_mean": -0.8516883850097656, "KL/mean": -3.230681896209717, "KL/rejected_KL_mean": -5.60968017578125, "KL/std": 5.340993881225586, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.096573106944561, "fcm_dpo/delta": -0.11157047003507614, "fcm_dpo/margin": 4.757992267608643, "fcm_dpo/q_t": 0.3874128460884094, "grad_norm": 26.96648597717285, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.4559105634689331, "logits/rejected": -0.062162164598703384, "logps/chosen": -87.23001098632812, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.71745300292969, "loss": 1.0519, "margin_dpo/margin_mean": 4.757991790771484, "margin_dpo/margin_std": 5.928530216217041, "step": 79 }, { "KL/chosen_KL_mean": -1.5661354064941406, "KL/mean": -3.4469351768493652, "KL/rejected_KL_mean": -5.327735900878906, "KL/std": 4.70152473449707, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.09817157685756683, "fcm_dpo/delta": 0.04759423807263374, "fcm_dpo/margin": 3.7616002559661865, "fcm_dpo/q_t": 0.41906654834747314, "grad_norm": 25.286014556884766, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.631805419921875, "logits/rejected": -0.26980793476104736, "logps/chosen": -95.53645324707031, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.90215301513672, "loss": 1.1079, "margin_dpo/margin_mean": 3.7615997791290283, "margin_dpo/margin_std": 5.109871864318848, "step": 80 }, { "KL/chosen_KL_mean": -1.1272392272949219, "KL/mean": -3.7496933937072754, "KL/rejected_KL_mean": -6.372142791748047, "KL/std": 5.805624961853027, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.08225585520267487, "fcm_dpo/delta": -0.08061912655830383, "fcm_dpo/margin": 5.244898796081543, "fcm_dpo/q_t": 0.3973659873008728, "grad_norm": 25.511486053466797, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6071520447731018, "logits/rejected": -0.1843399852514267, "logps/chosen": -100.95736694335938, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.04461669921875, "loss": 1.0727, "margin_dpo/margin_mean": 5.244898796081543, "margin_dpo/margin_std": 6.860940933227539, "step": 81 }, { "KL/chosen_KL_mean": -1.9464454650878906, "KL/mean": -3.8053197860717773, "KL/rejected_KL_mean": -5.664192199707031, "KL/std": 4.5491180419921875, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.09726786613464355, "fcm_dpo/delta": 0.041504621505737305, "fcm_dpo/margin": 3.717745780944824, "fcm_dpo/q_t": 0.4179716408252716, "grad_norm": 25.725452423095703, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.3996829390525818, "logits/rejected": -0.1878240704536438, "logps/chosen": -84.992431640625, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.19013977050781, "loss": 1.1179, "margin_dpo/margin_mean": 3.717745304107666, "margin_dpo/margin_std": 4.97337532043457, "step": 82 }, { "KL/chosen_KL_mean": -2.303119659423828, "KL/mean": -4.41403865814209, "KL/rejected_KL_mean": -6.524959564208984, "KL/std": 5.163552761077881, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.09340175986289978, "fcm_dpo/delta": 0.009366540238261223, "fcm_dpo/margin": 4.2218451499938965, "fcm_dpo/q_t": 0.4130074381828308, "grad_norm": 25.119962692260742, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6331052184104919, "logits/rejected": -0.21508915722370148, "logps/chosen": -94.47933959960938, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.26960754394531, "loss": 1.1149, "margin_dpo/margin_mean": 4.221845626831055, "margin_dpo/margin_std": 6.553589820861816, "step": 83 }, { "KL/chosen_KL_mean": -2.957317352294922, "KL/mean": -4.249142646789551, "KL/rejected_KL_mean": -5.5409698486328125, "KL/std": 4.580222129821777, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.09570898115634918, "fcm_dpo/delta": 0.009214779362082481, "fcm_dpo/margin": 2.58364200592041, "fcm_dpo/q_t": 0.44408926367759705, "grad_norm": 27.248920440673828, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.47079262137413025, "logits/rejected": -0.0846925899386406, "logps/chosen": -104.48381042480469, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.96705627441406, "loss": 1.2106, "margin_dpo/margin_mean": 2.5836429595947266, "margin_dpo/margin_std": 4.866208076477051, "step": 84 }, { "KL/chosen_KL_mean": -2.583587646484375, "KL/mean": -4.313380241394043, "KL/rejected_KL_mean": -6.043170928955078, "KL/std": 5.026597023010254, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.10699915885925293, "fcm_dpo/delta": 0.03522737696766853, "fcm_dpo/margin": 3.4595859050750732, "fcm_dpo/q_t": 0.4191376268863678, "grad_norm": 29.348041534423828, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.19802260398864746, "logits/rejected": 0.14651966094970703, "logps/chosen": -98.71096801757812, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.09835815429688, "loss": 1.1525, "margin_dpo/margin_mean": 3.4595859050750732, "margin_dpo/margin_std": 6.121627330780029, "step": 85 }, { "KL/chosen_KL_mean": -2.015239715576172, "KL/mean": -4.2334442138671875, "KL/rejected_KL_mean": -6.451656341552734, "KL/std": 5.765687942504883, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.09496334940195084, "fcm_dpo/delta": -0.03938116133213043, "fcm_dpo/margin": 4.436418533325195, "fcm_dpo/q_t": 0.407038152217865, "grad_norm": 21.422603607177734, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.3676757216453552, "logits/rejected": -0.11725524067878723, "logps/chosen": -83.71949768066406, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.1071548461914, "loss": 1.105, "margin_dpo/margin_mean": 4.436418533325195, "margin_dpo/margin_std": 7.103404521942139, "step": 86 }, { "KL/chosen_KL_mean": -2.288616180419922, "KL/mean": -4.72440242767334, "KL/rejected_KL_mean": -7.160186767578125, "KL/std": 6.443795204162598, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.08640536665916443, "fcm_dpo/delta": -0.04145876318216324, "fcm_dpo/margin": 4.871577262878418, "fcm_dpo/q_t": 0.4013766646385193, "grad_norm": 23.141870498657227, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.5837273597717285, "logits/rejected": -0.20109756290912628, "logps/chosen": -94.97457885742188, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.078369140625, "loss": 1.1074, "margin_dpo/margin_mean": 4.871577262878418, "margin_dpo/margin_std": 7.593893527984619, "step": 87 }, { "KL/chosen_KL_mean": -2.8838882446289062, "KL/mean": -4.816040992736816, "KL/rejected_KL_mean": -6.748191833496094, "KL/std": 6.213381767272949, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.09165433794260025, "fcm_dpo/delta": 0.06771846115589142, "fcm_dpo/margin": 3.864302396774292, "fcm_dpo/q_t": 0.4247264266014099, "grad_norm": 23.00444221496582, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.1479032039642334, "logits/rejected": 0.04087015613913536, "logps/chosen": -103.64686584472656, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.8985595703125, "loss": 1.1495, "margin_dpo/margin_mean": 3.864302635192871, "margin_dpo/margin_std": 6.318605422973633, "step": 88 }, { "KL/chosen_KL_mean": -2.39990234375, "KL/mean": -4.45469856262207, "KL/rejected_KL_mean": -6.509494781494141, "KL/std": 5.354462623596191, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.09682156145572662, "fcm_dpo/delta": 0.0004574395716190338, "fcm_dpo/margin": 4.1095871925354, "fcm_dpo/q_t": 0.41555190086364746, "grad_norm": 23.261980056762695, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7489761710166931, "logits/rejected": -0.415987104177475, "logps/chosen": -92.09098815917969, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.07782745361328, "loss": 1.1268, "margin_dpo/margin_mean": 4.109586715698242, "margin_dpo/margin_std": 7.129188537597656, "step": 89 }, { "KL/chosen_KL_mean": -2.181621551513672, "KL/mean": -4.906613349914551, "KL/rejected_KL_mean": -7.631603240966797, "KL/std": 6.271796226501465, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.08614469319581985, "fcm_dpo/delta": -0.12924645841121674, "fcm_dpo/margin": 5.449975967407227, "fcm_dpo/q_t": 0.38764768838882446, "grad_norm": 24.51807403564453, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6348592638969421, "logits/rejected": -0.5053714513778687, "logps/chosen": -83.56417846679688, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.40874481201172, "loss": 1.0668, "margin_dpo/margin_mean": 5.449975967407227, "margin_dpo/margin_std": 7.5158891677856445, "step": 90 }, { "KL/chosen_KL_mean": -2.440296173095703, "KL/mean": -4.734485149383545, "KL/rejected_KL_mean": -7.0286712646484375, "KL/std": 6.132182598114014, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.08230043947696686, "fcm_dpo/delta": 0.034939397126436234, "fcm_dpo/margin": 4.588379859924316, "fcm_dpo/q_t": 0.418811172246933, "grad_norm": 21.96123504638672, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.39139896631240845, "logits/rejected": 0.023991946130990982, "logps/chosen": -99.66217803955078, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.00541687011719, "loss": 1.1213, "margin_dpo/margin_mean": 4.588379383087158, "margin_dpo/margin_std": 7.170990943908691, "step": 91 }, { "KL/chosen_KL_mean": -2.775127410888672, "KL/mean": -4.742379188537598, "KL/rejected_KL_mean": -6.709625244140625, "KL/std": 6.117337226867676, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.09213550388813019, "fcm_dpo/delta": 0.05504574626684189, "fcm_dpo/margin": 3.934500217437744, "fcm_dpo/q_t": 0.42271173000335693, "grad_norm": 22.343942642211914, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.9041601419448853, "logits/rejected": -0.5409650802612305, "logps/chosen": -87.30540466308594, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.21094512939453, "loss": 1.1382, "margin_dpo/margin_mean": 3.934500217437744, "margin_dpo/margin_std": 6.585569381713867, "step": 92 }, { "KL/chosen_KL_mean": -2.9287033081054688, "KL/mean": -4.876253128051758, "KL/rejected_KL_mean": -6.823810577392578, "KL/std": 5.769142150878906, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.09825095534324646, "fcm_dpo/delta": 0.025926675647497177, "fcm_dpo/margin": 3.895108699798584, "fcm_dpo/q_t": 0.4176117777824402, "grad_norm": 26.708560943603516, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.8640522956848145, "logits/rejected": -0.28465545177459717, "logps/chosen": -93.17913818359375, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.91802978515625, "loss": 1.1478, "margin_dpo/margin_mean": 3.895108938217163, "margin_dpo/margin_std": 6.972263813018799, "step": 93 }, { "KL/chosen_KL_mean": -3.0681533813476562, "KL/mean": -5.963137626647949, "KL/rejected_KL_mean": -8.85811996459961, "KL/std": 6.777492523193359, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.07449446618556976, "fcm_dpo/delta": -0.10843983292579651, "fcm_dpo/margin": 5.789971351623535, "fcm_dpo/q_t": 0.38898053765296936, "grad_norm": 21.819231033325195, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.6281750202178955, "logits/rejected": -0.3216380178928375, "logps/chosen": -95.52236938476562, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.10540771484375, "loss": 1.0858, "margin_dpo/margin_mean": 5.789970874786377, "margin_dpo/margin_std": 7.836843490600586, "step": 94 }, { "KL/chosen_KL_mean": -2.521240234375, "KL/mean": -4.840521812438965, "KL/rejected_KL_mean": -7.159797668457031, "KL/std": 5.73303747177124, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.08374801278114319, "fcm_dpo/delta": 0.01243816688656807, "fcm_dpo/margin": 4.63856315612793, "fcm_dpo/q_t": 0.41510820388793945, "grad_norm": 20.15675163269043, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.9239981174468994, "logits/rejected": -0.5899873971939087, "logps/chosen": -81.39494323730469, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.75100708007812, "loss": 1.1167, "margin_dpo/margin_mean": 4.638563632965088, "margin_dpo/margin_std": 7.088174819946289, "step": 95 }, { "KL/chosen_KL_mean": -2.2003860473632812, "KL/mean": -5.071413993835449, "KL/rejected_KL_mean": -7.942447662353516, "KL/std": 6.824997901916504, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.07336930930614471, "fcm_dpo/delta": -0.04164620861411095, "fcm_dpo/margin": 5.742067337036133, "fcm_dpo/q_t": 0.4016120433807373, "grad_norm": 19.50255584716797, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.8199041485786438, "logits/rejected": -0.37996238470077515, "logps/chosen": -108.79254913330078, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.71047973632812, "loss": 1.0793, "margin_dpo/margin_mean": 5.742067813873291, "margin_dpo/margin_std": 7.533588886260986, "step": 96 }, { "KL/chosen_KL_mean": -3.301239013671875, "KL/mean": -5.845839023590088, "KL/rejected_KL_mean": -8.390438079833984, "KL/std": 5.996274948120117, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.07868689298629761, "fcm_dpo/delta": -0.01612304151058197, "fcm_dpo/margin": 5.089199542999268, "fcm_dpo/q_t": 0.4053194224834442, "grad_norm": 22.824743270874023, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.32762932777404785, "logits/rejected": 0.09243258088827133, "logps/chosen": -91.98112487792969, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.20272827148438, "loss": 1.0945, "margin_dpo/margin_mean": 5.089199542999268, "margin_dpo/margin_std": 6.374382495880127, "step": 97 }, { "KL/chosen_KL_mean": -2.5313796997070312, "KL/mean": -5.4299702644348145, "KL/rejected_KL_mean": -8.328559875488281, "KL/std": 5.71711540222168, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.06853002309799194, "fcm_dpo/delta": -0.0008508451282978058, "fcm_dpo/margin": 5.797178268432617, "fcm_dpo/q_t": 0.40739738941192627, "grad_norm": 20.01097869873047, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.7634233832359314, "logits/rejected": -0.50087571144104, "logps/chosen": -94.77388000488281, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.8480453491211, "loss": 1.0756, "margin_dpo/margin_mean": 5.797178268432617, "margin_dpo/margin_std": 6.429886817932129, "step": 98 }, { "KL/chosen_KL_mean": -1.8896255493164062, "KL/mean": -5.149999618530273, "KL/rejected_KL_mean": -8.41037368774414, "KL/std": 6.963126182556152, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.06554701179265976, "fcm_dpo/delta": -0.04992110654711723, "fcm_dpo/margin": 6.520757675170898, "fcm_dpo/q_t": 0.4018503427505493, "grad_norm": 19.862838745117188, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.7117747068405151, "logits/rejected": -0.4982157349586487, "logps/chosen": -74.07427978515625, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.29169464111328, "loss": 1.0755, "margin_dpo/margin_mean": 6.520757675170898, "margin_dpo/margin_std": 8.837644577026367, "step": 99 }, { "KL/chosen_KL_mean": -2.253368377685547, "KL/mean": -4.568122386932373, "KL/rejected_KL_mean": -6.882881164550781, "KL/std": 5.245757102966309, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.07298842817544937, "fcm_dpo/delta": 0.09458544850349426, "fcm_dpo/margin": 4.629512786865234, "fcm_dpo/q_t": 0.4275546669960022, "grad_norm": 17.889829635620117, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.673098087310791, "logits/rejected": -0.18285231292247772, "logps/chosen": -91.76719665527344, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -88.10000610351562, "loss": 1.1202, "margin_dpo/margin_mean": 4.629512310028076, "margin_dpo/margin_std": 5.659436225891113, "step": 100 }, { "KL/chosen_KL_mean": -3.587017059326172, "KL/mean": -5.697845458984375, "KL/rejected_KL_mean": -7.808677673339844, "KL/std": 6.820420742034912, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.08356277644634247, "fcm_dpo/delta": 0.07195214927196503, "fcm_dpo/margin": 4.2216620445251465, "fcm_dpo/q_t": 0.4244743585586548, "grad_norm": 23.93421173095703, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.7810462117195129, "logits/rejected": -0.5088837146759033, "logps/chosen": -97.9902114868164, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.71868896484375, "loss": 1.1597, "margin_dpo/margin_mean": 4.221662998199463, "margin_dpo/margin_std": 7.278441905975342, "step": 101 }, { "KL/chosen_KL_mean": -4.171089172363281, "KL/mean": -5.98170280456543, "KL/rejected_KL_mean": -7.7923126220703125, "KL/std": 6.601472854614258, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.08549235761165619, "fcm_dpo/delta": -0.05695316195487976, "fcm_dpo/margin": 3.621224880218506, "fcm_dpo/q_t": 0.4275171160697937, "grad_norm": 30.952787399291992, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.8864800930023193, "logits/rejected": -0.5173342227935791, "logps/chosen": -95.780517578125, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.56123352050781, "loss": 1.2068, "margin_dpo/margin_mean": 3.621225357055664, "margin_dpo/margin_std": 7.491289138793945, "step": 102 }, { "KL/chosen_KL_mean": -3.27850341796875, "KL/mean": -6.320806980133057, "KL/rejected_KL_mean": -9.363113403320312, "KL/std": 6.95861291885376, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.07105561345815659, "fcm_dpo/delta": -0.06375257670879364, "fcm_dpo/margin": 6.084613800048828, "fcm_dpo/q_t": 0.3953350782394409, "grad_norm": 20.111555099487305, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.9166337847709656, "logits/rejected": -0.5314723253250122, "logps/chosen": -100.68089294433594, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.34480285644531, "loss": 1.0737, "margin_dpo/margin_mean": 6.084613800048828, "margin_dpo/margin_std": 7.855329513549805, "step": 103 }, { "KL/chosen_KL_mean": -2.510406494140625, "KL/mean": -5.807180881500244, "KL/rejected_KL_mean": -9.103954315185547, "KL/std": 7.632823467254639, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.06462949514389038, "fcm_dpo/delta": -0.0488460548222065, "fcm_dpo/margin": 6.5935468673706055, "fcm_dpo/q_t": 0.40189439058303833, "grad_norm": 16.286401748657227, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.1864449977874756, "logits/rejected": -0.8508994579315186, "logps/chosen": -97.36676788330078, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -106.13064575195312, "loss": 1.0787, "margin_dpo/margin_mean": 6.5935468673706055, "margin_dpo/margin_std": 9.117274284362793, "step": 104 }, { "KL/chosen_KL_mean": -3.354045867919922, "KL/mean": -6.631870269775391, "KL/rejected_KL_mean": -9.909698486328125, "KL/std": 6.9054059982299805, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.06405629217624664, "fcm_dpo/delta": -0.038040630519390106, "fcm_dpo/margin": 6.55565071105957, "fcm_dpo/q_t": 0.4008115530014038, "grad_norm": 19.590778350830078, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.1231505870819092, "logits/rejected": -0.6537094116210938, "logps/chosen": -100.11194610595703, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.48719024658203, "loss": 1.0849, "margin_dpo/margin_mean": 6.55565071105957, "margin_dpo/margin_std": 8.63304328918457, "step": 105 }, { "KL/chosen_KL_mean": -3.3176002502441406, "KL/mean": -6.45479154586792, "KL/rejected_KL_mean": -9.59197998046875, "KL/std": 7.140379428863525, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.06256508827209473, "fcm_dpo/delta": 0.008652102202177048, "fcm_dpo/margin": 6.274376392364502, "fcm_dpo/q_t": 0.4120129346847534, "grad_norm": 16.581937789916992, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.6825619339942932, "logits/rejected": -0.2394205629825592, "logps/chosen": -95.75550842285156, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.6016845703125, "loss": 1.1002, "margin_dpo/margin_mean": 6.274375915527344, "margin_dpo/margin_std": 8.72793197631836, "step": 106 }, { "KL/chosen_KL_mean": -2.509490966796875, "KL/mean": -6.172967910766602, "KL/rejected_KL_mean": -9.836444854736328, "KL/std": 9.088972091674805, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.05734935402870178, "fcm_dpo/delta": -0.03532847389578819, "fcm_dpo/margin": 7.3269572257995605, "fcm_dpo/q_t": 0.40803956985473633, "grad_norm": 16.3721923828125, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.8719568848609924, "logits/rejected": -0.3563387393951416, "logps/chosen": -107.6344985961914, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -111.0189437866211, "loss": 1.0956, "margin_dpo/margin_mean": 7.326956748962402, "margin_dpo/margin_std": 11.279912948608398, "step": 107 }, { "KL/chosen_KL_mean": -3.6375465393066406, "KL/mean": -7.143028259277344, "KL/rejected_KL_mean": -10.648506164550781, "KL/std": 8.483004570007324, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.05558161064982414, "fcm_dpo/delta": 0.013727772980928421, "fcm_dpo/margin": 7.010951995849609, "fcm_dpo/q_t": 0.4124237298965454, "grad_norm": 17.105913162231445, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.1664843559265137, "logits/rejected": -0.6710296869277954, "logps/chosen": -92.84690856933594, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -112.48635864257812, "loss": 1.0977, "margin_dpo/margin_mean": 7.010951519012451, "margin_dpo/margin_std": 9.290668487548828, "step": 108 }, { "KL/chosen_KL_mean": -3.01806640625, "KL/mean": -7.291872978210449, "KL/rejected_KL_mean": -11.565685272216797, "KL/std": 8.555709838867188, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.05080903321504593, "fcm_dpo/delta": -0.06821153312921524, "fcm_dpo/margin": 8.547622680664062, "fcm_dpo/q_t": 0.3951244056224823, "grad_norm": 15.48365592956543, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.6874870657920837, "logits/rejected": -0.45433491468429565, "logps/chosen": -103.20515441894531, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -115.10308074951172, "loss": 1.0644, "margin_dpo/margin_mean": 8.547621726989746, "margin_dpo/margin_std": 10.667652130126953, "step": 109 }, { "KL/chosen_KL_mean": -3.960887908935547, "KL/mean": -6.505597114562988, "KL/rejected_KL_mean": -9.050308227539062, "KL/std": 7.199162483215332, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.053726520389318466, "fcm_dpo/delta": 0.028826363384723663, "fcm_dpo/margin": 5.089425086975098, "fcm_dpo/q_t": 0.43810567259788513, "grad_norm": 15.963815689086914, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.36268898844718933, "logits/rejected": 0.05319015681743622, "logps/chosen": -110.85660552978516, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.86151123046875, "loss": 1.1782, "margin_dpo/margin_mean": 5.089425086975098, "margin_dpo/margin_std": 8.119990348815918, "step": 110 }, { "KL/chosen_KL_mean": -3.4428558349609375, "KL/mean": -8.529221534729004, "KL/rejected_KL_mean": -13.615596771240234, "KL/std": 10.005598068237305, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.043472886085510254, "fcm_dpo/delta": -0.09939664602279663, "fcm_dpo/margin": 10.172739028930664, "fcm_dpo/q_t": 0.38764622807502747, "grad_norm": 14.34097671508789, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.9756143689155579, "logits/rejected": -0.6840360760688782, "logps/chosen": -96.77222442626953, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -117.79308319091797, "loss": 1.049, "margin_dpo/margin_mean": 10.172739028930664, "margin_dpo/margin_std": 11.215787887573242, "step": 111 }, { "KL/chosen_KL_mean": -4.026702880859375, "KL/mean": -6.584530830383301, "KL/rejected_KL_mean": -9.142356872558594, "KL/std": 5.938052654266357, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.05586817488074303, "fcm_dpo/delta": 0.1577061116695404, "fcm_dpo/margin": 5.115651607513428, "fcm_dpo/q_t": 0.44224512577056885, "grad_norm": 15.161541938781738, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.3096747398376465, "logits/rejected": -0.7946876883506775, "logps/chosen": -86.77374267578125, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -80.7008056640625, "loss": 1.1725, "margin_dpo/margin_mean": 5.115652561187744, "margin_dpo/margin_std": 8.12076187133789, "step": 112 }, { "KL/chosen_KL_mean": -5.2009735107421875, "KL/mean": -7.954853057861328, "KL/rejected_KL_mean": -10.708736419677734, "KL/std": 7.948338985443115, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.066313236951828, "fcm_dpo/delta": 0.05253490433096886, "fcm_dpo/margin": 5.507757186889648, "fcm_dpo/q_t": 0.41995012760162354, "grad_norm": 21.64862060546875, "learning_rate": 4.939343162231841e-07, "logits/chosen": -1.0535515546798706, "logits/rejected": -0.5135017037391663, "logps/chosen": -103.1004638671875, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -95.12833404541016, "loss": 1.1194, "margin_dpo/margin_mean": 5.507758140563965, "margin_dpo/margin_std": 7.776273727416992, "step": 113 }, { "KL/chosen_KL_mean": -4.860542297363281, "KL/mean": -8.802154541015625, "KL/rejected_KL_mean": -12.743770599365234, "KL/std": 9.23103141784668, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.05586257204413414, "fcm_dpo/delta": -0.08835941553115845, "fcm_dpo/margin": 7.883230209350586, "fcm_dpo/q_t": 0.3976415693759918, "grad_norm": 17.95973777770996, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.2103087902069092, "logits/rejected": -0.6274843811988831, "logps/chosen": -101.35565948486328, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -104.7591323852539, "loss": 1.0965, "margin_dpo/margin_mean": 7.883230209350586, "margin_dpo/margin_std": 12.131425857543945, "step": 114 }, { "KL/chosen_KL_mean": -4.974128723144531, "KL/mean": -8.982112884521484, "KL/rejected_KL_mean": -12.990097045898438, "KL/std": 9.756691932678223, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.053034596145153046, "fcm_dpo/delta": -0.043645843863487244, "fcm_dpo/margin": 8.015962600708008, "fcm_dpo/q_t": 0.40094977617263794, "grad_norm": 15.409124374389648, "learning_rate": 4.933595135901732e-07, "logits/chosen": -1.0849108695983887, "logits/rejected": -0.6720914840698242, "logps/chosen": -108.15425109863281, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -118.7060546875, "loss": 1.0804, "margin_dpo/margin_mean": 8.015962600708008, "margin_dpo/margin_std": 10.678518295288086, "step": 115 }, { "KL/chosen_KL_mean": -4.48321533203125, "KL/mean": -7.827206611633301, "KL/rejected_KL_mean": -11.171195983886719, "KL/std": 7.723655700683594, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.05637132748961449, "fcm_dpo/delta": 0.028978653252124786, "fcm_dpo/margin": 6.687981605529785, "fcm_dpo/q_t": 0.41488900780677795, "grad_norm": 15.676658630371094, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.037459135055542, "logits/rejected": -0.597579300403595, "logps/chosen": -85.64158630371094, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -97.47705078125, "loss": 1.1036, "margin_dpo/margin_mean": 6.687982082366943, "margin_dpo/margin_std": 8.76745891571045, "step": 116 }, { "KL/chosen_KL_mean": -5.44281005859375, "KL/mean": -8.042337417602539, "KL/rejected_KL_mean": -10.641864776611328, "KL/std": 7.842460632324219, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.0556965172290802, "fcm_dpo/delta": 0.0101877236738801, "fcm_dpo/margin": 5.1990509033203125, "fcm_dpo/q_t": 0.43383270502090454, "grad_norm": 22.66619873046875, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.9247353076934814, "logits/rejected": -0.5050001740455627, "logps/chosen": -102.52376556396484, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -85.78178405761719, "loss": 1.1886, "margin_dpo/margin_mean": 5.1990509033203125, "margin_dpo/margin_std": 9.762657165527344, "step": 117 }, { "KL/chosen_KL_mean": -4.044807434082031, "KL/mean": -7.908073425292969, "KL/rejected_KL_mean": -11.77133560180664, "KL/std": 8.614623069763184, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.053251735866069794, "fcm_dpo/delta": -0.020551854744553566, "fcm_dpo/margin": 7.726527214050293, "fcm_dpo/q_t": 0.4028622806072235, "grad_norm": 14.594022750854492, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.2256286144256592, "logits/rejected": -0.885383129119873, "logps/chosen": -90.58103942871094, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -94.70697784423828, "loss": 1.0714, "margin_dpo/margin_mean": 7.726527214050293, "margin_dpo/margin_std": 9.139314651489258, "step": 118 }, { "KL/chosen_KL_mean": -5.904273986816406, "KL/mean": -10.019554138183594, "KL/rejected_KL_mean": -14.13482666015625, "KL/std": 10.321023941040039, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.050140902400016785, "fcm_dpo/delta": -0.02478652074933052, "fcm_dpo/margin": 8.230552673339844, "fcm_dpo/q_t": 0.40361571311950684, "grad_norm": 15.639044761657715, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.3471566438674927, "logits/rejected": -1.0207691192626953, "logps/chosen": -110.51335144042969, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -123.29263305664062, "loss": 1.1124, "margin_dpo/margin_mean": 8.230552673339844, "margin_dpo/margin_std": 13.031841278076172, "step": 119 }, { "KL/chosen_KL_mean": -4.916339874267578, "KL/mean": -10.255768775939941, "KL/rejected_KL_mean": -15.595199584960938, "KL/std": 12.216974258422852, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.042514435946941376, "fcm_dpo/delta": -0.10630162805318832, "fcm_dpo/margin": 10.678858757019043, "fcm_dpo/q_t": 0.3900836110115051, "grad_norm": 15.349175453186035, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.3326067924499512, "logits/rejected": -0.7651596069335938, "logps/chosen": -119.60376739501953, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -126.78106689453125, "loss": 1.0797, "margin_dpo/margin_mean": 10.678857803344727, "margin_dpo/margin_std": 15.547449111938477, "step": 120 }, { "KL/chosen_KL_mean": -4.476264953613281, "KL/mean": -9.392608642578125, "KL/rejected_KL_mean": -14.308952331542969, "KL/std": 9.628084182739258, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.040236130356788635, "fcm_dpo/delta": 0.005083698779344559, "fcm_dpo/margin": 9.832690238952637, "fcm_dpo/q_t": 0.40753328800201416, "grad_norm": 15.026124000549316, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.026558518409729, "logits/rejected": -0.6895172595977783, "logps/chosen": -96.59292602539062, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -129.2818603515625, "loss": 1.0745, "margin_dpo/margin_mean": 9.832690238952637, "margin_dpo/margin_std": 10.782829284667969, "step": 121 }, { "KL/chosen_KL_mean": -4.757698059082031, "KL/mean": -10.397573471069336, "KL/rejected_KL_mean": -16.03744125366211, "KL/std": 10.39601993560791, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.03893639147281647, "fcm_dpo/delta": -0.06997775286436081, "fcm_dpo/margin": 11.279745101928711, "fcm_dpo/q_t": 0.391292929649353, "grad_norm": 15.583998680114746, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.0844228267669678, "logits/rejected": -0.9956216812133789, "logps/chosen": -89.63526916503906, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -127.12042236328125, "loss": 1.0506, "margin_dpo/margin_mean": 11.279744148254395, "margin_dpo/margin_std": 12.197562217712402, "step": 122 }, { "KL/chosen_KL_mean": -5.5025177001953125, "KL/mean": -9.58072280883789, "KL/rejected_KL_mean": -13.658920288085938, "KL/std": 9.666169166564941, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.03678719326853752, "fcm_dpo/delta": 0.009150171652436256, "fcm_dpo/margin": 8.156402587890625, "fcm_dpo/q_t": 0.42984551191329956, "grad_norm": 12.242349624633789, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.368896484375, "logits/rejected": -1.0506664514541626, "logps/chosen": -103.98126220703125, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -107.51309204101562, "loss": 1.1518, "margin_dpo/margin_mean": 8.156400680541992, "margin_dpo/margin_std": 11.327975273132324, "step": 123 }, { "KL/chosen_KL_mean": -5.303264617919922, "KL/mean": -10.395425796508789, "KL/rejected_KL_mean": -15.487586975097656, "KL/std": 12.138213157653809, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.03810625150799751, "fcm_dpo/delta": 0.019090991467237473, "fcm_dpo/margin": 10.184322357177734, "fcm_dpo/q_t": 0.4156780242919922, "grad_norm": 12.89729118347168, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.2245172262191772, "logits/rejected": -0.9596558809280396, "logps/chosen": -108.066162109375, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -138.49319458007812, "loss": 1.1154, "margin_dpo/margin_mean": 10.184322357177734, "margin_dpo/margin_std": 15.758091926574707, "step": 124 }, { "KL/chosen_KL_mean": -6.696804046630859, "KL/mean": -10.318450927734375, "KL/rejected_KL_mean": -13.94009780883789, "KL/std": 9.99661636352539, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.04502631723880768, "fcm_dpo/delta": 0.1080935001373291, "fcm_dpo/margin": 7.243290424346924, "fcm_dpo/q_t": 0.4325026571750641, "grad_norm": 15.055885314941406, "learning_rate": 4.90102044194588e-07, "logits/chosen": -1.0587656497955322, "logits/rejected": -0.8336589336395264, "logps/chosen": -97.12307739257812, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -103.18911743164062, "loss": 1.1618, "margin_dpo/margin_mean": 7.243290901184082, "margin_dpo/margin_std": 12.319543838500977, "step": 125 }, { "KL/chosen_KL_mean": -5.79815673828125, "KL/mean": -11.07423210144043, "KL/rejected_KL_mean": -16.350311279296875, "KL/std": 10.801017761230469, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.04195898771286011, "fcm_dpo/delta": -0.07875122129917145, "fcm_dpo/margin": 10.552156448364258, "fcm_dpo/q_t": 0.3898536264896393, "grad_norm": 13.708854675292969, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.1427385807037354, "logits/rejected": -0.8579765558242798, "logps/chosen": -100.22103881835938, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -120.23187255859375, "loss": 1.0455, "margin_dpo/margin_mean": 10.552155494689941, "margin_dpo/margin_std": 11.419673919677734, "step": 126 }, { "KL/chosen_KL_mean": -5.795806884765625, "KL/mean": -9.664770126342773, "KL/rejected_KL_mean": -13.533729553222656, "KL/std": 9.839605331420898, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.04595526307821274, "fcm_dpo/delta": 0.06270049512386322, "fcm_dpo/margin": 7.737924575805664, "fcm_dpo/q_t": 0.42338302731513977, "grad_norm": 15.248723030090332, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.8338533639907837, "logits/rejected": -0.46623650193214417, "logps/chosen": -90.54685974121094, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -97.07896423339844, "loss": 1.1368, "margin_dpo/margin_mean": 7.737924575805664, "margin_dpo/margin_std": 12.432262420654297, "step": 127 }, { "KL/chosen_KL_mean": -5.760974884033203, "KL/mean": -9.829865455627441, "KL/rejected_KL_mean": -13.898752212524414, "KL/std": 12.265727043151855, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.04878094047307968, "fcm_dpo/delta": -0.006075307726860046, "fcm_dpo/margin": 8.137781143188477, "fcm_dpo/q_t": 0.41133010387420654, "grad_norm": 18.62059211730957, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7571243643760681, "logits/rejected": -0.2513189911842346, "logps/chosen": -96.380615234375, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -97.00222778320312, "loss": 1.1517, "margin_dpo/margin_mean": 8.137781143188477, "margin_dpo/margin_std": 15.487457275390625, "step": 128 }, { "KL/chosen_KL_mean": -6.416015625, "KL/mean": -10.719934463500977, "KL/rejected_KL_mean": -15.023857116699219, "KL/std": 10.065567016601562, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.04658256098628044, "fcm_dpo/delta": -0.004061141982674599, "fcm_dpo/margin": 8.607840538024902, "fcm_dpo/q_t": 0.41314199566841125, "grad_norm": 14.07802677154541, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.9190844297409058, "logits/rejected": -0.6032007932662964, "logps/chosen": -105.33472442626953, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -118.30805206298828, "loss": 1.1228, "margin_dpo/margin_mean": 8.607840538024902, "margin_dpo/margin_std": 13.97426986694336, "step": 129 }, { "KL/chosen_KL_mean": -4.968711853027344, "KL/mean": -10.5032958984375, "KL/rejected_KL_mean": -16.037883758544922, "KL/std": 11.347689628601074, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.04155845195055008, "fcm_dpo/delta": -0.11511799693107605, "fcm_dpo/margin": 11.06917667388916, "fcm_dpo/q_t": 0.3876907527446747, "grad_norm": 12.400670051574707, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.5239005088806152, "logits/rejected": -1.2929034233093262, "logps/chosen": -78.15240478515625, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -104.0594253540039, "loss": 1.0643, "margin_dpo/margin_mean": 11.06917667388916, "margin_dpo/margin_std": 13.685577392578125, "step": 130 }, { "KL/chosen_KL_mean": -4.372795104980469, "KL/mean": -9.515098571777344, "KL/rejected_KL_mean": -14.65740966796875, "KL/std": 11.49531364440918, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.03821235150098801, "fcm_dpo/delta": 0.0009900815784931183, "fcm_dpo/margin": 10.2846097946167, "fcm_dpo/q_t": 0.4084089994430542, "grad_norm": 14.637885093688965, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.2388590574264526, "logits/rejected": -1.027995228767395, "logps/chosen": -78.32923889160156, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -109.11323547363281, "loss": 1.0985, "margin_dpo/margin_mean": 10.284608840942383, "margin_dpo/margin_std": 13.239768981933594, "step": 131 }, { "KL/chosen_KL_mean": -6.347705841064453, "KL/mean": -11.329395294189453, "KL/rejected_KL_mean": -16.311073303222656, "KL/std": 12.13582992553711, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.03723342716693878, "fcm_dpo/delta": 0.04552838206291199, "fcm_dpo/margin": 9.963367462158203, "fcm_dpo/q_t": 0.41745656728744507, "grad_norm": 13.612761497497559, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.5247846841812134, "logits/rejected": -1.1042265892028809, "logps/chosen": -98.78718566894531, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -110.44358825683594, "loss": 1.1186, "margin_dpo/margin_mean": 9.963367462158203, "margin_dpo/margin_std": 14.255733489990234, "step": 132 }, { "KL/chosen_KL_mean": -5.533168792724609, "KL/mean": -10.78897476196289, "KL/rejected_KL_mean": -16.044780731201172, "KL/std": 12.062196731567383, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.03879861533641815, "fcm_dpo/delta": -0.013259888626635075, "fcm_dpo/margin": 10.511611938476562, "fcm_dpo/q_t": 0.4055735766887665, "grad_norm": 12.96779727935791, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.7443186044692993, "logits/rejected": -1.351607322692871, "logps/chosen": -98.64982604980469, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -118.13969421386719, "loss": 1.0767, "margin_dpo/margin_mean": 10.511611938476562, "margin_dpo/margin_std": 12.886077880859375, "step": 133 }, { "KL/chosen_KL_mean": -5.651973724365234, "KL/mean": -11.295485496520996, "KL/rejected_KL_mean": -16.938995361328125, "KL/std": 11.309652328491211, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.0380212664604187, "fcm_dpo/delta": -0.060371000319719315, "fcm_dpo/margin": 11.287027359008789, "fcm_dpo/q_t": 0.3944806456565857, "grad_norm": 12.810723304748535, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.9120519161224365, "logits/rejected": -1.5820496082305908, "logps/chosen": -89.20210266113281, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -110.82056427001953, "loss": 1.0685, "margin_dpo/margin_mean": 11.287027359008789, "margin_dpo/margin_std": 12.427013397216797, "step": 134 }, { "KL/chosen_KL_mean": -5.205713272094727, "KL/mean": -11.02509880065918, "KL/rejected_KL_mean": -16.84449005126953, "KL/std": 12.661831855773926, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.03450964391231537, "fcm_dpo/delta": -0.004188090562820435, "fcm_dpo/margin": 11.638765335083008, "fcm_dpo/q_t": 0.40880876779556274, "grad_norm": 15.700349807739258, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.5839893817901611, "logits/rejected": -1.4693738222122192, "logps/chosen": -82.78833770751953, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -137.90097045898438, "loss": 1.0839, "margin_dpo/margin_mean": 11.63876724243164, "margin_dpo/margin_std": 14.422992706298828, "step": 135 }, { "KL/chosen_KL_mean": -6.113304138183594, "KL/mean": -11.65345573425293, "KL/rejected_KL_mean": -17.193607330322266, "KL/std": 12.955411911010742, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.03477206081151962, "fcm_dpo/delta": 0.02389763481914997, "fcm_dpo/margin": 11.080307006835938, "fcm_dpo/q_t": 0.4129774868488312, "grad_norm": 11.775751113891602, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.3690800666809082, "logits/rejected": -0.9234004020690918, "logps/chosen": -100.28604125976562, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -120.45790100097656, "loss": 1.0883, "margin_dpo/margin_mean": 11.080307006835938, "margin_dpo/margin_std": 13.611349105834961, "step": 136 }, { "KL/chosen_KL_mean": -4.842937469482422, "KL/mean": -13.751588821411133, "KL/rejected_KL_mean": -22.660240173339844, "KL/std": 16.79273223876953, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.025849711149930954, "fcm_dpo/delta": -0.1551552712917328, "fcm_dpo/margin": 17.817310333251953, "fcm_dpo/q_t": 0.37635451555252075, "grad_norm": 13.438129425048828, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.374749779701233, "logits/rejected": -1.1677782535552979, "logps/chosen": -82.96167755126953, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -140.58261108398438, "loss": 1.0308, "margin_dpo/margin_mean": 17.817310333251953, "margin_dpo/margin_std": 18.643211364746094, "step": 137 }, { "KL/chosen_KL_mean": -8.034114837646484, "KL/mean": -13.342679977416992, "KL/rejected_KL_mean": -18.6512451171875, "KL/std": 12.085899353027344, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.03047574684023857, "fcm_dpo/delta": 0.11220944672822952, "fcm_dpo/margin": 10.617134094238281, "fcm_dpo/q_t": 0.4314655065536499, "grad_norm": 10.797048568725586, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.1327121257781982, "logits/rejected": -0.8610665798187256, "logps/chosen": -100.86405944824219, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -117.29965209960938, "loss": 1.1313, "margin_dpo/margin_mean": 10.617134094238281, "margin_dpo/margin_std": 13.671524047851562, "step": 138 }, { "KL/chosen_KL_mean": -5.431125640869141, "KL/mean": -11.679722785949707, "KL/rejected_KL_mean": -17.928321838378906, "KL/std": 12.955331802368164, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.03182750567793846, "fcm_dpo/delta": 0.0027314499020576477, "fcm_dpo/margin": 12.497198104858398, "fcm_dpo/q_t": 0.4064083397388458, "grad_norm": 11.478351593017578, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.4794654846191406, "logits/rejected": -1.1529626846313477, "logps/chosen": -84.43414306640625, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -103.75565338134766, "loss": 1.0858, "margin_dpo/margin_mean": 12.497198104858398, "margin_dpo/margin_std": 15.660654067993164, "step": 139 }, { "KL/chosen_KL_mean": -6.115200042724609, "KL/mean": -11.783124923706055, "KL/rejected_KL_mean": -17.451045989990234, "KL/std": 12.226581573486328, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.03398571163415909, "fcm_dpo/delta": 0.023316429927945137, "fcm_dpo/margin": 11.33585262298584, "fcm_dpo/q_t": 0.41158053278923035, "grad_norm": 11.591765403747559, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.6476720571517944, "logits/rejected": -1.2275609970092773, "logps/chosen": -94.4676513671875, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -108.02581787109375, "loss": 1.0856, "margin_dpo/margin_mean": 11.335851669311523, "margin_dpo/margin_std": 13.271905899047852, "step": 140 }, { "KL/chosen_KL_mean": -8.513824462890625, "KL/mean": -14.797035217285156, "KL/rejected_KL_mean": -21.080245971679688, "KL/std": 16.665454864501953, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.03299643099308014, "fcm_dpo/delta": -0.024980634450912476, "fcm_dpo/margin": 12.566429138183594, "fcm_dpo/q_t": 0.4018610715866089, "grad_norm": 14.086519241333008, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.5442805290222168, "logits/rejected": -1.1464200019836426, "logps/chosen": -115.7339859008789, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -125.79248046875, "loss": 1.0863, "margin_dpo/margin_mean": 12.566429138183594, "margin_dpo/margin_std": 17.020280838012695, "step": 141 }, { "KL/chosen_KL_mean": -8.151092529296875, "KL/mean": -14.464200019836426, "KL/rejected_KL_mean": -20.77730941772461, "KL/std": 15.21985149383545, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.031740929931402206, "fcm_dpo/delta": -0.00217352993786335, "fcm_dpo/margin": 12.626214981079102, "fcm_dpo/q_t": 0.4086623191833496, "grad_norm": 10.850831985473633, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.2176506519317627, "logits/rejected": -0.9520937204360962, "logps/chosen": -96.88003540039062, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -120.5076904296875, "loss": 1.0827, "margin_dpo/margin_mean": 12.626214981079102, "margin_dpo/margin_std": 15.937889099121094, "step": 142 }, { "KL/chosen_KL_mean": -6.4573822021484375, "KL/mean": -11.523691177368164, "KL/rejected_KL_mean": -16.58999252319336, "KL/std": 12.487195014953613, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.03482381999492645, "fcm_dpo/delta": 0.07284322381019592, "fcm_dpo/margin": 10.132607460021973, "fcm_dpo/q_t": 0.4231829345226288, "grad_norm": 12.200576782226562, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.993595838546753, "logits/rejected": -1.5552196502685547, "logps/chosen": -91.96195983886719, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -102.3956527709961, "loss": 1.1211, "margin_dpo/margin_mean": 10.132606506347656, "margin_dpo/margin_std": 13.62544059753418, "step": 143 }, { "KL/chosen_KL_mean": -8.030925750732422, "KL/mean": -12.933025360107422, "KL/rejected_KL_mean": -17.835132598876953, "KL/std": 13.279560089111328, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.04014896601438522, "fcm_dpo/delta": -0.004362419247627258, "fcm_dpo/margin": 9.804203033447266, "fcm_dpo/q_t": 0.41037437319755554, "grad_norm": 19.410930633544922, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.4917770624160767, "logits/rejected": -0.9209951162338257, "logps/chosen": -112.718017578125, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -105.55421447753906, "loss": 1.141, "margin_dpo/margin_mean": 9.804203033447266, "margin_dpo/margin_std": 16.516376495361328, "step": 144 }, { "KL/chosen_KL_mean": -6.489265441894531, "KL/mean": -12.535322189331055, "KL/rejected_KL_mean": -18.581382751464844, "KL/std": 12.56440544128418, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.03485546261072159, "fcm_dpo/delta": -0.037631649523973465, "fcm_dpo/margin": 12.092121124267578, "fcm_dpo/q_t": 0.3992162346839905, "grad_norm": 12.388144493103027, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.3073782920837402, "logits/rejected": -0.7306171655654907, "logps/chosen": -95.0091552734375, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -102.34652709960938, "loss": 1.0658, "margin_dpo/margin_mean": 12.092122077941895, "margin_dpo/margin_std": 14.239847183227539, "step": 145 }, { "KL/chosen_KL_mean": -7.197425842285156, "KL/mean": -13.812812805175781, "KL/rejected_KL_mean": -20.42819595336914, "KL/std": 13.7945556640625, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.031101997941732407, "fcm_dpo/delta": -0.03021891415119171, "fcm_dpo/margin": 13.230766296386719, "fcm_dpo/q_t": 0.4005948305130005, "grad_norm": 11.479935646057129, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.8465049266815186, "logits/rejected": -1.1958274841308594, "logps/chosen": -110.10328674316406, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -109.3775634765625, "loss": 1.0731, "margin_dpo/margin_mean": 13.230766296386719, "margin_dpo/margin_std": 15.447964668273926, "step": 146 }, { "KL/chosen_KL_mean": -6.8497314453125, "KL/mean": -13.66101360321045, "KL/rejected_KL_mean": -20.472305297851562, "KL/std": 13.01979923248291, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.030813829973340034, "fcm_dpo/delta": -0.034091509878635406, "fcm_dpo/margin": 13.622573852539062, "fcm_dpo/q_t": 0.3990829586982727, "grad_norm": 10.852170944213867, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.5063129663467407, "logits/rejected": -0.9073103070259094, "logps/chosen": -96.11192321777344, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -96.97241973876953, "loss": 1.055, "margin_dpo/margin_mean": 13.622573852539062, "margin_dpo/margin_std": 14.30087661743164, "step": 147 }, { "KL/chosen_KL_mean": -5.16876220703125, "KL/mean": -11.777588844299316, "KL/rejected_KL_mean": -18.38642120361328, "KL/std": 14.05976676940918, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.03130657970905304, "fcm_dpo/delta": -0.039390355348587036, "fcm_dpo/margin": 13.217658996582031, "fcm_dpo/q_t": 0.4031597673892975, "grad_norm": 11.478275299072266, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.7876176834106445, "logits/rejected": -1.3963440656661987, "logps/chosen": -96.931640625, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -128.77194213867188, "loss": 1.0857, "margin_dpo/margin_mean": 13.217658042907715, "margin_dpo/margin_std": 16.112396240234375, "step": 148 }, { "KL/chosen_KL_mean": -8.569469451904297, "KL/mean": -14.707597732543945, "KL/rejected_KL_mean": -20.845718383789062, "KL/std": 14.904071807861328, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.029475510120391846, "fcm_dpo/delta": 0.059270065277814865, "fcm_dpo/margin": 12.276247024536133, "fcm_dpo/q_t": 0.4234876334667206, "grad_norm": 11.138805389404297, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.343320608139038, "logits/rejected": -0.999596118927002, "logps/chosen": -102.05348205566406, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -126.48941802978516, "loss": 1.1305, "margin_dpo/margin_mean": 12.276248931884766, "margin_dpo/margin_std": 18.941082000732422, "step": 149 }, { "KL/chosen_KL_mean": -6.826591491699219, "KL/mean": -13.331616401672363, "KL/rejected_KL_mean": -19.836639404296875, "KL/std": 15.675052642822266, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.030585436150431633, "fcm_dpo/delta": 0.0013244133442640305, "fcm_dpo/margin": 13.010046005249023, "fcm_dpo/q_t": 0.4092606008052826, "grad_norm": 10.162281036376953, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.4606035947799683, "logits/rejected": -1.2021255493164062, "logps/chosen": -82.60527801513672, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -109.8634033203125, "loss": 1.0932, "margin_dpo/margin_mean": 13.01004695892334, "margin_dpo/margin_std": 17.498172760009766, "step": 150 }, { "KL/chosen_KL_mean": -8.99380874633789, "KL/mean": -15.274971008300781, "KL/rejected_KL_mean": -21.556129455566406, "KL/std": 15.084770202636719, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.031588368117809296, "fcm_dpo/delta": 0.005246463697403669, "fcm_dpo/margin": 12.562313079833984, "fcm_dpo/q_t": 0.4102116823196411, "grad_norm": 12.747697830200195, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.1038832664489746, "logits/rejected": -1.4997131824493408, "logps/chosen": -111.72802734375, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -127.59394073486328, "loss": 1.0962, "margin_dpo/margin_mean": 12.562313079833984, "margin_dpo/margin_std": 17.452117919921875, "step": 151 }, { "KL/chosen_KL_mean": -6.131141662597656, "KL/mean": -13.75864028930664, "KL/rejected_KL_mean": -21.386138916015625, "KL/std": 15.154170989990234, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.02828652411699295, "fcm_dpo/delta": -0.06081906333565712, "fcm_dpo/margin": 15.255000114440918, "fcm_dpo/q_t": 0.39411404728889465, "grad_norm": 10.697059631347656, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.6519602537155151, "logits/rejected": -1.2613155841827393, "logps/chosen": -103.72188568115234, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -131.88526916503906, "loss": 1.0609, "margin_dpo/margin_mean": 15.255001068115234, "margin_dpo/margin_std": 18.035564422607422, "step": 152 }, { "KL/chosen_KL_mean": -8.44509506225586, "KL/mean": -13.947599411010742, "KL/rejected_KL_mean": -19.45010757446289, "KL/std": 14.692800521850586, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.03181755170226097, "fcm_dpo/delta": 0.07464361190795898, "fcm_dpo/margin": 11.005021095275879, "fcm_dpo/q_t": 0.4243382215499878, "grad_norm": 11.707709312438965, "learning_rate": 4.776497044244016e-07, "logits/chosen": -2.0307271480560303, "logits/rejected": -1.7184091806411743, "logps/chosen": -95.47069549560547, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -110.51864624023438, "loss": 1.1371, "margin_dpo/margin_mean": 11.005021095275879, "margin_dpo/margin_std": 17.232816696166992, "step": 153 }, { "KL/chosen_KL_mean": -6.800849914550781, "KL/mean": -15.63389778137207, "KL/rejected_KL_mean": -24.466949462890625, "KL/std": 15.966745376586914, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.027500268071889877, "fcm_dpo/delta": -0.16386540234088898, "fcm_dpo/margin": 17.666109085083008, "fcm_dpo/q_t": 0.3729284107685089, "grad_norm": 10.446661949157715, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.5727264881134033, "logits/rejected": -1.2519495487213135, "logps/chosen": -97.9065170288086, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -135.83932495117188, "loss": 1.0254, "margin_dpo/margin_mean": 17.666109085083008, "margin_dpo/margin_std": 17.381061553955078, "step": 154 }, { "KL/chosen_KL_mean": -9.000953674316406, "KL/mean": -15.439199447631836, "KL/rejected_KL_mean": -21.877437591552734, "KL/std": 16.38964080810547, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.02633526362478733, "fcm_dpo/delta": 0.09288912266492844, "fcm_dpo/margin": 12.87648868560791, "fcm_dpo/q_t": 0.42582815885543823, "grad_norm": 11.882445335388184, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.5320229530334473, "logits/rejected": -1.1350492238998413, "logps/chosen": -103.56192779541016, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -106.65103149414062, "loss": 1.1476, "margin_dpo/margin_mean": 12.87648868560791, "margin_dpo/margin_std": 20.559865951538086, "step": 155 }, { "KL/chosen_KL_mean": -10.72906494140625, "KL/mean": -16.497547149658203, "KL/rejected_KL_mean": -22.26602554321289, "KL/std": 14.699932098388672, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.03034614771604538, "fcm_dpo/delta": 0.07536019384860992, "fcm_dpo/margin": 11.536965370178223, "fcm_dpo/q_t": 0.4264217019081116, "grad_norm": 12.852429389953613, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.8345410823822021, "logits/rejected": -1.340245246887207, "logps/chosen": -105.38445281982422, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -106.68536376953125, "loss": 1.1453, "margin_dpo/margin_mean": 11.536964416503906, "margin_dpo/margin_std": 18.39851188659668, "step": 156 }, { "KL/chosen_KL_mean": -10.01668930053711, "KL/mean": -18.387117385864258, "KL/rejected_KL_mean": -26.757545471191406, "KL/std": 15.990899085998535, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.02758251316845417, "fcm_dpo/delta": -0.1183539479970932, "fcm_dpo/margin": 16.74085235595703, "fcm_dpo/q_t": 0.3811105787754059, "grad_norm": 11.679708480834961, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.165421485900879, "logits/rejected": -1.7144455909729004, "logps/chosen": -107.02908325195312, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -138.45889282226562, "loss": 1.029, "margin_dpo/margin_mean": 16.74085235595703, "margin_dpo/margin_std": 17.07858657836914, "step": 157 }, { "KL/chosen_KL_mean": -9.084854125976562, "KL/mean": -17.78292465209961, "KL/rejected_KL_mean": -26.480995178222656, "KL/std": 17.29114532470703, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.024575084447860718, "fcm_dpo/delta": -0.047846537083387375, "fcm_dpo/margin": 17.396137237548828, "fcm_dpo/q_t": 0.3968709409236908, "grad_norm": 10.020377159118652, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.4800779819488525, "logits/rejected": -1.2207670211791992, "logps/chosen": -103.79782104492188, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -147.4183349609375, "loss": 1.0539, "margin_dpo/margin_mean": 17.396137237548828, "margin_dpo/margin_std": 18.91985511779785, "step": 158 }, { "KL/chosen_KL_mean": -8.816139221191406, "KL/mean": -16.44505500793457, "KL/rejected_KL_mean": -24.073974609375, "KL/std": 16.652427673339844, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.024910463020205498, "fcm_dpo/delta": 0.03175995498895645, "fcm_dpo/margin": 15.257843971252441, "fcm_dpo/q_t": 0.4138708710670471, "grad_norm": 9.468279838562012, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.217350959777832, "logits/rejected": -1.7310891151428223, "logps/chosen": -117.03437805175781, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -118.91299438476562, "loss": 1.0956, "margin_dpo/margin_mean": 15.257843971252441, "margin_dpo/margin_std": 19.288772583007812, "step": 159 }, { "KL/chosen_KL_mean": -7.454814910888672, "KL/mean": -15.06037425994873, "KL/rejected_KL_mean": -22.665939331054688, "KL/std": 14.998775482177734, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.025489624589681625, "fcm_dpo/delta": 0.019632302224636078, "fcm_dpo/margin": 15.211118698120117, "fcm_dpo/q_t": 0.40979471802711487, "grad_norm": 9.854111671447754, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.7819815874099731, "logits/rejected": -1.2395319938659668, "logps/chosen": -105.61822509765625, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -124.94691467285156, "loss": 1.077, "margin_dpo/margin_mean": 15.211118698120117, "margin_dpo/margin_std": 16.20426368713379, "step": 160 }, { "KL/chosen_KL_mean": -9.543537139892578, "KL/mean": -16.292755126953125, "KL/rejected_KL_mean": -23.041961669921875, "KL/std": 13.750003814697266, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.027319878339767456, "fcm_dpo/delta": 0.04872329160571098, "fcm_dpo/margin": 13.498433113098145, "fcm_dpo/q_t": 0.41782480478286743, "grad_norm": 10.580819129943848, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.8971996307373047, "logits/rejected": -1.4011120796203613, "logps/chosen": -96.13540649414062, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -112.05839538574219, "loss": 1.1125, "margin_dpo/margin_mean": 13.498434066772461, "margin_dpo/margin_std": 18.154470443725586, "step": 161 }, { "KL/chosen_KL_mean": -8.603645324707031, "KL/mean": -16.233898162841797, "KL/rejected_KL_mean": -23.86414337158203, "KL/std": 16.122514724731445, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.026868203654885292, "fcm_dpo/delta": -0.022779015824198723, "fcm_dpo/margin": 15.26049518585205, "fcm_dpo/q_t": 0.40174761414527893, "grad_norm": 9.987943649291992, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.236636996269226, "logits/rejected": -0.8209042549133301, "logps/chosen": -106.00653076171875, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -119.7333755493164, "loss": 1.0707, "margin_dpo/margin_mean": 15.26049518585205, "margin_dpo/margin_std": 17.47882843017578, "step": 162 }, { "KL/chosen_KL_mean": -8.207584381103516, "KL/mean": -17.12823486328125, "KL/rejected_KL_mean": -26.048892974853516, "KL/std": 18.119428634643555, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.024195577949285507, "fcm_dpo/delta": -0.06415713578462601, "fcm_dpo/margin": 17.841312408447266, "fcm_dpo/q_t": 0.39417868852615356, "grad_norm": 8.57342529296875, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.6413637399673462, "logits/rejected": -1.2333698272705078, "logps/chosen": -90.38298034667969, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -108.91363525390625, "loss": 1.0519, "margin_dpo/margin_mean": 17.841312408447266, "margin_dpo/margin_std": 19.627460479736328, "step": 163 }, { "KL/chosen_KL_mean": -12.027351379394531, "KL/mean": -18.198223114013672, "KL/rejected_KL_mean": -24.36908721923828, "KL/std": 14.378446578979492, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.024008017033338547, "fcm_dpo/delta": -0.0059819892048835754, "fcm_dpo/margin": 12.341739654541016, "fcm_dpo/q_t": 0.4288352429866791, "grad_norm": 9.909667015075684, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.5571308135986328, "logits/rejected": -1.086966633796692, "logps/chosen": -102.50161743164062, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -111.39717864990234, "loss": 1.1519, "margin_dpo/margin_mean": 12.341739654541016, "margin_dpo/margin_std": 16.404850006103516, "step": 164 }, { "KL/chosen_KL_mean": -8.5146484375, "KL/mean": -17.281166076660156, "KL/rejected_KL_mean": -26.04767608642578, "KL/std": 16.74502182006836, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.0235980823636055, "fcm_dpo/delta": -0.02368142455816269, "fcm_dpo/margin": 17.533031463623047, "fcm_dpo/q_t": 0.40267863869667053, "grad_norm": 8.153257369995117, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.8125383853912354, "logits/rejected": -1.3737589120864868, "logps/chosen": -95.827880859375, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -107.95025634765625, "loss": 1.0629, "margin_dpo/margin_mean": 17.533029556274414, "margin_dpo/margin_std": 19.308509826660156, "step": 165 }, { "KL/chosen_KL_mean": -13.219406127929688, "KL/mean": -22.047931671142578, "KL/rejected_KL_mean": -30.87645721435547, "KL/std": 18.93154525756836, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.022754576057195663, "fcm_dpo/delta": -0.0029918462969362736, "fcm_dpo/margin": 17.65704345703125, "fcm_dpo/q_t": 0.4093131422996521, "grad_norm": 9.330409049987793, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.6635102033615112, "logits/rejected": -1.4681181907653809, "logps/chosen": -94.80547332763672, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -134.0416259765625, "loss": 1.0845, "margin_dpo/margin_mean": 17.65704345703125, "margin_dpo/margin_std": 23.060565948486328, "step": 166 }, { "KL/chosen_KL_mean": -11.875022888183594, "KL/mean": -21.213232040405273, "KL/rejected_KL_mean": -30.551441192626953, "KL/std": 17.212291717529297, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.02144010365009308, "fcm_dpo/delta": -0.007777787744998932, "fcm_dpo/margin": 18.676414489746094, "fcm_dpo/q_t": 0.4039514362812042, "grad_norm": 8.40523910522461, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.1510562896728516, "logits/rejected": -0.9839199781417847, "logps/chosen": -95.43669128417969, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -129.6044921875, "loss": 1.0636, "margin_dpo/margin_mean": 18.676414489746094, "margin_dpo/margin_std": 17.982219696044922, "step": 167 }, { "KL/chosen_KL_mean": -11.284164428710938, "KL/mean": -19.301692962646484, "KL/rejected_KL_mean": -27.31922149658203, "KL/std": 16.896108627319336, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.023806516081094742, "fcm_dpo/delta": 0.0276129599660635, "fcm_dpo/margin": 16.035051345825195, "fcm_dpo/q_t": 0.4120893180370331, "grad_norm": 9.610755920410156, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.845472812652588, "logits/rejected": -1.512012004852295, "logps/chosen": -91.21540069580078, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -107.42138671875, "loss": 1.0806, "margin_dpo/margin_mean": 16.035053253173828, "margin_dpo/margin_std": 17.188899993896484, "step": 168 }, { "KL/chosen_KL_mean": -9.733543395996094, "KL/mean": -19.049922943115234, "KL/rejected_KL_mean": -28.366294860839844, "KL/std": 18.481298446655273, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.022372495383024216, "fcm_dpo/delta": -0.030057338997721672, "fcm_dpo/margin": 18.63275718688965, "fcm_dpo/q_t": 0.4002898633480072, "grad_norm": 9.091026306152344, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.7211588621139526, "logits/rejected": -1.501706600189209, "logps/chosen": -92.63725280761719, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -118.48820495605469, "loss": 1.0578, "margin_dpo/margin_mean": 18.632755279541016, "margin_dpo/margin_std": 19.776887893676758, "step": 169 }, { "KL/chosen_KL_mean": -11.261974334716797, "KL/mean": -19.294782638549805, "KL/rejected_KL_mean": -27.327598571777344, "KL/std": 15.223163604736328, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.022995423525571823, "fcm_dpo/delta": 0.046721845865249634, "fcm_dpo/margin": 16.065629959106445, "fcm_dpo/q_t": 0.41515278816223145, "grad_norm": 9.735004425048828, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.4363036155700684, "logits/rejected": -1.220444679260254, "logps/chosen": -86.24394226074219, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -109.49320220947266, "loss": 1.0855, "margin_dpo/margin_mean": 16.065628051757812, "margin_dpo/margin_std": 15.186777114868164, "step": 170 }, { "KL/chosen_KL_mean": -14.918010711669922, "KL/mean": -23.821044921875, "KL/rejected_KL_mean": -32.72408676147461, "KL/std": 22.598228454589844, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.023546839132905006, "fcm_dpo/delta": -0.0330895259976387, "fcm_dpo/margin": 17.806079864501953, "fcm_dpo/q_t": 0.40662771463394165, "grad_norm": 11.752325057983398, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.8043212890625, "logits/rejected": -1.2537471055984497, "logps/chosen": -133.89654541015625, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -148.49339294433594, "loss": 1.1001, "margin_dpo/margin_mean": 17.806079864501953, "margin_dpo/margin_std": 27.594646453857422, "step": 171 }, { "KL/chosen_KL_mean": -15.78631591796875, "KL/mean": -23.564720153808594, "KL/rejected_KL_mean": -31.343124389648438, "KL/std": 15.939697265625, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.02397545427083969, "fcm_dpo/delta": 0.04312243312597275, "fcm_dpo/margin": 15.55681037902832, "fcm_dpo/q_t": 0.4138123691082001, "grad_norm": 9.72866439819336, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.396320343017578, "logits/rejected": -1.6208810806274414, "logps/chosen": -116.94010925292969, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -122.34880828857422, "loss": 1.0737, "margin_dpo/margin_mean": 15.556809425354004, "margin_dpo/margin_std": 13.068378448486328, "step": 172 }, { "KL/chosen_KL_mean": -13.997161865234375, "KL/mean": -20.885957717895508, "KL/rejected_KL_mean": -27.77475357055664, "KL/std": 15.009626388549805, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.027104847133159637, "fcm_dpo/delta": 0.03942152485251427, "fcm_dpo/margin": 13.777596473693848, "fcm_dpo/q_t": 0.41481730341911316, "grad_norm": 10.16706657409668, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.663690209388733, "logits/rejected": -1.1736788749694824, "logps/chosen": -113.05384826660156, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -111.62548828125, "loss": 1.0917, "margin_dpo/margin_mean": 13.777597427368164, "margin_dpo/margin_std": 15.93252944946289, "step": 173 }, { "KL/chosen_KL_mean": -13.586797714233398, "KL/mean": -23.341487884521484, "KL/rejected_KL_mean": -33.09618377685547, "KL/std": 20.859647750854492, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.023345882073044777, "fcm_dpo/delta": -0.10292532294988632, "fcm_dpo/margin": 19.509384155273438, "fcm_dpo/q_t": 0.3854006826877594, "grad_norm": 10.459673881530762, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.8551496267318726, "logits/rejected": -1.8205994367599487, "logps/chosen": -93.9957046508789, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -141.2978973388672, "loss": 1.0386, "margin_dpo/margin_mean": 19.509382247924805, "margin_dpo/margin_std": 21.330007553100586, "step": 174 }, { "KL/chosen_KL_mean": -17.114715576171875, "KL/mean": -26.405258178710938, "KL/rejected_KL_mean": -35.69580078125, "KL/std": 18.35416030883789, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.02201925218105316, "fcm_dpo/delta": -0.016497474163770676, "fcm_dpo/margin": 18.581085205078125, "fcm_dpo/q_t": 0.40131399035453796, "grad_norm": 9.981956481933594, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.1819151639938354, "logits/rejected": -0.9083249568939209, "logps/chosen": -99.83120727539062, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -132.36293029785156, "loss": 1.0563, "margin_dpo/margin_mean": 18.581085205078125, "margin_dpo/margin_std": 17.706998825073242, "step": 175 }, { "KL/chosen_KL_mean": -14.183273315429688, "KL/mean": -23.811195373535156, "KL/rejected_KL_mean": -33.43911361694336, "KL/std": 19.889118194580078, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.020982995629310608, "fcm_dpo/delta": -0.0068867821246385574, "fcm_dpo/margin": 19.25583839416504, "fcm_dpo/q_t": 0.405659556388855, "grad_norm": 9.701800346374512, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.8585162162780762, "logits/rejected": -1.7051494121551514, "logps/chosen": -93.81257629394531, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -136.12771606445312, "loss": 1.0772, "margin_dpo/margin_mean": 19.25583839416504, "margin_dpo/margin_std": 23.06543731689453, "step": 176 }, { "KL/chosen_KL_mean": -16.10700225830078, "KL/mean": -27.260700225830078, "KL/rejected_KL_mean": -38.41439437866211, "KL/std": 22.958951950073242, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.01969003491103649, "fcm_dpo/delta": -0.07011548429727554, "fcm_dpo/margin": 22.307397842407227, "fcm_dpo/q_t": 0.39351096749305725, "grad_norm": 8.654058456420898, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.4021286964416504, "logits/rejected": -1.0483670234680176, "logps/chosen": -108.66810607910156, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -141.75277709960938, "loss": 1.0471, "margin_dpo/margin_mean": 22.307397842407227, "margin_dpo/margin_std": 23.89608383178711, "step": 177 }, { "KL/chosen_KL_mean": -14.707077026367188, "KL/mean": -28.038732528686523, "KL/rejected_KL_mean": -41.37037658691406, "KL/std": 22.295166015625, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.016337813809514046, "fcm_dpo/delta": -0.0657539889216423, "fcm_dpo/margin": 26.663299560546875, "fcm_dpo/q_t": 0.39108985662460327, "grad_norm": 8.64277172088623, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.7497937679290771, "logits/rejected": -1.5437966585159302, "logps/chosen": -98.97286987304688, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -152.16769409179688, "loss": 1.0331, "margin_dpo/margin_mean": 26.663299560546875, "margin_dpo/margin_std": 24.052696228027344, "step": 178 }, { "KL/chosen_KL_mean": -18.87744903564453, "KL/mean": -27.24344825744629, "KL/rejected_KL_mean": -35.60944747924805, "KL/std": 21.517364501953125, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.018926918506622314, "fcm_dpo/delta": 0.12263308465480804, "fcm_dpo/margin": 16.731998443603516, "fcm_dpo/q_t": 0.4344087541103363, "grad_norm": 9.026203155517578, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.5580730438232422, "logits/rejected": -1.1288669109344482, "logps/chosen": -112.32542419433594, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -129.8648681640625, "loss": 1.1525, "margin_dpo/margin_mean": 16.731998443603516, "margin_dpo/margin_std": 25.16201400756836, "step": 179 }, { "KL/chosen_KL_mean": -15.595783233642578, "KL/mean": -26.67315673828125, "KL/rejected_KL_mean": -37.750526428222656, "KL/std": 20.63238525390625, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.01882864162325859, "fcm_dpo/delta": -0.03411019593477249, "fcm_dpo/margin": 22.154743194580078, "fcm_dpo/q_t": 0.3986380100250244, "grad_norm": 10.869185447692871, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.3728485107421875, "logits/rejected": -1.1380829811096191, "logps/chosen": -104.43132019042969, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -141.65750122070312, "loss": 1.0488, "margin_dpo/margin_mean": 22.154743194580078, "margin_dpo/margin_std": 21.194141387939453, "step": 180 }, { "KL/chosen_KL_mean": -15.612541198730469, "KL/mean": -23.85936737060547, "KL/rejected_KL_mean": -32.10619354248047, "KL/std": 19.77715301513672, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.021465249359607697, "fcm_dpo/delta": 0.06854017823934555, "fcm_dpo/margin": 16.4936580657959, "fcm_dpo/q_t": 0.42275896668434143, "grad_norm": 9.174457550048828, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.8468579053878784, "logits/rejected": -1.3972632884979248, "logps/chosen": -94.84650421142578, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -101.2806167602539, "loss": 1.1116, "margin_dpo/margin_mean": 16.493656158447266, "margin_dpo/margin_std": 21.36290740966797, "step": 181 }, { "KL/chosen_KL_mean": -19.280807495117188, "KL/mean": -30.26536750793457, "KL/rejected_KL_mean": -41.24992370605469, "KL/std": 22.911975860595703, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.019391227513551712, "fcm_dpo/delta": -0.05416266620159149, "fcm_dpo/margin": 21.969112396240234, "fcm_dpo/q_t": 0.3958855867385864, "grad_norm": 9.1427001953125, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.3486740589141846, "logits/rejected": -1.1394412517547607, "logps/chosen": -95.36299133300781, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -136.21853637695312, "loss": 1.0611, "margin_dpo/margin_mean": 21.969114303588867, "margin_dpo/margin_std": 25.212018966674805, "step": 182 }, { "KL/chosen_KL_mean": -20.70539093017578, "KL/mean": -31.127582550048828, "KL/rejected_KL_mean": -41.54977798461914, "KL/std": 20.84680938720703, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.018978679552674294, "fcm_dpo/delta": 0.0031061843037605286, "fcm_dpo/margin": 20.844385147094727, "fcm_dpo/q_t": 0.4062252640724182, "grad_norm": 9.663652420043945, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.8480618000030518, "logits/rejected": -1.2622904777526855, "logps/chosen": -114.61329650878906, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -125.11039733886719, "loss": 1.0715, "margin_dpo/margin_mean": 20.84438705444336, "margin_dpo/margin_std": 21.24587631225586, "step": 183 }, { "KL/chosen_KL_mean": -18.32046890258789, "KL/mean": -25.504440307617188, "KL/rejected_KL_mean": -32.688411712646484, "KL/std": 21.518394470214844, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.02325909025967121, "fcm_dpo/delta": 0.0935337096452713, "fcm_dpo/margin": 14.367938995361328, "fcm_dpo/q_t": 0.42768406867980957, "grad_norm": 10.157526969909668, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.882833480834961, "logits/rejected": -1.6070349216461182, "logps/chosen": -105.1883773803711, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -116.97296905517578, "loss": 1.148, "margin_dpo/margin_mean": 14.367938995361328, "margin_dpo/margin_std": 23.293241500854492, "step": 184 }, { "KL/chosen_KL_mean": -19.31169891357422, "KL/mean": -27.80594253540039, "KL/rejected_KL_mean": -36.30018997192383, "KL/std": 21.042705535888672, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.024065982550382614, "fcm_dpo/delta": -0.01710895635187626, "fcm_dpo/margin": 16.988487243652344, "fcm_dpo/q_t": 0.40628746151924133, "grad_norm": 11.896303176879883, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.12465500831604, "logits/rejected": -1.4937121868133545, "logps/chosen": -113.33312225341797, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -117.75686645507812, "loss": 1.0987, "margin_dpo/margin_mean": 16.988489151000977, "margin_dpo/margin_std": 24.361576080322266, "step": 185 }, { "KL/chosen_KL_mean": -24.265796661376953, "KL/mean": -33.5286979675293, "KL/rejected_KL_mean": -42.791595458984375, "KL/std": 20.634559631347656, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.021783214062452316, "fcm_dpo/delta": -0.01106051355600357, "fcm_dpo/margin": 18.52579116821289, "fcm_dpo/q_t": 0.40456879138946533, "grad_norm": 11.700699806213379, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.3390684127807617, "logits/rejected": -1.4157017469406128, "logps/chosen": -138.45748901367188, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -136.13072204589844, "loss": 1.0814, "margin_dpo/margin_mean": 18.52579116821289, "margin_dpo/margin_std": 22.561355590820312, "step": 186 }, { "KL/chosen_KL_mean": -20.37403106689453, "KL/mean": -29.430946350097656, "KL/rejected_KL_mean": -38.487857818603516, "KL/std": 23.026081085205078, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.022689610719680786, "fcm_dpo/delta": -0.02110946737229824, "fcm_dpo/margin": 18.113828659057617, "fcm_dpo/q_t": 0.4048634171485901, "grad_norm": 10.307825088500977, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.7366230487823486, "logits/rejected": -1.4996798038482666, "logps/chosen": -97.24357604980469, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -126.65280151367188, "loss": 1.0951, "margin_dpo/margin_mean": 18.113828659057617, "margin_dpo/margin_std": 25.21162223815918, "step": 187 }, { "KL/chosen_KL_mean": -15.868736267089844, "KL/mean": -26.20212173461914, "KL/rejected_KL_mean": -36.53550720214844, "KL/std": 22.594837188720703, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.019885893911123276, "fcm_dpo/delta": -0.026033148169517517, "fcm_dpo/margin": 20.666770935058594, "fcm_dpo/q_t": 0.399419903755188, "grad_norm": 9.820740699768066, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -1.9278550148010254, "logits/rejected": -1.6836109161376953, "logps/chosen": -95.06353759765625, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -133.36441040039062, "loss": 1.0655, "margin_dpo/margin_mean": 20.666770935058594, "margin_dpo/margin_std": 21.501262664794922, "step": 188 }, { "KL/chosen_KL_mean": -20.85760498046875, "KL/mean": -31.276973724365234, "KL/rejected_KL_mean": -41.69634246826172, "KL/std": 22.421960830688477, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.020366977900266647, "fcm_dpo/delta": -0.04783949255943298, "fcm_dpo/margin": 20.838741302490234, "fcm_dpo/q_t": 0.39896976947784424, "grad_norm": 9.133624076843262, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.7952313423156738, "logits/rejected": -1.1472108364105225, "logps/chosen": -104.75040435791016, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -121.16223907470703, "loss": 1.0727, "margin_dpo/margin_mean": 20.838741302490234, "margin_dpo/margin_std": 25.02562713623047, "step": 189 }, { "KL/chosen_KL_mean": -16.732723236083984, "KL/mean": -29.478778839111328, "KL/rejected_KL_mean": -42.22483825683594, "KL/std": 25.066465377807617, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.016983970999717712, "fcm_dpo/delta": -0.05930017679929733, "fcm_dpo/margin": 25.49211883544922, "fcm_dpo/q_t": 0.3924791216850281, "grad_norm": 8.655037879943848, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -2.053420066833496, "logits/rejected": -1.4726388454437256, "logps/chosen": -120.1741943359375, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -151.4398651123047, "loss": 1.0386, "margin_dpo/margin_mean": 25.49211883544922, "margin_dpo/margin_std": 23.887008666992188, "step": 190 }, { "KL/chosen_KL_mean": -16.08112335205078, "KL/mean": -30.620588302612305, "KL/rejected_KL_mean": -45.160057067871094, "KL/std": 27.34158706665039, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.015087387524545193, "fcm_dpo/delta": -0.0689755380153656, "fcm_dpo/margin": 29.07893180847168, "fcm_dpo/q_t": 0.39101529121398926, "grad_norm": 8.983642578125, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.5372636318206787, "logits/rejected": -1.2257533073425293, "logps/chosen": -115.86663818359375, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -161.16033935546875, "loss": 1.0372, "margin_dpo/margin_mean": 29.07893180847168, "margin_dpo/margin_std": 28.02654266357422, "step": 191 }, { "KL/chosen_KL_mean": -25.61743927001953, "KL/mean": -35.63984298706055, "KL/rejected_KL_mean": -45.6622428894043, "KL/std": 21.656509399414062, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.016704712063074112, "fcm_dpo/delta": 0.09662871062755585, "fcm_dpo/margin": 20.0447998046875, "fcm_dpo/q_t": 0.4280630946159363, "grad_norm": 9.679155349731445, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.6831111907958984, "logits/rejected": -1.434988021850586, "logps/chosen": -126.47776794433594, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -155.10723876953125, "loss": 1.1229, "margin_dpo/margin_mean": 20.0447998046875, "margin_dpo/margin_std": 25.390371322631836, "step": 192 }, { "KL/chosen_KL_mean": -21.39752960205078, "KL/mean": -29.51102066040039, "KL/rejected_KL_mean": -37.62451171875, "KL/std": 23.228696823120117, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.020698182284832, "fcm_dpo/delta": 0.08719652146100998, "fcm_dpo/margin": 16.226978302001953, "fcm_dpo/q_t": 0.4272039234638214, "grad_norm": 11.110200881958008, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.099287509918213, "logits/rejected": -1.5109379291534424, "logps/chosen": -114.33584594726562, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -120.73368835449219, "loss": 1.1597, "margin_dpo/margin_mean": 16.226978302001953, "margin_dpo/margin_std": 28.20010757446289, "step": 193 }, { "KL/chosen_KL_mean": -21.422264099121094, "KL/mean": -30.94770050048828, "KL/rejected_KL_mean": -40.473140716552734, "KL/std": 23.50460433959961, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.02126256376504898, "fcm_dpo/delta": -0.011447856202721596, "fcm_dpo/margin": 19.050880432128906, "fcm_dpo/q_t": 0.40607666969299316, "grad_norm": 9.406710624694824, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.115494728088379, "logits/rejected": -1.7120332717895508, "logps/chosen": -85.21383666992188, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -109.19979858398438, "loss": 1.0883, "margin_dpo/margin_mean": 19.050880432128906, "margin_dpo/margin_std": 24.800376892089844, "step": 194 }, { "KL/chosen_KL_mean": -20.765716552734375, "KL/mean": -33.62364196777344, "KL/rejected_KL_mean": -46.481571197509766, "KL/std": 24.00739288330078, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.01777392067015171, "fcm_dpo/delta": -0.10469934344291687, "fcm_dpo/margin": 25.715852737426758, "fcm_dpo/q_t": 0.3839297890663147, "grad_norm": 8.603767395019531, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.4391239881515503, "logits/rejected": -1.3286839723587036, "logps/chosen": -104.65117645263672, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -150.48281860351562, "loss": 1.0382, "margin_dpo/margin_mean": 25.715850830078125, "margin_dpo/margin_std": 27.31378746032715, "step": 195 }, { "KL/chosen_KL_mean": -24.249774932861328, "KL/mean": -36.186302185058594, "KL/rejected_KL_mean": -48.12282180786133, "KL/std": 25.707731246948242, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.016629699617624283, "fcm_dpo/delta": 0.004971545189619064, "fcm_dpo/margin": 23.873046875, "fcm_dpo/q_t": 0.40573468804359436, "grad_norm": 8.097493171691895, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.3754769563674927, "logits/rejected": -0.9225541353225708, "logps/chosen": -113.65113830566406, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -131.4701690673828, "loss": 1.0642, "margin_dpo/margin_mean": 23.873046875, "margin_dpo/margin_std": 23.32028579711914, "step": 196 }, { "KL/chosen_KL_mean": -24.560890197753906, "KL/mean": -31.811309814453125, "KL/rejected_KL_mean": -39.061729431152344, "KL/std": 20.62317657470703, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.02075611799955368, "fcm_dpo/delta": 0.13691161572933197, "fcm_dpo/margin": 14.500829696655273, "fcm_dpo/q_t": 0.4370638430118561, "grad_norm": 9.920583724975586, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.4133331775665283, "logits/rejected": -1.0779114961624146, "logps/chosen": -111.40159606933594, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -121.85462951660156, "loss": 1.157, "margin_dpo/margin_mean": 14.500828742980957, "margin_dpo/margin_std": 21.999858856201172, "step": 197 }, { "KL/chosen_KL_mean": -27.95855712890625, "KL/mean": -40.11628341674805, "KL/rejected_KL_mean": -52.274009704589844, "KL/std": 24.69515609741211, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.018661998212337494, "fcm_dpo/delta": -0.10239750891923904, "fcm_dpo/margin": 24.31545639038086, "fcm_dpo/q_t": 0.3852927088737488, "grad_norm": 10.50479793548584, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7679970264434814, "logits/rejected": -1.6012274026870728, "logps/chosen": -113.00680541992188, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -156.37765502929688, "loss": 1.034, "margin_dpo/margin_mean": 24.315458297729492, "margin_dpo/margin_std": 25.273727416992188, "step": 198 }, { "KL/chosen_KL_mean": -27.379043579101562, "KL/mean": -37.234519958496094, "KL/rejected_KL_mean": -47.08999252319336, "KL/std": 24.014957427978516, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.019137974828481674, "fcm_dpo/delta": 0.03476147726178169, "fcm_dpo/margin": 19.710954666137695, "fcm_dpo/q_t": 0.4146728813648224, "grad_norm": 9.465925216674805, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.8649623394012451, "logits/rejected": -1.6128031015396118, "logps/chosen": -109.16574096679688, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -137.88076782226562, "loss": 1.1002, "margin_dpo/margin_mean": 19.710954666137695, "margin_dpo/margin_std": 25.389015197753906, "step": 199 }, { "KL/chosen_KL_mean": -26.745758056640625, "KL/mean": -38.42354202270508, "KL/rejected_KL_mean": -50.101314544677734, "KL/std": 25.53433609008789, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.017533782869577408, "fcm_dpo/delta": -0.025411851704120636, "fcm_dpo/margin": 23.355552673339844, "fcm_dpo/q_t": 0.4025859832763672, "grad_norm": 9.019886016845703, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.7981500625610352, "logits/rejected": -1.260772466659546, "logps/chosen": -111.01385498046875, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -137.06918334960938, "loss": 1.0847, "margin_dpo/margin_mean": 23.355552673339844, "margin_dpo/margin_std": 29.66939353942871, "step": 200 }, { "KL/chosen_KL_mean": -29.172733306884766, "KL/mean": -40.46955871582031, "KL/rejected_KL_mean": -51.766395568847656, "KL/std": 27.96529769897461, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.018232036381959915, "fcm_dpo/delta": -0.022126559168100357, "fcm_dpo/margin": 22.59366226196289, "fcm_dpo/q_t": 0.4037489593029022, "grad_norm": 9.55221176147461, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.3308377265930176, "logits/rejected": -0.9431394338607788, "logps/chosen": -136.94522094726562, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -171.55886840820312, "loss": 1.089, "margin_dpo/margin_mean": 22.59366226196289, "margin_dpo/margin_std": 30.581981658935547, "step": 201 }, { "KL/chosen_KL_mean": -26.64809799194336, "KL/mean": -37.24383544921875, "KL/rejected_KL_mean": -47.83956527709961, "KL/std": 23.01310157775879, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.018232014030218124, "fcm_dpo/delta": 0.021169865503907204, "fcm_dpo/margin": 21.191463470458984, "fcm_dpo/q_t": 0.41185954213142395, "grad_norm": 9.631424903869629, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.7855966091156006, "logits/rejected": -1.6636284589767456, "logps/chosen": -102.62055206298828, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -142.29954528808594, "loss": 1.0919, "margin_dpo/margin_mean": 21.191463470458984, "margin_dpo/margin_std": 26.069576263427734, "step": 202 }, { "KL/chosen_KL_mean": -23.96137237548828, "KL/mean": -35.787696838378906, "KL/rejected_KL_mean": -47.61402130126953, "KL/std": 23.899282455444336, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.017664147540926933, "fcm_dpo/delta": -0.030632970854640007, "fcm_dpo/margin": 23.652647018432617, "fcm_dpo/q_t": 0.39925500750541687, "grad_norm": 8.61573600769043, "learning_rate": 4.439630306414758e-07, "logits/chosen": -2.234851837158203, "logits/rejected": -1.8041458129882812, "logps/chosen": -118.92852783203125, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -140.50169372558594, "loss": 1.0585, "margin_dpo/margin_mean": 23.652647018432617, "margin_dpo/margin_std": 25.093517303466797, "step": 203 }, { "KL/chosen_KL_mean": -30.41234588623047, "KL/mean": -40.591392517089844, "KL/rejected_KL_mean": -50.77043914794922, "KL/std": 24.1739501953125, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.018527427688241005, "fcm_dpo/delta": 0.03206801414489746, "fcm_dpo/margin": 20.358089447021484, "fcm_dpo/q_t": 0.41469165682792664, "grad_norm": 9.88554859161377, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.994116187095642, "logits/rejected": -1.3403141498565674, "logps/chosen": -146.76953125, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -144.16802978515625, "loss": 1.1115, "margin_dpo/margin_mean": 20.358089447021484, "margin_dpo/margin_std": 28.08462142944336, "step": 204 }, { "KL/chosen_KL_mean": -28.21560287475586, "KL/mean": -41.38969802856445, "KL/rejected_KL_mean": -54.56379699707031, "KL/std": 26.504756927490234, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.016262756660580635, "fcm_dpo/delta": -0.05583757907152176, "fcm_dpo/margin": 26.34819984436035, "fcm_dpo/q_t": 0.3920491337776184, "grad_norm": 8.155563354492188, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.240129232406616, "logits/rejected": -1.8025469779968262, "logps/chosen": -117.074951171875, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -146.4182891845703, "loss": 1.0353, "margin_dpo/margin_mean": 26.34819984436035, "margin_dpo/margin_std": 22.5654296875, "step": 205 }, { "KL/chosen_KL_mean": -28.73775863647461, "KL/mean": -40.928428649902344, "KL/rejected_KL_mean": -53.11910629272461, "KL/std": 25.63739013671875, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.01604914478957653, "fcm_dpo/delta": 0.011859940364956856, "fcm_dpo/margin": 24.381336212158203, "fcm_dpo/q_t": 0.4069425165653229, "grad_norm": 9.253057479858398, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.9383594989776611, "logits/rejected": -1.7979950904846191, "logps/chosen": -105.28410339355469, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -156.07223510742188, "loss": 1.0648, "margin_dpo/margin_mean": 24.381336212158203, "margin_dpo/margin_std": 21.7706241607666, "step": 206 }, { "KL/chosen_KL_mean": -29.988906860351562, "KL/mean": -43.711524963378906, "KL/rejected_KL_mean": -57.43413162231445, "KL/std": 31.82073211669922, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.015550734475255013, "fcm_dpo/delta": -0.047171495854854584, "fcm_dpo/margin": 27.44522476196289, "fcm_dpo/q_t": 0.3995243310928345, "grad_norm": 8.938323020935059, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.0010626316070557, "logits/rejected": -1.5707554817199707, "logps/chosen": -116.22055053710938, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -148.08924865722656, "loss": 1.0765, "margin_dpo/margin_mean": 27.44522476196289, "margin_dpo/margin_std": 36.156280517578125, "step": 207 }, { "KL/chosen_KL_mean": -31.522262573242188, "KL/mean": -45.483638763427734, "KL/rejected_KL_mean": -59.44500732421875, "KL/std": 28.65509033203125, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.015024326741695404, "fcm_dpo/delta": -0.03589870035648346, "fcm_dpo/margin": 27.9227352142334, "fcm_dpo/q_t": 0.39664125442504883, "grad_norm": 9.048262596130371, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.851772665977478, "logits/rejected": -1.502152442932129, "logps/chosen": -125.68180847167969, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -160.4073486328125, "loss": 1.0509, "margin_dpo/margin_mean": 27.92273712158203, "margin_dpo/margin_std": 26.070087432861328, "step": 208 }, { "KL/chosen_KL_mean": -26.613052368164062, "KL/mean": -38.91889190673828, "KL/rejected_KL_mean": -51.2247200012207, "KL/std": 25.670520782470703, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.014887316152453423, "fcm_dpo/delta": 0.05284200236201286, "fcm_dpo/margin": 24.61166763305664, "fcm_dpo/q_t": 0.41784700751304626, "grad_norm": 8.055638313293457, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.1662590503692627, "logits/rejected": -1.6927576065063477, "logps/chosen": -110.78361511230469, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -138.84426879882812, "loss": 1.1006, "margin_dpo/margin_mean": 24.61166763305664, "margin_dpo/margin_std": 29.15436553955078, "step": 209 }, { "KL/chosen_KL_mean": -30.306766510009766, "KL/mean": -42.0172119140625, "KL/rejected_KL_mean": -53.727664947509766, "KL/std": 27.017227172851562, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.016281170770525932, "fcm_dpo/delta": 0.03027649223804474, "fcm_dpo/margin": 23.4209041595459, "fcm_dpo/q_t": 0.41308555006980896, "grad_norm": 8.027063369750977, "learning_rate": 4.381713366536311e-07, "logits/chosen": -2.0255231857299805, "logits/rejected": -1.6418468952178955, "logps/chosen": -111.47793579101562, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -137.90245056152344, "loss": 1.0878, "margin_dpo/margin_mean": 23.42090606689453, "margin_dpo/margin_std": 27.242446899414062, "step": 210 }, { "KL/chosen_KL_mean": -34.55240249633789, "KL/mean": -45.765567779541016, "KL/rejected_KL_mean": -56.97873306274414, "KL/std": 28.742122650146484, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.01745748147368431, "fcm_dpo/delta": 0.011423172429203987, "fcm_dpo/margin": 22.42633628845215, "fcm_dpo/q_t": 0.4105288088321686, "grad_norm": 10.549016952514648, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.0074880123138428, "logits/rejected": -1.3655481338500977, "logps/chosen": -143.26510620117188, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -150.53439331054688, "loss": 1.0933, "margin_dpo/margin_mean": 22.426334381103516, "margin_dpo/margin_std": 29.201011657714844, "step": 211 }, { "KL/chosen_KL_mean": -24.582042694091797, "KL/mean": -41.39060974121094, "KL/rejected_KL_mean": -58.19917297363281, "KL/std": 33.02555465698242, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.013583431020379066, "fcm_dpo/delta": -0.1228347197175026, "fcm_dpo/margin": 33.61712646484375, "fcm_dpo/q_t": 0.38104936480522156, "grad_norm": 7.759756088256836, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.9238014221191406, "logits/rejected": -1.515088438987732, "logps/chosen": -122.9439926147461, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -168.08917236328125, "loss": 1.0276, "margin_dpo/margin_mean": 33.61712646484375, "margin_dpo/margin_std": 33.154151916503906, "step": 212 }, { "KL/chosen_KL_mean": -28.56222915649414, "KL/mean": -41.264549255371094, "KL/rejected_KL_mean": -53.96685791015625, "KL/std": 29.90005111694336, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.01434034388512373, "fcm_dpo/delta": 0.056417763233184814, "fcm_dpo/margin": 25.404632568359375, "fcm_dpo/q_t": 0.41861870884895325, "grad_norm": 9.785001754760742, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.946784496307373, "logits/rejected": -1.1499534845352173, "logps/chosen": -136.61753845214844, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -154.1110076904297, "loss": 1.1087, "margin_dpo/margin_mean": 25.40462875366211, "margin_dpo/margin_std": 32.844154357910156, "step": 213 }, { "KL/chosen_KL_mean": -30.311744689941406, "KL/mean": -42.30327224731445, "KL/rejected_KL_mean": -54.2947998046875, "KL/std": 32.09174728393555, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.015848934650421143, "fcm_dpo/delta": 0.03100607730448246, "fcm_dpo/margin": 23.983062744140625, "fcm_dpo/q_t": 0.4155980944633484, "grad_norm": 8.477694511413574, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.893712043762207, "logits/rejected": -1.7721482515335083, "logps/chosen": -108.11648559570312, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -138.57058715820312, "loss": 1.1146, "margin_dpo/margin_mean": 23.983064651489258, "margin_dpo/margin_std": 35.357486724853516, "step": 214 }, { "KL/chosen_KL_mean": -26.257171630859375, "KL/mean": -42.51734161376953, "KL/rejected_KL_mean": -58.77750778198242, "KL/std": 31.517927169799805, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.013798212632536888, "fcm_dpo/delta": -0.09119100868701935, "fcm_dpo/margin": 32.520328521728516, "fcm_dpo/q_t": 0.3866243362426758, "grad_norm": 7.775142192840576, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.7595620155334473, "logits/rejected": -1.2868529558181763, "logps/chosen": -111.37106323242188, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -155.63902282714844, "loss": 1.031, "margin_dpo/margin_mean": 32.520328521728516, "margin_dpo/margin_std": 31.44477081298828, "step": 215 }, { "KL/chosen_KL_mean": -28.483539581298828, "KL/mean": -42.55894470214844, "KL/rejected_KL_mean": -56.63434600830078, "KL/std": 28.258106231689453, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.013986124657094479, "fcm_dpo/delta": 0.0044418275356292725, "fcm_dpo/margin": 28.150814056396484, "fcm_dpo/q_t": 0.40690600872039795, "grad_norm": 7.455711364746094, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.6336688995361328, "logits/rejected": -1.209580898284912, "logps/chosen": -109.07584381103516, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -138.05418395996094, "loss": 1.0742, "margin_dpo/margin_mean": 28.150814056396484, "margin_dpo/margin_std": 29.290985107421875, "step": 216 }, { "KL/chosen_KL_mean": -34.820125579833984, "KL/mean": -46.223609924316406, "KL/rejected_KL_mean": -57.62709426879883, "KL/std": 29.31645965576172, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.01529780589044094, "fcm_dpo/delta": 0.07361626625061035, "fcm_dpo/margin": 22.806964874267578, "fcm_dpo/q_t": 0.4244721531867981, "grad_norm": 9.84107494354248, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.008674144744873, "logits/rejected": -1.701724886894226, "logps/chosen": -128.29270935058594, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -161.11561584472656, "loss": 1.1297, "margin_dpo/margin_mean": 22.806962966918945, "margin_dpo/margin_std": 33.87489318847656, "step": 217 }, { "KL/chosen_KL_mean": -34.021018981933594, "KL/mean": -48.00786590576172, "KL/rejected_KL_mean": -61.994720458984375, "KL/std": 29.727773666381836, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.014622284099459648, "fcm_dpo/delta": -0.01876235380768776, "fcm_dpo/margin": 27.97368621826172, "fcm_dpo/q_t": 0.40361082553863525, "grad_norm": 8.199453353881836, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.5939587354660034, "logits/rejected": -1.2205890417099, "logps/chosen": -123.07984924316406, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -156.30152893066406, "loss": 1.0849, "margin_dpo/margin_mean": 27.97368621826172, "margin_dpo/margin_std": 36.03280258178711, "step": 218 }, { "KL/chosen_KL_mean": -30.657360076904297, "KL/mean": -43.098426818847656, "KL/rejected_KL_mean": -55.53949737548828, "KL/std": 34.49659729003906, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.015904774889349937, "fcm_dpo/delta": -0.004683688282966614, "fcm_dpo/margin": 24.882137298583984, "fcm_dpo/q_t": 0.40850234031677246, "grad_norm": 10.271955490112305, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.6828067302703857, "logits/rejected": -1.214310646057129, "logps/chosen": -131.66470336914062, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -148.0074462890625, "loss": 1.1173, "margin_dpo/margin_mean": 24.882137298583984, "margin_dpo/margin_std": 37.04552459716797, "step": 219 }, { "KL/chosen_KL_mean": -31.62796401977539, "KL/mean": -43.2686767578125, "KL/rejected_KL_mean": -54.90937042236328, "KL/std": 28.620471954345703, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.015971828252077103, "fcm_dpo/delta": 0.04333730787038803, "fcm_dpo/margin": 23.281421661376953, "fcm_dpo/q_t": 0.4156688153743744, "grad_norm": 10.237257957458496, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5916625261306763, "logits/rejected": -1.2483887672424316, "logps/chosen": -127.16517639160156, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -149.21640014648438, "loss": 1.0968, "margin_dpo/margin_mean": 23.281421661376953, "margin_dpo/margin_std": 28.00597381591797, "step": 220 }, { "KL/chosen_KL_mean": -29.36585235595703, "KL/mean": -47.527591705322266, "KL/rejected_KL_mean": -65.6893310546875, "KL/std": 35.70523452758789, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.013429783284664154, "fcm_dpo/delta": -0.16814537346363068, "fcm_dpo/margin": 36.32347869873047, "fcm_dpo/q_t": 0.3704163134098053, "grad_norm": 9.146479606628418, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.4244792461395264, "logits/rejected": -1.292663812637329, "logps/chosen": -112.2107162475586, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -176.50112915039062, "loss": 1.0088, "margin_dpo/margin_mean": 36.32347869873047, "margin_dpo/margin_std": 31.43674659729004, "step": 221 }, { "KL/chosen_KL_mean": -37.56632995605469, "KL/mean": -52.21084213256836, "KL/rejected_KL_mean": -66.85535430908203, "KL/std": 34.300140380859375, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.012726984918117523, "fcm_dpo/delta": 0.03495822846889496, "fcm_dpo/margin": 29.289012908935547, "fcm_dpo/q_t": 0.41509485244750977, "grad_norm": 8.32919979095459, "learning_rate": 4.276818137766118e-07, "logits/chosen": -2.0493507385253906, "logits/rejected": -1.6233474016189575, "logps/chosen": -132.70831298828125, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -173.65975952148438, "loss": 1.112, "margin_dpo/margin_mean": 29.289012908935547, "margin_dpo/margin_std": 40.071533203125, "step": 222 }, { "KL/chosen_KL_mean": -36.74808883666992, "KL/mean": -47.09906768798828, "KL/rejected_KL_mean": -57.45003890991211, "KL/std": 30.32532501220703, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.014959942549467087, "fcm_dpo/delta": 0.12989695370197296, "fcm_dpo/margin": 20.701961517333984, "fcm_dpo/q_t": 0.43589895963668823, "grad_norm": 9.502055168151855, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.367314338684082, "logits/rejected": -1.9893834590911865, "logps/chosen": -122.32319641113281, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -143.9024200439453, "loss": 1.1599, "margin_dpo/margin_mean": 20.701961517333984, "margin_dpo/margin_std": 33.15089416503906, "step": 223 }, { "KL/chosen_KL_mean": -33.36845779418945, "KL/mean": -48.267364501953125, "KL/rejected_KL_mean": -63.16627502441406, "KL/std": 33.512718200683594, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.014448115602135658, "fcm_dpo/delta": -0.05688474327325821, "fcm_dpo/margin": 29.797821044921875, "fcm_dpo/q_t": 0.39574962854385376, "grad_norm": 8.814480781555176, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.9550702571868896, "logits/rejected": -1.5077335834503174, "logps/chosen": -116.0922622680664, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -145.76165771484375, "loss": 1.0616, "margin_dpo/margin_mean": 29.797821044921875, "margin_dpo/margin_std": 35.16073226928711, "step": 224 }, { "KL/chosen_KL_mean": -37.670101165771484, "KL/mean": -52.994712829589844, "KL/rejected_KL_mean": -68.31932830810547, "KL/std": 35.124595642089844, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.013900795951485634, "fcm_dpo/delta": -0.046865276992321014, "fcm_dpo/margin": 30.649215698242188, "fcm_dpo/q_t": 0.3987656235694885, "grad_norm": 9.368855476379395, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.166888475418091, "logits/rejected": -1.7030830383300781, "logps/chosen": -133.3477783203125, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -173.41619873046875, "loss": 1.0778, "margin_dpo/margin_mean": 30.649215698242188, "margin_dpo/margin_std": 39.681060791015625, "step": 225 }, { "KL/chosen_KL_mean": -31.060943603515625, "KL/mean": -45.80016326904297, "KL/rejected_KL_mean": -60.53938293457031, "KL/std": 34.38666534423828, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.012924063950777054, "fcm_dpo/delta": 0.02641749382019043, "fcm_dpo/margin": 29.47844123840332, "fcm_dpo/q_t": 0.4116349220275879, "grad_norm": 7.631762981414795, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.9054889678955078, "logits/rejected": -1.3718931674957275, "logps/chosen": -124.52186584472656, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -147.24114990234375, "loss": 1.0829, "margin_dpo/margin_mean": 29.47844123840332, "margin_dpo/margin_std": 30.5301513671875, "step": 226 }, { "KL/chosen_KL_mean": -41.78410720825195, "KL/mean": -57.13151931762695, "KL/rejected_KL_mean": -72.47891998291016, "KL/std": 33.61286926269531, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.013396022841334343, "fcm_dpo/delta": -0.01897718757390976, "fcm_dpo/margin": 30.694812774658203, "fcm_dpo/q_t": 0.40383145213127136, "grad_norm": 8.617940902709961, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.4279125928878784, "logits/rejected": -1.0750735998153687, "logps/chosen": -134.59837341308594, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -177.21585083007812, "loss": 1.0844, "margin_dpo/margin_mean": 30.694812774658203, "margin_dpo/margin_std": 39.89923858642578, "step": 227 }, { "KL/chosen_KL_mean": -39.06369400024414, "KL/mean": -50.657264709472656, "KL/rejected_KL_mean": -62.2508430480957, "KL/std": 31.4700984954834, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.014958096668124199, "fcm_dpo/delta": 0.08011473715305328, "fcm_dpo/margin": 23.1871395111084, "fcm_dpo/q_t": 0.42464399337768555, "grad_norm": 9.758392333984375, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.7402262687683105, "logits/rejected": -1.2977312803268433, "logps/chosen": -133.10081481933594, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -158.27236938476562, "loss": 1.1366, "margin_dpo/margin_mean": 23.18714141845703, "margin_dpo/margin_std": 35.606361389160156, "step": 228 }, { "KL/chosen_KL_mean": -34.97439193725586, "KL/mean": -53.907447814941406, "KL/rejected_KL_mean": -72.84049987792969, "KL/std": 37.880043029785156, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.012412037700414658, "fcm_dpo/delta": -0.14040081202983856, "fcm_dpo/margin": 37.86611557006836, "fcm_dpo/q_t": 0.3796314597129822, "grad_norm": 8.227317810058594, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.90834379196167, "logits/rejected": -1.2805614471435547, "logps/chosen": -130.55206298828125, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -162.01429748535156, "loss": 1.0318, "margin_dpo/margin_mean": 37.866119384765625, "margin_dpo/margin_std": 41.496620178222656, "step": 229 }, { "KL/chosen_KL_mean": -35.98126220703125, "KL/mean": -55.691490173339844, "KL/rejected_KL_mean": -75.40171813964844, "KL/std": 37.497276306152344, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.010833067819476128, "fcm_dpo/delta": -0.04822303354740143, "fcm_dpo/margin": 39.420448303222656, "fcm_dpo/q_t": 0.39439505338668823, "grad_norm": 8.7562837600708, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.861053466796875, "logits/rejected": -1.804795742034912, "logps/chosen": -99.69488525390625, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -167.3104248046875, "loss": 1.0427, "margin_dpo/margin_mean": 39.420448303222656, "margin_dpo/margin_std": 37.27819061279297, "step": 230 }, { "KL/chosen_KL_mean": -39.63923645019531, "KL/mean": -56.79326629638672, "KL/rejected_KL_mean": -73.9472885131836, "KL/std": 36.21617889404297, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.010920818895101547, "fcm_dpo/delta": 0.039366140961647034, "fcm_dpo/margin": 34.30805587768555, "fcm_dpo/q_t": 0.41346484422683716, "grad_norm": 6.904531002044678, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -2.0363216400146484, "logits/rejected": -1.4934704303741455, "logps/chosen": -135.09490966796875, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -154.90296936035156, "loss": 1.0776, "margin_dpo/margin_mean": 34.30805587768555, "margin_dpo/margin_std": 31.39630699157715, "step": 231 }, { "KL/chosen_KL_mean": -42.49223327636719, "KL/mean": -56.569908142089844, "KL/rejected_KL_mean": -70.64759826660156, "KL/std": 33.45750427246094, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.012690041214227676, "fcm_dpo/delta": 0.06600663810968399, "fcm_dpo/margin": 28.155364990234375, "fcm_dpo/q_t": 0.42038047313690186, "grad_norm": 8.677007675170898, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.5225317478179932, "logits/rejected": -1.0358479022979736, "logps/chosen": -139.38949584960938, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -160.41221618652344, "loss": 1.1032, "margin_dpo/margin_mean": 28.15536117553711, "margin_dpo/margin_std": 33.1614875793457, "step": 232 }, { "KL/chosen_KL_mean": -44.02593231201172, "KL/mean": -63.88432693481445, "KL/rejected_KL_mean": -83.74270629882812, "KL/std": 42.500877380371094, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.011198405176401138, "fcm_dpo/delta": -0.08836612105369568, "fcm_dpo/margin": 39.71678161621094, "fcm_dpo/q_t": 0.38725996017456055, "grad_norm": 8.500791549682617, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.8786520957946777, "logits/rejected": -1.6874920129776, "logps/chosen": -133.07626342773438, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -196.5018768310547, "loss": 1.0397, "margin_dpo/margin_mean": 39.71678161621094, "margin_dpo/margin_std": 41.10282516479492, "step": 233 }, { "KL/chosen_KL_mean": -43.2431640625, "KL/mean": -64.59178161621094, "KL/rejected_KL_mean": -85.9404067993164, "KL/std": 40.70171356201172, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.009655490517616272, "fcm_dpo/delta": -0.039936766028404236, "fcm_dpo/margin": 42.697235107421875, "fcm_dpo/q_t": 0.39797458052635193, "grad_norm": 7.022353649139404, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.297795295715332, "logits/rejected": -1.2353768348693848, "logps/chosen": -117.5615234375, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -192.3280029296875, "loss": 1.0647, "margin_dpo/margin_mean": 42.697235107421875, "margin_dpo/margin_std": 44.680580139160156, "step": 234 }, { "KL/chosen_KL_mean": -52.29008865356445, "KL/mean": -68.25447845458984, "KL/rejected_KL_mean": -84.21886444091797, "KL/std": 36.69854736328125, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.011297399178147316, "fcm_dpo/delta": 0.05838266387581825, "fcm_dpo/margin": 31.928775787353516, "fcm_dpo/q_t": 0.41837528347969055, "grad_norm": 8.112325668334961, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.5562790632247925, "logits/rejected": -0.9920419454574585, "logps/chosen": -150.50714111328125, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -181.46563720703125, "loss": 1.0942, "margin_dpo/margin_mean": 31.928773880004883, "margin_dpo/margin_std": 34.59777069091797, "step": 235 }, { "KL/chosen_KL_mean": -43.66767883300781, "KL/mean": -64.72691345214844, "KL/rejected_KL_mean": -85.78614807128906, "KL/std": 46.164283752441406, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.010431657545268536, "fcm_dpo/delta": -0.06996703892946243, "fcm_dpo/margin": 42.11846923828125, "fcm_dpo/q_t": 0.39521899819374084, "grad_norm": 8.315152168273926, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.378066062927246, "logits/rejected": -1.3865348100662231, "logps/chosen": -122.50540924072266, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -194.84957885742188, "loss": 1.0542, "margin_dpo/margin_mean": 42.11846923828125, "margin_dpo/margin_std": 49.86656188964844, "step": 236 }, { "KL/chosen_KL_mean": -52.87141036987305, "KL/mean": -67.88642883300781, "KL/rejected_KL_mean": -82.90142822265625, "KL/std": 40.57427978515625, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.011392922140657902, "fcm_dpo/delta": 0.08566456288099289, "fcm_dpo/margin": 30.030014038085938, "fcm_dpo/q_t": 0.42848214507102966, "grad_norm": 8.070027351379395, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.5541038513183594, "logits/rejected": -1.2086238861083984, "logps/chosen": -138.08270263671875, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -182.81143188476562, "loss": 1.1548, "margin_dpo/margin_mean": 30.030014038085938, "margin_dpo/margin_std": 52.57080078125, "step": 237 }, { "KL/chosen_KL_mean": -57.90274429321289, "KL/mean": -75.58332824707031, "KL/rejected_KL_mean": -93.26392364501953, "KL/std": 42.26802062988281, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.011448966339230537, "fcm_dpo/delta": -0.009010070003569126, "fcm_dpo/margin": 35.36119079589844, "fcm_dpo/q_t": 0.4059496521949768, "grad_norm": 7.730382442474365, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2824382781982422, "logits/rejected": -1.1367621421813965, "logps/chosen": -136.63397216796875, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -188.6823272705078, "loss": 1.0804, "margin_dpo/margin_mean": 35.36119079589844, "margin_dpo/margin_std": 43.897003173828125, "step": 238 }, { "KL/chosen_KL_mean": -52.539432525634766, "KL/mean": -68.958984375, "KL/rejected_KL_mean": -85.37854766845703, "KL/std": 37.600669860839844, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.011430753394961357, "fcm_dpo/delta": 0.03199122101068497, "fcm_dpo/margin": 32.839111328125, "fcm_dpo/q_t": 0.41403454542160034, "grad_norm": 8.733816146850586, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.9408204555511475, "logits/rejected": -1.471644639968872, "logps/chosen": -145.140380859375, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -185.896240234375, "loss": 1.1108, "margin_dpo/margin_mean": 32.839111328125, "margin_dpo/margin_std": 41.699771881103516, "step": 239 }, { "KL/chosen_KL_mean": -54.71290969848633, "KL/mean": -69.02798461914062, "KL/rejected_KL_mean": -83.34306335449219, "KL/std": 38.050743103027344, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.0129498690366745, "fcm_dpo/delta": 0.04650016874074936, "fcm_dpo/margin": 28.630151748657227, "fcm_dpo/q_t": 0.41770845651626587, "grad_norm": 10.670819282531738, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.0044283866882324, "logits/rejected": -1.633946418762207, "logps/chosen": -140.8726806640625, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -163.79873657226562, "loss": 1.1181, "margin_dpo/margin_mean": 28.63015365600586, "margin_dpo/margin_std": 41.03307342529297, "step": 240 }, { "KL/chosen_KL_mean": -50.717533111572266, "KL/mean": -76.59806823730469, "KL/rejected_KL_mean": -102.47859191894531, "KL/std": 43.300086975097656, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.009885473176836967, "fcm_dpo/delta": -0.23193351924419403, "fcm_dpo/margin": 51.761051177978516, "fcm_dpo/q_t": 0.35864895582199097, "grad_norm": 7.766709327697754, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.5603175163269043, "logits/rejected": -0.852818489074707, "logps/chosen": -151.7634735107422, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -196.5279541015625, "loss": 0.999, "margin_dpo/margin_mean": 51.76105499267578, "margin_dpo/margin_std": 48.65491485595703, "step": 241 }, { "KL/chosen_KL_mean": -59.60043716430664, "KL/mean": -78.2728271484375, "KL/rejected_KL_mean": -96.94520568847656, "KL/std": 39.201316833496094, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.009607050567865372, "fcm_dpo/delta": 0.05769995599985123, "fcm_dpo/margin": 37.344764709472656, "fcm_dpo/q_t": 0.4182150661945343, "grad_norm": 7.82089376449585, "learning_rate": 4.087222918524807e-07, "logits/chosen": -2.090040445327759, "logits/rejected": -1.4473483562469482, "logps/chosen": -155.27310180664062, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -187.59976196289062, "loss": 1.1021, "margin_dpo/margin_mean": 37.344764709472656, "margin_dpo/margin_std": 43.499420166015625, "step": 242 }, { "KL/chosen_KL_mean": -59.01629638671875, "KL/mean": -77.67515563964844, "KL/rejected_KL_mean": -96.33403015136719, "KL/std": 41.97687911987305, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.010426703840494156, "fcm_dpo/delta": -0.0011812672019004822, "fcm_dpo/margin": 37.31774139404297, "fcm_dpo/q_t": 0.4080137610435486, "grad_norm": 8.348605155944824, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.7883131504058838, "logits/rejected": -0.9382839202880859, "logps/chosen": -157.04769897460938, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -179.5220947265625, "loss": 1.1109, "margin_dpo/margin_mean": 37.31774139404297, "margin_dpo/margin_std": 50.22868347167969, "step": 243 }, { "KL/chosen_KL_mean": -61.70500183105469, "KL/mean": -81.40685272216797, "KL/rejected_KL_mean": -101.10868835449219, "KL/std": 46.01466369628906, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.00976712629199028, "fcm_dpo/delta": 0.024592578411102295, "fcm_dpo/margin": 39.40367889404297, "fcm_dpo/q_t": 0.4116143584251404, "grad_norm": 7.544835567474365, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.2850565910339355, "logits/rejected": -1.961534023284912, "logps/chosen": -150.5989227294922, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -203.6814727783203, "loss": 1.0849, "margin_dpo/margin_mean": 39.40367889404297, "margin_dpo/margin_std": 45.42003631591797, "step": 244 }, { "KL/chosen_KL_mean": -53.3755989074707, "KL/mean": -73.92649841308594, "KL/rejected_KL_mean": -94.4773941040039, "KL/std": 42.19940948486328, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.00995391234755516, "fcm_dpo/delta": -0.01583714969456196, "fcm_dpo/margin": 41.10179138183594, "fcm_dpo/q_t": 0.4025821387767792, "grad_norm": 7.236560344696045, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.206695556640625, "logits/rejected": -1.3517184257507324, "logps/chosen": -157.56959533691406, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -187.13385009765625, "loss": 1.0642, "margin_dpo/margin_mean": 41.1017951965332, "margin_dpo/margin_std": 43.74834060668945, "step": 245 }, { "KL/chosen_KL_mean": -68.9504623413086, "KL/mean": -85.34588623046875, "KL/rejected_KL_mean": -101.74131774902344, "KL/std": 41.66215515136719, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.010647161863744259, "fcm_dpo/delta": 0.07868749648332596, "fcm_dpo/margin": 32.790863037109375, "fcm_dpo/q_t": 0.42459893226623535, "grad_norm": 8.191699981689453, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.7274425029754639, "logits/rejected": -1.1654466390609741, "logps/chosen": -172.1695098876953, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -192.73353576660156, "loss": 1.1265, "margin_dpo/margin_mean": 32.790863037109375, "margin_dpo/margin_std": 45.25007629394531, "step": 246 }, { "KL/chosen_KL_mean": -62.83671188354492, "KL/mean": -90.07368469238281, "KL/rejected_KL_mean": -117.31065368652344, "KL/std": 46.90167236328125, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.008427303284406662, "fcm_dpo/delta": -0.14904604852199554, "fcm_dpo/margin": 54.47394561767578, "fcm_dpo/q_t": 0.3777352571487427, "grad_norm": 7.19643497467041, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.1096835136413574, "logits/rejected": -1.9387943744659424, "logps/chosen": -149.83108520507812, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -228.648681640625, "loss": 1.0272, "margin_dpo/margin_mean": 54.47394561767578, "margin_dpo/margin_std": 53.149330139160156, "step": 247 }, { "KL/chosen_KL_mean": -72.24562072753906, "KL/mean": -90.23958587646484, "KL/rejected_KL_mean": -108.23355102539062, "KL/std": 41.048492431640625, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.00970226526260376, "fcm_dpo/delta": 0.07043702900409698, "fcm_dpo/margin": 35.98794937133789, "fcm_dpo/q_t": 0.42163190245628357, "grad_norm": 8.040066719055176, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.7089741230010986, "logits/rejected": -1.6894145011901855, "logps/chosen": -147.0311279296875, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -206.51043701171875, "loss": 1.1079, "margin_dpo/margin_mean": 35.98794937133789, "margin_dpo/margin_std": 42.24217224121094, "step": 248 }, { "KL/chosen_KL_mean": -80.46951293945312, "KL/mean": -96.70954132080078, "KL/rejected_KL_mean": -112.9495849609375, "KL/std": 44.331207275390625, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.01064964197576046, "fcm_dpo/delta": 0.08309750258922577, "fcm_dpo/margin": 32.480072021484375, "fcm_dpo/q_t": 0.4260251522064209, "grad_norm": 9.767303466796875, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.2541074752807617, "logits/rejected": -1.7130029201507568, "logps/chosen": -176.17330932617188, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -200.09605407714844, "loss": 1.1462, "margin_dpo/margin_mean": 32.480072021484375, "margin_dpo/margin_std": 50.92515182495117, "step": 249 }, { "KL/chosen_KL_mean": -71.84210205078125, "KL/mean": -90.30519104003906, "KL/rejected_KL_mean": -108.76828002929688, "KL/std": 44.679054260253906, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.01075826957821846, "fcm_dpo/delta": -0.004653561860322952, "fcm_dpo/margin": 36.926185607910156, "fcm_dpo/q_t": 0.4063282608985901, "grad_norm": 9.055630683898926, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.216327428817749, "logits/rejected": -1.312872290611267, "logps/chosen": -185.89431762695312, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -189.85595703125, "loss": 1.0962, "margin_dpo/margin_mean": 36.926185607910156, "margin_dpo/margin_std": 47.95946502685547, "step": 250 }, { "KL/chosen_KL_mean": -71.90447998046875, "KL/mean": -88.15071868896484, "KL/rejected_KL_mean": -104.39695739746094, "KL/std": 41.83228302001953, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.011398296803236008, "fcm_dpo/delta": 0.04113733768463135, "fcm_dpo/margin": 32.49248504638672, "fcm_dpo/q_t": 0.41805654764175415, "grad_norm": 10.314250946044922, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -2.158214807510376, "logits/rejected": -1.4886447191238403, "logps/chosen": -169.6157684326172, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -187.92437744140625, "loss": 1.1378, "margin_dpo/margin_mean": 32.49248504638672, "margin_dpo/margin_std": 50.15900802612305, "step": 251 }, { "KL/chosen_KL_mean": -72.24581146240234, "KL/mean": -88.44799041748047, "KL/rejected_KL_mean": -104.65017700195312, "KL/std": 44.73468017578125, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.012491932138800621, "fcm_dpo/delta": -0.009644491598010063, "fcm_dpo/margin": 32.40435791015625, "fcm_dpo/q_t": 0.4062234163284302, "grad_norm": 11.036243438720703, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.140951156616211, "logits/rejected": -1.6016391515731812, "logps/chosen": -181.23191833496094, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -196.21441650390625, "loss": 1.1087, "margin_dpo/margin_mean": 32.40435791015625, "margin_dpo/margin_std": 47.295799255371094, "step": 252 }, { "KL/chosen_KL_mean": -79.60952758789062, "KL/mean": -94.69136047363281, "KL/rejected_KL_mean": -109.77320098876953, "KL/std": 48.00592803955078, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.013044758699834347, "fcm_dpo/delta": -0.006036818027496338, "fcm_dpo/margin": 30.16366958618164, "fcm_dpo/q_t": 0.4070701599121094, "grad_norm": 11.234423637390137, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.0504040718078613, "logits/rejected": -1.8417129516601562, "logps/chosen": -179.82583618164062, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -215.4499053955078, "loss": 1.1133, "margin_dpo/margin_mean": 30.163673400878906, "margin_dpo/margin_std": 41.97046661376953, "step": 253 }, { "KL/chosen_KL_mean": -77.02665710449219, "KL/mean": -92.86161804199219, "KL/rejected_KL_mean": -108.6965560913086, "KL/std": 45.403053283691406, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.011966011486947536, "fcm_dpo/delta": 0.03048786334693432, "fcm_dpo/margin": 31.669902801513672, "fcm_dpo/q_t": 0.4141252636909485, "grad_norm": 10.577949523925781, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.9924665689468384, "logits/rejected": -1.4619767665863037, "logps/chosen": -175.7618408203125, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -202.434814453125, "loss": 1.1291, "margin_dpo/margin_mean": 31.669902801513672, "margin_dpo/margin_std": 48.812103271484375, "step": 254 }, { "KL/chosen_KL_mean": -78.15425109863281, "KL/mean": -102.44584655761719, "KL/rejected_KL_mean": -126.73743438720703, "KL/std": 50.72254943847656, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.009860595688223839, "fcm_dpo/delta": -0.1616874486207962, "fcm_dpo/margin": 48.58319854736328, "fcm_dpo/q_t": 0.37331393361091614, "grad_norm": 9.03993034362793, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.9481287002563477, "logits/rejected": -1.6093533039093018, "logps/chosen": -168.5368194580078, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -223.81369018554688, "loss": 1.0256, "margin_dpo/margin_mean": 48.583194732666016, "margin_dpo/margin_std": 51.81832504272461, "step": 255 }, { "KL/chosen_KL_mean": -76.50236511230469, "KL/mean": -97.74116516113281, "KL/rejected_KL_mean": -118.97996520996094, "KL/std": 53.09275817871094, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.009415676817297935, "fcm_dpo/delta": -0.0017725415527820587, "fcm_dpo/margin": 42.47760009765625, "fcm_dpo/q_t": 0.40830373764038086, "grad_norm": 8.699026107788086, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.7076398134231567, "logits/rejected": -1.3639167547225952, "logps/chosen": -165.2551727294922, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -217.4738006591797, "loss": 1.1084, "margin_dpo/margin_mean": 42.47760009765625, "margin_dpo/margin_std": 62.763710021972656, "step": 256 }, { "KL/chosen_KL_mean": -78.20892333984375, "KL/mean": -99.98008728027344, "KL/rejected_KL_mean": -121.75125122070312, "KL/std": 54.494873046875, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.00870074238628149, "fcm_dpo/delta": 0.018506944179534912, "fcm_dpo/margin": 43.542335510253906, "fcm_dpo/q_t": 0.4112655520439148, "grad_norm": 7.773326396942139, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.136134147644043, "logits/rejected": -1.7473636865615845, "logps/chosen": -161.012451171875, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -207.61904907226562, "loss": 1.1058, "margin_dpo/margin_mean": 43.542335510253906, "margin_dpo/margin_std": 52.872398376464844, "step": 257 }, { "KL/chosen_KL_mean": -81.66576385498047, "KL/mean": -101.5562744140625, "KL/rejected_KL_mean": -121.44678497314453, "KL/std": 51.85133743286133, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.010053161531686783, "fcm_dpo/delta": -0.01215844601392746, "fcm_dpo/margin": 39.7810173034668, "fcm_dpo/q_t": 0.40453779697418213, "grad_norm": 9.036176681518555, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.2034924030303955, "logits/rejected": -1.852813720703125, "logps/chosen": -171.853515625, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -224.91748046875, "loss": 1.0938, "margin_dpo/margin_mean": 39.7810173034668, "margin_dpo/margin_std": 49.63108825683594, "step": 258 }, { "KL/chosen_KL_mean": -87.57511138916016, "KL/mean": -104.64314270019531, "KL/rejected_KL_mean": -121.711181640625, "KL/std": 52.2846794128418, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.009097885340452194, "fcm_dpo/delta": -0.00943857990205288, "fcm_dpo/margin": 34.136070251464844, "fcm_dpo/q_t": 0.4263812303543091, "grad_norm": 8.248490333557129, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.3247981071472168, "logits/rejected": -1.069736123085022, "logps/chosen": -178.34765625, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -216.29934692382812, "loss": 1.1716, "margin_dpo/margin_mean": 34.13607406616211, "margin_dpo/margin_std": 59.79408264160156, "step": 259 }, { "KL/chosen_KL_mean": -90.15082550048828, "KL/mean": -104.41741943359375, "KL/rejected_KL_mean": -118.68400573730469, "KL/std": 52.27936553955078, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.011000007390975952, "fcm_dpo/delta": 0.12306459248065948, "fcm_dpo/margin": 28.533180236816406, "fcm_dpo/q_t": 0.4352329969406128, "grad_norm": 11.24988079071045, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.6315704584121704, "logits/rejected": -1.6683452129364014, "logps/chosen": -165.7435302734375, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -219.529541015625, "loss": 1.1905, "margin_dpo/margin_mean": 28.53318214416504, "margin_dpo/margin_std": 57.25318145751953, "step": 260 }, { "KL/chosen_KL_mean": -69.11610412597656, "KL/mean": -94.44135284423828, "KL/rejected_KL_mean": -119.7666015625, "KL/std": 53.59440612792969, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.008800452575087547, "fcm_dpo/delta": -0.12712539732456207, "fcm_dpo/margin": 50.65049743652344, "fcm_dpo/q_t": 0.3812870383262634, "grad_norm": 7.751087188720703, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.979121446609497, "logits/rejected": -1.6095151901245117, "logps/chosen": -163.37100219726562, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -213.423583984375, "loss": 1.0416, "margin_dpo/margin_mean": 50.65049743652344, "margin_dpo/margin_std": 52.38161087036133, "step": 261 }, { "KL/chosen_KL_mean": -75.12834167480469, "KL/mean": -98.7419662475586, "KL/rejected_KL_mean": -122.35557556152344, "KL/std": 56.35432815551758, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.008549867197871208, "fcm_dpo/delta": -0.010947544127702713, "fcm_dpo/margin": 47.22724533081055, "fcm_dpo/q_t": 0.4043210744857788, "grad_norm": 7.569308757781982, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.6712672710418701, "logits/rejected": -1.129181146621704, "logps/chosen": -160.39564514160156, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -213.18167114257812, "loss": 1.0798, "margin_dpo/margin_mean": 47.22724533081055, "margin_dpo/margin_std": 56.550193786621094, "step": 262 }, { "KL/chosen_KL_mean": -61.24101257324219, "KL/mean": -93.20952606201172, "KL/rejected_KL_mean": -125.17803955078125, "KL/std": 61.35432815551758, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.007313176989555359, "fcm_dpo/delta": -0.12959256768226624, "fcm_dpo/margin": 63.93701934814453, "fcm_dpo/q_t": 0.37808120250701904, "grad_norm": 6.793945789337158, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.949862003326416, "logits/rejected": -1.3766202926635742, "logps/chosen": -156.99891662597656, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -227.01181030273438, "loss": 1.0282, "margin_dpo/margin_mean": 63.93701934814453, "margin_dpo/margin_std": 65.72269439697266, "step": 263 }, { "KL/chosen_KL_mean": -70.86907958984375, "KL/mean": -98.05549621582031, "KL/rejected_KL_mean": -125.24191284179688, "KL/std": 57.78061294555664, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.007049550302326679, "fcm_dpo/delta": 0.027134904637932777, "fcm_dpo/margin": 54.37283706665039, "fcm_dpo/q_t": 0.41129663586616516, "grad_norm": 6.799558162689209, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -1.8689451217651367, "logits/rejected": -1.5743999481201172, "logps/chosen": -161.49172973632812, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -225.5674591064453, "loss": 1.0807, "margin_dpo/margin_mean": 54.37283706665039, "margin_dpo/margin_std": 58.874488830566406, "step": 264 }, { "KL/chosen_KL_mean": -74.09158325195312, "KL/mean": -97.90462493896484, "KL/rejected_KL_mean": -121.71766662597656, "KL/std": 59.067657470703125, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.007614785805344582, "fcm_dpo/delta": 0.05832373723387718, "fcm_dpo/margin": 47.62608337402344, "fcm_dpo/q_t": 0.41985946893692017, "grad_norm": 7.385326385498047, "learning_rate": 3.848656339557562e-07, "logits/chosen": -1.8767725229263306, "logits/rejected": -1.4533321857452393, "logps/chosen": -166.4639129638672, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -216.34524536132812, "loss": 1.1158, "margin_dpo/margin_mean": 47.62608337402344, "margin_dpo/margin_std": 63.99040985107422, "step": 265 }, { "KL/chosen_KL_mean": -85.69625854492188, "KL/mean": -107.37635803222656, "KL/rejected_KL_mean": -129.0564422607422, "KL/std": 57.640655517578125, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.008548152633011341, "fcm_dpo/delta": 0.046835727989673615, "fcm_dpo/margin": 43.36018371582031, "fcm_dpo/q_t": 0.417877197265625, "grad_norm": 9.484822273254395, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.3835809230804443, "logits/rejected": -1.5656896829605103, "logps/chosen": -187.8962860107422, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -203.42286682128906, "loss": 1.1148, "margin_dpo/margin_mean": 43.36018371582031, "margin_dpo/margin_std": 60.85496520996094, "step": 266 }, { "KL/chosen_KL_mean": -76.37699890136719, "KL/mean": -101.5779800415039, "KL/rejected_KL_mean": -126.7789535522461, "KL/std": 57.627471923828125, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.00815313309431076, "fcm_dpo/delta": -0.026394978165626526, "fcm_dpo/margin": 50.401947021484375, "fcm_dpo/q_t": 0.401265025138855, "grad_norm": 8.165326118469238, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.7807798385620117, "logits/rejected": -1.1493260860443115, "logps/chosen": -180.66299438476562, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -222.76614379882812, "loss": 1.0674, "margin_dpo/margin_mean": 50.401947021484375, "margin_dpo/margin_std": 55.57642364501953, "step": 267 }, { "KL/chosen_KL_mean": -75.14181518554688, "KL/mean": -104.7325439453125, "KL/rejected_KL_mean": -134.32327270507812, "KL/std": 57.9452018737793, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.007221899461001158, "fcm_dpo/delta": -0.06403975933790207, "fcm_dpo/margin": 59.181453704833984, "fcm_dpo/q_t": 0.3915325105190277, "grad_norm": 7.930283069610596, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.7432785034179688, "logits/rejected": -1.5955947637557983, "logps/chosen": -161.83804321289062, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -243.51510620117188, "loss": 1.0471, "margin_dpo/margin_mean": 59.18145751953125, "margin_dpo/margin_std": 57.53593063354492, "step": 268 }, { "KL/chosen_KL_mean": -79.7818603515625, "KL/mean": -108.68545532226562, "KL/rejected_KL_mean": -137.58905029296875, "KL/std": 62.23661422729492, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.007246762048453093, "fcm_dpo/delta": -0.03315407410264015, "fcm_dpo/margin": 57.80718994140625, "fcm_dpo/q_t": 0.3984869122505188, "grad_norm": 7.207725524902344, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.6059300899505615, "logits/rejected": -1.1359739303588867, "logps/chosen": -184.753662109375, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -250.0655059814453, "loss": 1.0556, "margin_dpo/margin_mean": 57.807186126708984, "margin_dpo/margin_std": 59.212318420410156, "step": 269 }, { "KL/chosen_KL_mean": -89.27076721191406, "KL/mean": -113.37016296386719, "KL/rejected_KL_mean": -137.46957397460938, "KL/std": 61.21848678588867, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.007665836252272129, "fcm_dpo/delta": 0.042005572468042374, "fcm_dpo/margin": 48.198787689208984, "fcm_dpo/q_t": 0.4161429703235626, "grad_norm": 8.438031196594238, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.6312403678894043, "logits/rejected": -2.2320382595062256, "logps/chosen": -190.3446044921875, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -255.22247314453125, "loss": 1.1196, "margin_dpo/margin_mean": 48.19878387451172, "margin_dpo/margin_std": 69.4327392578125, "step": 270 }, { "KL/chosen_KL_mean": -72.98890686035156, "KL/mean": -103.23493957519531, "KL/rejected_KL_mean": -133.48095703125, "KL/std": 57.474639892578125, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0069838170893490314, "fcm_dpo/delta": -0.04091693088412285, "fcm_dpo/margin": 60.492042541503906, "fcm_dpo/q_t": 0.39607155323028564, "grad_norm": 8.002872467041016, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.4680789709091187, "logits/rejected": -1.0105818510055542, "logps/chosen": -169.7135009765625, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -232.00537109375, "loss": 1.0587, "margin_dpo/margin_mean": 60.492042541503906, "margin_dpo/margin_std": 65.67672729492188, "step": 271 }, { "KL/chosen_KL_mean": -63.21324920654297, "KL/mean": -92.207763671875, "KL/rejected_KL_mean": -121.2022705078125, "KL/std": 63.72555160522461, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.0069214170798659325, "fcm_dpo/delta": -0.002284811809659004, "fcm_dpo/margin": 57.989013671875, "fcm_dpo/q_t": 0.40608519315719604, "grad_norm": 7.296811580657959, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.394051194190979, "logits/rejected": -1.2873725891113281, "logps/chosen": -144.19046020507812, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -229.75762939453125, "loss": 1.0794, "margin_dpo/margin_mean": 57.989013671875, "margin_dpo/margin_std": 69.31120300292969, "step": 272 }, { "KL/chosen_KL_mean": -85.55570983886719, "KL/mean": -115.26547241210938, "KL/rejected_KL_mean": -144.97523498535156, "KL/std": 65.25542449951172, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.006530684418976307, "fcm_dpo/delta": 0.007129177451133728, "fcm_dpo/margin": 59.419525146484375, "fcm_dpo/q_t": 0.4080367088317871, "grad_norm": 7.227160930633545, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.7198429107666016, "logits/rejected": -1.3132685422897339, "logps/chosen": -177.78030395507812, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -242.33828735351562, "loss": 1.1004, "margin_dpo/margin_mean": 59.419525146484375, "margin_dpo/margin_std": 74.32960510253906, "step": 273 }, { "KL/chosen_KL_mean": -87.88787841796875, "KL/mean": -110.52479553222656, "KL/rejected_KL_mean": -133.16171264648438, "KL/std": 64.29875183105469, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.007743429392576218, "fcm_dpo/delta": 0.07657352089881897, "fcm_dpo/margin": 45.273841857910156, "fcm_dpo/q_t": 0.42358309030532837, "grad_norm": 12.762664794921875, "learning_rate": 3.75e-07, "logits/chosen": -1.9282622337341309, "logits/rejected": -1.5335572957992554, "logps/chosen": -170.14877319335938, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -217.0386962890625, "loss": 1.1185, "margin_dpo/margin_mean": 45.27383804321289, "margin_dpo/margin_std": 59.143035888671875, "step": 274 }, { "KL/chosen_KL_mean": -74.35482788085938, "KL/mean": -100.41793823242188, "KL/rejected_KL_mean": -126.48104858398438, "KL/std": 63.125179290771484, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.008153165690600872, "fcm_dpo/delta": -0.05063331872224808, "fcm_dpo/margin": 52.126197814941406, "fcm_dpo/q_t": 0.39810723066329956, "grad_norm": 9.26767635345459, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.0538885593414307, "logits/rejected": -1.7311184406280518, "logps/chosen": -154.0417938232422, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -225.23202514648438, "loss": 1.1027, "margin_dpo/margin_mean": 52.12620162963867, "margin_dpo/margin_std": 75.4040298461914, "step": 275 }, { "KL/chosen_KL_mean": -87.99552917480469, "KL/mean": -116.18345642089844, "KL/rejected_KL_mean": -144.3713836669922, "KL/std": 63.77820587158203, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.00693746842443943, "fcm_dpo/delta": 0.002303972840309143, "fcm_dpo/margin": 56.37583923339844, "fcm_dpo/q_t": 0.4080579876899719, "grad_norm": 8.848185539245605, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.1868884563446045, "logits/rejected": -1.6166386604309082, "logps/chosen": -174.53524780273438, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -222.22532653808594, "loss": 1.1023, "margin_dpo/margin_mean": 56.37583923339844, "margin_dpo/margin_std": 72.99510955810547, "step": 276 }, { "KL/chosen_KL_mean": -95.13591003417969, "KL/mean": -115.3011474609375, "KL/rejected_KL_mean": -135.46636962890625, "KL/std": 62.129554748535156, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.008342960849404335, "fcm_dpo/delta": 0.09657222032546997, "fcm_dpo/margin": 40.3304443359375, "fcm_dpo/q_t": 0.4289149343967438, "grad_norm": 9.546941757202148, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.0544233322143555, "logits/rejected": -1.6384103298187256, "logps/chosen": -179.38003540039062, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -234.30056762695312, "loss": 1.1511, "margin_dpo/margin_mean": 40.3304443359375, "margin_dpo/margin_std": 64.20411682128906, "step": 277 }, { "KL/chosen_KL_mean": -91.01161193847656, "KL/mean": -116.58341979980469, "KL/rejected_KL_mean": -142.15521240234375, "KL/std": 65.84242248535156, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.008238095790147781, "fcm_dpo/delta": -0.04111159220337868, "fcm_dpo/margin": 51.14360046386719, "fcm_dpo/q_t": 0.3982882797718048, "grad_norm": 12.335659980773926, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.259094715118408, "logits/rejected": -2.047288417816162, "logps/chosen": -173.44349670410156, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -240.0121307373047, "loss": 1.0821, "margin_dpo/margin_mean": 51.14360427856445, "margin_dpo/margin_std": 67.13626098632812, "step": 278 }, { "KL/chosen_KL_mean": -93.08076477050781, "KL/mean": -125.50665283203125, "KL/rejected_KL_mean": -157.93252563476562, "KL/std": 70.4123764038086, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.006816631183028221, "fcm_dpo/delta": -0.09438958764076233, "fcm_dpo/margin": 64.85176849365234, "fcm_dpo/q_t": 0.38738805055618286, "grad_norm": 9.860002517700195, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.862331509590149, "logits/rejected": -1.6523163318634033, "logps/chosen": -178.24876403808594, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -252.05917358398438, "loss": 1.0465, "margin_dpo/margin_mean": 64.85176849365234, "margin_dpo/margin_std": 70.2264404296875, "step": 279 }, { "KL/chosen_KL_mean": -96.8802719116211, "KL/mean": -129.1238555908203, "KL/rejected_KL_mean": -161.367431640625, "KL/std": 64.56246948242188, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.006577412132173777, "fcm_dpo/delta": -0.041864246129989624, "fcm_dpo/margin": 64.48716735839844, "fcm_dpo/q_t": 0.3965538740158081, "grad_norm": 7.720088958740234, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.7576899528503418, "logits/rejected": -1.5599184036254883, "logps/chosen": -182.7366943359375, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -265.48602294921875, "loss": 1.0555, "margin_dpo/margin_mean": 64.48716735839844, "margin_dpo/margin_std": 67.58592224121094, "step": 280 }, { "KL/chosen_KL_mean": -103.12272644042969, "KL/mean": -135.16384887695312, "KL/rejected_KL_mean": -167.20497131347656, "KL/std": 74.68962860107422, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0062049636617302895, "fcm_dpo/delta": 0.0027420073747634888, "fcm_dpo/margin": 64.08223724365234, "fcm_dpo/q_t": 0.4070470333099365, "grad_norm": 5.97775411605835, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -1.9057626724243164, "logits/rejected": -1.141242265701294, "logps/chosen": -189.67355346679688, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -255.83364868164062, "loss": 1.0811, "margin_dpo/margin_mean": 64.08224487304688, "margin_dpo/margin_std": 75.51350402832031, "step": 281 }, { "KL/chosen_KL_mean": -101.30918884277344, "KL/mean": -141.61265563964844, "KL/rejected_KL_mean": -181.9161376953125, "KL/std": 72.41990661621094, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.005512166768312454, "fcm_dpo/delta": -0.08416593819856644, "fcm_dpo/margin": 80.60694885253906, "fcm_dpo/q_t": 0.38654381036758423, "grad_norm": 7.747836112976074, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.815206527709961, "logits/rejected": -1.433701992034912, "logps/chosen": -206.414306640625, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -284.7695007324219, "loss": 1.0284, "margin_dpo/margin_mean": 80.60694885253906, "margin_dpo/margin_std": 72.90723419189453, "step": 282 }, { "KL/chosen_KL_mean": -118.03620910644531, "KL/mean": -146.22140502929688, "KL/rejected_KL_mean": -174.40658569335938, "KL/std": 71.47140502929688, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.006116765551269054, "fcm_dpo/delta": 0.08236735314130783, "fcm_dpo/margin": 56.37036895751953, "fcm_dpo/q_t": 0.4247892200946808, "grad_norm": 8.988859176635742, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.1113791465759277, "logits/rejected": -1.7681806087493896, "logps/chosen": -213.08880615234375, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -277.9511413574219, "loss": 1.1283, "margin_dpo/margin_mean": 56.37036895751953, "margin_dpo/margin_std": 79.92024993896484, "step": 283 }, { "KL/chosen_KL_mean": -108.63221740722656, "KL/mean": -143.81338500976562, "KL/rejected_KL_mean": -178.99456787109375, "KL/std": 70.65594482421875, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.005858813412487507, "fcm_dpo/delta": -0.026428356766700745, "fcm_dpo/margin": 70.36234283447266, "fcm_dpo/q_t": 0.3987847566604614, "grad_norm": 8.546588897705078, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.9221277236938477, "logits/rejected": -1.5922625064849854, "logps/chosen": -196.2987060546875, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -277.74560546875, "loss": 1.0503, "margin_dpo/margin_mean": 70.36234283447266, "margin_dpo/margin_std": 63.96889877319336, "step": 284 }, { "KL/chosen_KL_mean": -121.29149627685547, "KL/mean": -146.2021484375, "KL/rejected_KL_mean": -171.11277770996094, "KL/std": 65.45597839355469, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.006930316332727671, "fcm_dpo/delta": 0.07760372012853622, "fcm_dpo/margin": 49.82128143310547, "fcm_dpo/q_t": 0.4228743612766266, "grad_norm": 10.192597389221191, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.2866902351379395, "logits/rejected": -2.002204418182373, "logps/chosen": -215.7245330810547, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -275.1847229003906, "loss": 1.1158, "margin_dpo/margin_mean": 49.82128143310547, "margin_dpo/margin_std": 62.70260238647461, "step": 285 }, { "KL/chosen_KL_mean": -125.57996368408203, "KL/mean": -159.6600341796875, "KL/rejected_KL_mean": -193.74008178710938, "KL/std": 69.58126068115234, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.006307811476290226, "fcm_dpo/delta": -0.05636781081557274, "fcm_dpo/margin": 68.16011810302734, "fcm_dpo/q_t": 0.3938465118408203, "grad_norm": 7.766751766204834, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.0504117012023926, "logits/rejected": -1.5911575555801392, "logps/chosen": -229.30035400390625, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -289.9978332519531, "loss": 1.0527, "margin_dpo/margin_mean": 68.16011810302734, "margin_dpo/margin_std": 72.71003723144531, "step": 286 }, { "KL/chosen_KL_mean": -144.86009216308594, "KL/mean": -163.83120727539062, "KL/rejected_KL_mean": -182.80230712890625, "KL/std": 62.245540618896484, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.006535199470818043, "fcm_dpo/delta": 0.04180797562003136, "fcm_dpo/margin": 37.94221115112305, "fcm_dpo/q_t": 0.4445207715034485, "grad_norm": 7.637104511260986, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.0241904258728027, "logits/rejected": -1.3994685411453247, "logps/chosen": -238.74996948242188, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -266.135986328125, "loss": 1.2103, "margin_dpo/margin_mean": 37.94221115112305, "margin_dpo/margin_std": 73.16700744628906, "step": 287 }, { "KL/chosen_KL_mean": -137.07302856445312, "KL/mean": -172.24517822265625, "KL/rejected_KL_mean": -207.41732788085938, "KL/std": 70.39920806884766, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.006102034822106361, "fcm_dpo/delta": -0.056060925126075745, "fcm_dpo/margin": 70.34429931640625, "fcm_dpo/q_t": 0.3952332139015198, "grad_norm": 7.024817943572998, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.8364543914794922, "logits/rejected": -1.4212539196014404, "logps/chosen": -225.22903442382812, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -300.69927978515625, "loss": 1.0557, "margin_dpo/margin_mean": 70.34429931640625, "margin_dpo/margin_std": 78.51313781738281, "step": 288 }, { "KL/chosen_KL_mean": -133.52142333984375, "KL/mean": -168.92355346679688, "KL/rejected_KL_mean": -204.32571411132812, "KL/std": 73.35023498535156, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.005918733775615692, "fcm_dpo/delta": -0.03301646187901497, "fcm_dpo/margin": 70.80428314208984, "fcm_dpo/q_t": 0.40107160806655884, "grad_norm": 6.352198600769043, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.075766086578369, "logits/rejected": -1.875805139541626, "logps/chosen": -208.9143524169922, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -297.47998046875, "loss": 1.0703, "margin_dpo/margin_mean": 70.80429077148438, "margin_dpo/margin_std": 84.77729034423828, "step": 289 }, { "KL/chosen_KL_mean": -143.22689819335938, "KL/mean": -180.09906005859375, "KL/rejected_KL_mean": -216.97122192382812, "KL/std": 78.23715209960938, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0057178703136742115, "fcm_dpo/delta": -0.044641174376010895, "fcm_dpo/margin": 73.74433898925781, "fcm_dpo/q_t": 0.39854639768600464, "grad_norm": 7.593597888946533, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.3777105808258057, "logits/rejected": -1.4266893863677979, "logps/chosen": -231.26885986328125, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -340.1833801269531, "loss": 1.0677, "margin_dpo/margin_mean": 73.74433135986328, "margin_dpo/margin_std": 83.1454086303711, "step": 290 }, { "KL/chosen_KL_mean": -134.96804809570312, "KL/mean": -166.61239624023438, "KL/rejected_KL_mean": -198.25672912597656, "KL/std": 74.31089782714844, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.005756012164056301, "fcm_dpo/delta": 0.053068291395902634, "fcm_dpo/margin": 63.28868865966797, "fcm_dpo/q_t": 0.417277455329895, "grad_norm": 8.8091402053833, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.3709869384765625, "logits/rejected": -1.9075868129730225, "logps/chosen": -229.06329345703125, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -294.3067932128906, "loss": 1.0988, "margin_dpo/margin_mean": 63.288692474365234, "margin_dpo/margin_std": 74.5374984741211, "step": 291 }, { "KL/chosen_KL_mean": -138.90353393554688, "KL/mean": -171.26853942871094, "KL/rejected_KL_mean": -203.633544921875, "KL/std": 72.37640380859375, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.005980066489428282, "fcm_dpo/delta": 0.020991913974285126, "fcm_dpo/margin": 64.73001098632812, "fcm_dpo/q_t": 0.4101616442203522, "grad_norm": 7.874806880950928, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0173730850219727, "logits/rejected": -1.5826966762542725, "logps/chosen": -227.15394592285156, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -300.05120849609375, "loss": 1.0753, "margin_dpo/margin_mean": 64.73001098632812, "margin_dpo/margin_std": 68.11990356445312, "step": 292 }, { "KL/chosen_KL_mean": -138.3443603515625, "KL/mean": -164.57342529296875, "KL/rejected_KL_mean": -190.802490234375, "KL/std": 76.12394714355469, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.006862225942313671, "fcm_dpo/delta": 0.05196109786629677, "fcm_dpo/margin": 52.458126068115234, "fcm_dpo/q_t": 0.42075809836387634, "grad_norm": 8.108826637268066, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.471024513244629, "logits/rejected": -1.9145504236221313, "logps/chosen": -225.72091674804688, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -276.55828857421875, "loss": 1.1316, "margin_dpo/margin_mean": 52.4581298828125, "margin_dpo/margin_std": 80.86062622070312, "step": 293 }, { "KL/chosen_KL_mean": -140.26246643066406, "KL/mean": -161.78773498535156, "KL/rejected_KL_mean": -183.31301879882812, "KL/std": 71.9763412475586, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.007897116243839264, "fcm_dpo/delta": 0.07987435162067413, "fcm_dpo/margin": 43.05056381225586, "fcm_dpo/q_t": 0.4271060824394226, "grad_norm": 9.467598915100098, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.6267622709274292, "logits/rejected": -1.462005853652954, "logps/chosen": -213.77041625976562, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -271.40179443359375, "loss": 1.1825, "margin_dpo/margin_mean": 43.050567626953125, "margin_dpo/margin_std": 84.49092102050781, "step": 294 }, { "KL/chosen_KL_mean": -132.92127990722656, "KL/mean": -165.94041442871094, "KL/rejected_KL_mean": -198.95953369140625, "KL/std": 74.09613037109375, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.0065531134605407715, "fcm_dpo/delta": -0.07899242639541626, "fcm_dpo/margin": 66.03825378417969, "fcm_dpo/q_t": 0.39214441180229187, "grad_norm": 8.66662311553955, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.369492530822754, "logits/rejected": -1.2058568000793457, "logps/chosen": -210.686767578125, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -293.206787109375, "loss": 1.0658, "margin_dpo/margin_mean": 66.03825378417969, "margin_dpo/margin_std": 80.46630096435547, "step": 295 }, { "KL/chosen_KL_mean": -142.25599670410156, "KL/mean": -177.5230255126953, "KL/rejected_KL_mean": -212.79006958007812, "KL/std": 74.44865417480469, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.005980327725410461, "fcm_dpo/delta": -0.04734375327825546, "fcm_dpo/margin": 70.5340805053711, "fcm_dpo/q_t": 0.39555618166923523, "grad_norm": 11.163251876831055, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.6691079139709473, "logits/rejected": -2.1965138912200928, "logps/chosen": -240.11241149902344, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -313.60638427734375, "loss": 1.0581, "margin_dpo/margin_mean": 70.53408813476562, "margin_dpo/margin_std": 75.27764892578125, "step": 296 }, { "KL/chosen_KL_mean": -142.66671752929688, "KL/mean": -172.5174560546875, "KL/rejected_KL_mean": -202.36822509765625, "KL/std": 75.17344665527344, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.00645687198266387, "fcm_dpo/delta": 0.021310636773705482, "fcm_dpo/margin": 59.70153045654297, "fcm_dpo/q_t": 0.4129793345928192, "grad_norm": 10.755855560302734, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.1680850982666016, "logits/rejected": -0.8585318922996521, "logps/chosen": -231.2325439453125, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -303.9248046875, "loss": 1.1008, "margin_dpo/margin_mean": 59.70152282714844, "margin_dpo/margin_std": 80.69510650634766, "step": 297 }, { "KL/chosen_KL_mean": -119.81941223144531, "KL/mean": -158.08847045898438, "KL/rejected_KL_mean": -196.35752868652344, "KL/std": 81.08302307128906, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0058023747988045216, "fcm_dpo/delta": -0.07874768227338791, "fcm_dpo/margin": 76.53810119628906, "fcm_dpo/q_t": 0.3907705843448639, "grad_norm": 10.19721794128418, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.0512328147888184, "logits/rejected": -2.0359785556793213, "logps/chosen": -214.69985961914062, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -318.6685485839844, "loss": 1.0448, "margin_dpo/margin_mean": 76.5381088256836, "margin_dpo/margin_std": 82.40852355957031, "step": 298 }, { "KL/chosen_KL_mean": -124.30731201171875, "KL/mean": -162.330322265625, "KL/rejected_KL_mean": -200.35333251953125, "KL/std": 78.50457763671875, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.005344615317881107, "fcm_dpo/delta": -0.010926278308033943, "fcm_dpo/margin": 76.04600524902344, "fcm_dpo/q_t": 0.4034714996814728, "grad_norm": 9.035113334655762, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.0138659477233887, "logits/rejected": -0.7907317876815796, "logps/chosen": -204.71566772460938, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -289.8905029296875, "loss": 1.0662, "margin_dpo/margin_mean": 76.04600524902344, "margin_dpo/margin_std": 81.98774719238281, "step": 299 }, { "KL/chosen_KL_mean": -131.2325439453125, "KL/mean": -162.87997436523438, "KL/rejected_KL_mean": -194.52737426757812, "KL/std": 81.86282348632812, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.005670792888849974, "fcm_dpo/delta": 0.06372794508934021, "fcm_dpo/margin": 63.29485321044922, "fcm_dpo/q_t": 0.4207995533943176, "grad_norm": 7.633592128753662, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.6230599880218506, "logits/rejected": -1.412355661392212, "logps/chosen": -219.39144897460938, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -295.4665832519531, "loss": 1.1248, "margin_dpo/margin_mean": 63.29485321044922, "margin_dpo/margin_std": 88.53703308105469, "step": 300 }, { "KL/chosen_KL_mean": -120.6318130493164, "KL/mean": -159.8607177734375, "KL/rejected_KL_mean": -199.08963012695312, "KL/std": 80.09178161621094, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.00532924709841609, "fcm_dpo/delta": -0.0456632636487484, "fcm_dpo/margin": 78.45780181884766, "fcm_dpo/q_t": 0.3971927762031555, "grad_norm": 8.348127365112305, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.8194541931152344, "logits/rejected": -1.3939547538757324, "logps/chosen": -206.13671875, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -298.0064697265625, "loss": 1.0687, "margin_dpo/margin_mean": 78.45780944824219, "margin_dpo/margin_std": 90.18197631835938, "step": 301 }, { "KL/chosen_KL_mean": -115.84928131103516, "KL/mean": -149.8520965576172, "KL/rejected_KL_mean": -183.8549346923828, "KL/std": 82.57183074951172, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.005656576715409756, "fcm_dpo/delta": 0.024848731234669685, "fcm_dpo/margin": 68.00563049316406, "fcm_dpo/q_t": 0.4112318158149719, "grad_norm": 7.6754841804504395, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3087787628173828, "logits/rejected": -1.1348693370819092, "logps/chosen": -195.2996826171875, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -286.9140319824219, "loss": 1.0887, "margin_dpo/margin_mean": 68.00563049316406, "margin_dpo/margin_std": 80.90306091308594, "step": 302 }, { "KL/chosen_KL_mean": -109.30900573730469, "KL/mean": -146.0797882080078, "KL/rejected_KL_mean": -182.8505859375, "KL/std": 76.5225830078125, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.005683101713657379, "fcm_dpo/delta": -0.031322553753852844, "fcm_dpo/margin": 73.54158020019531, "fcm_dpo/q_t": 0.39893293380737305, "grad_norm": 7.924332618713379, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.8019849061965942, "logits/rejected": -1.335475206375122, "logps/chosen": -204.90728759765625, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -274.9151611328125, "loss": 1.0655, "margin_dpo/margin_mean": 73.54158020019531, "margin_dpo/margin_std": 81.73271942138672, "step": 303 }, { "KL/chosen_KL_mean": -108.677734375, "KL/mean": -146.6348876953125, "KL/rejected_KL_mean": -184.59201049804688, "KL/std": 79.30681610107422, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.005362863652408123, "fcm_dpo/delta": -0.012009890750050545, "fcm_dpo/margin": 75.914306640625, "fcm_dpo/q_t": 0.4029920995235443, "grad_norm": 8.072406768798828, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.0955557823181152, "logits/rejected": -1.6733598709106445, "logps/chosen": -208.13088989257812, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -282.6742858886719, "loss": 1.0666, "margin_dpo/margin_mean": 75.914306640625, "margin_dpo/margin_std": 82.62706756591797, "step": 304 }, { "KL/chosen_KL_mean": -119.12056732177734, "KL/mean": -155.30160522460938, "KL/rejected_KL_mean": -191.48263549804688, "KL/std": 82.82929229736328, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.005267108324915171, "fcm_dpo/delta": 0.025592684745788574, "fcm_dpo/margin": 72.362060546875, "fcm_dpo/q_t": 0.41171693801879883, "grad_norm": 6.989083290100098, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.9993151426315308, "logits/rejected": -1.5025957822799683, "logps/chosen": -222.38397216796875, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -289.31158447265625, "loss": 1.1003, "margin_dpo/margin_mean": 72.362060546875, "margin_dpo/margin_std": 90.61041259765625, "step": 305 }, { "KL/chosen_KL_mean": -106.37397766113281, "KL/mean": -148.51177978515625, "KL/rejected_KL_mean": -190.64959716796875, "KL/std": 78.14771270751953, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.005036830436438322, "fcm_dpo/delta": -0.0488281324505806, "fcm_dpo/margin": 84.27560424804688, "fcm_dpo/q_t": 0.39326998591423035, "grad_norm": 8.833101272583008, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.2793540954589844, "logits/rejected": -1.7246590852737427, "logps/chosen": -206.67782592773438, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -282.8804626464844, "loss": 1.0392, "margin_dpo/margin_mean": 84.27560424804688, "margin_dpo/margin_std": 72.43910217285156, "step": 306 }, { "KL/chosen_KL_mean": -111.98723602294922, "KL/mean": -143.8970947265625, "KL/rejected_KL_mean": -175.8069305419922, "KL/std": 79.33708953857422, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.005522524937987328, "fcm_dpo/delta": 0.0737767368555069, "fcm_dpo/margin": 63.81969451904297, "fcm_dpo/q_t": 0.42346447706222534, "grad_norm": 8.614710807800293, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.032927989959717, "logits/rejected": -1.6877312660217285, "logps/chosen": -208.63699340820312, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -269.155517578125, "loss": 1.135, "margin_dpo/margin_mean": 63.81969451904297, "margin_dpo/margin_std": 94.41084289550781, "step": 307 }, { "KL/chosen_KL_mean": -117.37681579589844, "KL/mean": -149.37599182128906, "KL/rejected_KL_mean": -181.37518310546875, "KL/std": 80.8201904296875, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.005835318937897682, "fcm_dpo/delta": 0.034761447459459305, "fcm_dpo/margin": 63.99835205078125, "fcm_dpo/q_t": 0.416228711605072, "grad_norm": 10.915842056274414, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7092130184173584, "logits/rejected": -1.424241542816162, "logps/chosen": -203.1014862060547, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -267.79833984375, "loss": 1.127, "margin_dpo/margin_mean": 63.99834442138672, "margin_dpo/margin_std": 94.40786743164062, "step": 308 }, { "KL/chosen_KL_mean": -131.66928100585938, "KL/mean": -160.21629333496094, "KL/rejected_KL_mean": -188.76333618164062, "KL/std": 86.45205688476562, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.006669000722467899, "fcm_dpo/delta": 0.030254226177930832, "fcm_dpo/margin": 57.09407043457031, "fcm_dpo/q_t": 0.41490092873573303, "grad_norm": 12.990861892700195, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.6675835847854614, "logits/rejected": -1.231377124786377, "logps/chosen": -214.82073974609375, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -275.349365234375, "loss": 1.1233, "margin_dpo/margin_mean": 57.09407043457031, "margin_dpo/margin_std": 86.33448028564453, "step": 309 }, { "KL/chosen_KL_mean": -119.68170166015625, "KL/mean": -151.29469299316406, "KL/rejected_KL_mean": -182.90768432617188, "KL/std": 78.17030334472656, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006458095274865627, "fcm_dpo/delta": -0.015126347541809082, "fcm_dpo/margin": 63.22599792480469, "fcm_dpo/q_t": 0.4043968915939331, "grad_norm": 9.842387199401855, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.3783397674560547, "logits/rejected": -1.9735199213027954, "logps/chosen": -209.41969299316406, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -269.978515625, "loss": 1.0865, "margin_dpo/margin_mean": 63.22599792480469, "margin_dpo/margin_std": 82.84346008300781, "step": 310 }, { "KL/chosen_KL_mean": -136.98794555664062, "KL/mean": -178.02337646484375, "KL/rejected_KL_mean": -219.0587921142578, "KL/std": 79.81135559082031, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.005540521815419197, "fcm_dpo/delta": -0.10382238030433655, "fcm_dpo/margin": 82.07086181640625, "fcm_dpo/q_t": 0.38441652059555054, "grad_norm": 8.882184028625488, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.7298040390014648, "logits/rejected": -1.0063247680664062, "logps/chosen": -225.51437377929688, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -301.32489013671875, "loss": 1.0298, "margin_dpo/margin_mean": 82.07085418701172, "margin_dpo/margin_std": 81.08486938476562, "step": 311 }, { "KL/chosen_KL_mean": -133.03199768066406, "KL/mean": -166.96292114257812, "KL/rejected_KL_mean": -200.89385986328125, "KL/std": 87.51390075683594, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.005662238225340843, "fcm_dpo/delta": 0.020663822069764137, "fcm_dpo/margin": 67.86186981201172, "fcm_dpo/q_t": 0.4110272526741028, "grad_norm": 9.671820640563965, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.7496552467346191, "logits/rejected": -1.2369227409362793, "logps/chosen": -213.4326934814453, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -283.34100341796875, "loss": 1.0943, "margin_dpo/margin_mean": 67.86186981201172, "margin_dpo/margin_std": 84.20599365234375, "step": 312 }, { "KL/chosen_KL_mean": -126.95478057861328, "KL/mean": -160.23040771484375, "KL/rejected_KL_mean": -193.50604248046875, "KL/std": 85.92005920410156, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.00584026426076889, "fcm_dpo/delta": 0.014166075736284256, "fcm_dpo/margin": 66.55126190185547, "fcm_dpo/q_t": 0.41052669286727905, "grad_norm": 9.97815990447998, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.430640459060669, "logits/rejected": -2.0300726890563965, "logps/chosen": -211.0966796875, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -286.9328918457031, "loss": 1.1043, "margin_dpo/margin_mean": 66.55125427246094, "margin_dpo/margin_std": 91.82222747802734, "step": 313 }, { "KL/chosen_KL_mean": -130.34268188476562, "KL/mean": -162.76739501953125, "KL/rejected_KL_mean": -195.192138671875, "KL/std": 86.12786865234375, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.005816853605210781, "fcm_dpo/delta": 0.03581969439983368, "fcm_dpo/margin": 64.84945678710938, "fcm_dpo/q_t": 0.4157375991344452, "grad_norm": 10.02777099609375, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.920820951461792, "logits/rejected": -1.7030256986618042, "logps/chosen": -219.15240478515625, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -299.35107421875, "loss": 1.1078, "margin_dpo/margin_mean": 64.84945678710938, "margin_dpo/margin_std": 88.28302001953125, "step": 314 }, { "KL/chosen_KL_mean": -124.9246826171875, "KL/mean": -158.84100341796875, "KL/rejected_KL_mean": -192.75732421875, "KL/std": 90.37235260009766, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.00596240721642971, "fcm_dpo/delta": -0.007844885811209679, "fcm_dpo/margin": 67.83265686035156, "fcm_dpo/q_t": 0.4078792631626129, "grad_norm": 9.914691925048828, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.3414465188980103, "logits/rejected": -1.166211485862732, "logps/chosen": -205.7183837890625, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -284.8944396972656, "loss": 1.1129, "margin_dpo/margin_mean": 67.8326644897461, "margin_dpo/margin_std": 104.65972137451172, "step": 315 }, { "KL/chosen_KL_mean": -127.85499572753906, "KL/mean": -164.584228515625, "KL/rejected_KL_mean": -201.3134765625, "KL/std": 89.54456329345703, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.005742821842432022, "fcm_dpo/delta": -0.037673451006412506, "fcm_dpo/margin": 73.45846557617188, "fcm_dpo/q_t": 0.40031713247299194, "grad_norm": 9.18078327178955, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.81521475315094, "logits/rejected": -1.815087080001831, "logps/chosen": -212.03924560546875, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -307.0042724609375, "loss": 1.0762, "margin_dpo/margin_mean": 73.45846557617188, "margin_dpo/margin_std": 93.87252044677734, "step": 316 }, { "KL/chosen_KL_mean": -134.22824096679688, "KL/mean": -179.08355712890625, "KL/rejected_KL_mean": -223.93887329101562, "KL/std": 98.23494720458984, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.005019248928874731, "fcm_dpo/delta": -0.09062010794878006, "fcm_dpo/margin": 89.71064758300781, "fcm_dpo/q_t": 0.388563871383667, "grad_norm": 9.592621803283691, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -2.0090808868408203, "logits/rejected": -1.540313482284546, "logps/chosen": -222.1017303466797, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -332.2040710449219, "loss": 1.0535, "margin_dpo/margin_mean": 89.71064758300781, "margin_dpo/margin_std": 105.04151916503906, "step": 317 }, { "KL/chosen_KL_mean": -129.34405517578125, "KL/mean": -169.109619140625, "KL/rejected_KL_mean": -208.87518310546875, "KL/std": 82.23076629638672, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.0048066191375255585, "fcm_dpo/delta": 0.028766699135303497, "fcm_dpo/margin": 79.5311279296875, "fcm_dpo/q_t": 0.41211366653442383, "grad_norm": 8.622135162353516, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.3892154693603516, "logits/rejected": -1.6928386688232422, "logps/chosen": -227.54959106445312, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -297.011474609375, "loss": 1.0864, "margin_dpo/margin_mean": 79.5311279296875, "margin_dpo/margin_std": 91.41517639160156, "step": 318 }, { "KL/chosen_KL_mean": -111.46063232421875, "KL/mean": -155.43270874023438, "KL/rejected_KL_mean": -199.40480041503906, "KL/std": 85.68540954589844, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.004675729665905237, "fcm_dpo/delta": -0.020688775926828384, "fcm_dpo/margin": 87.94416809082031, "fcm_dpo/q_t": 0.4013651907444, "grad_norm": 8.984302520751953, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.671229362487793, "logits/rejected": -1.6070188283920288, "logps/chosen": -193.96400451660156, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -318.05548095703125, "loss": 1.0651, "margin_dpo/margin_mean": 87.94416809082031, "margin_dpo/margin_std": 96.52432250976562, "step": 319 }, { "KL/chosen_KL_mean": -114.83718872070312, "KL/mean": -150.49081420898438, "KL/rejected_KL_mean": -186.14443969726562, "KL/std": 87.14494323730469, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0050945947878062725, "fcm_dpo/delta": 0.05782891437411308, "fcm_dpo/margin": 71.3072509765625, "fcm_dpo/q_t": 0.41906067728996277, "grad_norm": 7.899081707000732, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.152372360229492, "logits/rejected": -1.9679925441741943, "logps/chosen": -201.97442626953125, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -289.3685302734375, "loss": 1.1021, "margin_dpo/margin_mean": 71.3072509765625, "margin_dpo/margin_std": 85.63368225097656, "step": 320 }, { "KL/chosen_KL_mean": -128.232177734375, "KL/mean": -169.60940551757812, "KL/rejected_KL_mean": -210.98663330078125, "KL/std": 89.21060943603516, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.004935946315526962, "fcm_dpo/delta": -0.020057110115885735, "fcm_dpo/margin": 82.75444793701172, "fcm_dpo/q_t": 0.40090078115463257, "grad_norm": 8.30031967163086, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.874639868736267, "logits/rejected": -1.4788427352905273, "logps/chosen": -221.3368377685547, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -308.45477294921875, "loss": 1.0675, "margin_dpo/margin_mean": 82.75444793701172, "margin_dpo/margin_std": 89.81394958496094, "step": 321 }, { "KL/chosen_KL_mean": -120.0365219116211, "KL/mean": -164.35853576660156, "KL/rejected_KL_mean": -208.68055725097656, "KL/std": 88.38485717773438, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0045737409964203835, "fcm_dpo/delta": -0.02514272928237915, "fcm_dpo/margin": 88.64405822753906, "fcm_dpo/q_t": 0.4005265235900879, "grad_norm": 11.764183044433594, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.571209192276001, "logits/rejected": -1.2052593231201172, "logps/chosen": -212.22943115234375, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -309.57379150390625, "loss": 1.0731, "margin_dpo/margin_mean": 88.64405822753906, "margin_dpo/margin_std": 95.80950927734375, "step": 322 }, { "KL/chosen_KL_mean": -138.45901489257812, "KL/mean": -175.59133911132812, "KL/rejected_KL_mean": -212.7236785888672, "KL/std": 88.88581085205078, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.005056221038103104, "fcm_dpo/delta": 0.03935381397604942, "fcm_dpo/margin": 74.2646484375, "fcm_dpo/q_t": 0.4156927466392517, "grad_norm": 10.556021690368652, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0375118255615234, "logits/rejected": -1.6604541540145874, "logps/chosen": -218.73110961914062, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -293.02703857421875, "loss": 1.0987, "margin_dpo/margin_mean": 74.26465606689453, "margin_dpo/margin_std": 93.21639251708984, "step": 323 }, { "KL/chosen_KL_mean": -138.38348388671875, "KL/mean": -177.0481414794922, "KL/rejected_KL_mean": -215.71279907226562, "KL/std": 89.76466369628906, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.005248438101261854, "fcm_dpo/delta": -0.010489151813089848, "fcm_dpo/margin": 77.32931518554688, "fcm_dpo/q_t": 0.4033399820327759, "grad_norm": 9.112119674682617, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.6951037645339966, "logits/rejected": -1.0471745729446411, "logps/chosen": -234.1128387451172, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -309.88201904296875, "loss": 1.0695, "margin_dpo/margin_mean": 77.32930755615234, "margin_dpo/margin_std": 85.30317687988281, "step": 324 }, { "KL/chosen_KL_mean": -125.67424011230469, "KL/mean": -167.6588134765625, "KL/rejected_KL_mean": -209.64340209960938, "KL/std": 85.18707275390625, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.004960745573043823, "fcm_dpo/delta": -0.028291627764701843, "fcm_dpo/margin": 83.96918487548828, "fcm_dpo/q_t": 0.3993198573589325, "grad_norm": 7.516818046569824, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.8027971982955933, "logits/rejected": -1.0680700540542603, "logps/chosen": -224.381103515625, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -286.051513671875, "loss": 1.0599, "margin_dpo/margin_mean": 83.96918487548828, "margin_dpo/margin_std": 88.60990142822266, "step": 325 }, { "KL/chosen_KL_mean": -140.87615966796875, "KL/mean": -178.54632568359375, "KL/rejected_KL_mean": -216.21646118164062, "KL/std": 84.9094009399414, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.005057758651673794, "fcm_dpo/delta": 0.030559197068214417, "fcm_dpo/margin": 75.34030151367188, "fcm_dpo/q_t": 0.41249316930770874, "grad_norm": 7.292680740356445, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.5664820671081543, "logits/rejected": -1.9061024188995361, "logps/chosen": -245.5374755859375, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -303.0418701171875, "loss": 1.0929, "margin_dpo/margin_mean": 75.34030151367188, "margin_dpo/margin_std": 91.9469985961914, "step": 326 }, { "KL/chosen_KL_mean": -136.6942138671875, "KL/mean": -172.9864501953125, "KL/rejected_KL_mean": -209.2786865234375, "KL/std": 77.61909484863281, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005244665779173374, "fcm_dpo/delta": 0.03040998801589012, "fcm_dpo/margin": 72.58446502685547, "fcm_dpo/q_t": 0.4136658310890198, "grad_norm": 8.428020477294922, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.8614349365234375, "logits/rejected": -1.2843204736709595, "logps/chosen": -244.1344451904297, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -326.3319091796875, "loss": 1.093, "margin_dpo/margin_mean": 72.58447265625, "margin_dpo/margin_std": 88.01046752929688, "step": 327 }, { "KL/chosen_KL_mean": -132.56954956054688, "KL/mean": -180.3650665283203, "KL/rejected_KL_mean": -228.16058349609375, "KL/std": 91.98912048339844, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.004718562122434378, "fcm_dpo/delta": -0.09572428464889526, "fcm_dpo/margin": 95.59103393554688, "fcm_dpo/q_t": 0.38455888628959656, "grad_norm": 9.45504379272461, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6441705226898193, "logits/rejected": -1.3435397148132324, "logps/chosen": -219.18023681640625, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -325.5624694824219, "loss": 1.0278, "margin_dpo/margin_mean": 95.59102630615234, "margin_dpo/margin_std": 88.99942016601562, "step": 328 }, { "KL/chosen_KL_mean": -138.77880859375, "KL/mean": -176.19931030273438, "KL/rejected_KL_mean": -213.6198272705078, "KL/std": 89.1434326171875, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.004806600511074066, "fcm_dpo/delta": 0.06268124282360077, "fcm_dpo/margin": 74.84102630615234, "fcm_dpo/q_t": 0.42049404978752136, "grad_norm": 9.077881813049316, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.5159385204315186, "logits/rejected": -1.0158560276031494, "logps/chosen": -239.39187622070312, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -318.57354736328125, "loss": 1.1136, "margin_dpo/margin_mean": 74.84102630615234, "margin_dpo/margin_std": 96.241455078125, "step": 329 }, { "KL/chosen_KL_mean": -117.56141662597656, "KL/mean": -157.85287475585938, "KL/rejected_KL_mean": -198.14431762695312, "KL/std": 86.5996322631836, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005001050420105457, "fcm_dpo/delta": -0.005748318508267403, "fcm_dpo/margin": 80.58289337158203, "fcm_dpo/q_t": 0.40291112661361694, "grad_norm": 8.345678329467773, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.8934423923492432, "logits/rejected": -1.816645622253418, "logps/chosen": -194.5198516845703, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -290.9592590332031, "loss": 1.0608, "margin_dpo/margin_mean": 80.58289337158203, "margin_dpo/margin_std": 77.0096664428711, "step": 330 }, { "KL/chosen_KL_mean": -127.35227966308594, "KL/mean": -172.7540740966797, "KL/rejected_KL_mean": -218.1558837890625, "KL/std": 84.83792114257812, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.004785456694662571, "fcm_dpo/delta": -0.06124322488903999, "fcm_dpo/margin": 90.80361938476562, "fcm_dpo/q_t": 0.39184629917144775, "grad_norm": 9.262165069580078, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.7697722911834717, "logits/rejected": -1.4616000652313232, "logps/chosen": -210.41168212890625, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -312.2870178222656, "loss": 1.0418, "margin_dpo/margin_mean": 90.80361938476562, "margin_dpo/margin_std": 85.6999282836914, "step": 331 }, { "KL/chosen_KL_mean": -136.5724334716797, "KL/mean": -167.06793212890625, "KL/rejected_KL_mean": -197.56341552734375, "KL/std": 93.20266723632812, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005346943624317646, "fcm_dpo/delta": 0.10385388880968094, "fcm_dpo/margin": 60.990997314453125, "fcm_dpo/q_t": 0.42972856760025024, "grad_norm": 8.439253807067871, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.1632633209228516, "logits/rejected": -1.7369685173034668, "logps/chosen": -218.50332641601562, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -280.4093322753906, "loss": 1.143, "margin_dpo/margin_mean": 60.990997314453125, "margin_dpo/margin_std": 91.30017852783203, "step": 332 }, { "KL/chosen_KL_mean": -156.37936401367188, "KL/mean": -200.17327880859375, "KL/rejected_KL_mean": -243.96717834472656, "KL/std": 98.14936828613281, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.004764959216117859, "fcm_dpo/delta": -0.049977146089076996, "fcm_dpo/margin": 87.58784484863281, "fcm_dpo/q_t": 0.3974137008190155, "grad_norm": 7.541940689086914, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5582157373428345, "logits/rejected": -1.2176024913787842, "logps/chosen": -244.91192626953125, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -346.4331970214844, "loss": 1.0754, "margin_dpo/margin_mean": 87.58783721923828, "margin_dpo/margin_std": 105.15007019042969, "step": 333 }, { "KL/chosen_KL_mean": -142.09259033203125, "KL/mean": -181.12710571289062, "KL/rejected_KL_mean": -220.16162109375, "KL/std": 92.58718872070312, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.004874119069427252, "fcm_dpo/delta": 0.0238705612719059, "fcm_dpo/margin": 78.06904602050781, "fcm_dpo/q_t": 0.41126853227615356, "grad_norm": 8.47435474395752, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.3775663375854492, "logits/rejected": -1.0792407989501953, "logps/chosen": -239.85037231445312, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -320.1674499511719, "loss": 1.1113, "margin_dpo/margin_mean": 78.06904602050781, "margin_dpo/margin_std": 103.42625427246094, "step": 334 }, { "KL/chosen_KL_mean": -141.42141723632812, "KL/mean": -174.60887145996094, "KL/rejected_KL_mean": -207.79632568359375, "KL/std": 91.83039855957031, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005617561750113964, "fcm_dpo/delta": 0.04229723662137985, "fcm_dpo/margin": 66.37489318847656, "fcm_dpo/q_t": 0.41690993309020996, "grad_norm": 13.305623054504395, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.241384506225586, "logits/rejected": -1.8093868494033813, "logps/chosen": -237.79745483398438, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -302.7203369140625, "loss": 1.124, "margin_dpo/margin_mean": 66.37489318847656, "margin_dpo/margin_std": 101.1884765625, "step": 335 }, { "KL/chosen_KL_mean": -144.22120666503906, "KL/mean": -186.97958374023438, "KL/rejected_KL_mean": -229.73794555664062, "KL/std": 94.35333251953125, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005184105597436428, "fcm_dpo/delta": -0.0772881954908371, "fcm_dpo/margin": 85.51676940917969, "fcm_dpo/q_t": 0.39063888788223267, "grad_norm": 10.315840721130371, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.4495594501495361, "logits/rejected": -1.2979142665863037, "logps/chosen": -232.750732421875, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -339.2937316894531, "loss": 1.0551, "margin_dpo/margin_mean": 85.51676940917969, "margin_dpo/margin_std": 100.03225708007812, "step": 336 }, { "KL/chosen_KL_mean": -141.9661865234375, "KL/mean": -182.85537719726562, "KL/rejected_KL_mean": -223.74452209472656, "KL/std": 95.3763198852539, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.004924943670630455, "fcm_dpo/delta": -0.0052871135994791985, "fcm_dpo/margin": 81.77833557128906, "fcm_dpo/q_t": 0.40678083896636963, "grad_norm": 8.4593505859375, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.428349018096924, "logits/rejected": -2.1489925384521484, "logps/chosen": -239.27857971191406, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -333.8758544921875, "loss": 1.0962, "margin_dpo/margin_mean": 81.77833557128906, "margin_dpo/margin_std": 112.36693572998047, "step": 337 }, { "KL/chosen_KL_mean": -147.97787475585938, "KL/mean": -185.20068359375, "KL/rejected_KL_mean": -222.42349243164062, "KL/std": 96.07821655273438, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.005069919861853123, "fcm_dpo/delta": 0.03627874329686165, "fcm_dpo/margin": 74.44561767578125, "fcm_dpo/q_t": 0.4139079451560974, "grad_norm": 8.702380180358887, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7598867416381836, "logits/rejected": -1.6071659326553345, "logps/chosen": -231.6079864501953, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -316.1158447265625, "loss": 1.1001, "margin_dpo/margin_mean": 74.44561767578125, "margin_dpo/margin_std": 95.22329711914062, "step": 338 }, { "KL/chosen_KL_mean": -157.33897399902344, "KL/mean": -197.40542602539062, "KL/rejected_KL_mean": -237.47183227539062, "KL/std": 95.40558624267578, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005266932770609856, "fcm_dpo/delta": -0.050593409687280655, "fcm_dpo/margin": 80.13288116455078, "fcm_dpo/q_t": 0.3975127041339874, "grad_norm": 8.386322021484375, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -3.0357718467712402, "logits/rejected": -2.618709087371826, "logps/chosen": -257.1907958984375, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -346.54620361328125, "loss": 1.0967, "margin_dpo/margin_mean": 80.13287353515625, "margin_dpo/margin_std": 106.0298843383789, "step": 339 }, { "KL/chosen_KL_mean": -169.05628967285156, "KL/mean": -203.800537109375, "KL/rejected_KL_mean": -238.54476928710938, "KL/std": 87.52046203613281, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0051825279369950294, "fcm_dpo/delta": 0.06076701730489731, "fcm_dpo/margin": 69.48846435546875, "fcm_dpo/q_t": 0.41989991068840027, "grad_norm": 9.654882431030273, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.5756580829620361, "logits/rejected": -1.1013648509979248, "logps/chosen": -254.03025817871094, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -317.3450927734375, "loss": 1.1161, "margin_dpo/margin_mean": 69.48846435546875, "margin_dpo/margin_std": 96.29338073730469, "step": 340 }, { "KL/chosen_KL_mean": -176.08062744140625, "KL/mean": -209.83151245117188, "KL/rejected_KL_mean": -243.58241271972656, "KL/std": 91.63997650146484, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.005516769364476204, "fcm_dpo/delta": 0.04376038536429405, "fcm_dpo/margin": 67.50175476074219, "fcm_dpo/q_t": 0.4181392192840576, "grad_norm": 10.35293197631836, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.3414604663848877, "logits/rejected": -1.0919972658157349, "logps/chosen": -261.3446044921875, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -349.6683654785156, "loss": 1.1298, "margin_dpo/margin_mean": 67.50175476074219, "margin_dpo/margin_std": 105.7359848022461, "step": 341 }, { "KL/chosen_KL_mean": -141.23577880859375, "KL/mean": -190.37640380859375, "KL/rejected_KL_mean": -239.5170440673828, "KL/std": 91.96182250976562, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.004491261672228575, "fcm_dpo/delta": -0.11229302734136581, "fcm_dpo/margin": 98.28126525878906, "fcm_dpo/q_t": 0.38067924976348877, "grad_norm": 7.022151947021484, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.8105194568634033, "logits/rejected": -1.0907869338989258, "logps/chosen": -242.27438354492188, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -338.4155578613281, "loss": 1.0456, "margin_dpo/margin_mean": 98.28126525878906, "margin_dpo/margin_std": 98.15878295898438, "step": 342 }, { "KL/chosen_KL_mean": -153.82176208496094, "KL/mean": -194.6729278564453, "KL/rejected_KL_mean": -235.52410888671875, "KL/std": 95.52540588378906, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.00489017553627491, "fcm_dpo/delta": -0.011595658957958221, "fcm_dpo/margin": 81.70234680175781, "fcm_dpo/q_t": 0.4044443666934967, "grad_norm": 9.836390495300293, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1067421436309814, "logits/rejected": -2.000286817550659, "logps/chosen": -237.56912231445312, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -352.10321044921875, "loss": 1.093, "margin_dpo/margin_mean": 81.70234680175781, "margin_dpo/margin_std": 100.31512451171875, "step": 343 }, { "KL/chosen_KL_mean": -152.23423767089844, "KL/mean": -196.07675170898438, "KL/rejected_KL_mean": -239.91928100585938, "KL/std": 101.6594009399414, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.004651868250221014, "fcm_dpo/delta": -0.020795173943042755, "fcm_dpo/margin": 87.68505859375, "fcm_dpo/q_t": 0.40290772914886475, "grad_norm": 9.91162395477295, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.7674915790557861, "logits/rejected": -1.5272550582885742, "logps/chosen": -230.4882049560547, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -335.3479919433594, "loss": 1.0839, "margin_dpo/margin_mean": 87.68505859375, "margin_dpo/margin_std": 107.86272430419922, "step": 344 }, { "KL/chosen_KL_mean": -158.42086791992188, "KL/mean": -203.90975952148438, "KL/rejected_KL_mean": -249.3986358642578, "KL/std": 101.72346496582031, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.004332138225436211, "fcm_dpo/delta": 0.009422138333320618, "fcm_dpo/margin": 90.97775268554688, "fcm_dpo/q_t": 0.40836280584335327, "grad_norm": 8.374910354614258, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.36847984790802, "logits/rejected": -1.0656843185424805, "logps/chosen": -256.5003662109375, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -353.43341064453125, "loss": 1.0802, "margin_dpo/margin_mean": 90.97774505615234, "margin_dpo/margin_std": 105.74385833740234, "step": 345 }, { "KL/chosen_KL_mean": -176.19046020507812, "KL/mean": -211.84561157226562, "KL/rejected_KL_mean": -247.50076293945312, "KL/std": 94.43191528320312, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.004936929326504469, "fcm_dpo/delta": 0.0728856697678566, "fcm_dpo/margin": 71.31031036376953, "fcm_dpo/q_t": 0.4237104654312134, "grad_norm": 8.207139015197754, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.7262773513793945, "logits/rejected": -1.4096572399139404, "logps/chosen": -256.8739318847656, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -333.39337158203125, "loss": 1.1326, "margin_dpo/margin_mean": 71.310302734375, "margin_dpo/margin_std": 108.79135131835938, "step": 346 }, { "KL/chosen_KL_mean": -166.5946502685547, "KL/mean": -200.9240264892578, "KL/rejected_KL_mean": -235.25338745117188, "KL/std": 93.81053161621094, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.005539868958294392, "fcm_dpo/delta": 0.024451635777950287, "fcm_dpo/margin": 68.65873718261719, "fcm_dpo/q_t": 0.4140353798866272, "grad_norm": 9.176746368408203, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.84743332862854, "logits/rejected": -1.5264726877212524, "logps/chosen": -260.2740173339844, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -332.7958984375, "loss": 1.1194, "margin_dpo/margin_mean": 68.65873718261719, "margin_dpo/margin_std": 101.87372589111328, "step": 347 }, { "KL/chosen_KL_mean": -143.98399353027344, "KL/mean": -182.97328186035156, "KL/rejected_KL_mean": -221.96258544921875, "KL/std": 88.13381958007812, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.004418687429279089, "fcm_dpo/delta": -0.09629756212234497, "fcm_dpo/margin": 77.97859954833984, "fcm_dpo/q_t": 0.40730705857276917, "grad_norm": 7.224355697631836, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.9611477851867676, "logits/rejected": -1.6888054609298706, "logps/chosen": -234.23623657226562, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -317.73284912109375, "loss": 1.1142, "margin_dpo/margin_mean": 77.97859954833984, "margin_dpo/margin_std": 90.14311218261719, "step": 348 }, { "KL/chosen_KL_mean": -155.7855682373047, "KL/mean": -191.64532470703125, "KL/rejected_KL_mean": -227.5050811767578, "KL/std": 97.03779602050781, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.00501466728746891, "fcm_dpo/delta": 0.05312078446149826, "fcm_dpo/margin": 71.7195053100586, "fcm_dpo/q_t": 0.41924625635147095, "grad_norm": 9.917169570922852, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.0494327545166016, "logits/rejected": -1.824891209602356, "logps/chosen": -247.84365844726562, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -331.4703369140625, "loss": 1.1225, "margin_dpo/margin_mean": 71.7195053100586, "margin_dpo/margin_std": 101.27935791015625, "step": 349 }, { "KL/chosen_KL_mean": -163.74227905273438, "KL/mean": -194.71780395507812, "KL/rejected_KL_mean": -225.69329833984375, "KL/std": 89.30181121826172, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005554020404815674, "fcm_dpo/delta": 0.08499324321746826, "fcm_dpo/margin": 61.951026916503906, "fcm_dpo/q_t": 0.4262838363647461, "grad_norm": 11.68614673614502, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8125853538513184, "logits/rejected": -1.5856201648712158, "logps/chosen": -250.79647827148438, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -321.41522216796875, "loss": 1.1444, "margin_dpo/margin_mean": 61.951019287109375, "margin_dpo/margin_std": 97.80931091308594, "step": 350 }, { "KL/chosen_KL_mean": -164.5549774169922, "KL/mean": -207.9117431640625, "KL/rejected_KL_mean": -251.26849365234375, "KL/std": 100.05374908447266, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.005165339447557926, "fcm_dpo/delta": -0.0870525985956192, "fcm_dpo/margin": 86.71351623535156, "fcm_dpo/q_t": 0.3891069293022156, "grad_norm": 8.415233612060547, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.6477341651916504, "logits/rejected": -1.3467531204223633, "logps/chosen": -253.72401428222656, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -359.5665283203125, "loss": 1.0613, "margin_dpo/margin_mean": 86.71351623535156, "margin_dpo/margin_std": 107.152587890625, "step": 351 }, { "KL/chosen_KL_mean": -150.9803009033203, "KL/mean": -192.03981018066406, "KL/rejected_KL_mean": -233.09930419921875, "KL/std": 98.10633850097656, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.004877937957644463, "fcm_dpo/delta": -0.000964876264333725, "fcm_dpo/margin": 82.11900329589844, "fcm_dpo/q_t": 0.40813249349594116, "grad_norm": 8.331216812133789, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.9250929355621338, "logits/rejected": -1.37711501121521, "logps/chosen": -247.79177856445312, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -317.504150390625, "loss": 1.0893, "margin_dpo/margin_mean": 82.11900329589844, "margin_dpo/margin_std": 106.06199645996094, "step": 352 }, { "KL/chosen_KL_mean": -172.656494140625, "KL/mean": -198.89688110351562, "KL/rejected_KL_mean": -225.13723754882812, "KL/std": 86.63299560546875, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.005277459509670734, "fcm_dpo/delta": 0.03870457783341408, "fcm_dpo/margin": 52.48075485229492, "fcm_dpo/q_t": 0.4373767375946045, "grad_norm": 9.852267265319824, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.1173386573791504, "logits/rejected": -1.728187084197998, "logps/chosen": -258.01898193359375, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -312.50457763671875, "loss": 1.1949, "margin_dpo/margin_mean": 52.48075866699219, "margin_dpo/margin_std": 98.10032653808594, "step": 353 }, { "KL/chosen_KL_mean": -150.485107421875, "KL/mean": -192.18359375, "KL/rejected_KL_mean": -233.88209533691406, "KL/std": 87.98265075683594, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.005084910895675421, "fcm_dpo/delta": -0.04195880517363548, "fcm_dpo/margin": 83.39698791503906, "fcm_dpo/q_t": 0.39652591943740845, "grad_norm": 7.970493793487549, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8477108478546143, "logits/rejected": -1.3759740591049194, "logps/chosen": -246.28628540039062, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -328.74822998046875, "loss": 1.0488, "margin_dpo/margin_mean": 83.39698028564453, "margin_dpo/margin_std": 81.77227783203125, "step": 354 }, { "KL/chosen_KL_mean": -149.34368896484375, "KL/mean": -187.77423095703125, "KL/rejected_KL_mean": -226.20477294921875, "KL/std": 89.85858154296875, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.004932297393679619, "fcm_dpo/delta": 0.03230014443397522, "fcm_dpo/margin": 76.86109924316406, "fcm_dpo/q_t": 0.41329216957092285, "grad_norm": 9.578350067138672, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4671366214752197, "logits/rejected": -1.9849226474761963, "logps/chosen": -235.52923583984375, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -314.6663513183594, "loss": 1.0982, "margin_dpo/margin_mean": 76.86109924316406, "margin_dpo/margin_std": 95.00485229492188, "step": 355 }, { "KL/chosen_KL_mean": -175.90048217773438, "KL/mean": -205.43350219726562, "KL/rejected_KL_mean": -234.966552734375, "KL/std": 87.7643051147461, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005948870442807674, "fcm_dpo/delta": 0.06626837700605392, "fcm_dpo/margin": 59.06607437133789, "fcm_dpo/q_t": 0.42321956157684326, "grad_norm": 9.909618377685547, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.9188305139541626, "logits/rejected": -1.4568400382995605, "logps/chosen": -266.71856689453125, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -317.0938720703125, "loss": 1.1494, "margin_dpo/margin_mean": 59.06607437133789, "margin_dpo/margin_std": 99.92112731933594, "step": 356 }, { "KL/chosen_KL_mean": -148.53494262695312, "KL/mean": -195.39776611328125, "KL/rejected_KL_mean": -242.26058959960938, "KL/std": 95.83121490478516, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.004628303460776806, "fcm_dpo/delta": -0.10139767825603485, "fcm_dpo/margin": 93.72566223144531, "fcm_dpo/q_t": 0.384423166513443, "grad_norm": 7.728548526763916, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.991597056388855, "logits/rejected": -1.6252827644348145, "logps/chosen": -236.03970336914062, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -334.50994873046875, "loss": 1.0447, "margin_dpo/margin_mean": 93.72565460205078, "margin_dpo/margin_std": 89.18292236328125, "step": 357 }, { "KL/chosen_KL_mean": -159.04208374023438, "KL/mean": -193.9208984375, "KL/rejected_KL_mean": -228.7997283935547, "KL/std": 89.45537567138672, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.005235760472714901, "fcm_dpo/delta": 0.05436001718044281, "fcm_dpo/margin": 69.75764465332031, "fcm_dpo/q_t": 0.4200412631034851, "grad_norm": 8.853779792785645, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.5999598503112793, "logits/rejected": -1.1286380290985107, "logps/chosen": -254.09754943847656, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -325.2758483886719, "loss": 1.1123, "margin_dpo/margin_mean": 69.75764465332031, "margin_dpo/margin_std": 95.80384826660156, "step": 358 }, { "KL/chosen_KL_mean": -178.33157348632812, "KL/mean": -211.00900268554688, "KL/rejected_KL_mean": -243.68643188476562, "KL/std": 83.64961242675781, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.005540061742067337, "fcm_dpo/delta": 0.05544459819793701, "fcm_dpo/margin": 65.3548583984375, "fcm_dpo/q_t": 0.42039650678634644, "grad_norm": 10.893966674804688, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.7206437587738037, "logits/rejected": -1.1644620895385742, "logps/chosen": -265.70489501953125, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -332.45831298828125, "loss": 1.1336, "margin_dpo/margin_mean": 65.3548583984375, "margin_dpo/margin_std": 97.57875061035156, "step": 359 }, { "KL/chosen_KL_mean": -163.54183959960938, "KL/mean": -205.091796875, "KL/rejected_KL_mean": -246.64178466796875, "KL/std": 84.52761840820312, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.005267015658318996, "fcm_dpo/delta": -0.07250785827636719, "fcm_dpo/margin": 83.09994506835938, "fcm_dpo/q_t": 0.3900858163833618, "grad_norm": 15.112351417541504, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.7374799251556396, "logits/rejected": -1.4006633758544922, "logps/chosen": -238.76528930664062, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -322.73577880859375, "loss": 1.0484, "margin_dpo/margin_mean": 83.09994506835938, "margin_dpo/margin_std": 89.52436828613281, "step": 360 }, { "KL/chosen_KL_mean": -166.44239807128906, "KL/mean": -202.9842529296875, "KL/rejected_KL_mean": -239.526123046875, "KL/std": 96.23770141601562, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005352683365345001, "fcm_dpo/delta": 0.014076105318963528, "fcm_dpo/margin": 73.08374786376953, "fcm_dpo/q_t": 0.41177117824554443, "grad_norm": 10.72282600402832, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.6961123943328857, "logits/rejected": -1.453178882598877, "logps/chosen": -248.39572143554688, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -336.205322265625, "loss": 1.1039, "margin_dpo/margin_mean": 73.083740234375, "margin_dpo/margin_std": 103.05294799804688, "step": 361 }, { "KL/chosen_KL_mean": -172.02316284179688, "KL/mean": -210.51968383789062, "KL/rejected_KL_mean": -249.01620483398438, "KL/std": 91.55164337158203, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.005179382860660553, "fcm_dpo/delta": -0.0006334614008665085, "fcm_dpo/margin": 76.99303436279297, "fcm_dpo/q_t": 0.40670424699783325, "grad_norm": 9.05645751953125, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.7500807046890259, "logits/rejected": -1.5464097261428833, "logps/chosen": -261.36285400390625, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -359.5181884765625, "loss": 1.09, "margin_dpo/margin_mean": 76.99303436279297, "margin_dpo/margin_std": 98.41926574707031, "step": 362 }, { "KL/chosen_KL_mean": -160.5657958984375, "KL/mean": -210.23556518554688, "KL/rejected_KL_mean": -259.90533447265625, "KL/std": 95.75578308105469, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.004669893532991409, "fcm_dpo/delta": -0.11772938817739487, "fcm_dpo/margin": 99.33952331542969, "fcm_dpo/q_t": 0.38202327489852905, "grad_norm": 10.127141952514648, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.9037559032440186, "logits/rejected": -1.4790892601013184, "logps/chosen": -241.34078979492188, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -339.3380126953125, "loss": 1.0367, "margin_dpo/margin_mean": 99.33952331542969, "margin_dpo/margin_std": 104.3455810546875, "step": 363 }, { "KL/chosen_KL_mean": -159.18710327148438, "KL/mean": -198.87255859375, "KL/rejected_KL_mean": -238.55801391601562, "KL/std": 88.62889099121094, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.004715713206678629, "fcm_dpo/delta": 0.024468008428812027, "fcm_dpo/margin": 79.37092590332031, "fcm_dpo/q_t": 0.41108155250549316, "grad_norm": 11.656632423400879, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0119457244873047, "logits/rejected": -1.8716355562210083, "logps/chosen": -235.71670532226562, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -336.1197509765625, "loss": 1.0934, "margin_dpo/margin_mean": 79.37092590332031, "margin_dpo/margin_std": 85.44772338867188, "step": 364 }, { "KL/chosen_KL_mean": -154.18173217773438, "KL/mean": -198.01905822753906, "KL/rejected_KL_mean": -241.8563690185547, "KL/std": 95.9437255859375, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.004577017389237881, "fcm_dpo/delta": -0.0061031728982925415, "fcm_dpo/margin": 87.67462158203125, "fcm_dpo/q_t": 0.40661463141441345, "grad_norm": 8.368146896362305, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.8039417266845703, "logits/rejected": -1.6841681003570557, "logps/chosen": -234.72662353515625, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -341.1795349121094, "loss": 1.0854, "margin_dpo/margin_mean": 87.67462158203125, "margin_dpo/margin_std": 109.43146514892578, "step": 365 }, { "KL/chosen_KL_mean": -155.93832397460938, "KL/mean": -192.54891967773438, "KL/rejected_KL_mean": -229.15948486328125, "KL/std": 99.65890502929688, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.004862410016357899, "fcm_dpo/delta": 0.06722716987133026, "fcm_dpo/margin": 73.22116088867188, "fcm_dpo/q_t": 0.42281490564346313, "grad_norm": 8.244962692260742, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.997748851776123, "logits/rejected": -2.032345771789551, "logps/chosen": -228.1763916015625, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -320.8313293457031, "loss": 1.1136, "margin_dpo/margin_mean": 73.22116088867188, "margin_dpo/margin_std": 97.96697998046875, "step": 366 }, { "KL/chosen_KL_mean": -142.0294189453125, "KL/mean": -185.6251220703125, "KL/rejected_KL_mean": -229.2208251953125, "KL/std": 87.5277328491211, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.004797385074198246, "fcm_dpo/delta": -0.031446754932403564, "fcm_dpo/margin": 87.19140625, "fcm_dpo/q_t": 0.3989282250404358, "grad_norm": 7.934207916259766, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.7657058238983154, "logits/rejected": -1.4161231517791748, "logps/chosen": -227.45143127441406, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -312.7386169433594, "loss": 1.0546, "margin_dpo/margin_mean": 87.19140625, "margin_dpo/margin_std": 90.02625274658203, "step": 367 }, { "KL/chosen_KL_mean": -131.94058227539062, "KL/mean": -180.40225219726562, "KL/rejected_KL_mean": -228.86390686035156, "KL/std": 96.75106811523438, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0044775125570595264, "fcm_dpo/delta": -0.06072847545146942, "fcm_dpo/margin": 96.92330932617188, "fcm_dpo/q_t": 0.3919161260128021, "grad_norm": 8.534385681152344, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.3111026287078857, "logits/rejected": -1.280817985534668, "logps/chosen": -205.78170776367188, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -333.5592041015625, "loss": 1.0509, "margin_dpo/margin_mean": 96.92330932617188, "margin_dpo/margin_std": 97.85261535644531, "step": 368 }, { "KL/chosen_KL_mean": -146.64479064941406, "KL/mean": -192.08819580078125, "KL/rejected_KL_mean": -237.53163146972656, "KL/std": 89.51618957519531, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.004316114820539951, "fcm_dpo/delta": 0.011442071758210659, "fcm_dpo/margin": 90.88682556152344, "fcm_dpo/q_t": 0.4079192876815796, "grad_norm": 8.034127235412598, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.616666555404663, "logits/rejected": -1.2470760345458984, "logps/chosen": -245.10675048828125, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -336.6883544921875, "loss": 1.0686, "margin_dpo/margin_mean": 90.88682556152344, "margin_dpo/margin_std": 90.60661315917969, "step": 369 }, { "KL/chosen_KL_mean": -163.76638793945312, "KL/mean": -193.4920196533203, "KL/rejected_KL_mean": -223.21763610839844, "KL/std": 85.49427795410156, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.005225845612585545, "fcm_dpo/delta": 0.12399961054325104, "fcm_dpo/margin": 59.45125961303711, "fcm_dpo/q_t": 0.4342229664325714, "grad_norm": 9.312982559204102, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -2.008328437805176, "logits/rejected": -1.813949465751648, "logps/chosen": -260.5421142578125, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -325.96173095703125, "loss": 1.1636, "margin_dpo/margin_mean": 59.451263427734375, "margin_dpo/margin_std": 99.54501342773438, "step": 370 }, { "KL/chosen_KL_mean": -155.26034545898438, "KL/mean": -189.02142333984375, "KL/rejected_KL_mean": -222.78253173828125, "KL/std": 86.614013671875, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005776134319603443, "fcm_dpo/delta": 0.011809393763542175, "fcm_dpo/margin": 67.52218627929688, "fcm_dpo/q_t": 0.40996047854423523, "grad_norm": 13.134374618530273, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.7723586559295654, "logits/rejected": -1.4006497859954834, "logps/chosen": -258.25054931640625, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -328.1331481933594, "loss": 1.1071, "margin_dpo/margin_mean": 67.52218627929688, "margin_dpo/margin_std": 94.3348159790039, "step": 371 }, { "KL/chosen_KL_mean": -131.38095092773438, "KL/mean": -181.34506225585938, "KL/rejected_KL_mean": -231.30917358398438, "KL/std": 100.54611206054688, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.004642265848815441, "fcm_dpo/delta": -0.1230599582195282, "fcm_dpo/margin": 99.92823028564453, "fcm_dpo/q_t": 0.3803677558898926, "grad_norm": 7.2212324142456055, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.960278034210205, "logits/rejected": -1.3313850164413452, "logps/chosen": -222.40719604492188, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -316.9194030761719, "loss": 1.0289, "margin_dpo/margin_mean": 99.92823028564453, "margin_dpo/margin_std": 103.13169860839844, "step": 372 }, { "KL/chosen_KL_mean": -153.4341583251953, "KL/mean": -190.20083618164062, "KL/rejected_KL_mean": -226.96749877929688, "KL/std": 96.31011962890625, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.004824399948120117, "fcm_dpo/delta": 0.07009799778461456, "fcm_dpo/margin": 73.53334045410156, "fcm_dpo/q_t": 0.4221888780593872, "grad_norm": 10.094091415405273, "learning_rate": 2.538498388222517e-07, "logits/chosen": -1.9386470317840576, "logits/rejected": -1.3960031270980835, "logps/chosen": -253.50723266601562, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -318.0838623046875, "loss": 1.1231, "margin_dpo/margin_mean": 73.53333282470703, "margin_dpo/margin_std": 103.16032409667969, "step": 373 }, { "KL/chosen_KL_mean": -132.370361328125, "KL/mean": -173.37899780273438, "KL/rejected_KL_mean": -214.38763427734375, "KL/std": 90.96646881103516, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.004872763995081186, "fcm_dpo/delta": -0.0018004514276981354, "fcm_dpo/margin": 82.01726531982422, "fcm_dpo/q_t": 0.40591293573379517, "grad_norm": 7.844433784484863, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.3017139434814453, "logits/rejected": -1.9193270206451416, "logps/chosen": -230.38270568847656, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -315.139892578125, "loss": 1.0698, "margin_dpo/margin_mean": 82.01726531982422, "margin_dpo/margin_std": 87.47737121582031, "step": 374 }, { "KL/chosen_KL_mean": -133.60255432128906, "KL/mean": -180.385986328125, "KL/rejected_KL_mean": -227.16943359375, "KL/std": 87.99430847167969, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0044007194228470325, "fcm_dpo/delta": -0.0391390286386013, "fcm_dpo/margin": 93.56686401367188, "fcm_dpo/q_t": 0.3968508541584015, "grad_norm": 8.355687141418457, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.6268527507781982, "logits/rejected": -1.3933794498443604, "logps/chosen": -217.99884033203125, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -322.8882751464844, "loss": 1.0625, "margin_dpo/margin_mean": 93.56686401367188, "margin_dpo/margin_std": 93.41703033447266, "step": 375 }, { "KL/chosen_KL_mean": -128.77108764648438, "KL/mean": -180.58831787109375, "KL/rejected_KL_mean": -232.40554809570312, "KL/std": 101.49803161621094, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004166179336607456, "fcm_dpo/delta": -0.05922778695821762, "fcm_dpo/margin": 103.63444519042969, "fcm_dpo/q_t": 0.39305081963539124, "grad_norm": 8.149178504943848, "learning_rate": 2.5e-07, "logits/chosen": -1.8320562839508057, "logits/rejected": -1.5225492715835571, "logps/chosen": -224.5087890625, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -344.37884521484375, "loss": 1.049, "margin_dpo/margin_mean": 103.63444519042969, "margin_dpo/margin_std": 108.79814147949219, "step": 376 }, { "KL/chosen_KL_mean": -136.67376708984375, "KL/mean": -190.44203186035156, "KL/rejected_KL_mean": -244.2102813720703, "KL/std": 101.85166931152344, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.0038235196843743324, "fcm_dpo/delta": -0.024394672363996506, "fcm_dpo/margin": 107.53649139404297, "fcm_dpo/q_t": 0.39938563108444214, "grad_norm": 6.649849891662598, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.362177848815918, "logits/rejected": -1.245814561843872, "logps/chosen": -215.60072326660156, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -350.10919189453125, "loss": 1.0537, "margin_dpo/margin_mean": 107.5364990234375, "margin_dpo/margin_std": 100.8247299194336, "step": 377 }, { "KL/chosen_KL_mean": -133.52047729492188, "KL/mean": -175.71469116210938, "KL/rejected_KL_mean": -217.90890502929688, "KL/std": 96.64350128173828, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004371408373117447, "fcm_dpo/delta": 0.039767660200595856, "fcm_dpo/margin": 84.3884506225586, "fcm_dpo/q_t": 0.4141680598258972, "grad_norm": 8.924489974975586, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.9796288013458252, "logits/rejected": -1.7012921571731567, "logps/chosen": -208.8928680419922, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -311.6130065917969, "loss": 1.0996, "margin_dpo/margin_mean": 84.3884506225586, "margin_dpo/margin_std": 100.00442504882812, "step": 378 }, { "KL/chosen_KL_mean": -156.3795166015625, "KL/mean": -193.59693908691406, "KL/rejected_KL_mean": -230.81436157226562, "KL/std": 99.44560241699219, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004739839117974043, "fcm_dpo/delta": 0.07199611514806747, "fcm_dpo/margin": 74.43482971191406, "fcm_dpo/q_t": 0.4238220453262329, "grad_norm": 9.035327911376953, "learning_rate": 2.461501611777483e-07, "logits/chosen": -2.005770683288574, "logits/rejected": -1.9334874153137207, "logps/chosen": -245.23519897460938, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -352.19549560546875, "loss": 1.1339, "margin_dpo/margin_mean": 74.43482971191406, "margin_dpo/margin_std": 114.83609008789062, "step": 379 }, { "KL/chosen_KL_mean": -170.99432373046875, "KL/mean": -216.66058349609375, "KL/rejected_KL_mean": -262.32684326171875, "KL/std": 102.63862609863281, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.00461345911026001, "fcm_dpo/delta": -0.037516046315431595, "fcm_dpo/margin": 91.33250427246094, "fcm_dpo/q_t": 0.4006444215774536, "grad_norm": 11.879865646362305, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.2965553998947144, "logits/rejected": -1.2731271982192993, "logps/chosen": -250.82752990722656, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -373.07159423828125, "loss": 1.0757, "margin_dpo/margin_mean": 91.33250427246094, "margin_dpo/margin_std": 117.3880615234375, "step": 380 }, { "KL/chosen_KL_mean": -149.98377990722656, "KL/mean": -189.67340087890625, "KL/rejected_KL_mean": -229.36306762695312, "KL/std": 98.19049835205078, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0047920080833137035, "fcm_dpo/delta": 0.03114820271730423, "fcm_dpo/margin": 79.3792724609375, "fcm_dpo/q_t": 0.41297948360443115, "grad_norm": 10.622302055358887, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.8291049003601074, "logits/rejected": -1.2836849689483643, "logps/chosen": -236.55059814453125, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -313.7798156738281, "loss": 1.0988, "margin_dpo/margin_mean": 79.3792724609375, "margin_dpo/margin_std": 102.00825500488281, "step": 381 }, { "KL/chosen_KL_mean": -146.85629272460938, "KL/mean": -191.8480682373047, "KL/rejected_KL_mean": -236.83984375, "KL/std": 96.07752227783203, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004448979627341032, "fcm_dpo/delta": -0.013537604361772537, "fcm_dpo/margin": 89.98355102539062, "fcm_dpo/q_t": 0.40356171131134033, "grad_norm": 8.437421798706055, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.8222088813781738, "logits/rejected": -1.5566939115524292, "logps/chosen": -251.31809997558594, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -351.44207763671875, "loss": 1.0771, "margin_dpo/margin_mean": 89.98355102539062, "margin_dpo/margin_std": 98.89371490478516, "step": 382 }, { "KL/chosen_KL_mean": -161.33767700195312, "KL/mean": -199.90020751953125, "KL/rejected_KL_mean": -238.46273803710938, "KL/std": 99.63267517089844, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0048899599350988865, "fcm_dpo/delta": 0.036691129207611084, "fcm_dpo/margin": 77.12507629394531, "fcm_dpo/q_t": 0.41463643312454224, "grad_norm": 9.19774341583252, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.8063138723373413, "logits/rejected": -1.6873232126235962, "logps/chosen": -243.45375061035156, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -337.2398986816406, "loss": 1.1005, "margin_dpo/margin_mean": 77.12507629394531, "margin_dpo/margin_std": 98.01612091064453, "step": 383 }, { "KL/chosen_KL_mean": -171.69021606445312, "KL/mean": -209.97122192382812, "KL/rejected_KL_mean": -248.25222778320312, "KL/std": 103.49443054199219, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.005328080616891384, "fcm_dpo/delta": -0.030480392277240753, "fcm_dpo/margin": 76.56201171875, "fcm_dpo/q_t": 0.4018878936767578, "grad_norm": 11.26003360748291, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.754657506942749, "logits/rejected": -1.5304888486862183, "logps/chosen": -252.2754669189453, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -339.70013427734375, "loss": 1.1323, "margin_dpo/margin_mean": 76.56201171875, "margin_dpo/margin_std": 115.1290283203125, "step": 384 }, { "KL/chosen_KL_mean": -142.41969299316406, "KL/mean": -192.1409912109375, "KL/rejected_KL_mean": -241.86227416992188, "KL/std": 92.68511962890625, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004196967929601669, "fcm_dpo/delta": -0.04493045434355736, "fcm_dpo/margin": 99.44258880615234, "fcm_dpo/q_t": 0.3967437744140625, "grad_norm": 8.83307933807373, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.0936384201049805, "logits/rejected": -1.4705524444580078, "logps/chosen": -235.17987060546875, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -326.322265625, "loss": 1.0628, "margin_dpo/margin_mean": 99.44258117675781, "margin_dpo/margin_std": 106.60604858398438, "step": 385 }, { "KL/chosen_KL_mean": -151.29489135742188, "KL/mean": -198.89401245117188, "KL/rejected_KL_mean": -246.4931182861328, "KL/std": 94.94862365722656, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0042707063257694244, "fcm_dpo/delta": -0.011195244267582893, "fcm_dpo/margin": 95.19820404052734, "fcm_dpo/q_t": 0.4021121859550476, "grad_norm": 7.02681303024292, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.5935578346252441, "logits/rejected": -1.3066604137420654, "logps/chosen": -236.65386962890625, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -343.4837646484375, "loss": 1.0676, "margin_dpo/margin_mean": 95.19820404052734, "margin_dpo/margin_std": 103.40359497070312, "step": 386 }, { "KL/chosen_KL_mean": -176.50975036621094, "KL/mean": -218.89956665039062, "KL/rejected_KL_mean": -261.28936767578125, "KL/std": 92.18296813964844, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004516792483627796, "fcm_dpo/delta": 0.02580871805548668, "fcm_dpo/margin": 84.77963256835938, "fcm_dpo/q_t": 0.41131681203842163, "grad_norm": 10.362727165222168, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.584172248840332, "logits/rejected": -1.9172459840774536, "logps/chosen": -279.1978759765625, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -360.71795654296875, "loss": 1.0831, "margin_dpo/margin_mean": 84.77963256835938, "margin_dpo/margin_std": 94.3072280883789, "step": 387 }, { "KL/chosen_KL_mean": -178.92124938964844, "KL/mean": -225.64920043945312, "KL/rejected_KL_mean": -272.37713623046875, "KL/std": 102.50267028808594, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0043545677326619625, "fcm_dpo/delta": -0.012389106675982475, "fcm_dpo/margin": 93.45589447021484, "fcm_dpo/q_t": 0.40298885107040405, "grad_norm": 8.365912437438965, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.797644853591919, "logits/rejected": -1.6841622591018677, "logps/chosen": -262.5064697265625, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -382.4775390625, "loss": 1.0703, "margin_dpo/margin_mean": 93.45589447021484, "margin_dpo/margin_std": 105.01012420654297, "step": 388 }, { "KL/chosen_KL_mean": -184.31829833984375, "KL/mean": -224.34149169921875, "KL/rejected_KL_mean": -264.3646240234375, "KL/std": 104.21456909179688, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004658023826777935, "fcm_dpo/delta": 0.04320334643125534, "fcm_dpo/margin": 80.04634094238281, "fcm_dpo/q_t": 0.41654300689697266, "grad_norm": 11.520211219787598, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.4679157733917236, "logits/rejected": -2.025216579437256, "logps/chosen": -274.14056396484375, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -349.2184143066406, "loss": 1.1125, "margin_dpo/margin_mean": 80.04634094238281, "margin_dpo/margin_std": 112.41036987304688, "step": 389 }, { "KL/chosen_KL_mean": -184.16677856445312, "KL/mean": -222.81753540039062, "KL/rejected_KL_mean": -261.46832275390625, "KL/std": 101.71823120117188, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004812168888747692, "fcm_dpo/delta": 0.039968959987163544, "fcm_dpo/margin": 77.30155181884766, "fcm_dpo/q_t": 0.4184502363204956, "grad_norm": 9.133197784423828, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.504925012588501, "logits/rejected": -1.2968418598175049, "logps/chosen": -261.69921875, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -354.05224609375, "loss": 1.1272, "margin_dpo/margin_mean": 77.30155181884766, "margin_dpo/margin_std": 116.90959167480469, "step": 390 }, { "KL/chosen_KL_mean": -167.3387451171875, "KL/mean": -209.18458557128906, "KL/rejected_KL_mean": -251.03045654296875, "KL/std": 97.683837890625, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004852045327425003, "fcm_dpo/delta": -0.015781115740537643, "fcm_dpo/margin": 83.69171142578125, "fcm_dpo/q_t": 0.4034882187843323, "grad_norm": 9.065732955932617, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.9890871047973633, "logits/rejected": -1.5430749654769897, "logps/chosen": -255.57167053222656, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -344.03802490234375, "loss": 1.0828, "margin_dpo/margin_mean": 83.69171142578125, "margin_dpo/margin_std": 102.24632263183594, "step": 391 }, { "KL/chosen_KL_mean": -170.35867309570312, "KL/mean": -221.60076904296875, "KL/rejected_KL_mean": -272.8428955078125, "KL/std": 110.13011169433594, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0044526103883981705, "fcm_dpo/delta": -0.10250647366046906, "fcm_dpo/margin": 102.48419189453125, "fcm_dpo/q_t": 0.38508322834968567, "grad_norm": 8.954988479614258, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.4457604885101318, "logits/rejected": -1.3295844793319702, "logps/chosen": -258.10308837890625, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -381.41357421875, "loss": 1.0516, "margin_dpo/margin_mean": 102.48418426513672, "margin_dpo/margin_std": 119.2347183227539, "step": 392 }, { "KL/chosen_KL_mean": -152.1531982421875, "KL/mean": -197.7803497314453, "KL/rejected_KL_mean": -243.40750122070312, "KL/std": 103.11881256103516, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004219436086714268, "fcm_dpo/delta": 0.02433878555893898, "fcm_dpo/margin": 91.25431823730469, "fcm_dpo/q_t": 0.4110582768917084, "grad_norm": 7.779191017150879, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.766125202178955, "logits/rejected": -1.4481041431427002, "logps/chosen": -228.71701049804688, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -323.1336975097656, "loss": 1.0817, "margin_dpo/margin_mean": 91.25431823730469, "margin_dpo/margin_std": 101.48300170898438, "step": 393 }, { "KL/chosen_KL_mean": -160.64463806152344, "KL/mean": -210.6321258544922, "KL/rejected_KL_mean": -260.61962890625, "KL/std": 98.72900390625, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004136508796364069, "fcm_dpo/delta": -0.023224610835313797, "fcm_dpo/margin": 99.9749526977539, "fcm_dpo/q_t": 0.3996431529521942, "grad_norm": 7.699851989746094, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6143742799758911, "logits/rejected": -1.3850247859954834, "logps/chosen": -245.3433074951172, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -358.077392578125, "loss": 1.0538, "margin_dpo/margin_mean": 99.97494506835938, "margin_dpo/margin_std": 97.37299346923828, "step": 394 }, { "KL/chosen_KL_mean": -166.45114135742188, "KL/mean": -205.01739501953125, "KL/rejected_KL_mean": -243.5836181640625, "KL/std": 102.94894409179688, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004474648740142584, "fcm_dpo/delta": 0.08386095613241196, "fcm_dpo/margin": 77.13246154785156, "fcm_dpo/q_t": 0.42753124237060547, "grad_norm": 8.854455947875977, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5290987491607666, "logits/rejected": -2.126862049102783, "logps/chosen": -255.33877563476562, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -334.629150390625, "loss": 1.1692, "margin_dpo/margin_mean": 77.1324691772461, "margin_dpo/margin_std": 139.1849365234375, "step": 395 }, { "KL/chosen_KL_mean": -175.59585571289062, "KL/mean": -210.22373962402344, "KL/rejected_KL_mean": -244.85165405273438, "KL/std": 91.3704605102539, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.005267119966447353, "fcm_dpo/delta": 0.05479075759649277, "fcm_dpo/margin": 69.25578308105469, "fcm_dpo/q_t": 0.41995906829833984, "grad_norm": 12.198870658874512, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.9704898595809937, "logits/rejected": -1.4841680526733398, "logps/chosen": -278.06719970703125, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -338.08642578125, "loss": 1.1312, "margin_dpo/margin_mean": 69.25578308105469, "margin_dpo/margin_std": 107.99999237060547, "step": 396 }, { "KL/chosen_KL_mean": -157.98046875, "KL/mean": -219.48223876953125, "KL/rejected_KL_mean": -280.9840393066406, "KL/std": 106.53456115722656, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.003884212113916874, "fcm_dpo/delta": -0.18341916799545288, "fcm_dpo/margin": 123.00354766845703, "fcm_dpo/q_t": 0.36872291564941406, "grad_norm": 7.332563400268555, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.3599181175231934, "logits/rejected": -1.7708492279052734, "logps/chosen": -259.02490234375, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -389.835205078125, "loss": 1.0215, "margin_dpo/margin_mean": 123.00355529785156, "margin_dpo/margin_std": 127.05548095703125, "step": 397 }, { "KL/chosen_KL_mean": -167.18234252929688, "KL/mean": -211.37245178222656, "KL/rejected_KL_mean": -255.5625762939453, "KL/std": 104.31248474121094, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004078170284628868, "fcm_dpo/delta": 0.06188623234629631, "fcm_dpo/margin": 88.38023376464844, "fcm_dpo/q_t": 0.42055463790893555, "grad_norm": 7.488183498382568, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.058173418045044, "logits/rejected": -1.7777199745178223, "logps/chosen": -262.2315673828125, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -369.5682067871094, "loss": 1.1196, "margin_dpo/margin_mean": 88.38024139404297, "margin_dpo/margin_std": 123.79286193847656, "step": 398 }, { "KL/chosen_KL_mean": -145.90306091308594, "KL/mean": -202.34832763671875, "KL/rejected_KL_mean": -258.7936096191406, "KL/std": 93.80574035644531, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0038810078985989094, "fcm_dpo/delta": -0.06761468946933746, "fcm_dpo/margin": 112.8905258178711, "fcm_dpo/q_t": 0.38991186022758484, "grad_norm": 7.6356611251831055, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.0084075927734375, "logits/rejected": -1.192576289176941, "logps/chosen": -242.54159545898438, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -342.3300476074219, "loss": 1.0335, "margin_dpo/margin_mean": 112.8905258178711, "margin_dpo/margin_std": 101.35710144042969, "step": 399 }, { "KL/chosen_KL_mean": -152.0984344482422, "KL/mean": -209.37957763671875, "KL/rejected_KL_mean": -266.66070556640625, "KL/std": 103.83226776123047, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0035920217633247375, "fcm_dpo/delta": -0.019619688391685486, "fcm_dpo/margin": 114.56228637695312, "fcm_dpo/q_t": 0.40111827850341797, "grad_norm": 10.05203914642334, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.21094012260437, "logits/rejected": -1.732450246810913, "logps/chosen": -257.4015808105469, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -375.31622314453125, "loss": 1.0626, "margin_dpo/margin_mean": 114.56228637695312, "margin_dpo/margin_std": 120.2484130859375, "step": 400 }, { "KL/chosen_KL_mean": -167.85890197753906, "KL/mean": -203.14907836914062, "KL/rejected_KL_mean": -238.43927001953125, "KL/std": 105.22209167480469, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.004309911746531725, "fcm_dpo/delta": 0.1384097784757614, "fcm_dpo/margin": 70.58036804199219, "fcm_dpo/q_t": 0.43961992859840393, "grad_norm": 10.694751739501953, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.4215683937072754, "logits/rejected": -1.175071358680725, "logps/chosen": -252.1480712890625, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -329.38238525390625, "loss": 1.1802, "margin_dpo/margin_mean": 70.58036804199219, "margin_dpo/margin_std": 129.00228881835938, "step": 401 }, { "KL/chosen_KL_mean": -148.88027954101562, "KL/mean": -215.65234375, "KL/rejected_KL_mean": -282.4244384765625, "KL/std": 112.36537170410156, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0036211651749908924, "fcm_dpo/delta": -0.1703907698392868, "fcm_dpo/margin": 133.5441436767578, "fcm_dpo/q_t": 0.3676212430000305, "grad_norm": 7.127365589141846, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -2.0082826614379883, "logits/rejected": -1.5479857921600342, "logps/chosen": -232.47340393066406, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -370.23468017578125, "loss": 1.0011, "margin_dpo/margin_mean": 133.5441436767578, "margin_dpo/margin_std": 110.00785827636719, "step": 402 }, { "KL/chosen_KL_mean": -152.83416748046875, "KL/mean": -202.25161743164062, "KL/rejected_KL_mean": -251.6690673828125, "KL/std": 107.68571472167969, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0036315941251814365, "fcm_dpo/delta": 0.06432539969682693, "fcm_dpo/margin": 98.83491516113281, "fcm_dpo/q_t": 0.4198303520679474, "grad_norm": 7.529103755950928, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.180328130722046, "logits/rejected": -1.776517391204834, "logps/chosen": -226.58724975585938, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -335.5892028808594, "loss": 1.1033, "margin_dpo/margin_mean": 98.83491516113281, "margin_dpo/margin_std": 115.40177917480469, "step": 403 }, { "KL/chosen_KL_mean": -179.7040252685547, "KL/mean": -222.43936157226562, "KL/rejected_KL_mean": -265.1746826171875, "KL/std": 103.97486877441406, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0041496893391013145, "fcm_dpo/delta": 0.07065935432910919, "fcm_dpo/margin": 85.47064208984375, "fcm_dpo/q_t": 0.4227173626422882, "grad_norm": 11.441384315490723, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.0066380500793457, "logits/rejected": -1.6690661907196045, "logps/chosen": -259.38018798828125, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -349.4552001953125, "loss": 1.1302, "margin_dpo/margin_mean": 85.47064971923828, "margin_dpo/margin_std": 126.88984680175781, "step": 404 }, { "KL/chosen_KL_mean": -182.75387573242188, "KL/mean": -226.0937957763672, "KL/rejected_KL_mean": -269.4336853027344, "KL/std": 98.4761734008789, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004595717880874872, "fcm_dpo/delta": -0.005296044051647186, "fcm_dpo/margin": 86.6798095703125, "fcm_dpo/q_t": 0.4063471555709839, "grad_norm": 8.78022575378418, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.4723773002624512, "logits/rejected": -0.9900224208831787, "logps/chosen": -279.2158203125, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -362.30438232421875, "loss": 1.086, "margin_dpo/margin_mean": 86.6798095703125, "margin_dpo/margin_std": 104.23223876953125, "step": 405 }, { "KL/chosen_KL_mean": -175.27471923828125, "KL/mean": -214.13243103027344, "KL/rejected_KL_mean": -252.9901580810547, "KL/std": 104.8065185546875, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004705079831182957, "fcm_dpo/delta": 0.053307823836803436, "fcm_dpo/margin": 77.71544647216797, "fcm_dpo/q_t": 0.41950565576553345, "grad_norm": 15.054417610168457, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -2.016731023788452, "logits/rejected": -1.8524185419082642, "logps/chosen": -253.40867614746094, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -351.27374267578125, "loss": 1.1322, "margin_dpo/margin_mean": 77.71543884277344, "margin_dpo/margin_std": 122.106201171875, "step": 406 }, { "KL/chosen_KL_mean": -180.1064453125, "KL/mean": -214.5044403076172, "KL/rejected_KL_mean": -248.90243530273438, "KL/std": 103.11662292480469, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.00527278333902359, "fcm_dpo/delta": 0.057411614805459976, "fcm_dpo/margin": 68.79596710205078, "fcm_dpo/q_t": 0.4207257032394409, "grad_norm": 11.159727096557617, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.26931095123291, "logits/rejected": -1.7588486671447754, "logps/chosen": -264.1192932128906, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -331.6834716796875, "loss": 1.1341, "margin_dpo/margin_mean": 68.79596710205078, "margin_dpo/margin_std": 110.22282409667969, "step": 407 }, { "KL/chosen_KL_mean": -173.29946899414062, "KL/mean": -207.7612762451172, "KL/rejected_KL_mean": -242.22308349609375, "KL/std": 98.42083740234375, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.005551498383283615, "fcm_dpo/delta": 0.0281324852257967, "fcm_dpo/margin": 68.92359924316406, "fcm_dpo/q_t": 0.4167592525482178, "grad_norm": 11.504765510559082, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.333503484725952, "logits/rejected": -1.7187881469726562, "logps/chosen": -277.76123046875, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -338.59527587890625, "loss": 1.1472, "margin_dpo/margin_mean": 68.92359924316406, "margin_dpo/margin_std": 122.54326629638672, "step": 408 }, { "KL/chosen_KL_mean": -182.81329345703125, "KL/mean": -219.31846618652344, "KL/rejected_KL_mean": -255.8236541748047, "KL/std": 101.0455322265625, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.005588524043560028, "fcm_dpo/delta": -0.013597136363387108, "fcm_dpo/margin": 73.0103759765625, "fcm_dpo/q_t": 0.4038846492767334, "grad_norm": 10.248709678649902, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5433369874954224, "logits/rejected": -1.1056166887283325, "logps/chosen": -280.4815979003906, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -345.8695068359375, "loss": 1.115, "margin_dpo/margin_mean": 73.0103759765625, "margin_dpo/margin_std": 111.819091796875, "step": 409 }, { "KL/chosen_KL_mean": -177.62811279296875, "KL/mean": -221.84169006347656, "KL/rejected_KL_mean": -266.0552673339844, "KL/std": 106.99247741699219, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0049283262342214584, "fcm_dpo/delta": -0.06745325028896332, "fcm_dpo/margin": 88.4271469116211, "fcm_dpo/q_t": 0.3916129767894745, "grad_norm": 8.896446228027344, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.6460766792297363, "logits/rejected": -1.5142734050750732, "logps/chosen": -254.09735107421875, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -354.6959228515625, "loss": 1.063, "margin_dpo/margin_mean": 88.4271469116211, "margin_dpo/margin_std": 107.10031127929688, "step": 410 }, { "KL/chosen_KL_mean": -189.69943237304688, "KL/mean": -230.718505859375, "KL/rejected_KL_mean": -271.73760986328125, "KL/std": 100.30302429199219, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004761087242513895, "fcm_dpo/delta": 0.012248404324054718, "fcm_dpo/margin": 82.03816986083984, "fcm_dpo/q_t": 0.4112668037414551, "grad_norm": 8.837950706481934, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.3662893772125244, "logits/rejected": -1.9626891613006592, "logps/chosen": -276.86572265625, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -358.8336181640625, "loss": 1.0976, "margin_dpo/margin_mean": 82.03817749023438, "margin_dpo/margin_std": 108.24765014648438, "step": 411 }, { "KL/chosen_KL_mean": -179.90597534179688, "KL/mean": -220.33294677734375, "KL/rejected_KL_mean": -260.7598876953125, "KL/std": 100.2435073852539, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004883988760411739, "fcm_dpo/delta": 0.006576664745807648, "fcm_dpo/margin": 80.85392761230469, "fcm_dpo/q_t": 0.41150540113449097, "grad_norm": 10.899422645568848, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.3223915100097656, "logits/rejected": -2.1068882942199707, "logps/chosen": -258.85333251953125, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -366.8654479980469, "loss": 1.1224, "margin_dpo/margin_mean": 80.85394287109375, "margin_dpo/margin_std": 128.37335205078125, "step": 412 }, { "KL/chosen_KL_mean": -192.58847045898438, "KL/mean": -237.19407653808594, "KL/rejected_KL_mean": -281.7996826171875, "KL/std": 106.85282897949219, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004614253528416157, "fcm_dpo/delta": -0.027992550283670425, "fcm_dpo/margin": 89.2112045288086, "fcm_dpo/q_t": 0.40089648962020874, "grad_norm": 9.790851593017578, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.3124396800994873, "logits/rejected": -1.0585637092590332, "logps/chosen": -288.283203125, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -389.76055908203125, "loss": 1.0983, "margin_dpo/margin_mean": 89.2112045288086, "margin_dpo/margin_std": 127.17318725585938, "step": 413 }, { "KL/chosen_KL_mean": -178.53021240234375, "KL/mean": -220.6641845703125, "KL/rejected_KL_mean": -262.79815673828125, "KL/std": 102.38829040527344, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0047956788912415504, "fcm_dpo/delta": -0.00751829706132412, "fcm_dpo/margin": 84.26795959472656, "fcm_dpo/q_t": 0.40749505162239075, "grad_norm": 12.249616622924805, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.7038583755493164, "logits/rejected": -1.3551607131958008, "logps/chosen": -266.806884765625, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -355.668212890625, "loss": 1.1146, "margin_dpo/margin_mean": 84.26795959472656, "margin_dpo/margin_std": 130.97010803222656, "step": 414 }, { "KL/chosen_KL_mean": -194.88992309570312, "KL/mean": -230.1785888671875, "KL/rejected_KL_mean": -265.46728515625, "KL/std": 93.6524658203125, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.005161660257726908, "fcm_dpo/delta": 0.05407894402742386, "fcm_dpo/margin": 70.57735443115234, "fcm_dpo/q_t": 0.4202832877635956, "grad_norm": 11.753730773925781, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.6724467277526855, "logits/rejected": -1.2966456413269043, "logps/chosen": -303.8058166503906, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -372.93865966796875, "loss": 1.1385, "margin_dpo/margin_mean": 70.57734680175781, "margin_dpo/margin_std": 115.16864776611328, "step": 415 }, { "KL/chosen_KL_mean": -194.14801025390625, "KL/mean": -242.9152374267578, "KL/rejected_KL_mean": -291.6824645996094, "KL/std": 108.26374816894531, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004779009148478508, "fcm_dpo/delta": -0.12810039520263672, "fcm_dpo/margin": 97.53443908691406, "fcm_dpo/q_t": 0.3821769058704376, "grad_norm": 9.709710121154785, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8006792068481445, "logits/rejected": -1.2953376770019531, "logps/chosen": -287.5469055175781, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -389.3554382324219, "loss": 1.0599, "margin_dpo/margin_mean": 97.53443908691406, "margin_dpo/margin_std": 114.06884765625, "step": 416 }, { "KL/chosen_KL_mean": -181.09359741210938, "KL/mean": -226.26687622070312, "KL/rejected_KL_mean": -271.4401550292969, "KL/std": 100.17672729492188, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.004256817977875471, "fcm_dpo/delta": 0.02383759245276451, "fcm_dpo/margin": 90.34654235839844, "fcm_dpo/q_t": 0.4110073447227478, "grad_norm": 8.41218376159668, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.2666587829589844, "logits/rejected": -1.3610515594482422, "logps/chosen": -264.6289367675781, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -345.8819885253906, "loss": 1.0839, "margin_dpo/margin_mean": 90.3465347290039, "margin_dpo/margin_std": 102.33303833007812, "step": 417 }, { "KL/chosen_KL_mean": -173.88331604003906, "KL/mean": -230.87852478027344, "KL/rejected_KL_mean": -287.87371826171875, "KL/std": 107.85359191894531, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004006189294159412, "fcm_dpo/delta": -0.1134781613945961, "fcm_dpo/margin": 113.99041748046875, "fcm_dpo/q_t": 0.38223546743392944, "grad_norm": 10.0100736618042, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.269974946975708, "logits/rejected": -1.6248165369033813, "logps/chosen": -282.1048583984375, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -399.7383728027344, "loss": 1.0445, "margin_dpo/margin_mean": 113.99043273925781, "margin_dpo/margin_std": 106.09822082519531, "step": 418 }, { "KL/chosen_KL_mean": -181.53469848632812, "KL/mean": -231.10244750976562, "KL/rejected_KL_mean": -280.67022705078125, "KL/std": 96.02542114257812, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0037487971130758524, "fcm_dpo/delta": 0.03628823161125183, "fcm_dpo/margin": 99.135498046875, "fcm_dpo/q_t": 0.41365084052085876, "grad_norm": 9.430407524108887, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.334961175918579, "logits/rejected": -0.8795088529586792, "logps/chosen": -278.7179870605469, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -378.85552978515625, "loss": 1.0889, "margin_dpo/margin_mean": 99.135498046875, "margin_dpo/margin_std": 105.69784545898438, "step": 419 }, { "KL/chosen_KL_mean": -186.0679168701172, "KL/mean": -230.18678283691406, "KL/rejected_KL_mean": -274.3056640625, "KL/std": 101.33537292480469, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.004093095660209656, "fcm_dpo/delta": 0.05367731675505638, "fcm_dpo/margin": 88.23770904541016, "fcm_dpo/q_t": 0.4186694025993347, "grad_norm": 13.177937507629395, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.054335117340088, "logits/rejected": -0.918046236038208, "logps/chosen": -300.37640380859375, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -349.98919677734375, "loss": 1.1125, "margin_dpo/margin_mean": 88.23771667480469, "margin_dpo/margin_std": 116.67931365966797, "step": 420 }, { "KL/chosen_KL_mean": -176.91961669921875, "KL/mean": -226.0680694580078, "KL/rejected_KL_mean": -275.216552734375, "KL/std": 104.53178405761719, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0040306467562913895, "fcm_dpo/delta": 0.005903397686779499, "fcm_dpo/margin": 98.29690551757812, "fcm_dpo/q_t": 0.4071115255355835, "grad_norm": 7.2168707847595215, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.9147285223007202, "logits/rejected": -1.546878457069397, "logps/chosen": -262.7994689941406, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -371.55303955078125, "loss": 1.0811, "margin_dpo/margin_mean": 98.29691314697266, "margin_dpo/margin_std": 115.43807983398438, "step": 421 }, { "KL/chosen_KL_mean": -175.11614990234375, "KL/mean": -222.0814971923828, "KL/rejected_KL_mean": -269.04681396484375, "KL/std": 96.419677734375, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004206541925668716, "fcm_dpo/delta": 0.0072818538174033165, "fcm_dpo/margin": 93.9306869506836, "fcm_dpo/q_t": 0.4079977571964264, "grad_norm": 11.74337387084961, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.5421252250671387, "logits/rejected": -2.1893234252929688, "logps/chosen": -275.59674072265625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -363.455078125, "loss": 1.0789, "margin_dpo/margin_mean": 93.93067932128906, "margin_dpo/margin_std": 108.5113296508789, "step": 422 }, { "KL/chosen_KL_mean": -162.89402770996094, "KL/mean": -210.56072998046875, "KL/rejected_KL_mean": -258.22747802734375, "KL/std": 115.6995849609375, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.004260974936187267, "fcm_dpo/delta": -0.014800310134887695, "fcm_dpo/margin": 95.33343505859375, "fcm_dpo/q_t": 0.4055958390235901, "grad_norm": 9.505404472351074, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9443423748016357, "logits/rejected": -1.8277565240859985, "logps/chosen": -241.34396362304688, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -350.27398681640625, "loss": 1.0983, "margin_dpo/margin_mean": 95.33343505859375, "margin_dpo/margin_std": 131.62005615234375, "step": 423 }, { "KL/chosen_KL_mean": -183.3204345703125, "KL/mean": -229.90501403808594, "KL/rejected_KL_mean": -276.4896240234375, "KL/std": 102.44046020507812, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.004155920818448067, "fcm_dpo/delta": 0.02088252082467079, "fcm_dpo/margin": 93.1691665649414, "fcm_dpo/q_t": 0.4100187420845032, "grad_norm": 8.343191146850586, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.6766777038574219, "logits/rejected": -1.0773122310638428, "logps/chosen": -270.9627685546875, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -348.85528564453125, "loss": 1.0863, "margin_dpo/margin_mean": 93.16915893554688, "margin_dpo/margin_std": 110.10171508789062, "step": 424 }, { "KL/chosen_KL_mean": -189.5157928466797, "KL/mean": -221.34750366210938, "KL/rejected_KL_mean": -253.17921447753906, "KL/std": 106.3233642578125, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004990538582205772, "fcm_dpo/delta": 0.12081054598093033, "fcm_dpo/margin": 63.66339874267578, "fcm_dpo/q_t": 0.4372677505016327, "grad_norm": 11.375186920166016, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.716882586479187, "logits/rejected": -1.6539833545684814, "logps/chosen": -268.03558349609375, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -355.9278564453125, "loss": 1.1986, "margin_dpo/margin_mean": 63.66339874267578, "margin_dpo/margin_std": 132.87921142578125, "step": 425 }, { "KL/chosen_KL_mean": -163.41897583007812, "KL/mean": -209.47048950195312, "KL/rejected_KL_mean": -255.52203369140625, "KL/std": 97.9679183959961, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.004857394844293594, "fcm_dpo/delta": -0.08500692248344421, "fcm_dpo/margin": 92.10308837890625, "fcm_dpo/q_t": 0.3896920084953308, "grad_norm": 11.810247421264648, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.7764379978179932, "logits/rejected": -1.0434608459472656, "logps/chosen": -271.9248046875, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -343.82208251953125, "loss": 1.0635, "margin_dpo/margin_mean": 92.10308837890625, "margin_dpo/margin_std": 114.77928924560547, "step": 426 }, { "KL/chosen_KL_mean": -134.11309814453125, "KL/mean": -177.22207641601562, "KL/rejected_KL_mean": -220.33103942871094, "KL/std": 93.32369995117188, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0046097468584775925, "fcm_dpo/delta": 0.003671671263873577, "fcm_dpo/margin": 86.21794128417969, "fcm_dpo/q_t": 0.4069150686264038, "grad_norm": 8.87088394165039, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.342360019683838, "logits/rejected": -1.7008912563323975, "logps/chosen": -233.23355102539062, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -306.0556640625, "loss": 1.0789, "margin_dpo/margin_mean": 86.21794128417969, "margin_dpo/margin_std": 100.27278900146484, "step": 427 }, { "KL/chosen_KL_mean": -190.5924835205078, "KL/mean": -227.04598999023438, "KL/rejected_KL_mean": -263.49945068359375, "KL/std": 105.06121826171875, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.004846968688070774, "fcm_dpo/delta": 0.07074836641550064, "fcm_dpo/margin": 72.906982421875, "fcm_dpo/q_t": 0.4236881732940674, "grad_norm": 10.94521713256836, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.8032784461975098, "logits/rejected": -1.548487901687622, "logps/chosen": -296.5617370605469, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -372.6016540527344, "loss": 1.1484, "margin_dpo/margin_mean": 72.906982421875, "margin_dpo/margin_std": 116.56391906738281, "step": 428 }, { "KL/chosen_KL_mean": -175.35250854492188, "KL/mean": -222.66241455078125, "KL/rejected_KL_mean": -269.97235107421875, "KL/std": 110.11876678466797, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.004865257069468498, "fcm_dpo/delta": -0.11719562858343124, "fcm_dpo/margin": 94.61983489990234, "fcm_dpo/q_t": 0.3866245150566101, "grad_norm": 9.078190803527832, "learning_rate": 1.828194884925749e-07, "logits/chosen": -2.0975754261016846, "logits/rejected": -1.398925542831421, "logps/chosen": -288.8973693847656, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -368.21435546875, "loss": 1.0846, "margin_dpo/margin_mean": 94.61984252929688, "margin_dpo/margin_std": 125.21287536621094, "step": 429 }, { "KL/chosen_KL_mean": -166.0338897705078, "KL/mean": -202.96798706054688, "KL/rejected_KL_mean": -239.90203857421875, "KL/std": 101.92799377441406, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.00476789940148592, "fcm_dpo/delta": 0.06470806896686554, "fcm_dpo/margin": 73.86817932128906, "fcm_dpo/q_t": 0.4227973222732544, "grad_norm": 9.421457290649414, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.2553670406341553, "logits/rejected": -1.2192316055297852, "logps/chosen": -257.353271484375, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -350.0116882324219, "loss": 1.141, "margin_dpo/margin_mean": 73.86817932128906, "margin_dpo/margin_std": 118.4478530883789, "step": 430 }, { "KL/chosen_KL_mean": -167.86351013183594, "KL/mean": -211.3416748046875, "KL/rejected_KL_mean": -254.81982421875, "KL/std": 104.25059509277344, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004610789008438587, "fcm_dpo/delta": -0.0027859490364789963, "fcm_dpo/margin": 86.95631408691406, "fcm_dpo/q_t": 0.40672099590301514, "grad_norm": 10.194257736206055, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.9516189098358154, "logits/rejected": -1.5697989463806152, "logps/chosen": -261.04473876953125, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -352.95208740234375, "loss": 1.0963, "margin_dpo/margin_mean": 86.95631408691406, "margin_dpo/margin_std": 118.56509399414062, "step": 431 }, { "KL/chosen_KL_mean": -145.94134521484375, "KL/mean": -198.55746459960938, "KL/rejected_KL_mean": -251.17355346679688, "KL/std": 108.05368041992188, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.004260255489498377, "fcm_dpo/delta": -0.08680972456932068, "fcm_dpo/margin": 105.23225402832031, "fcm_dpo/q_t": 0.38689985871315, "grad_norm": 8.809940338134766, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.6157441139221191, "logits/rejected": -1.0725090503692627, "logps/chosen": -250.37611389160156, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -356.26312255859375, "loss": 1.0373, "margin_dpo/margin_mean": 105.23226165771484, "margin_dpo/margin_std": 104.21809387207031, "step": 432 }, { "KL/chosen_KL_mean": -175.45770263671875, "KL/mean": -216.27476501464844, "KL/rejected_KL_mean": -257.0918273925781, "KL/std": 102.27630615234375, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004458767361938953, "fcm_dpo/delta": 0.04540761932730675, "fcm_dpo/margin": 81.6341323852539, "fcm_dpo/q_t": 0.4174819588661194, "grad_norm": 8.810248374938965, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.7001516819000244, "logits/rejected": -1.3830372095108032, "logps/chosen": -265.3009033203125, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -358.8252868652344, "loss": 1.122, "margin_dpo/margin_mean": 81.6341323852539, "margin_dpo/margin_std": 115.78897094726562, "step": 433 }, { "KL/chosen_KL_mean": -151.13812255859375, "KL/mean": -197.9150390625, "KL/rejected_KL_mean": -244.69195556640625, "KL/std": 114.12715911865234, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004322323948144913, "fcm_dpo/delta": -0.007504099048674107, "fcm_dpo/margin": 93.55381774902344, "fcm_dpo/q_t": 0.40848463773727417, "grad_norm": 11.34160327911377, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.4890990257263184, "logits/rejected": -1.8233656883239746, "logps/chosen": -248.83065795898438, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -328.78326416015625, "loss": 1.0968, "margin_dpo/margin_mean": 93.55381774902344, "margin_dpo/margin_std": 133.07388305664062, "step": 434 }, { "KL/chosen_KL_mean": -149.54263305664062, "KL/mean": -197.25633239746094, "KL/rejected_KL_mean": -244.97003173828125, "KL/std": 105.13895416259766, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.00425665732473135, "fcm_dpo/delta": -0.0107885105535388, "fcm_dpo/margin": 95.42740631103516, "fcm_dpo/q_t": 0.40506988763809204, "grad_norm": 9.601332664489746, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.3978500366210938, "logits/rejected": -1.9519944190979004, "logps/chosen": -235.71456909179688, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -338.72174072265625, "loss": 1.0797, "margin_dpo/margin_mean": 95.42740631103516, "margin_dpo/margin_std": 117.4110107421875, "step": 435 }, { "KL/chosen_KL_mean": -123.91187286376953, "KL/mean": -174.17019653320312, "KL/rejected_KL_mean": -224.42852783203125, "KL/std": 89.50836181640625, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004046005662530661, "fcm_dpo/delta": -0.011956536211073399, "fcm_dpo/margin": 100.51663970947266, "fcm_dpo/q_t": 0.40188825130462646, "grad_norm": 9.175294876098633, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.7359514236450195, "logits/rejected": -0.8693200945854187, "logps/chosen": -229.698974609375, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -313.05322265625, "loss": 1.0539, "margin_dpo/margin_mean": 100.51664733886719, "margin_dpo/margin_std": 92.1972885131836, "step": 436 }, { "KL/chosen_KL_mean": -144.34274291992188, "KL/mean": -184.70896911621094, "KL/rejected_KL_mean": -225.0751953125, "KL/std": 95.95875549316406, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004505590070039034, "fcm_dpo/delta": 0.053700223565101624, "fcm_dpo/margin": 80.73246765136719, "fcm_dpo/q_t": 0.4168490469455719, "grad_norm": 10.32386302947998, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.970510721206665, "logits/rejected": -1.3135467767715454, "logps/chosen": -240.40478515625, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -314.08740234375, "loss": 1.1004, "margin_dpo/margin_mean": 80.73246765136719, "margin_dpo/margin_std": 95.95643615722656, "step": 437 }, { "KL/chosen_KL_mean": -148.440185546875, "KL/mean": -189.84994506835938, "KL/rejected_KL_mean": -231.2597198486328, "KL/std": 96.81468200683594, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.004559430293738842, "fcm_dpo/delta": 0.032795362174510956, "fcm_dpo/margin": 82.81953430175781, "fcm_dpo/q_t": 0.4135589599609375, "grad_norm": 9.748353004455566, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -1.9794278144836426, "logits/rejected": -1.1435534954071045, "logps/chosen": -250.29556274414062, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -320.7073974609375, "loss": 1.1023, "margin_dpo/margin_mean": 82.81953430175781, "margin_dpo/margin_std": 103.70001983642578, "step": 438 }, { "KL/chosen_KL_mean": -159.73666381835938, "KL/mean": -200.69808959960938, "KL/rejected_KL_mean": -241.6595458984375, "KL/std": 104.9482421875, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.00463654613122344, "fcm_dpo/delta": 0.019627489149570465, "fcm_dpo/margin": 81.9228744506836, "fcm_dpo/q_t": 0.41171377897262573, "grad_norm": 10.349353790283203, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.7122106552124023, "logits/rejected": -1.4814281463623047, "logps/chosen": -241.49229431152344, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -332.24591064453125, "loss": 1.109, "margin_dpo/margin_mean": 81.9228744506836, "margin_dpo/margin_std": 102.98176574707031, "step": 439 }, { "KL/chosen_KL_mean": -143.01638793945312, "KL/mean": -193.98068237304688, "KL/rejected_KL_mean": -244.9449920654297, "KL/std": 106.21638488769531, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.004190884530544281, "fcm_dpo/delta": -0.07287675142288208, "fcm_dpo/margin": 101.92862701416016, "fcm_dpo/q_t": 0.39218002557754517, "grad_norm": 7.27903938293457, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.654543876647949, "logits/rejected": -2.1770920753479004, "logps/chosen": -248.65745544433594, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -348.34600830078125, "loss": 1.0654, "margin_dpo/margin_mean": 101.92861938476562, "margin_dpo/margin_std": 117.64850616455078, "step": 440 }, { "KL/chosen_KL_mean": -184.28952026367188, "KL/mean": -227.08975219726562, "KL/rejected_KL_mean": -269.8899841308594, "KL/std": 100.72953796386719, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.004480929579585791, "fcm_dpo/delta": 0.02669834904372692, "fcm_dpo/margin": 85.6004638671875, "fcm_dpo/q_t": 0.4139643907546997, "grad_norm": 9.028122901916504, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.6215391159057617, "logits/rejected": -1.2592815160751343, "logps/chosen": -275.81903076171875, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -373.5093688964844, "loss": 1.0962, "margin_dpo/margin_mean": 85.6004638671875, "margin_dpo/margin_std": 111.03690338134766, "step": 441 }, { "KL/chosen_KL_mean": -178.9105224609375, "KL/mean": -233.7051239013672, "KL/rejected_KL_mean": -288.4997253417969, "KL/std": 106.39604187011719, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.00405229814350605, "fcm_dpo/delta": -0.08114978671073914, "fcm_dpo/margin": 109.58921813964844, "fcm_dpo/q_t": 0.3877112567424774, "grad_norm": 9.92836856842041, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.625436782836914, "logits/rejected": -1.150752067565918, "logps/chosen": -263.6880798339844, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -372.3238830566406, "loss": 1.036, "margin_dpo/margin_mean": 109.58921813964844, "margin_dpo/margin_std": 107.75190734863281, "step": 442 }, { "KL/chosen_KL_mean": -175.0455780029297, "KL/mean": -208.82803344726562, "KL/rejected_KL_mean": -242.61048889160156, "KL/std": 103.93907165527344, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0046254713088274, "fcm_dpo/delta": 0.12850916385650635, "fcm_dpo/margin": 67.56490325927734, "fcm_dpo/q_t": 0.4353976547718048, "grad_norm": 11.169327735900879, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.919511079788208, "logits/rejected": -1.3677531480789185, "logps/chosen": -277.6948547363281, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -335.6485595703125, "loss": 1.1599, "margin_dpo/margin_mean": 67.56490325927734, "margin_dpo/margin_std": 101.4620361328125, "step": 443 }, { "KL/chosen_KL_mean": -178.73696899414062, "KL/mean": -218.7532501220703, "KL/rejected_KL_mean": -258.76953125, "KL/std": 101.40406799316406, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.005048186518251896, "fcm_dpo/delta": -0.006932250689715147, "fcm_dpo/margin": 80.03257751464844, "fcm_dpo/q_t": 0.4075952172279358, "grad_norm": 11.716943740844727, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.728158950805664, "logits/rejected": -1.5662286281585693, "logps/chosen": -266.65667724609375, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -362.093017578125, "loss": 1.1046, "margin_dpo/margin_mean": 80.03257751464844, "margin_dpo/margin_std": 117.46232604980469, "step": 444 }, { "KL/chosen_KL_mean": -156.82369995117188, "KL/mean": -203.46446228027344, "KL/rejected_KL_mean": -250.10523986816406, "KL/std": 97.83328247070312, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0044675590470433235, "fcm_dpo/delta": -0.038991086184978485, "fcm_dpo/margin": 93.28153991699219, "fcm_dpo/q_t": 0.39787137508392334, "grad_norm": 8.604227066040039, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.8339993953704834, "logits/rejected": -1.4113221168518066, "logps/chosen": -258.22454833984375, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -349.14312744140625, "loss": 1.0649, "margin_dpo/margin_mean": 93.28154754638672, "margin_dpo/margin_std": 103.1292724609375, "step": 445 }, { "KL/chosen_KL_mean": -151.64605712890625, "KL/mean": -193.40594482421875, "KL/rejected_KL_mean": -235.16583251953125, "KL/std": 103.88375854492188, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.004742743447422981, "fcm_dpo/delta": 0.004942053928971291, "fcm_dpo/margin": 83.51976776123047, "fcm_dpo/q_t": 0.40773141384124756, "grad_norm": 13.81721305847168, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.8588109016418457, "logits/rejected": -1.719390630722046, "logps/chosen": -239.0684051513672, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -341.8665771484375, "loss": 1.0812, "margin_dpo/margin_mean": 83.51976013183594, "margin_dpo/margin_std": 98.7499771118164, "step": 446 }, { "KL/chosen_KL_mean": -148.78799438476562, "KL/mean": -203.45889282226562, "KL/rejected_KL_mean": -258.1297912597656, "KL/std": 108.67035675048828, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004052440635859966, "fcm_dpo/delta": -0.08156979084014893, "fcm_dpo/margin": 109.34180450439453, "fcm_dpo/q_t": 0.3899604082107544, "grad_norm": 8.002371788024902, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.119405746459961, "logits/rejected": -2.076201915740967, "logps/chosen": -232.40325927734375, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -363.04217529296875, "loss": 1.0535, "margin_dpo/margin_mean": 109.34180450439453, "margin_dpo/margin_std": 124.98095703125, "step": 447 }, { "KL/chosen_KL_mean": -161.07196044921875, "KL/mean": -213.84481811523438, "KL/rejected_KL_mean": -266.6177062988281, "KL/std": 113.26884460449219, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.00395313510671258, "fcm_dpo/delta": -0.033412255346775055, "fcm_dpo/margin": 105.54573059082031, "fcm_dpo/q_t": 0.4000400900840759, "grad_norm": 9.063913345336914, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.7182838916778564, "logits/rejected": -1.448482632637024, "logps/chosen": -253.64773559570312, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -374.3074645996094, "loss": 1.0832, "margin_dpo/margin_mean": 105.54572296142578, "margin_dpo/margin_std": 132.21575927734375, "step": 448 }, { "KL/chosen_KL_mean": -158.5665283203125, "KL/mean": -213.28146362304688, "KL/rejected_KL_mean": -267.99639892578125, "KL/std": 105.68087768554688, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.003674742067232728, "fcm_dpo/delta": -0.0035585779696702957, "fcm_dpo/margin": 109.42988586425781, "fcm_dpo/q_t": 0.4047185480594635, "grad_norm": 7.683147430419922, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4760916233062744, "logits/rejected": -1.0314109325408936, "logps/chosen": -260.9654541015625, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -363.145263671875, "loss": 1.0715, "margin_dpo/margin_mean": 109.42988586425781, "margin_dpo/margin_std": 121.295654296875, "step": 449 }, { "KL/chosen_KL_mean": -140.26527404785156, "KL/mean": -192.14407348632812, "KL/rejected_KL_mean": -244.02288818359375, "KL/std": 106.91035461425781, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0037433048710227013, "fcm_dpo/delta": 0.018990162760019302, "fcm_dpo/margin": 103.75762939453125, "fcm_dpo/q_t": 0.40951135754585266, "grad_norm": 9.310491561889648, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.4720988273620605, "logits/rejected": -2.284257650375366, "logps/chosen": -227.2581329345703, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -352.554931640625, "loss": 1.0813, "margin_dpo/margin_mean": 103.75761413574219, "margin_dpo/margin_std": 117.09740447998047, "step": 450 }, { "KL/chosen_KL_mean": -160.70419311523438, "KL/mean": -200.97280883789062, "KL/rejected_KL_mean": -241.24139404296875, "KL/std": 106.73225402832031, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.004343983717262745, "fcm_dpo/delta": 0.0726906955242157, "fcm_dpo/margin": 80.53719329833984, "fcm_dpo/q_t": 0.42443740367889404, "grad_norm": 13.81638240814209, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.4379723072052, "logits/rejected": -1.9357476234436035, "logps/chosen": -247.51547241210938, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -321.09698486328125, "loss": 1.1483, "margin_dpo/margin_mean": 80.53719329833984, "margin_dpo/margin_std": 135.08676147460938, "step": 451 }, { "KL/chosen_KL_mean": -139.84133911132812, "KL/mean": -185.8059539794922, "KL/rejected_KL_mean": -231.7705841064453, "KL/std": 100.86477661132812, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004175640642642975, "fcm_dpo/delta": 0.012559719383716583, "fcm_dpo/margin": 91.92924499511719, "fcm_dpo/q_t": 0.40863674879074097, "grad_norm": 8.557513236999512, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.9087903499603271, "logits/rejected": -1.8527933359146118, "logps/chosen": -219.22097778320312, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -335.4859619140625, "loss": 1.0851, "margin_dpo/margin_mean": 91.92925262451172, "margin_dpo/margin_std": 93.28313446044922, "step": 452 }, { "KL/chosen_KL_mean": -167.53453063964844, "KL/mean": -224.1717529296875, "KL/rejected_KL_mean": -280.8089599609375, "KL/std": 103.6968994140625, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.00395984435454011, "fcm_dpo/delta": -0.08942581713199615, "fcm_dpo/margin": 113.27445983886719, "fcm_dpo/q_t": 0.38706135749816895, "grad_norm": 10.809945106506348, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -2.0151002407073975, "logits/rejected": -1.5501811504364014, "logps/chosen": -255.22972106933594, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -370.8671875, "loss": 1.0342, "margin_dpo/margin_mean": 113.27445983886719, "margin_dpo/margin_std": 109.57632446289062, "step": 453 }, { "KL/chosen_KL_mean": -176.50994873046875, "KL/mean": -221.919189453125, "KL/rejected_KL_mean": -267.32843017578125, "KL/std": 106.3758544921875, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004159209784120321, "fcm_dpo/delta": 0.024521756917238235, "fcm_dpo/margin": 90.81847381591797, "fcm_dpo/q_t": 0.41380155086517334, "grad_norm": 8.709529876708984, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.039666175842285, "logits/rejected": -1.5942840576171875, "logps/chosen": -266.076171875, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -360.2494812011719, "loss": 1.1262, "margin_dpo/margin_mean": 90.81847381591797, "margin_dpo/margin_std": 138.0836181640625, "step": 454 }, { "KL/chosen_KL_mean": -166.71087646484375, "KL/mean": -209.4121856689453, "KL/rejected_KL_mean": -252.11346435546875, "KL/std": 104.2428970336914, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.004364639520645142, "fcm_dpo/delta": 0.03674519062042236, "fcm_dpo/margin": 85.40260314941406, "fcm_dpo/q_t": 0.41532665491104126, "grad_norm": 9.605944633483887, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -2.136798620223999, "logits/rejected": -2.0993053913116455, "logps/chosen": -244.3408660888672, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -371.0914306640625, "loss": 1.1137, "margin_dpo/margin_mean": 85.40260314941406, "margin_dpo/margin_std": 119.05706787109375, "step": 455 }, { "KL/chosen_KL_mean": -168.18087768554688, "KL/mean": -210.7486572265625, "KL/rejected_KL_mean": -253.31642150878906, "KL/std": 105.94180297851562, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.004523547366261482, "fcm_dpo/delta": 0.021607208997011185, "fcm_dpo/margin": 85.13553619384766, "fcm_dpo/q_t": 0.4115739166736603, "grad_norm": 11.971829414367676, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.1685986518859863, "logits/rejected": -2.0423097610473633, "logps/chosen": -257.7977294921875, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -362.8761291503906, "loss": 1.0951, "margin_dpo/margin_mean": 85.13553619384766, "margin_dpo/margin_std": 108.1827392578125, "step": 456 }, { "KL/chosen_KL_mean": -137.1604461669922, "KL/mean": -192.81851196289062, "KL/rejected_KL_mean": -248.4765625, "KL/std": 105.62006378173828, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0037295669317245483, "fcm_dpo/delta": -0.05863155424594879, "fcm_dpo/margin": 111.3161392211914, "fcm_dpo/q_t": 0.3920668065547943, "grad_norm": 9.642630577087402, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.0477685928344727, "logits/rejected": -1.7290433645248413, "logps/chosen": -224.4821319580078, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -339.2431640625, "loss": 1.0511, "margin_dpo/margin_mean": 111.31613159179688, "margin_dpo/margin_std": 93.54414367675781, "step": 457 }, { "KL/chosen_KL_mean": -159.44627380371094, "KL/mean": -200.7682647705078, "KL/rejected_KL_mean": -242.09027099609375, "KL/std": 105.59513854980469, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004410221241414547, "fcm_dpo/delta": 0.05078205466270447, "fcm_dpo/margin": 82.64398193359375, "fcm_dpo/q_t": 0.41964423656463623, "grad_norm": 9.40721607208252, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.8018229007720947, "logits/rejected": -1.5734291076660156, "logps/chosen": -252.9667205810547, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -345.4592590332031, "loss": 1.1233, "margin_dpo/margin_mean": 82.64398193359375, "margin_dpo/margin_std": 122.85063171386719, "step": 458 }, { "KL/chosen_KL_mean": -152.69473266601562, "KL/mean": -204.36691284179688, "KL/rejected_KL_mean": -256.03912353515625, "KL/std": 110.60247039794922, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.003967789933085442, "fcm_dpo/delta": -0.02914269268512726, "fcm_dpo/margin": 103.34439086914062, "fcm_dpo/q_t": 0.4010574519634247, "grad_norm": 6.6724019050598145, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6315174102783203, "logits/rejected": -1.381065845489502, "logps/chosen": -228.3829345703125, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -348.2095947265625, "loss": 1.0735, "margin_dpo/margin_mean": 103.34439086914062, "margin_dpo/margin_std": 118.76679992675781, "step": 459 }, { "KL/chosen_KL_mean": -155.52664184570312, "KL/mean": -203.31259155273438, "KL/rejected_KL_mean": -251.0985107421875, "KL/std": 107.5677490234375, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.00406930036842823, "fcm_dpo/delta": 0.01469976082444191, "fcm_dpo/margin": 95.57188415527344, "fcm_dpo/q_t": 0.4100395441055298, "grad_norm": 10.259289741516113, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0149388313293457, "logits/rejected": -2.0433106422424316, "logps/chosen": -236.64453125, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -361.410888671875, "loss": 1.0985, "margin_dpo/margin_mean": 95.57189178466797, "margin_dpo/margin_std": 122.98133087158203, "step": 460 }, { "KL/chosen_KL_mean": -193.4769744873047, "KL/mean": -231.97410583496094, "KL/rejected_KL_mean": -270.47125244140625, "KL/std": 94.12167358398438, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004632891155779362, "fcm_dpo/delta": 0.06771949678659439, "fcm_dpo/margin": 76.99427795410156, "fcm_dpo/q_t": 0.42272934317588806, "grad_norm": 9.476213455200195, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.40522301197052, "logits/rejected": -1.2098404169082642, "logps/chosen": -275.06048583984375, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -364.3483581542969, "loss": 1.1314, "margin_dpo/margin_mean": 76.99427032470703, "margin_dpo/margin_std": 114.23277282714844, "step": 461 }, { "KL/chosen_KL_mean": -146.39410400390625, "KL/mean": -196.26129150390625, "KL/rejected_KL_mean": -246.12850952148438, "KL/std": 101.52086639404297, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004426286555826664, "fcm_dpo/delta": -0.07409730553627014, "fcm_dpo/margin": 99.73442077636719, "fcm_dpo/q_t": 0.3909256160259247, "grad_norm": 8.858579635620117, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.467555046081543, "logits/rejected": -1.9808671474456787, "logps/chosen": -240.5926513671875, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -331.7601318359375, "loss": 1.0485, "margin_dpo/margin_mean": 99.73442077636719, "margin_dpo/margin_std": 108.26769256591797, "step": 462 }, { "KL/chosen_KL_mean": -143.1199493408203, "KL/mean": -202.0547637939453, "KL/rejected_KL_mean": -260.98956298828125, "KL/std": 109.03651428222656, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0038711370434612036, "fcm_dpo/delta": -0.10507725924253464, "fcm_dpo/margin": 117.86962890625, "fcm_dpo/q_t": 0.3826623857021332, "grad_norm": 13.261340141296387, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.8112142086029053, "logits/rejected": -1.3834519386291504, "logps/chosen": -229.04470825195312, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -357.89141845703125, "loss": 1.0467, "margin_dpo/margin_mean": 117.86962890625, "margin_dpo/margin_std": 115.90103149414062, "step": 463 }, { "KL/chosen_KL_mean": -175.66976928710938, "KL/mean": -223.53787231445312, "KL/rejected_KL_mean": -271.406005859375, "KL/std": 108.9599838256836, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0037259021773934364, "fcm_dpo/delta": 0.06764520704746246, "fcm_dpo/margin": 95.73623657226562, "fcm_dpo/q_t": 0.4212005138397217, "grad_norm": 10.644168853759766, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.9061057567596436, "logits/rejected": -1.9865224361419678, "logps/chosen": -255.3589630126953, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -378.69830322265625, "loss": 1.1134, "margin_dpo/margin_mean": 95.73623657226562, "margin_dpo/margin_std": 124.52812194824219, "step": 464 }, { "KL/chosen_KL_mean": -167.60714721679688, "KL/mean": -223.8770294189453, "KL/rejected_KL_mean": -280.1469421386719, "KL/std": 117.35865783691406, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0036897514946758747, "fcm_dpo/delta": -0.02762109413743019, "fcm_dpo/margin": 112.539794921875, "fcm_dpo/q_t": 0.39902204275131226, "grad_norm": 8.167524337768555, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.7697234153747559, "logits/rejected": -1.473779559135437, "logps/chosen": -259.46734619140625, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -398.85693359375, "loss": 1.054, "margin_dpo/margin_mean": 112.539794921875, "margin_dpo/margin_std": 111.62767791748047, "step": 465 }, { "KL/chosen_KL_mean": -148.68186950683594, "KL/mean": -206.63079833984375, "KL/rejected_KL_mean": -264.5797119140625, "KL/std": 111.83316040039062, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.003548400942236185, "fcm_dpo/delta": -0.019896645098924637, "fcm_dpo/margin": 115.89786529541016, "fcm_dpo/q_t": 0.4008832275867462, "grad_norm": 6.166109561920166, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.5463011264801025, "logits/rejected": -1.2276811599731445, "logps/chosen": -233.38327026367188, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -360.63055419921875, "loss": 1.0585, "margin_dpo/margin_mean": 115.89785766601562, "margin_dpo/margin_std": 118.12859344482422, "step": 466 }, { "KL/chosen_KL_mean": -191.1156005859375, "KL/mean": -239.16915893554688, "KL/rejected_KL_mean": -287.22271728515625, "KL/std": 117.21507263183594, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.003751535899937153, "fcm_dpo/delta": 0.06088443845510483, "fcm_dpo/margin": 96.10710906982422, "fcm_dpo/q_t": 0.41987118124961853, "grad_norm": 11.548589706420898, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.4588465690612793, "logits/rejected": -1.8937785625457764, "logps/chosen": -300.4139404296875, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -396.06634521484375, "loss": 1.1224, "margin_dpo/margin_mean": 96.10710906982422, "margin_dpo/margin_std": 131.92352294921875, "step": 467 }, { "KL/chosen_KL_mean": -148.42849731445312, "KL/mean": -192.81204223632812, "KL/rejected_KL_mean": -237.195556640625, "KL/std": 113.6472396850586, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.004226831253618002, "fcm_dpo/delta": 0.03979168087244034, "fcm_dpo/margin": 88.76704406738281, "fcm_dpo/q_t": 0.4172515273094177, "grad_norm": 9.76155948638916, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.9352748394012451, "logits/rejected": -1.733092188835144, "logps/chosen": -246.75015258789062, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -343.87603759765625, "loss": 1.1177, "margin_dpo/margin_mean": 88.76704406738281, "margin_dpo/margin_std": 130.5291748046875, "step": 468 }, { "KL/chosen_KL_mean": -182.78546142578125, "KL/mean": -237.8021240234375, "KL/rejected_KL_mean": -292.8188171386719, "KL/std": 114.89578247070312, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0037884791381657124, "fcm_dpo/delta": -0.046975091099739075, "fcm_dpo/margin": 110.0333480834961, "fcm_dpo/q_t": 0.3969864547252655, "grad_norm": 9.44655704498291, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.2797884941101074, "logits/rejected": -1.8643840551376343, "logps/chosen": -279.5496826171875, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -402.413818359375, "loss": 1.0684, "margin_dpo/margin_mean": 110.03335571289062, "margin_dpo/margin_std": 124.08121490478516, "step": 469 }, { "KL/chosen_KL_mean": -167.97726440429688, "KL/mean": -217.93173217773438, "KL/rejected_KL_mean": -267.8861999511719, "KL/std": 105.97056579589844, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.003974203485995531, "fcm_dpo/delta": 0.0048775034956634045, "fcm_dpo/margin": 99.90892028808594, "fcm_dpo/q_t": 0.40664148330688477, "grad_norm": 9.04902172088623, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.7260205745697021, "logits/rejected": -1.3102350234985352, "logps/chosen": -251.4739227294922, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -356.3719787597656, "loss": 1.0705, "margin_dpo/margin_mean": 99.90892028808594, "margin_dpo/margin_std": 106.15619659423828, "step": 470 }, { "KL/chosen_KL_mean": -192.7497100830078, "KL/mean": -240.27554321289062, "KL/rejected_KL_mean": -287.8013916015625, "KL/std": 105.72528076171875, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004102043807506561, "fcm_dpo/delta": 0.016440996900200844, "fcm_dpo/margin": 95.05167388916016, "fcm_dpo/q_t": 0.4095396101474762, "grad_norm": 8.993461608886719, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6866495609283447, "logits/rejected": -1.27484130859375, "logps/chosen": -281.2239990234375, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -378.2830810546875, "loss": 1.0794, "margin_dpo/margin_mean": 95.05168151855469, "margin_dpo/margin_std": 107.0554428100586, "step": 471 }, { "KL/chosen_KL_mean": -181.86192321777344, "KL/mean": -237.57009887695312, "KL/rejected_KL_mean": -293.27825927734375, "KL/std": 106.19841003417969, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.003762502921745181, "fcm_dpo/delta": -0.038854680955410004, "fcm_dpo/margin": 111.41636657714844, "fcm_dpo/q_t": 0.39807412028312683, "grad_norm": 7.71476411819458, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.2670388221740723, "logits/rejected": -1.5799387693405151, "logps/chosen": -307.09564208984375, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -414.331787109375, "loss": 1.054, "margin_dpo/margin_mean": 111.41636657714844, "margin_dpo/margin_std": 112.93126678466797, "step": 472 }, { "KL/chosen_KL_mean": -205.76402282714844, "KL/mean": -240.4404296875, "KL/rejected_KL_mean": -275.11688232421875, "KL/std": 107.12960815429688, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.004614626988768578, "fcm_dpo/delta": 0.11143109202384949, "fcm_dpo/margin": 69.35285186767578, "fcm_dpo/q_t": 0.43345674872398376, "grad_norm": 11.181159019470215, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8183441162109375, "logits/rejected": -1.327343463897705, "logps/chosen": -301.3753967285156, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -363.26800537109375, "loss": 1.1675, "margin_dpo/margin_mean": 69.35285186767578, "margin_dpo/margin_std": 124.60063171386719, "step": 473 }, { "KL/chosen_KL_mean": -191.34152221679688, "KL/mean": -227.0296630859375, "KL/rejected_KL_mean": -262.7178039550781, "KL/std": 101.69308471679688, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.004570808261632919, "fcm_dpo/delta": -0.04423840716481209, "fcm_dpo/margin": 71.37625885009766, "fcm_dpo/q_t": 0.42263293266296387, "grad_norm": 15.797048568725586, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.5207116603851318, "logits/rejected": -1.3319883346557617, "logps/chosen": -272.8212890625, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -359.18341064453125, "loss": 1.1727, "margin_dpo/margin_mean": 71.37625122070312, "margin_dpo/margin_std": 124.67689514160156, "step": 474 }, { "KL/chosen_KL_mean": -145.18341064453125, "KL/mean": -201.0333251953125, "KL/rejected_KL_mean": -256.88323974609375, "KL/std": 108.72150421142578, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.003929138649255037, "fcm_dpo/delta": -0.07037956267595291, "fcm_dpo/margin": 111.69982147216797, "fcm_dpo/q_t": 0.39185622334480286, "grad_norm": 7.910090923309326, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.2782387733459473, "logits/rejected": -2.2266132831573486, "logps/chosen": -219.37939453125, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -351.57568359375, "loss": 1.0506, "margin_dpo/margin_mean": 111.6998291015625, "margin_dpo/margin_std": 123.81658935546875, "step": 475 }, { "KL/chosen_KL_mean": -168.28453063964844, "KL/mean": -218.2811279296875, "KL/rejected_KL_mean": -268.2777404785156, "KL/std": 111.14730834960938, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.003834768198430538, "fcm_dpo/delta": 0.025721672922372818, "fcm_dpo/margin": 99.99321746826172, "fcm_dpo/q_t": 0.41254156827926636, "grad_norm": 9.708074569702148, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.7751457691192627, "logits/rejected": -1.3865535259246826, "logps/chosen": -240.25563049316406, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -348.53997802734375, "loss": 1.091, "margin_dpo/margin_mean": 99.99321746826172, "margin_dpo/margin_std": 120.21827697753906, "step": 476 }, { "KL/chosen_KL_mean": -170.09632873535156, "KL/mean": -219.28900146484375, "KL/rejected_KL_mean": -268.4817199707031, "KL/std": 97.5474853515625, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004053044598549604, "fcm_dpo/delta": 0.001660497859120369, "fcm_dpo/margin": 98.3853759765625, "fcm_dpo/q_t": 0.40637117624282837, "grad_norm": 9.090182304382324, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -1.9611761569976807, "logits/rejected": -1.005052924156189, "logps/chosen": -275.10186767578125, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -350.3601379394531, "loss": 1.0848, "margin_dpo/margin_mean": 98.38536834716797, "margin_dpo/margin_std": 121.54266357421875, "step": 477 }, { "KL/chosen_KL_mean": -165.187255859375, "KL/mean": -216.29705810546875, "KL/rejected_KL_mean": -267.4068603515625, "KL/std": 105.95360565185547, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.003954238258302212, "fcm_dpo/delta": -0.007119315210729837, "fcm_dpo/margin": 102.21961212158203, "fcm_dpo/q_t": 0.4044448435306549, "grad_norm": 9.185165405273438, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.6938798427581787, "logits/rejected": -1.5197014808654785, "logps/chosen": -241.97544860839844, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -357.8468017578125, "loss": 1.0734, "margin_dpo/margin_mean": 102.21961975097656, "margin_dpo/margin_std": 117.00482177734375, "step": 478 }, { "KL/chosen_KL_mean": -167.66067504882812, "KL/mean": -219.04627990722656, "KL/rejected_KL_mean": -270.431884765625, "KL/std": 107.3389892578125, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004000389948487282, "fcm_dpo/delta": -0.02257903292775154, "fcm_dpo/margin": 102.77120971679688, "fcm_dpo/q_t": 0.4006846249103546, "grad_norm": 11.252288818359375, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.4802839756011963, "logits/rejected": -1.3384814262390137, "logps/chosen": -249.02297973632812, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -371.5230407714844, "loss": 1.067, "margin_dpo/margin_mean": 102.77120971679688, "margin_dpo/margin_std": 110.45889282226562, "step": 479 }, { "KL/chosen_KL_mean": -194.994384765625, "KL/mean": -235.3463592529297, "KL/rejected_KL_mean": -275.69830322265625, "KL/std": 102.23946380615234, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004299666732549667, "fcm_dpo/delta": 0.07867898046970367, "fcm_dpo/margin": 80.70393371582031, "fcm_dpo/q_t": 0.42419540882110596, "grad_norm": 10.34135627746582, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.6615073680877686, "logits/rejected": -0.9717175960540771, "logps/chosen": -291.768310546875, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -362.1030578613281, "loss": 1.1251, "margin_dpo/margin_mean": 80.70392608642578, "margin_dpo/margin_std": 112.63233184814453, "step": 480 }, { "KL/chosen_KL_mean": -191.00640869140625, "KL/mean": -237.99830627441406, "KL/rejected_KL_mean": -284.990234375, "KL/std": 106.76126861572266, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004320221953094006, "fcm_dpo/delta": -0.010452356189489365, "fcm_dpo/margin": 93.98381805419922, "fcm_dpo/q_t": 0.40441155433654785, "grad_norm": 11.982155799865723, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.9980521202087402, "logits/rejected": -1.5677520036697388, "logps/chosen": -282.67657470703125, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -383.68511962890625, "loss": 1.0827, "margin_dpo/margin_mean": 93.98381042480469, "margin_dpo/margin_std": 118.41268920898438, "step": 481 }, { "KL/chosen_KL_mean": -184.3730926513672, "KL/mean": -230.38760375976562, "KL/rejected_KL_mean": -276.402099609375, "KL/std": 105.1009521484375, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004266241565346718, "fcm_dpo/delta": 0.010154308751225471, "fcm_dpo/margin": 92.02899169921875, "fcm_dpo/q_t": 0.40831708908081055, "grad_norm": 12.659887313842773, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.6617851257324219, "logits/rejected": -1.3088536262512207, "logps/chosen": -282.8931884765625, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -371.2315673828125, "loss": 1.0935, "margin_dpo/margin_mean": 92.02899169921875, "margin_dpo/margin_std": 117.93003845214844, "step": 482 }, { "KL/chosen_KL_mean": -190.11099243164062, "KL/mean": -239.60009765625, "KL/rejected_KL_mean": -289.08917236328125, "KL/std": 111.11929321289062, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004351481795310974, "fcm_dpo/delta": -0.060581520199775696, "fcm_dpo/margin": 98.97822570800781, "fcm_dpo/q_t": 0.3934367299079895, "grad_norm": 8.268973350524902, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.0628910064697266, "logits/rejected": -1.422515869140625, "logps/chosen": -297.2296142578125, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -390.2041931152344, "loss": 1.0678, "margin_dpo/margin_mean": 98.97822570800781, "margin_dpo/margin_std": 110.85212707519531, "step": 483 }, { "KL/chosen_KL_mean": -173.43426513671875, "KL/mean": -228.42660522460938, "KL/rejected_KL_mean": -283.4189453125, "KL/std": 112.64073944091797, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0038229217752814293, "fcm_dpo/delta": -0.03603435307741165, "fcm_dpo/margin": 109.98468017578125, "fcm_dpo/q_t": 0.39796584844589233, "grad_norm": 7.810755252838135, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.413437843322754, "logits/rejected": -1.9362444877624512, "logps/chosen": -260.4141845703125, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -374.14263916015625, "loss": 1.0647, "margin_dpo/margin_mean": 109.98468017578125, "margin_dpo/margin_std": 123.97660827636719, "step": 484 }, { "KL/chosen_KL_mean": -164.9792938232422, "KL/mean": -224.58958435058594, "KL/rejected_KL_mean": -284.19989013671875, "KL/std": 117.58708190917969, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0034957611933350563, "fcm_dpo/delta": -0.028806108981370926, "fcm_dpo/margin": 119.2205810546875, "fcm_dpo/q_t": 0.3990354835987091, "grad_norm": 7.691453456878662, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.656134843826294, "logits/rejected": -1.2342376708984375, "logps/chosen": -259.0380554199219, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -392.76287841796875, "loss": 1.0539, "margin_dpo/margin_mean": 119.22058868408203, "margin_dpo/margin_std": 119.400634765625, "step": 485 }, { "KL/chosen_KL_mean": -189.89102172851562, "KL/mean": -241.38479614257812, "KL/rejected_KL_mean": -292.87860107421875, "KL/std": 112.89820861816406, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0036974630784243345, "fcm_dpo/delta": 0.02676502615213394, "fcm_dpo/margin": 102.98757934570312, "fcm_dpo/q_t": 0.41167178750038147, "grad_norm": 9.562994003295898, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.9815372228622437, "logits/rejected": -1.273094892501831, "logps/chosen": -283.63690185546875, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -390.94921875, "loss": 1.0902, "margin_dpo/margin_mean": 102.98757934570312, "margin_dpo/margin_std": 120.64112854003906, "step": 486 }, { "KL/chosen_KL_mean": -183.45567321777344, "KL/mean": -230.52969360351562, "KL/rejected_KL_mean": -277.60369873046875, "KL/std": 107.55680084228516, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.003910684492439032, "fcm_dpo/delta": 0.04999777674674988, "fcm_dpo/margin": 94.14802551269531, "fcm_dpo/q_t": 0.4171021282672882, "grad_norm": 10.369401931762695, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.2891812324523926, "logits/rejected": -1.9871735572814941, "logps/chosen": -271.4892578125, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -376.0758056640625, "loss": 1.0995, "margin_dpo/margin_mean": 94.14801788330078, "margin_dpo/margin_std": 114.8535385131836, "step": 487 }, { "KL/chosen_KL_mean": -165.28195190429688, "KL/mean": -217.646728515625, "KL/rejected_KL_mean": -270.011474609375, "KL/std": 113.80177307128906, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.003941092640161514, "fcm_dpo/delta": -0.021855227649211884, "fcm_dpo/margin": 104.72949981689453, "fcm_dpo/q_t": 0.40007248520851135, "grad_norm": 8.776195526123047, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.7987802028656006, "logits/rejected": -1.5444393157958984, "logps/chosen": -250.0716094970703, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -360.2849426269531, "loss": 1.0564, "margin_dpo/margin_mean": 104.72949981689453, "margin_dpo/margin_std": 101.12307739257812, "step": 488 }, { "KL/chosen_KL_mean": -187.0088348388672, "KL/mean": -234.9036407470703, "KL/rejected_KL_mean": -282.7984313964844, "KL/std": 104.3592529296875, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.004011179320514202, "fcm_dpo/delta": 0.025131162256002426, "fcm_dpo/margin": 95.78959655761719, "fcm_dpo/q_t": 0.41197842359542847, "grad_norm": 9.431554794311523, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.636871337890625, "logits/rejected": -1.147147297859192, "logps/chosen": -277.4781494140625, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -369.196044921875, "loss": 1.0902, "margin_dpo/margin_mean": 95.78959655761719, "margin_dpo/margin_std": 116.59144592285156, "step": 489 }, { "KL/chosen_KL_mean": -187.03050231933594, "KL/mean": -232.18243408203125, "KL/rejected_KL_mean": -277.3343505859375, "KL/std": 112.99996185302734, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004363939166069031, "fcm_dpo/delta": -0.009347416460514069, "fcm_dpo/margin": 90.30384063720703, "fcm_dpo/q_t": 0.4091726541519165, "grad_norm": 9.109635353088379, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1888318061828613, "logits/rejected": -2.2696468830108643, "logps/chosen": -272.35064697265625, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -393.3282165527344, "loss": 1.143, "margin_dpo/margin_mean": 90.30384063720703, "margin_dpo/margin_std": 149.23362731933594, "step": 490 }, { "KL/chosen_KL_mean": -190.674072265625, "KL/mean": -232.76231384277344, "KL/rejected_KL_mean": -274.8505554199219, "KL/std": 115.24931335449219, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0043132067658007145, "fcm_dpo/delta": 0.057637959718704224, "fcm_dpo/margin": 84.17648315429688, "fcm_dpo/q_t": 0.4192957878112793, "grad_norm": 11.090476989746094, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8444252014160156, "logits/rejected": -1.353452444076538, "logps/chosen": -276.8175964355469, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -355.530029296875, "loss": 1.117, "margin_dpo/margin_mean": 84.17647552490234, "margin_dpo/margin_std": 116.955810546875, "step": 491 }, { "KL/chosen_KL_mean": -179.6365966796875, "KL/mean": -213.23187255859375, "KL/rejected_KL_mean": -246.82713317871094, "KL/std": 113.39543914794922, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.005053139291703701, "fcm_dpo/delta": 0.09164711087942123, "fcm_dpo/margin": 67.19052124023438, "fcm_dpo/q_t": 0.430001437664032, "grad_norm": 16.156007766723633, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.0220508575439453, "logits/rejected": -1.6358906030654907, "logps/chosen": -260.7441711425781, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -327.5791015625, "loss": 1.1958, "margin_dpo/margin_mean": 67.19052124023438, "margin_dpo/margin_std": 141.38870239257812, "step": 492 }, { "KL/chosen_KL_mean": -176.832275390625, "KL/mean": -222.15625, "KL/rejected_KL_mean": -267.480224609375, "KL/std": 102.19290161132812, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004609157331287861, "fcm_dpo/delta": -0.05096413940191269, "fcm_dpo/margin": 90.64791870117188, "fcm_dpo/q_t": 0.39649590849876404, "grad_norm": 17.894441604614258, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.5181074142456055, "logits/rejected": -1.801941156387329, "logps/chosen": -289.03961181640625, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -361.08740234375, "loss": 1.076, "margin_dpo/margin_mean": 90.64791870117188, "margin_dpo/margin_std": 109.41395568847656, "step": 493 }, { "KL/chosen_KL_mean": -172.33987426757812, "KL/mean": -230.59078979492188, "KL/rejected_KL_mean": -288.8417053222656, "KL/std": 113.44667053222656, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.0037648226134479046, "fcm_dpo/delta": -0.10750997066497803, "fcm_dpo/margin": 116.50184631347656, "fcm_dpo/q_t": 0.38446539640426636, "grad_norm": 9.22940444946289, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.3738503456115723, "logits/rejected": -1.2996692657470703, "logps/chosen": -245.45477294921875, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -381.00469970703125, "loss": 1.0497, "margin_dpo/margin_mean": 116.50184631347656, "margin_dpo/margin_std": 123.53590393066406, "step": 494 }, { "KL/chosen_KL_mean": -177.78839111328125, "KL/mean": -227.76483154296875, "KL/rejected_KL_mean": -277.74127197265625, "KL/std": 107.70386505126953, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.003934459760785103, "fcm_dpo/delta": 0.011112788692116737, "fcm_dpo/margin": 99.95291900634766, "fcm_dpo/q_t": 0.4093318283557892, "grad_norm": 10.702784538269043, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.15327787399292, "logits/rejected": -1.6815271377563477, "logps/chosen": -277.3087158203125, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -375.67218017578125, "loss": 1.0921, "margin_dpo/margin_mean": 99.95291137695312, "margin_dpo/margin_std": 129.6333465576172, "step": 495 }, { "KL/chosen_KL_mean": -196.944091796875, "KL/mean": -240.24777221679688, "KL/rejected_KL_mean": -283.5514221191406, "KL/std": 108.44285583496094, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004332245327532291, "fcm_dpo/delta": 0.034652214497327805, "fcm_dpo/margin": 86.6073226928711, "fcm_dpo/q_t": 0.41368353366851807, "grad_norm": 9.694496154785156, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.041687488555908, "logits/rejected": -1.382910132408142, "logps/chosen": -304.80084228515625, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -376.3219909667969, "loss": 1.1022, "margin_dpo/margin_mean": 86.60731506347656, "margin_dpo/margin_std": 109.33544158935547, "step": 496 }, { "KL/chosen_KL_mean": -209.55130004882812, "KL/mean": -237.54856872558594, "KL/rejected_KL_mean": -265.54583740234375, "KL/std": 109.03726196289062, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004747485741972923, "fcm_dpo/delta": 0.0540403351187706, "fcm_dpo/margin": 55.99449920654297, "fcm_dpo/q_t": 0.44242340326309204, "grad_norm": 9.724367141723633, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.8368582725524902, "logits/rejected": -1.3960461616516113, "logps/chosen": -301.634521484375, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -347.3408508300781, "loss": 1.213, "margin_dpo/margin_mean": 55.99449920654297, "margin_dpo/margin_std": 116.81739807128906, "step": 497 }, { "KL/chosen_KL_mean": -178.7913055419922, "KL/mean": -225.8453369140625, "KL/rejected_KL_mean": -272.89935302734375, "KL/std": 108.29557800292969, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.004448693245649338, "fcm_dpo/delta": -0.033698126673698425, "fcm_dpo/margin": 94.10806274414062, "fcm_dpo/q_t": 0.39940178394317627, "grad_norm": 12.247204780578613, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.635292887687683, "logits/rejected": -1.1205748319625854, "logps/chosen": -276.9856872558594, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -363.5868225097656, "loss": 1.0772, "margin_dpo/margin_mean": 94.10806274414062, "margin_dpo/margin_std": 119.41586303710938, "step": 498 }, { "KL/chosen_KL_mean": -186.60202026367188, "KL/mean": -235.6619415283203, "KL/rejected_KL_mean": -284.72186279296875, "KL/std": 116.28143310546875, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.004265683237463236, "fcm_dpo/delta": -0.031816817820072174, "fcm_dpo/margin": 98.11983489990234, "fcm_dpo/q_t": 0.40056493878364563, "grad_norm": 9.570252418518066, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.0321011543273926, "logits/rejected": -1.9825952053070068, "logps/chosen": -267.6419372558594, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -391.6435546875, "loss": 1.0765, "margin_dpo/margin_mean": 98.11982727050781, "margin_dpo/margin_std": 123.97483825683594, "step": 499 }, { "KL/chosen_KL_mean": -207.5809326171875, "KL/mean": -249.44723510742188, "KL/rejected_KL_mean": -291.31353759765625, "KL/std": 111.60982513427734, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004422968253493309, "fcm_dpo/delta": 0.047275442630052567, "fcm_dpo/margin": 83.73258972167969, "fcm_dpo/q_t": 0.4191230237483978, "grad_norm": 12.331366539001465, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.6252048015594482, "logits/rejected": -1.4543970823287964, "logps/chosen": -296.8289794921875, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -391.7237548828125, "loss": 1.1236, "margin_dpo/margin_mean": 83.73258972167969, "margin_dpo/margin_std": 126.45248413085938, "step": 500 }, { "KL/chosen_KL_mean": -181.21331787109375, "KL/mean": -228.1404266357422, "KL/rejected_KL_mean": -275.0675354003906, "KL/std": 121.04946899414062, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004586878698319197, "fcm_dpo/delta": -0.06762456148862839, "fcm_dpo/margin": 93.85420227050781, "fcm_dpo/q_t": 0.39662468433380127, "grad_norm": 10.065316200256348, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.3139714002609253, "logits/rejected": -0.8577677011489868, "logps/chosen": -271.81024169921875, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -360.36480712890625, "loss": 1.1129, "margin_dpo/margin_mean": 93.85420227050781, "margin_dpo/margin_std": 136.49630737304688, "step": 501 }, { "KL/chosen_KL_mean": -182.559814453125, "KL/mean": -224.15399169921875, "KL/rejected_KL_mean": -265.74822998046875, "KL/std": 100.89479064941406, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0044504511170089245, "fcm_dpo/delta": 0.037441153079271317, "fcm_dpo/margin": 83.1883773803711, "fcm_dpo/q_t": 0.4155160188674927, "grad_norm": 11.313714027404785, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9658268690109253, "logits/rejected": -1.2364789247512817, "logps/chosen": -286.0305480957031, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -354.7923583984375, "loss": 1.1264, "margin_dpo/margin_mean": 83.18836975097656, "margin_dpo/margin_std": 124.42054748535156, "step": 502 }, { "KL/chosen_KL_mean": -180.94015502929688, "KL/mean": -227.8570556640625, "KL/rejected_KL_mean": -274.77398681640625, "KL/std": 107.9952621459961, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004384302534162998, "fcm_dpo/delta": -0.024553127586841583, "fcm_dpo/margin": 93.83379364013672, "fcm_dpo/q_t": 0.40212005376815796, "grad_norm": 10.181975364685059, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7843360900878906, "logits/rejected": -1.2898759841918945, "logps/chosen": -274.670166015625, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -362.3829345703125, "loss": 1.0854, "margin_dpo/margin_mean": 93.83380126953125, "margin_dpo/margin_std": 118.87229919433594, "step": 503 }, { "KL/chosen_KL_mean": -184.10269165039062, "KL/mean": -244.16500854492188, "KL/rejected_KL_mean": -304.227294921875, "KL/std": 112.89115905761719, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0036769802682101727, "fcm_dpo/delta": -0.07542788237333298, "fcm_dpo/margin": 120.12460327148438, "fcm_dpo/q_t": 0.3882262408733368, "grad_norm": 9.095620155334473, "learning_rate": 9.730107739932805e-08, "logits/chosen": -1.9918608665466309, "logits/rejected": -1.6554005146026611, "logps/chosen": -276.3945617675781, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -413.26495361328125, "loss": 1.035, "margin_dpo/margin_mean": 120.12459564208984, "margin_dpo/margin_std": 114.1951904296875, "step": 504 }, { "KL/chosen_KL_mean": -202.728515625, "KL/mean": -245.943359375, "KL/rejected_KL_mean": -289.158203125, "KL/std": 111.10302734375, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.003999015316367149, "fcm_dpo/delta": 0.08197946846485138, "fcm_dpo/margin": 86.42964172363281, "fcm_dpo/q_t": 0.4242320656776428, "grad_norm": 8.858912467956543, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.310492753982544, "logits/rejected": -1.7447282075881958, "logps/chosen": -309.0054626464844, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -383.36431884765625, "loss": 1.1165, "margin_dpo/margin_mean": 86.42964172363281, "margin_dpo/margin_std": 107.70947265625, "step": 505 }, { "KL/chosen_KL_mean": -195.20245361328125, "KL/mean": -244.12803649902344, "KL/rejected_KL_mean": -293.0536193847656, "KL/std": 105.05601501464844, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.004026284907013178, "fcm_dpo/delta": 0.005750983953475952, "fcm_dpo/margin": 97.85118103027344, "fcm_dpo/q_t": 0.4062432050704956, "grad_norm": 8.849411964416504, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9295623302459717, "logits/rejected": -1.567194938659668, "logps/chosen": -276.643310546875, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -378.718017578125, "loss": 1.0814, "margin_dpo/margin_mean": 97.8511734008789, "margin_dpo/margin_std": 110.69987487792969, "step": 506 }, { "KL/chosen_KL_mean": -203.35467529296875, "KL/mean": -252.71450805664062, "KL/rejected_KL_mean": -302.0743103027344, "KL/std": 110.57595825195312, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004128397908061743, "fcm_dpo/delta": -0.012763844802975655, "fcm_dpo/margin": 98.71964263916016, "fcm_dpo/q_t": 0.40491199493408203, "grad_norm": 8.903661727905273, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.1967501640319824, "logits/rejected": -1.7582318782806396, "logps/chosen": -302.7816162109375, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -404.3272705078125, "loss": 1.0902, "margin_dpo/margin_mean": 98.71963500976562, "margin_dpo/margin_std": 130.520263671875, "step": 507 }, { "KL/chosen_KL_mean": -197.0191192626953, "KL/mean": -238.8664093017578, "KL/rejected_KL_mean": -280.71368408203125, "KL/std": 118.57240295410156, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0044249193742871284, "fcm_dpo/delta": 0.046121466904878616, "fcm_dpo/margin": 83.694580078125, "fcm_dpo/q_t": 0.4189831614494324, "grad_norm": 16.515426635742188, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.091078281402588, "logits/rejected": -1.6141128540039062, "logps/chosen": -294.93475341796875, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -384.5115051269531, "loss": 1.14, "margin_dpo/margin_mean": 83.694580078125, "margin_dpo/margin_std": 138.35150146484375, "step": 508 }, { "KL/chosen_KL_mean": -183.7742919921875, "KL/mean": -221.78561401367188, "KL/rejected_KL_mean": -259.7969055175781, "KL/std": 92.2997055053711, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004895099438726902, "fcm_dpo/delta": 0.041128940880298615, "fcm_dpo/margin": 76.02261352539062, "fcm_dpo/q_t": 0.4154103994369507, "grad_norm": 16.632291793823242, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.028520107269287, "logits/rejected": -1.3638054132461548, "logps/chosen": -284.504150390625, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -346.7402648925781, "loss": 1.1346, "margin_dpo/margin_mean": 76.02261352539062, "margin_dpo/margin_std": 122.09099578857422, "step": 509 }, { "KL/chosen_KL_mean": -216.56671142578125, "KL/mean": -263.10125732421875, "KL/rejected_KL_mean": -309.63580322265625, "KL/std": 111.41972351074219, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004601519089192152, "fcm_dpo/delta": -0.04919745400547981, "fcm_dpo/margin": 93.06910705566406, "fcm_dpo/q_t": 0.3966670632362366, "grad_norm": 13.239019393920898, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.503027081489563, "logits/rejected": -1.3185627460479736, "logps/chosen": -305.736083984375, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -417.55517578125, "loss": 1.0696, "margin_dpo/margin_mean": 93.06910705566406, "margin_dpo/margin_std": 111.27007293701172, "step": 510 }, { "KL/chosen_KL_mean": -198.82598876953125, "KL/mean": -248.6855010986328, "KL/rejected_KL_mean": -298.54498291015625, "KL/std": 110.0887680053711, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004257123917341232, "fcm_dpo/delta": -0.04275636374950409, "fcm_dpo/margin": 99.71903228759766, "fcm_dpo/q_t": 0.3987162411212921, "grad_norm": 11.088542938232422, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.147007942199707, "logits/rejected": -1.5797722339630127, "logps/chosen": -288.5757751464844, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -396.27630615234375, "loss": 1.0715, "margin_dpo/margin_mean": 99.71903228759766, "margin_dpo/margin_std": 123.35247802734375, "step": 511 }, { "KL/chosen_KL_mean": -188.91165161132812, "KL/mean": -237.29119873046875, "KL/rejected_KL_mean": -285.6707458496094, "KL/std": 107.19219970703125, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004200418945401907, "fcm_dpo/delta": -0.018388740718364716, "fcm_dpo/margin": 96.75908660888672, "fcm_dpo/q_t": 0.4030628800392151, "grad_norm": 13.62331771850586, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.8013097047805786, "logits/rejected": -1.3579256534576416, "logps/chosen": -274.64678955078125, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -373.2956237792969, "loss": 1.0966, "margin_dpo/margin_mean": 96.75908660888672, "margin_dpo/margin_std": 127.86942291259766, "step": 512 }, { "KL/chosen_KL_mean": -201.0149383544922, "KL/mean": -241.0980224609375, "KL/rejected_KL_mean": -281.18115234375, "KL/std": 102.79104614257812, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004293826874345541, "fcm_dpo/delta": 0.0852707177400589, "fcm_dpo/margin": 80.16618347167969, "fcm_dpo/q_t": 0.4248065948486328, "grad_norm": 11.036299705505371, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.344414710998535, "logits/rejected": -1.652350902557373, "logps/chosen": -297.2054443359375, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -364.098876953125, "loss": 1.1413, "margin_dpo/margin_mean": 80.16618347167969, "margin_dpo/margin_std": 116.56480407714844, "step": 513 }, { "KL/chosen_KL_mean": -194.50048828125, "KL/mean": -238.89697265625, "KL/rejected_KL_mean": -283.29345703125, "KL/std": 103.21626281738281, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004580825101584196, "fcm_dpo/delta": -0.011357513256371021, "fcm_dpo/margin": 88.79296875, "fcm_dpo/q_t": 0.40631628036499023, "grad_norm": 11.923307418823242, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.8175849914550781, "logits/rejected": -1.2560988664627075, "logps/chosen": -292.3241271972656, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -368.66326904296875, "loss": 1.0945, "margin_dpo/margin_mean": 88.79296875, "margin_dpo/margin_std": 122.91987609863281, "step": 514 }, { "KL/chosen_KL_mean": -186.01718139648438, "KL/mean": -229.83596801757812, "KL/rejected_KL_mean": -273.6547546386719, "KL/std": 107.3678207397461, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.004543589428067207, "fcm_dpo/delta": 0.0029966672882437706, "fcm_dpo/margin": 87.63755798339844, "fcm_dpo/q_t": 0.4079955816268921, "grad_norm": 11.81691837310791, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.799559473991394, "logits/rejected": -1.5691996812820435, "logps/chosen": -267.95074462890625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -367.2313232421875, "loss": 1.0961, "margin_dpo/margin_mean": 87.63756561279297, "margin_dpo/margin_std": 117.6414794921875, "step": 515 }, { "KL/chosen_KL_mean": -192.21009826660156, "KL/mean": -234.13528442382812, "KL/rejected_KL_mean": -276.0604248046875, "KL/std": 123.33728790283203, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.004708443768322468, "fcm_dpo/delta": 0.0072814784944057465, "fcm_dpo/margin": 83.85032653808594, "fcm_dpo/q_t": 0.4127081334590912, "grad_norm": 14.483184814453125, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.173567056655884, "logits/rejected": -2.0016000270843506, "logps/chosen": -289.3017578125, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -394.61712646484375, "loss": 1.1395, "margin_dpo/margin_mean": 83.85033416748047, "margin_dpo/margin_std": 146.04052734375, "step": 516 }, { "KL/chosen_KL_mean": -202.74560546875, "KL/mean": -254.10745239257812, "KL/rejected_KL_mean": -305.4693603515625, "KL/std": 110.64585876464844, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.003999802283942699, "fcm_dpo/delta": -0.04342689365148544, "fcm_dpo/margin": 102.72373962402344, "fcm_dpo/q_t": 0.3982778787612915, "grad_norm": 9.2653226852417, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.887431263923645, "logits/rejected": -1.4618699550628662, "logps/chosen": -295.9321594238281, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -401.0621337890625, "loss": 1.0869, "margin_dpo/margin_mean": 102.72373962402344, "margin_dpo/margin_std": 125.35282135009766, "step": 517 }, { "KL/chosen_KL_mean": -212.26333618164062, "KL/mean": -250.4909210205078, "KL/rejected_KL_mean": -288.718505859375, "KL/std": 104.20194244384766, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004684789106249809, "fcm_dpo/delta": 0.06446320563554764, "fcm_dpo/margin": 76.45515441894531, "fcm_dpo/q_t": 0.4229106605052948, "grad_norm": 11.186882972717285, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.9539227485656738, "logits/rejected": -1.6276707649230957, "logps/chosen": -299.2846984863281, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -379.96307373046875, "loss": 1.1466, "margin_dpo/margin_mean": 76.45516204833984, "margin_dpo/margin_std": 130.1359100341797, "step": 518 }, { "KL/chosen_KL_mean": -197.18560791015625, "KL/mean": -239.16722106933594, "KL/rejected_KL_mean": -281.14886474609375, "KL/std": 97.1707763671875, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.004731203429400921, "fcm_dpo/delta": 0.002944929525256157, "fcm_dpo/margin": 83.9632339477539, "fcm_dpo/q_t": 0.4079910218715668, "grad_norm": 11.563436508178711, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.077796697616577, "logits/rejected": -1.7142317295074463, "logps/chosen": -297.8731689453125, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -390.17926025390625, "loss": 1.0826, "margin_dpo/margin_mean": 83.9632339477539, "margin_dpo/margin_std": 101.4863052368164, "step": 519 }, { "KL/chosen_KL_mean": -183.45443725585938, "KL/mean": -233.45059204101562, "KL/rejected_KL_mean": -283.44671630859375, "KL/std": 109.62330627441406, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0043082088232040405, "fcm_dpo/delta": -0.06034073978662491, "fcm_dpo/margin": 99.9923095703125, "fcm_dpo/q_t": 0.39322108030319214, "grad_norm": 9.259502410888672, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.5571348667144775, "logits/rejected": -1.1183112859725952, "logps/chosen": -282.2320556640625, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -390.3924560546875, "loss": 1.0526, "margin_dpo/margin_mean": 99.9923095703125, "margin_dpo/margin_std": 107.71607971191406, "step": 520 }, { "KL/chosen_KL_mean": -184.08534240722656, "KL/mean": -235.3127899169922, "KL/rejected_KL_mean": -286.5402526855469, "KL/std": 102.7373046875, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0041893720626831055, "fcm_dpo/delta": -0.05381395295262337, "fcm_dpo/margin": 102.45487976074219, "fcm_dpo/q_t": 0.39430904388427734, "grad_norm": 10.44382095336914, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9835575819015503, "logits/rejected": -1.6484732627868652, "logps/chosen": -270.4364318847656, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -376.93341064453125, "loss": 1.0536, "margin_dpo/margin_mean": 102.45487976074219, "margin_dpo/margin_std": 104.60745239257812, "step": 521 }, { "KL/chosen_KL_mean": -192.97076416015625, "KL/mean": -238.2750244140625, "KL/rejected_KL_mean": -283.57928466796875, "KL/std": 112.01185607910156, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.004166669212281704, "fcm_dpo/delta": 0.03294726833701134, "fcm_dpo/margin": 90.60853576660156, "fcm_dpo/q_t": 0.41386678814888, "grad_norm": 9.982280731201172, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.5451467037200928, "logits/rejected": -1.1697936058044434, "logps/chosen": -268.97882080078125, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -358.0205383300781, "loss": 1.1017, "margin_dpo/margin_mean": 90.60853576660156, "margin_dpo/margin_std": 117.73515319824219, "step": 522 }, { "KL/chosen_KL_mean": -213.6632537841797, "KL/mean": -246.56903076171875, "KL/rejected_KL_mean": -279.47479248046875, "KL/std": 110.78080749511719, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.004889430478215218, "fcm_dpo/delta": 0.11451803892850876, "fcm_dpo/margin": 65.81153106689453, "fcm_dpo/q_t": 0.4342994689941406, "grad_norm": 13.150151252746582, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.7962114810943604, "logits/rejected": -1.0739948749542236, "logps/chosen": -310.1595458984375, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -370.4900817871094, "loss": 1.1848, "margin_dpo/margin_mean": 65.81153106689453, "margin_dpo/margin_std": 128.51531982421875, "step": 523 }, { "KL/chosen_KL_mean": -194.20263671875, "KL/mean": -242.115966796875, "KL/rejected_KL_mean": -290.029296875, "KL/std": 110.95149230957031, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.00455843610689044, "fcm_dpo/delta": -0.07095936685800552, "fcm_dpo/margin": 95.82670593261719, "fcm_dpo/q_t": 0.3936810791492462, "grad_norm": 12.211541175842285, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.0521726608276367, "logits/rejected": -1.3391023874282837, "logps/chosen": -297.608154296875, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -377.939453125, "loss": 1.0632, "margin_dpo/margin_mean": 95.82670593261719, "margin_dpo/margin_std": 117.563232421875, "step": 524 }, { "KL/chosen_KL_mean": -183.75955200195312, "KL/mean": -237.47314453125, "KL/rejected_KL_mean": -291.1867370605469, "KL/std": 111.12457275390625, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.003767544636502862, "fcm_dpo/delta": -0.048858799040317535, "fcm_dpo/margin": 107.42718505859375, "fcm_dpo/q_t": 0.39929020404815674, "grad_norm": 10.244484901428223, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.5399448871612549, "logits/rejected": -1.4535471200942993, "logps/chosen": -267.9660339355469, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -397.4483337402344, "loss": 1.0909, "margin_dpo/margin_mean": 107.42718505859375, "margin_dpo/margin_std": 129.64991760253906, "step": 525 }, { "KL/chosen_KL_mean": -188.47080993652344, "KL/mean": -237.37387084960938, "KL/rejected_KL_mean": -286.27691650390625, "KL/std": 105.37904357910156, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.004090573638677597, "fcm_dpo/delta": -0.00014090235345065594, "fcm_dpo/margin": 97.80610656738281, "fcm_dpo/q_t": 0.40674149990081787, "grad_norm": 8.084105491638184, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.603651762008667, "logits/rejected": -0.9356840252876282, "logps/chosen": -291.3249206542969, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -379.9283752441406, "loss": 1.0872, "margin_dpo/margin_mean": 97.80610656738281, "margin_dpo/margin_std": 124.41708374023438, "step": 526 }, { "KL/chosen_KL_mean": -190.821044921875, "KL/mean": -240.45004272460938, "KL/rejected_KL_mean": -290.07904052734375, "KL/std": 110.35308837890625, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004052374511957169, "fcm_dpo/delta": -0.003769530914723873, "fcm_dpo/margin": 99.25804138183594, "fcm_dpo/q_t": 0.4038817286491394, "grad_norm": 10.284876823425293, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.0719895362854004, "logits/rejected": -2.2490463256835938, "logps/chosen": -287.58648681640625, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -375.4742431640625, "loss": 1.0611, "margin_dpo/margin_mean": 99.25804138183594, "margin_dpo/margin_std": 97.23628234863281, "step": 527 }, { "KL/chosen_KL_mean": -177.70693969726562, "KL/mean": -226.75709533691406, "KL/rejected_KL_mean": -275.8072814941406, "KL/std": 112.07632446289062, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004099573008716106, "fcm_dpo/delta": -0.004286906681954861, "fcm_dpo/margin": 98.10033416748047, "fcm_dpo/q_t": 0.406130313873291, "grad_norm": 9.14182186126709, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.1626100540161133, "logits/rejected": -1.6520543098449707, "logps/chosen": -271.8718566894531, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -376.1795959472656, "loss": 1.0889, "margin_dpo/margin_mean": 98.10033416748047, "margin_dpo/margin_std": 127.56358337402344, "step": 528 }, { "KL/chosen_KL_mean": -205.85780334472656, "KL/mean": -250.17393493652344, "KL/rejected_KL_mean": -294.4900817871094, "KL/std": 110.82950592041016, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.004305378533899784, "fcm_dpo/delta": 0.02623051404953003, "fcm_dpo/margin": 88.63227081298828, "fcm_dpo/q_t": 0.4143197536468506, "grad_norm": 11.192075729370117, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.9869022369384766, "logits/rejected": -1.814370036125183, "logps/chosen": -286.63458251953125, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -391.6812438964844, "loss": 1.1161, "margin_dpo/margin_mean": 88.63227081298828, "margin_dpo/margin_std": 131.3165740966797, "step": 529 }, { "KL/chosen_KL_mean": -189.63235473632812, "KL/mean": -234.03094482421875, "KL/rejected_KL_mean": -278.4295654296875, "KL/std": 107.01654052734375, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.004289058968424797, "fcm_dpo/delta": 0.02945873513817787, "fcm_dpo/margin": 88.79719543457031, "fcm_dpo/q_t": 0.4131714105606079, "grad_norm": 9.951581954956055, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.7517859935760498, "logits/rejected": -1.440185785293579, "logps/chosen": -269.4080810546875, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -370.48345947265625, "loss": 1.0903, "margin_dpo/margin_mean": 88.79719543457031, "margin_dpo/margin_std": 103.99673461914062, "step": 530 }, { "KL/chosen_KL_mean": -183.8838653564453, "KL/mean": -239.03482055664062, "KL/rejected_KL_mean": -294.185791015625, "KL/std": 112.97952270507812, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.0039543090388178825, "fcm_dpo/delta": -0.0695347860455513, "fcm_dpo/margin": 110.30188751220703, "fcm_dpo/q_t": 0.3900374174118042, "grad_norm": 10.811996459960938, "learning_rate": 7.141774982445147e-08, "logits/chosen": -2.127960681915283, "logits/rejected": -1.5549917221069336, "logps/chosen": -263.20306396484375, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -372.2940979003906, "loss": 1.0439, "margin_dpo/margin_mean": 110.30188751220703, "margin_dpo/margin_std": 112.67278289794922, "step": 531 }, { "KL/chosen_KL_mean": -198.05078125, "KL/mean": -258.1451110839844, "KL/rejected_KL_mean": -318.2394714355469, "KL/std": 108.17269897460938, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0036241617053747177, "fcm_dpo/delta": -0.06269079446792603, "fcm_dpo/margin": 120.18868255615234, "fcm_dpo/q_t": 0.39186882972717285, "grad_norm": 9.965243339538574, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.8069169521331787, "logits/rejected": -1.1937768459320068, "logps/chosen": -299.68768310546875, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -411.5848388671875, "loss": 1.0461, "margin_dpo/margin_mean": 120.18869018554688, "margin_dpo/margin_std": 120.50927734375, "step": 532 }, { "KL/chosen_KL_mean": -191.59500122070312, "KL/mean": -226.3450469970703, "KL/rejected_KL_mean": -261.0950927734375, "KL/std": 114.19988250732422, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004319524392485619, "fcm_dpo/delta": 0.13565778732299805, "fcm_dpo/margin": 69.50010681152344, "fcm_dpo/q_t": 0.4389488697052002, "grad_norm": 12.116378784179688, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.5358405113220215, "logits/rejected": -1.9493520259857178, "logps/chosen": -282.3416442871094, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -343.98358154296875, "loss": 1.1753, "margin_dpo/margin_mean": 69.50010681152344, "margin_dpo/margin_std": 122.92598724365234, "step": 533 }, { "KL/chosen_KL_mean": -170.67047119140625, "KL/mean": -222.377197265625, "KL/rejected_KL_mean": -274.08392333984375, "KL/std": 106.86367797851562, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004195000045001507, "fcm_dpo/delta": -0.05950738489627838, "fcm_dpo/margin": 103.41343688964844, "fcm_dpo/q_t": 0.39259523153305054, "grad_norm": 9.621932029724121, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.9384288787841797, "logits/rejected": -1.5390427112579346, "logps/chosen": -257.30133056640625, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -358.95587158203125, "loss": 1.047, "margin_dpo/margin_mean": 103.41343688964844, "margin_dpo/margin_std": 105.35342407226562, "step": 534 }, { "KL/chosen_KL_mean": -171.998779296875, "KL/mean": -226.56329345703125, "KL/rejected_KL_mean": -281.1278076171875, "KL/std": 108.13581848144531, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.003916206769645214, "fcm_dpo/delta": -0.050133973360061646, "fcm_dpo/margin": 109.12902069091797, "fcm_dpo/q_t": 0.39607101678848267, "grad_norm": 7.781865119934082, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.7835547924041748, "logits/rejected": -1.5995663404464722, "logps/chosen": -262.34417724609375, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -381.90576171875, "loss": 1.0638, "margin_dpo/margin_mean": 109.1290283203125, "margin_dpo/margin_std": 123.74038696289062, "step": 535 }, { "KL/chosen_KL_mean": -170.562744140625, "KL/mean": -224.71437072753906, "KL/rejected_KL_mean": -278.8659973144531, "KL/std": 108.64645385742188, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0036306194961071014, "fcm_dpo/delta": 0.010682567954063416, "fcm_dpo/margin": 108.30326843261719, "fcm_dpo/q_t": 0.4073899984359741, "grad_norm": 8.099604606628418, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.768435001373291, "logits/rejected": -1.5147953033447266, "logps/chosen": -249.52230834960938, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -371.95379638671875, "loss": 1.0731, "margin_dpo/margin_mean": 108.30327606201172, "margin_dpo/margin_std": 113.79376220703125, "step": 536 }, { "KL/chosen_KL_mean": -149.19468688964844, "KL/mean": -206.44505310058594, "KL/rejected_KL_mean": -263.6954345703125, "KL/std": 112.0416259765625, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.003700463566929102, "fcm_dpo/delta": -0.04728936776518822, "fcm_dpo/margin": 114.50071716308594, "fcm_dpo/q_t": 0.39483755826950073, "grad_norm": 9.056998252868652, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.6398684978485107, "logits/rejected": -0.95721036195755, "logps/chosen": -235.90408325195312, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -342.60308837890625, "loss": 1.0553, "margin_dpo/margin_mean": 114.50071716308594, "margin_dpo/margin_std": 109.29192352294922, "step": 537 }, { "KL/chosen_KL_mean": -203.49478149414062, "KL/mean": -250.30014038085938, "KL/rejected_KL_mean": -297.10546875, "KL/std": 100.41062927246094, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.003782902378588915, "fcm_dpo/delta": 0.06916332989931107, "fcm_dpo/margin": 93.6106948852539, "fcm_dpo/q_t": 0.42084649205207825, "grad_norm": 8.359902381896973, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.777895212173462, "logits/rejected": -1.2337672710418701, "logps/chosen": -301.9369201660156, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -388.20013427734375, "loss": 1.1025, "margin_dpo/margin_mean": 93.61070251464844, "margin_dpo/margin_std": 107.40973663330078, "step": 538 }, { "KL/chosen_KL_mean": -177.1366424560547, "KL/mean": -226.20803833007812, "KL/rejected_KL_mean": -275.2794494628906, "KL/std": 110.0771255493164, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.003986022435128689, "fcm_dpo/delta": 0.014190929010510445, "fcm_dpo/margin": 98.14280700683594, "fcm_dpo/q_t": 0.40952664613723755, "grad_norm": 8.376276016235352, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.345952033996582, "logits/rejected": -1.858872413635254, "logps/chosen": -268.0687561035156, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -372.72491455078125, "loss": 1.0921, "margin_dpo/margin_mean": 98.14280700683594, "margin_dpo/margin_std": 125.53030395507812, "step": 539 }, { "KL/chosen_KL_mean": -190.23329162597656, "KL/mean": -250.08102416992188, "KL/rejected_KL_mean": -309.9287414550781, "KL/std": 120.46733093261719, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0037117372266948223, "fcm_dpo/delta": -0.0798773467540741, "fcm_dpo/margin": 119.6954345703125, "fcm_dpo/q_t": 0.39113515615463257, "grad_norm": 9.580788612365723, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.0889296531677246, "logits/rejected": -1.8571369647979736, "logps/chosen": -298.10321044921875, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -431.322265625, "loss": 1.0593, "margin_dpo/margin_mean": 119.6954345703125, "margin_dpo/margin_std": 141.21328735351562, "step": 540 }, { "KL/chosen_KL_mean": -210.28025817871094, "KL/mean": -247.50820922851562, "KL/rejected_KL_mean": -284.7361755371094, "KL/std": 98.92278289794922, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.003567720064893365, "fcm_dpo/delta": 0.02386743761599064, "fcm_dpo/margin": 74.4559097290039, "fcm_dpo/q_t": 0.43867483735084534, "grad_norm": 8.41271686553955, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.1076836585998535, "logits/rejected": -1.6230944395065308, "logps/chosen": -316.8477478027344, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -393.8025207519531, "loss": 1.1773, "margin_dpo/margin_mean": 74.4559097290039, "margin_dpo/margin_std": 106.49676513671875, "step": 541 }, { "KL/chosen_KL_mean": -179.38189697265625, "KL/mean": -236.08987426757812, "KL/rejected_KL_mean": -292.7978515625, "KL/std": 111.81019592285156, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0035460018552839756, "fcm_dpo/delta": -0.003642420284450054, "fcm_dpo/margin": 113.41595458984375, "fcm_dpo/q_t": 0.4042140543460846, "grad_norm": 8.979308128356934, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.2128279209136963, "logits/rejected": -2.0900485515594482, "logps/chosen": -266.7511901855469, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -395.61822509765625, "loss": 1.068, "margin_dpo/margin_mean": 113.41595458984375, "margin_dpo/margin_std": 120.74441528320312, "step": 542 }, { "KL/chosen_KL_mean": -204.1962432861328, "KL/mean": -234.85421752929688, "KL/rejected_KL_mean": -265.51220703125, "KL/std": 118.56993103027344, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.003995827399194241, "fcm_dpo/delta": 0.06030438095331192, "fcm_dpo/margin": 61.315940856933594, "fcm_dpo/q_t": 0.4472350776195526, "grad_norm": 12.82109546661377, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.7317593097686768, "logits/rejected": -1.780052900314331, "logps/chosen": -291.34881591796875, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -377.0439147949219, "loss": 1.2365, "margin_dpo/margin_mean": 61.315948486328125, "margin_dpo/margin_std": 144.61422729492188, "step": 543 }, { "KL/chosen_KL_mean": -214.48928833007812, "KL/mean": -252.87509155273438, "KL/rejected_KL_mean": -291.2608642578125, "KL/std": 111.28385925292969, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.004508660174906254, "fcm_dpo/delta": 0.08181186020374298, "fcm_dpo/margin": 76.77159118652344, "fcm_dpo/q_t": 0.42579102516174316, "grad_norm": 9.708186149597168, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.6566474437713623, "logits/rejected": -1.563339114189148, "logps/chosen": -300.6361999511719, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -397.65478515625, "loss": 1.1349, "margin_dpo/margin_mean": 76.7715835571289, "margin_dpo/margin_std": 116.39035034179688, "step": 544 }, { "KL/chosen_KL_mean": -190.5648193359375, "KL/mean": -237.8396453857422, "KL/rejected_KL_mean": -285.1144714355469, "KL/std": 110.9161605834961, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004490881226956844, "fcm_dpo/delta": -0.04261649772524834, "fcm_dpo/margin": 94.54963684082031, "fcm_dpo/q_t": 0.3993746042251587, "grad_norm": 9.776100158691406, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4877209663391113, "logits/rejected": -1.059246301651001, "logps/chosen": -286.98907470703125, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -385.8485107421875, "loss": 1.0963, "margin_dpo/margin_mean": 94.54963684082031, "margin_dpo/margin_std": 136.65936279296875, "step": 545 }, { "KL/chosen_KL_mean": -195.68289184570312, "KL/mean": -237.80545043945312, "KL/rejected_KL_mean": -279.92803955078125, "KL/std": 112.18067169189453, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0044837528839707375, "fcm_dpo/delta": 0.03578674793243408, "fcm_dpo/margin": 84.24513244628906, "fcm_dpo/q_t": 0.4143354296684265, "grad_norm": 9.942118644714355, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.3523839712142944, "logits/rejected": -0.9882311224937439, "logps/chosen": -281.7040100097656, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -367.3006591796875, "loss": 1.1167, "margin_dpo/margin_mean": 84.24513244628906, "margin_dpo/margin_std": 123.15866088867188, "step": 546 }, { "KL/chosen_KL_mean": -189.1626739501953, "KL/mean": -239.9893798828125, "KL/rejected_KL_mean": -290.81610107421875, "KL/std": 109.18626403808594, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004315031226724386, "fcm_dpo/delta": -0.06843620538711548, "fcm_dpo/margin": 101.65339660644531, "fcm_dpo/q_t": 0.39034390449523926, "grad_norm": 9.558741569519043, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.783473014831543, "logits/rejected": -1.3153836727142334, "logps/chosen": -289.81365966796875, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -388.72784423828125, "loss": 1.0394, "margin_dpo/margin_mean": 101.65339660644531, "margin_dpo/margin_std": 97.69438934326172, "step": 547 }, { "KL/chosen_KL_mean": -184.96896362304688, "KL/mean": -226.81192016601562, "KL/rejected_KL_mean": -268.6549072265625, "KL/std": 111.80146789550781, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.004396004136651754, "fcm_dpo/delta": 0.04968549311161041, "fcm_dpo/margin": 83.68592834472656, "fcm_dpo/q_t": 0.41897696256637573, "grad_norm": 10.656475067138672, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.311931848526001, "logits/rejected": -1.9578964710235596, "logps/chosen": -280.99908447265625, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -367.068115234375, "loss": 1.1434, "margin_dpo/margin_mean": 83.68592834472656, "margin_dpo/margin_std": 140.76950073242188, "step": 548 }, { "KL/chosen_KL_mean": -176.72637939453125, "KL/mean": -219.50473022460938, "KL/rejected_KL_mean": -262.2830810546875, "KL/std": 106.44308471679688, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004692007787525654, "fcm_dpo/delta": -0.011916689574718475, "fcm_dpo/margin": 85.55670166015625, "fcm_dpo/q_t": 0.40551868081092834, "grad_norm": 12.381553649902344, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8679215908050537, "logits/rejected": -1.4529683589935303, "logps/chosen": -260.1773376464844, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -343.663330078125, "loss": 1.1267, "margin_dpo/margin_mean": 85.55670166015625, "margin_dpo/margin_std": 130.8936767578125, "step": 549 }, { "KL/chosen_KL_mean": -194.7747344970703, "KL/mean": -236.562255859375, "KL/rejected_KL_mean": -278.34979248046875, "KL/std": 100.19804382324219, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004614471923559904, "fcm_dpo/delta": 0.0209793783724308, "fcm_dpo/margin": 83.5750732421875, "fcm_dpo/q_t": 0.411903977394104, "grad_norm": 9.921513557434082, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.507566213607788, "logits/rejected": -1.2922874689102173, "logps/chosen": -284.36907958984375, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -379.6653137207031, "loss": 1.1004, "margin_dpo/margin_mean": 83.5750732421875, "margin_dpo/margin_std": 109.2386474609375, "step": 550 }, { "KL/chosen_KL_mean": -173.93869018554688, "KL/mean": -225.74862670898438, "KL/rejected_KL_mean": -277.5585632324219, "KL/std": 115.54156494140625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004018111154437065, "fcm_dpo/delta": -0.042948391288518906, "fcm_dpo/margin": 103.61990356445312, "fcm_dpo/q_t": 0.3986860513687134, "grad_norm": 10.77768611907959, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.561767816543579, "logits/rejected": -1.4202309846878052, "logps/chosen": -269.0208435058594, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -395.27783203125, "loss": 1.0671, "margin_dpo/margin_mean": 103.61990356445312, "margin_dpo/margin_std": 116.80528259277344, "step": 551 }, { "KL/chosen_KL_mean": -166.49801635742188, "KL/mean": -239.40354919433594, "KL/rejected_KL_mean": -312.30908203125, "KL/std": 124.28648376464844, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0030566842760890722, "fcm_dpo/delta": -0.14016100764274597, "fcm_dpo/margin": 145.81103515625, "fcm_dpo/q_t": 0.3754468560218811, "grad_norm": 9.511717796325684, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.1801297664642334, "logits/rejected": -1.8272068500518799, "logps/chosen": -267.9072265625, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -423.14752197265625, "loss": 1.0313, "margin_dpo/margin_mean": 145.81103515625, "margin_dpo/margin_std": 129.1879119873047, "step": 552 }, { "KL/chosen_KL_mean": -179.96188354492188, "KL/mean": -223.71241760253906, "KL/rejected_KL_mean": -267.46295166015625, "KL/std": 115.49626922607422, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.003753693774342537, "fcm_dpo/delta": 0.10090121626853943, "fcm_dpo/margin": 87.50106811523438, "fcm_dpo/q_t": 0.4283342957496643, "grad_norm": 8.757123947143555, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.5366982221603394, "logits/rejected": -1.2080976963043213, "logps/chosen": -277.92578125, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -371.0401611328125, "loss": 1.1361, "margin_dpo/margin_mean": 87.50106811523438, "margin_dpo/margin_std": 123.70553588867188, "step": 553 }, { "KL/chosen_KL_mean": -188.12274169921875, "KL/mean": -236.33200073242188, "KL/rejected_KL_mean": -284.5412902832031, "KL/std": 116.01214599609375, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.004078247118741274, "fcm_dpo/delta": 0.005324602127075195, "fcm_dpo/margin": 96.41854858398438, "fcm_dpo/q_t": 0.4094139039516449, "grad_norm": 8.836812019348145, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.9179351329803467, "logits/rejected": -1.1454757452011108, "logps/chosen": -291.8095397949219, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -377.53338623046875, "loss": 1.1218, "margin_dpo/margin_mean": 96.41854858398438, "margin_dpo/margin_std": 149.38800048828125, "step": 554 }, { "KL/chosen_KL_mean": -196.5872802734375, "KL/mean": -240.90084838867188, "KL/rejected_KL_mean": -285.2144470214844, "KL/std": 119.14089965820312, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.00419111642986536, "fcm_dpo/delta": 0.044848762452602386, "fcm_dpo/margin": 88.62718200683594, "fcm_dpo/q_t": 0.4183122515678406, "grad_norm": 9.570213317871094, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.7512118816375732, "logits/rejected": -1.6617116928100586, "logps/chosen": -282.7908935546875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -399.76165771484375, "loss": 1.1273, "margin_dpo/margin_mean": 88.62718200683594, "margin_dpo/margin_std": 137.94485473632812, "step": 555 }, { "KL/chosen_KL_mean": -177.55528259277344, "KL/mean": -226.089599609375, "KL/rejected_KL_mean": -274.6239013671875, "KL/std": 106.20394897460938, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004111986607313156, "fcm_dpo/delta": -0.0020339153707027435, "fcm_dpo/margin": 97.06863403320312, "fcm_dpo/q_t": 0.4063228666782379, "grad_norm": 11.195953369140625, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.658190369606018, "logits/rejected": -1.36137056350708, "logps/chosen": -265.9587707519531, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -381.2630310058594, "loss": 1.0863, "margin_dpo/margin_mean": 97.06863403320312, "margin_dpo/margin_std": 120.18802642822266, "step": 556 }, { "KL/chosen_KL_mean": -189.83895874023438, "KL/mean": -245.20994567871094, "KL/rejected_KL_mean": -300.5809326171875, "KL/std": 123.37799072265625, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.003773641772568226, "fcm_dpo/delta": -0.04335470870137215, "fcm_dpo/margin": 110.74199676513672, "fcm_dpo/q_t": 0.3984304368495941, "grad_norm": 7.461354732513428, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -2.1747303009033203, "logits/rejected": -2.006115674972534, "logps/chosen": -288.6741943359375, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -416.2298278808594, "loss": 1.0749, "margin_dpo/margin_mean": 110.74198913574219, "margin_dpo/margin_std": 134.37045288085938, "step": 557 }, { "KL/chosen_KL_mean": -185.5137939453125, "KL/mean": -245.38372802734375, "KL/rejected_KL_mean": -305.253662109375, "KL/std": 106.18901824951172, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.003535135183483362, "fcm_dpo/delta": -0.045551687479019165, "fcm_dpo/margin": 119.7398681640625, "fcm_dpo/q_t": 0.39468058943748474, "grad_norm": 9.610949516296387, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.449483871459961, "logits/rejected": -1.8492202758789062, "logps/chosen": -291.525634765625, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -413.6449890136719, "loss": 1.0439, "margin_dpo/margin_mean": 119.7398681640625, "margin_dpo/margin_std": 108.53416442871094, "step": 558 }, { "KL/chosen_KL_mean": -203.10147094726562, "KL/mean": -250.00540161132812, "KL/rejected_KL_mean": -296.9093322753906, "KL/std": 108.86219787597656, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.003906002501025796, "fcm_dpo/delta": 0.05056924745440483, "fcm_dpo/margin": 93.80784606933594, "fcm_dpo/q_t": 0.41686320304870605, "grad_norm": 10.129043579101562, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.424231767654419, "logits/rejected": -1.955519199371338, "logps/chosen": -303.5262145996094, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -397.322021484375, "loss": 1.109, "margin_dpo/margin_mean": 93.80784606933594, "margin_dpo/margin_std": 123.23207092285156, "step": 559 }, { "KL/chosen_KL_mean": -192.42794799804688, "KL/mean": -238.49147033691406, "KL/rejected_KL_mean": -284.5549621582031, "KL/std": 106.27796173095703, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0041178688406944275, "fcm_dpo/delta": 0.0332985557615757, "fcm_dpo/margin": 92.12702178955078, "fcm_dpo/q_t": 0.4140366315841675, "grad_norm": 11.510968208312988, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.6014184951782227, "logits/rejected": -1.171985149383545, "logps/chosen": -280.0634765625, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -371.934814453125, "loss": 1.1031, "margin_dpo/margin_mean": 92.12702178955078, "margin_dpo/margin_std": 122.46831512451172, "step": 560 }, { "KL/chosen_KL_mean": -201.8858184814453, "KL/mean": -244.60626220703125, "KL/rejected_KL_mean": -287.32672119140625, "KL/std": 104.89224243164062, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004486174322664738, "fcm_dpo/delta": 0.02395746484398842, "fcm_dpo/margin": 85.4408950805664, "fcm_dpo/q_t": 0.41181430220603943, "grad_norm": 9.245798110961914, "learning_rate": 4.669493178106432e-08, "logits/chosen": -2.038922071456909, "logits/rejected": -2.01111102104187, "logps/chosen": -276.7672119140625, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -393.37469482421875, "loss": 1.0885, "margin_dpo/margin_mean": 85.44090270996094, "margin_dpo/margin_std": 101.25171661376953, "step": 561 }, { "KL/chosen_KL_mean": -183.35128784179688, "KL/mean": -226.96429443359375, "KL/rejected_KL_mean": -270.5772705078125, "KL/std": 106.15071868896484, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0043993983417749405, "fcm_dpo/delta": 0.02348300628364086, "fcm_dpo/margin": 87.22602081298828, "fcm_dpo/q_t": 0.4134664535522461, "grad_norm": 11.735950469970703, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.1368260383605957, "logits/rejected": -1.684516429901123, "logps/chosen": -284.0144958496094, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -372.81817626953125, "loss": 1.1197, "margin_dpo/margin_mean": 87.22602081298828, "margin_dpo/margin_std": 131.24676513671875, "step": 562 }, { "KL/chosen_KL_mean": -193.5550537109375, "KL/mean": -229.55609130859375, "KL/rejected_KL_mean": -265.55712890625, "KL/std": 106.2152099609375, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0051466356962919235, "fcm_dpo/delta": 0.037241317331790924, "fcm_dpo/margin": 72.00206756591797, "fcm_dpo/q_t": 0.4166424870491028, "grad_norm": 12.581303596496582, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.7071254253387451, "logits/rejected": -1.370862364768982, "logps/chosen": -289.81427001953125, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -366.7621765136719, "loss": 1.1336, "margin_dpo/margin_mean": 72.00206756591797, "margin_dpo/margin_std": 113.54364013671875, "step": 563 }, { "KL/chosen_KL_mean": -190.25856018066406, "KL/mean": -230.56163024902344, "KL/rejected_KL_mean": -270.86468505859375, "KL/std": 107.1156005859375, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.005036159418523312, "fcm_dpo/delta": -0.011364789679646492, "fcm_dpo/margin": 80.60613250732422, "fcm_dpo/q_t": 0.4042383134365082, "grad_norm": 13.769268989562988, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.3881359100341797, "logits/rejected": -1.8219919204711914, "logps/chosen": -303.2189636230469, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -370.3927307128906, "loss": 1.079, "margin_dpo/margin_mean": 80.60614013671875, "margin_dpo/margin_std": 97.26882934570312, "step": 564 }, { "KL/chosen_KL_mean": -185.42236328125, "KL/mean": -235.73995971679688, "KL/rejected_KL_mean": -286.0575256347656, "KL/std": 116.46221923828125, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0043101017363369465, "fcm_dpo/delta": -0.06512085348367691, "fcm_dpo/margin": 100.63516235351562, "fcm_dpo/q_t": 0.3943363428115845, "grad_norm": 9.226898193359375, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.633779764175415, "logits/rejected": -1.1233911514282227, "logps/chosen": -280.7879638671875, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -378.27490234375, "loss": 1.0696, "margin_dpo/margin_mean": 100.63516235351562, "margin_dpo/margin_std": 127.49407958984375, "step": 565 }, { "KL/chosen_KL_mean": -195.03363037109375, "KL/mean": -242.14633178710938, "KL/rejected_KL_mean": -289.259033203125, "KL/std": 118.93144989013672, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004316807724535465, "fcm_dpo/delta": -0.012796195223927498, "fcm_dpo/margin": 94.22538757324219, "fcm_dpo/q_t": 0.4035506844520569, "grad_norm": 8.767983436584473, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.8149954080581665, "logits/rejected": -1.3792887926101685, "logps/chosen": -294.0068359375, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -400.0799560546875, "loss": 1.0847, "margin_dpo/margin_mean": 94.22538757324219, "margin_dpo/margin_std": 117.83792114257812, "step": 566 }, { "KL/chosen_KL_mean": -189.21368408203125, "KL/mean": -231.024658203125, "KL/rejected_KL_mean": -272.83563232421875, "KL/std": 106.90601348876953, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.004334107507020235, "fcm_dpo/delta": 0.056459441781044006, "fcm_dpo/margin": 83.6219482421875, "fcm_dpo/q_t": 0.41849285364151, "grad_norm": 10.315511703491211, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.2469425201416016, "logits/rejected": -1.5712857246398926, "logps/chosen": -281.91729736328125, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -364.0599670410156, "loss": 1.12, "margin_dpo/margin_mean": 83.6219482421875, "margin_dpo/margin_std": 112.88079071044922, "step": 567 }, { "KL/chosen_KL_mean": -178.92034912109375, "KL/mean": -235.05740356445312, "KL/rejected_KL_mean": -291.1944580078125, "KL/std": 113.07342529296875, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.00389849953353405, "fcm_dpo/delta": -0.08443574607372284, "fcm_dpo/margin": 112.27410888671875, "fcm_dpo/q_t": 0.3881864547729492, "grad_norm": 8.229316711425781, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3744077682495117, "logits/rejected": -1.2460823059082031, "logps/chosen": -252.0526123046875, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -386.1470947265625, "loss": 1.044, "margin_dpo/margin_mean": 112.27410888671875, "margin_dpo/margin_std": 114.69502258300781, "step": 568 }, { "KL/chosen_KL_mean": -189.89292907714844, "KL/mean": -235.1575927734375, "KL/rejected_KL_mean": -280.4222412109375, "KL/std": 114.31448364257812, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.00420395890250802, "fcm_dpo/delta": 0.02927519753575325, "fcm_dpo/margin": 90.52932739257812, "fcm_dpo/q_t": 0.4133378863334656, "grad_norm": 14.168577194213867, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.85007643699646, "logits/rejected": -1.465879201889038, "logps/chosen": -279.1100158691406, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -366.83331298828125, "loss": 1.1219, "margin_dpo/margin_mean": 90.52932739257812, "margin_dpo/margin_std": 137.29898071289062, "step": 569 }, { "KL/chosen_KL_mean": -170.21925354003906, "KL/mean": -229.12054443359375, "KL/rejected_KL_mean": -288.0218505859375, "KL/std": 108.46678924560547, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0037095341831445694, "fcm_dpo/delta": -0.0693393275141716, "fcm_dpo/margin": 117.80258178710938, "fcm_dpo/q_t": 0.389659583568573, "grad_norm": 8.516546249389648, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.402067184448242, "logits/rejected": -1.7316069602966309, "logps/chosen": -264.9820251464844, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -376.8687744140625, "loss": 1.0386, "margin_dpo/margin_mean": 117.80258178710938, "margin_dpo/margin_std": 113.78658294677734, "step": 570 }, { "KL/chosen_KL_mean": -177.45945739746094, "KL/mean": -229.21371459960938, "KL/rejected_KL_mean": -280.96795654296875, "KL/std": 106.96788024902344, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.00379068311303854, "fcm_dpo/delta": 0.010573504492640495, "fcm_dpo/margin": 103.50848388671875, "fcm_dpo/q_t": 0.4088497757911682, "grad_norm": 11.1614408493042, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.6163082122802734, "logits/rejected": -1.4560933113098145, "logps/chosen": -258.28076171875, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -377.9832458496094, "loss": 1.09, "margin_dpo/margin_mean": 103.50848388671875, "margin_dpo/margin_std": 129.3594970703125, "step": 571 }, { "KL/chosen_KL_mean": -182.0912628173828, "KL/mean": -234.409423828125, "KL/rejected_KL_mean": -286.72760009765625, "KL/std": 113.88053894042969, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.003750098403543234, "fcm_dpo/delta": 0.012412065640091896, "fcm_dpo/margin": 104.63632202148438, "fcm_dpo/q_t": 0.409495085477829, "grad_norm": 8.283276557922363, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.6165482997894287, "logits/rejected": -1.4440460205078125, "logps/chosen": -259.47119140625, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -377.2935791015625, "loss": 1.0815, "margin_dpo/margin_mean": 104.63632202148438, "margin_dpo/margin_std": 122.11529541015625, "step": 572 }, { "KL/chosen_KL_mean": -176.22048950195312, "KL/mean": -225.3522186279297, "KL/rejected_KL_mean": -274.48394775390625, "KL/std": 113.28913879394531, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.004054199904203415, "fcm_dpo/delta": -0.010627083480358124, "fcm_dpo/margin": 98.26344299316406, "fcm_dpo/q_t": 0.40413326025009155, "grad_norm": 8.750584602355957, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1331498622894287, "logits/rejected": -1.9029173851013184, "logps/chosen": -259.92657470703125, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -365.181396484375, "loss": 1.0926, "margin_dpo/margin_mean": 98.26344299316406, "margin_dpo/margin_std": 119.43943786621094, "step": 573 }, { "KL/chosen_KL_mean": -185.67889404296875, "KL/mean": -239.35939025878906, "KL/rejected_KL_mean": -293.0399169921875, "KL/std": 112.40690612792969, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.003788725705817342, "fcm_dpo/delta": -0.013874528929591179, "fcm_dpo/margin": 107.36102294921875, "fcm_dpo/q_t": 0.4031282365322113, "grad_norm": 9.050606727600098, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.2014827728271484, "logits/rejected": -1.879083514213562, "logps/chosen": -270.8083801269531, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -392.09393310546875, "loss": 1.0771, "margin_dpo/margin_mean": 107.36100769042969, "margin_dpo/margin_std": 126.37437438964844, "step": 574 }, { "KL/chosen_KL_mean": -192.13250732421875, "KL/mean": -241.02951049804688, "KL/rejected_KL_mean": -289.9265441894531, "KL/std": 114.93977355957031, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.003848564811050892, "fcm_dpo/delta": 0.037555992603302, "fcm_dpo/margin": 97.79402923583984, "fcm_dpo/q_t": 0.41513147950172424, "grad_norm": 10.929811477661133, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.2388700246810913, "logits/rejected": -0.7612457275390625, "logps/chosen": -297.9033508300781, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -390.8759765625, "loss": 1.1083, "margin_dpo/margin_mean": 97.79402160644531, "margin_dpo/margin_std": 134.4611053466797, "step": 575 }, { "KL/chosen_KL_mean": -191.68881225585938, "KL/mean": -238.84185791015625, "KL/rejected_KL_mean": -285.9949035644531, "KL/std": 106.36695861816406, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.004105234518647194, "fcm_dpo/delta": 0.019392866641283035, "fcm_dpo/margin": 94.30609130859375, "fcm_dpo/q_t": 0.41092538833618164, "grad_norm": 11.044134140014648, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.984339952468872, "logits/rejected": -1.925837755203247, "logps/chosen": -272.8933410644531, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -391.404296875, "loss": 1.1024, "margin_dpo/margin_mean": 94.30609130859375, "margin_dpo/margin_std": 127.72843170166016, "step": 576 }, { "KL/chosen_KL_mean": -170.7347412109375, "KL/mean": -229.9490509033203, "KL/rejected_KL_mean": -289.16339111328125, "KL/std": 115.85111999511719, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0035868389531970024, "fcm_dpo/delta": -0.053523242473602295, "fcm_dpo/margin": 118.42861938476562, "fcm_dpo/q_t": 0.39460986852645874, "grad_norm": 7.333293914794922, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.1363179683685303, "logits/rejected": -1.4547725915908813, "logps/chosen": -272.1989440917969, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -378.48974609375, "loss": 1.0605, "margin_dpo/margin_mean": 118.42861938476562, "margin_dpo/margin_std": 132.06484985351562, "step": 577 }, { "KL/chosen_KL_mean": -205.0172576904297, "KL/mean": -247.7278594970703, "KL/rejected_KL_mean": -290.4384460449219, "KL/std": 110.20349884033203, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.004097639117389917, "fcm_dpo/delta": 0.07617572695016861, "fcm_dpo/margin": 85.42117309570312, "fcm_dpo/q_t": 0.4236934185028076, "grad_norm": 13.900803565979004, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.4303010702133179, "logits/rejected": -1.1934425830841064, "logps/chosen": -282.3096923828125, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -378.6854553222656, "loss": 1.1312, "margin_dpo/margin_mean": 85.42117309570312, "margin_dpo/margin_std": 126.06173706054688, "step": 578 }, { "KL/chosen_KL_mean": -182.83689880371094, "KL/mean": -226.2441864013672, "KL/rejected_KL_mean": -269.6515197753906, "KL/std": 109.62467956542969, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004311924800276756, "fcm_dpo/delta": 0.03799785301089287, "fcm_dpo/margin": 86.81461334228516, "fcm_dpo/q_t": 0.41587913036346436, "grad_norm": 8.738679885864258, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.6388249397277832, "logits/rejected": -1.3418940305709839, "logps/chosen": -270.179931640625, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -371.77252197265625, "loss": 1.1078, "margin_dpo/margin_mean": 86.81462097167969, "margin_dpo/margin_std": 112.37191772460938, "step": 579 }, { "KL/chosen_KL_mean": -175.76275634765625, "KL/mean": -229.45626831054688, "KL/rejected_KL_mean": -283.14984130859375, "KL/std": 109.3326416015625, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.003956370521336794, "fcm_dpo/delta": -0.05976353585720062, "fcm_dpo/margin": 107.38706970214844, "fcm_dpo/q_t": 0.39575159549713135, "grad_norm": 9.425237655639648, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.5406163930892944, "logits/rejected": -1.2278809547424316, "logps/chosen": -264.6915283203125, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -371.846435546875, "loss": 1.0726, "margin_dpo/margin_mean": 107.38706970214844, "margin_dpo/margin_std": 132.49078369140625, "step": 580 }, { "KL/chosen_KL_mean": -185.35464477539062, "KL/mean": -233.89126586914062, "KL/rejected_KL_mean": -282.4278869628906, "KL/std": 110.55764770507812, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004120805766433477, "fcm_dpo/delta": -0.00038307905197143555, "fcm_dpo/margin": 97.07325744628906, "fcm_dpo/q_t": 0.40711236000061035, "grad_norm": 11.744544982910156, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.9513033628463745, "logits/rejected": -1.6031007766723633, "logps/chosen": -277.577880859375, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -381.81732177734375, "loss": 1.1033, "margin_dpo/margin_mean": 97.07325744628906, "margin_dpo/margin_std": 138.2708740234375, "step": 581 }, { "KL/chosen_KL_mean": -163.79339599609375, "KL/mean": -208.24020385742188, "KL/rejected_KL_mean": -252.68702697753906, "KL/std": 115.78497314453125, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0038165333680808544, "fcm_dpo/delta": -0.06712497770786285, "fcm_dpo/margin": 88.89362335205078, "fcm_dpo/q_t": 0.4174116253852844, "grad_norm": 9.185233116149902, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.205718517303467, "logits/rejected": -2.1975700855255127, "logps/chosen": -251.1290283203125, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -367.8414306640625, "loss": 1.1443, "margin_dpo/margin_mean": 88.89362335205078, "margin_dpo/margin_std": 127.58975982666016, "step": 582 }, { "KL/chosen_KL_mean": -196.99899291992188, "KL/mean": -245.7613525390625, "KL/rejected_KL_mean": -294.5237121582031, "KL/std": 108.63801574707031, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.003744625486433506, "fcm_dpo/delta": 0.05438258498907089, "fcm_dpo/margin": 97.52471923828125, "fcm_dpo/q_t": 0.41877394914627075, "grad_norm": 10.584507942199707, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5501468181610107, "logits/rejected": -1.3183159828186035, "logps/chosen": -280.716064453125, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -389.1083679199219, "loss": 1.1127, "margin_dpo/margin_mean": 97.52472686767578, "margin_dpo/margin_std": 128.62460327148438, "step": 583 }, { "KL/chosen_KL_mean": -175.5447998046875, "KL/mean": -227.7523193359375, "KL/rejected_KL_mean": -279.9598388671875, "KL/std": 113.04061889648438, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.004030222073197365, "fcm_dpo/delta": -0.0472341887652874, "fcm_dpo/margin": 104.41504669189453, "fcm_dpo/q_t": 0.39671194553375244, "grad_norm": 8.55087661743164, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9667150974273682, "logits/rejected": -1.5133857727050781, "logps/chosen": -269.8043518066406, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -384.4425354003906, "loss": 1.085, "margin_dpo/margin_mean": 104.41505432128906, "margin_dpo/margin_std": 128.18971252441406, "step": 584 }, { "KL/chosen_KL_mean": -186.87347412109375, "KL/mean": -244.79843139648438, "KL/rejected_KL_mean": -302.723388671875, "KL/std": 129.2334747314453, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.003583179786801338, "fcm_dpo/delta": -0.02720346860587597, "fcm_dpo/margin": 115.84993743896484, "fcm_dpo/q_t": 0.4017300009727478, "grad_norm": 8.345029830932617, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.9612243175506592, "logits/rejected": -1.6887824535369873, "logps/chosen": -272.5102233886719, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -393.00885009765625, "loss": 1.0859, "margin_dpo/margin_mean": 115.84992980957031, "margin_dpo/margin_std": 151.69676208496094, "step": 585 }, { "KL/chosen_KL_mean": -207.4397430419922, "KL/mean": -246.25892639160156, "KL/rejected_KL_mean": -285.078125, "KL/std": 112.6623306274414, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.004103472456336021, "fcm_dpo/delta": 0.11684402823448181, "fcm_dpo/margin": 77.63837432861328, "fcm_dpo/q_t": 0.43232160806655884, "grad_norm": 10.595845222473145, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.735818862915039, "logits/rejected": -1.4702496528625488, "logps/chosen": -291.1208801269531, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -383.4372863769531, "loss": 1.1417, "margin_dpo/margin_mean": 77.63837432861328, "margin_dpo/margin_std": 107.00849914550781, "step": 586 }, { "KL/chosen_KL_mean": -168.65676879882812, "KL/mean": -210.70912170410156, "KL/rejected_KL_mean": -252.76144409179688, "KL/std": 109.2750244140625, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.004557089414447546, "fcm_dpo/delta": 0.025876076892018318, "fcm_dpo/margin": 84.10467529296875, "fcm_dpo/q_t": 0.4130954146385193, "grad_norm": 11.859359741210938, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.329587459564209, "logits/rejected": -2.045229434967041, "logps/chosen": -246.23428344726562, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -342.081787109375, "loss": 1.1027, "margin_dpo/margin_mean": 84.10467529296875, "margin_dpo/margin_std": 113.89482116699219, "step": 587 }, { "KL/chosen_KL_mean": -182.6682586669922, "KL/mean": -222.12503051757812, "KL/rejected_KL_mean": -261.581787109375, "KL/std": 111.35823822021484, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004740839824080467, "fcm_dpo/delta": 0.04128327593207359, "fcm_dpo/margin": 78.91352844238281, "fcm_dpo/q_t": 0.419075071811676, "grad_norm": 16.442163467407227, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.657165765762329, "logits/rejected": -1.6207575798034668, "logps/chosen": -266.4725341796875, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -374.70123291015625, "loss": 1.1443, "margin_dpo/margin_mean": 78.91352844238281, "margin_dpo/margin_std": 134.62469482421875, "step": 588 }, { "KL/chosen_KL_mean": -195.0045166015625, "KL/mean": -247.41665649414062, "KL/rejected_KL_mean": -299.82879638671875, "KL/std": 109.36178588867188, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004278839565813541, "fcm_dpo/delta": -0.09115688502788544, "fcm_dpo/margin": 104.8243179321289, "fcm_dpo/q_t": 0.38624894618988037, "grad_norm": 9.08829116821289, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.5811209678649902, "logits/rejected": -1.1853132247924805, "logps/chosen": -291.46954345703125, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -415.86602783203125, "loss": 1.0357, "margin_dpo/margin_mean": 104.82431030273438, "margin_dpo/margin_std": 104.88018798828125, "step": 589 }, { "KL/chosen_KL_mean": -178.8616485595703, "KL/mean": -231.02783203125, "KL/rejected_KL_mean": -283.19403076171875, "KL/std": 102.16694641113281, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004050453659147024, "fcm_dpo/delta": -0.04140602797269821, "fcm_dpo/margin": 104.33235168457031, "fcm_dpo/q_t": 0.3971394896507263, "grad_norm": 11.486143112182617, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.9775739908218384, "logits/rejected": -1.8361643552780151, "logps/chosen": -267.2895202636719, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -393.39276123046875, "loss": 1.0546, "margin_dpo/margin_mean": 104.33235168457031, "margin_dpo/margin_std": 106.19613647460938, "step": 590 }, { "KL/chosen_KL_mean": -170.855712890625, "KL/mean": -226.00604248046875, "KL/rejected_KL_mean": -281.1563720703125, "KL/std": 109.72062683105469, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.003659013658761978, "fcm_dpo/delta": -0.007877502590417862, "fcm_dpo/margin": 110.3006591796875, "fcm_dpo/q_t": 0.4032720923423767, "grad_norm": 6.636234760284424, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9436073303222656, "logits/rejected": -1.3294568061828613, "logps/chosen": -258.70538330078125, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -370.7355651855469, "loss": 1.0606, "margin_dpo/margin_mean": 110.3006591796875, "margin_dpo/margin_std": 106.6264877319336, "step": 591 }, { "KL/chosen_KL_mean": -203.46163940429688, "KL/mean": -245.78981018066406, "KL/rejected_KL_mean": -288.11798095703125, "KL/std": 106.5216293334961, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.00418453011661768, "fcm_dpo/delta": 0.06869176030158997, "fcm_dpo/margin": 84.65632629394531, "fcm_dpo/q_t": 0.4220181703567505, "grad_norm": 11.439746856689453, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.8336644172668457, "logits/rejected": -1.5078234672546387, "logps/chosen": -278.4373779296875, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -369.67059326171875, "loss": 1.138, "margin_dpo/margin_mean": 84.65632629394531, "margin_dpo/margin_std": 134.10736083984375, "step": 592 }, { "KL/chosen_KL_mean": -186.12567138671875, "KL/mean": -222.360595703125, "KL/rejected_KL_mean": -258.5954895019531, "KL/std": 111.55308532714844, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004741748794913292, "fcm_dpo/delta": 0.08650164306163788, "fcm_dpo/margin": 72.46980285644531, "fcm_dpo/q_t": 0.42770031094551086, "grad_norm": 16.61159324645996, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.9083876609802246, "logits/rejected": -0.6083428859710693, "logps/chosen": -258.3647155761719, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -328.6920166015625, "loss": 1.1582, "margin_dpo/margin_mean": 72.46980285644531, "margin_dpo/margin_std": 123.86127471923828, "step": 593 }, { "KL/chosen_KL_mean": -166.18899536132812, "KL/mean": -222.37139892578125, "KL/rejected_KL_mean": -278.5538330078125, "KL/std": 108.43301391601562, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.004251548554748297, "fcm_dpo/delta": -0.14708656072616577, "fcm_dpo/margin": 112.36483001708984, "fcm_dpo/q_t": 0.3749922811985016, "grad_norm": 10.658546447753906, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9925775527954102, "logits/rejected": -1.4729397296905518, "logps/chosen": -256.626708984375, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -370.8214111328125, "loss": 1.0202, "margin_dpo/margin_mean": 112.36482238769531, "margin_dpo/margin_std": 105.16474914550781, "step": 594 }, { "KL/chosen_KL_mean": -179.59683227539062, "KL/mean": -225.38800048828125, "KL/rejected_KL_mean": -271.17919921875, "KL/std": 108.55419921875, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.004116406664252281, "fcm_dpo/delta": 0.03211667388677597, "fcm_dpo/margin": 91.58233642578125, "fcm_dpo/q_t": 0.41465529799461365, "grad_norm": 9.642860412597656, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.939357876777649, "logits/rejected": -1.5417799949645996, "logps/chosen": -261.11163330078125, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -357.69964599609375, "loss": 1.1079, "margin_dpo/margin_mean": 91.58233642578125, "margin_dpo/margin_std": 124.67848205566406, "step": 595 }, { "KL/chosen_KL_mean": -186.35610961914062, "KL/mean": -227.70437622070312, "KL/rejected_KL_mean": -269.0526428222656, "KL/std": 107.80686950683594, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.004428141750395298, "fcm_dpo/delta": 0.051403485238552094, "fcm_dpo/margin": 82.69654846191406, "fcm_dpo/q_t": 0.41986966133117676, "grad_norm": 12.868489265441895, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.4264780282974243, "logits/rejected": -0.825819194316864, "logps/chosen": -274.56414794921875, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -355.1463928222656, "loss": 1.139, "margin_dpo/margin_mean": 82.69654083251953, "margin_dpo/margin_std": 136.31443786621094, "step": 596 }, { "KL/chosen_KL_mean": -180.01622009277344, "KL/mean": -235.11688232421875, "KL/rejected_KL_mean": -290.217529296875, "KL/std": 116.17593383789062, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.003921789117157459, "fcm_dpo/delta": -0.06511155515909195, "fcm_dpo/margin": 110.20133972167969, "fcm_dpo/q_t": 0.39345961809158325, "grad_norm": 8.788663864135742, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.369748830795288, "logits/rejected": -1.2546484470367432, "logps/chosen": -263.606201171875, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -397.16192626953125, "loss": 1.0578, "margin_dpo/margin_mean": 110.20133209228516, "margin_dpo/margin_std": 126.08158874511719, "step": 597 }, { "KL/chosen_KL_mean": -171.85574340820312, "KL/mean": -218.7259063720703, "KL/rejected_KL_mean": -265.5960693359375, "KL/std": 108.24942016601562, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0040054707787930965, "fcm_dpo/delta": 0.0372554287314415, "fcm_dpo/margin": 93.74031829833984, "fcm_dpo/q_t": 0.4140588641166687, "grad_norm": 9.790456771850586, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.863771915435791, "logits/rejected": -1.5465221405029297, "logps/chosen": -262.03839111328125, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -360.3946228027344, "loss": 1.1094, "margin_dpo/margin_mean": 93.74031829833984, "margin_dpo/margin_std": 122.135009765625, "step": 598 }, { "KL/chosen_KL_mean": -173.68954467773438, "KL/mean": -214.694091796875, "KL/rejected_KL_mean": -255.69863891601562, "KL/std": 112.34767150878906, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.00440196692943573, "fcm_dpo/delta": 0.058074332773685455, "fcm_dpo/margin": 82.00910186767578, "fcm_dpo/q_t": 0.4198734760284424, "grad_norm": 10.022141456604004, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.788451910018921, "logits/rejected": -1.6696381568908691, "logps/chosen": -251.53179931640625, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -344.8962707519531, "loss": 1.1363, "margin_dpo/margin_mean": 82.00910949707031, "margin_dpo/margin_std": 124.0187759399414, "step": 599 }, { "KL/chosen_KL_mean": -194.92031860351562, "KL/mean": -241.35365295410156, "KL/rejected_KL_mean": -287.7869873046875, "KL/std": 106.65613555908203, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004531376529484987, "fcm_dpo/delta": -0.03596094250679016, "fcm_dpo/margin": 92.86663818359375, "fcm_dpo/q_t": 0.4002681374549866, "grad_norm": 10.223231315612793, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.4003381729125977, "logits/rejected": -2.056006669998169, "logps/chosen": -287.108154296875, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -380.74127197265625, "loss": 1.0758, "margin_dpo/margin_mean": 92.86663818359375, "margin_dpo/margin_std": 115.7651596069336, "step": 600 }, { "KL/chosen_KL_mean": -196.5718994140625, "KL/mean": -246.0014190673828, "KL/rejected_KL_mean": -295.43096923828125, "KL/std": 108.26984405517578, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.00412956066429615, "fcm_dpo/delta": -0.019921191036701202, "fcm_dpo/margin": 98.85905456542969, "fcm_dpo/q_t": 0.40172943472862244, "grad_norm": 9.046636581420898, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.6750414371490479, "logits/rejected": -1.346100091934204, "logps/chosen": -285.78802490234375, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -395.60150146484375, "loss": 1.0636, "margin_dpo/margin_mean": 98.85905456542969, "margin_dpo/margin_std": 103.0640869140625, "step": 601 }, { "KL/chosen_KL_mean": -156.61456298828125, "KL/mean": -220.8557586669922, "KL/rejected_KL_mean": -285.0969543457031, "KL/std": 118.22274017333984, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0035622185096144676, "fcm_dpo/delta": -0.11172901093959808, "fcm_dpo/margin": 128.48239135742188, "fcm_dpo/q_t": 0.3824708163738251, "grad_norm": 8.315542221069336, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.199807643890381, "logits/rejected": -1.9447989463806152, "logps/chosen": -236.67214965820312, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -378.2944641113281, "loss": 1.0345, "margin_dpo/margin_mean": 128.48240661621094, "margin_dpo/margin_std": 135.3699951171875, "step": 602 }, { "KL/chosen_KL_mean": -166.41217041015625, "KL/mean": -219.33334350585938, "KL/rejected_KL_mean": -272.2545166015625, "KL/std": 106.8939208984375, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0036015259101986885, "fcm_dpo/delta": 0.029317451640963554, "fcm_dpo/margin": 105.8423843383789, "fcm_dpo/q_t": 0.4124029874801636, "grad_norm": 7.329595565795898, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1257219314575195, "logits/rejected": -1.5442402362823486, "logps/chosen": -268.72174072265625, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -380.3233642578125, "loss": 1.0871, "margin_dpo/margin_mean": 105.84239196777344, "margin_dpo/margin_std": 122.98930358886719, "step": 603 }, { "KL/chosen_KL_mean": -174.13185119628906, "KL/mean": -228.56655883789062, "KL/rejected_KL_mean": -283.00128173828125, "KL/std": 112.04252624511719, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0035855977330356836, "fcm_dpo/delta": 0.01420794427394867, "fcm_dpo/margin": 108.86942291259766, "fcm_dpo/q_t": 0.407693088054657, "grad_norm": 7.88932991027832, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.6765320301055908, "logits/rejected": -1.5236759185791016, "logps/chosen": -252.30593872070312, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -382.49737548828125, "loss": 1.0764, "margin_dpo/margin_mean": 108.86941528320312, "margin_dpo/margin_std": 115.07293701171875, "step": 604 }, { "KL/chosen_KL_mean": -190.55160522460938, "KL/mean": -235.61837768554688, "KL/rejected_KL_mean": -280.6851806640625, "KL/std": 99.54632568359375, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004043128807097673, "fcm_dpo/delta": 0.05543963611125946, "fcm_dpo/margin": 90.133544921875, "fcm_dpo/q_t": 0.41860440373420715, "grad_norm": 9.374359130859375, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8060482740402222, "logits/rejected": -1.3085124492645264, "logps/chosen": -285.324951171875, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -387.99005126953125, "loss": 1.1036, "margin_dpo/margin_mean": 90.133544921875, "margin_dpo/margin_std": 112.09410858154297, "step": 605 }, { "KL/chosen_KL_mean": -183.50656127929688, "KL/mean": -230.38771057128906, "KL/rejected_KL_mean": -277.26885986328125, "KL/std": 110.0111312866211, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004320506006479263, "fcm_dpo/delta": -0.017580360174179077, "fcm_dpo/margin": 93.7623062133789, "fcm_dpo/q_t": 0.4027911424636841, "grad_norm": 8.749417304992676, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.8702247142791748, "logits/rejected": -1.4856966733932495, "logps/chosen": -271.25994873046875, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -381.5111083984375, "loss": 1.0888, "margin_dpo/margin_mean": 93.76231384277344, "margin_dpo/margin_std": 116.77613067626953, "step": 606 }, { "KL/chosen_KL_mean": -177.63978576660156, "KL/mean": -230.2085723876953, "KL/rejected_KL_mean": -282.7773742675781, "KL/std": 117.01425170898438, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.003913186024874449, "fcm_dpo/delta": -0.019374905154109, "fcm_dpo/margin": 105.13758850097656, "fcm_dpo/q_t": 0.40324097871780396, "grad_norm": 11.10448932647705, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7275084257125854, "logits/rejected": -1.196345567703247, "logps/chosen": -265.96881103515625, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -369.54547119140625, "loss": 1.0911, "margin_dpo/margin_mean": 105.13758850097656, "margin_dpo/margin_std": 143.46839904785156, "step": 607 }, { "KL/chosen_KL_mean": -162.0630645751953, "KL/mean": -217.26296997070312, "KL/rejected_KL_mean": -272.462890625, "KL/std": 115.47715759277344, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0035805744118988514, "fcm_dpo/delta": -0.005172066390514374, "fcm_dpo/margin": 110.39983367919922, "fcm_dpo/q_t": 0.4043319523334503, "grad_norm": 8.435405731201172, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.9465622901916504, "logits/rejected": -1.6531970500946045, "logps/chosen": -247.83242797851562, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -372.6957092285156, "loss": 1.0849, "margin_dpo/margin_mean": 110.39983367919922, "margin_dpo/margin_std": 125.81660461425781, "step": 608 }, { "KL/chosen_KL_mean": -182.21615600585938, "KL/mean": -233.79730224609375, "KL/rejected_KL_mean": -285.3784484863281, "KL/std": 116.7973403930664, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.003748733550310135, "fcm_dpo/delta": 0.017452098429203033, "fcm_dpo/margin": 103.16229248046875, "fcm_dpo/q_t": 0.41033506393432617, "grad_norm": 10.245551109313965, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.1670355796813965, "logits/rejected": -1.739976406097412, "logps/chosen": -275.1827392578125, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -381.296630859375, "loss": 1.0971, "margin_dpo/margin_mean": 103.16230010986328, "margin_dpo/margin_std": 122.95185852050781, "step": 609 }, { "KL/chosen_KL_mean": -166.59432983398438, "KL/mean": -221.85629272460938, "KL/rejected_KL_mean": -277.1182556152344, "KL/std": 110.46842956542969, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0037200618535280228, "fcm_dpo/delta": -0.0220570657402277, "fcm_dpo/margin": 110.52392578125, "fcm_dpo/q_t": 0.40001940727233887, "grad_norm": 8.26205825805664, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.737299919128418, "logits/rejected": -1.1926950216293335, "logps/chosen": -261.29461669921875, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -366.8056640625, "loss": 1.0629, "margin_dpo/margin_mean": 110.52392578125, "margin_dpo/margin_std": 117.19044494628906, "step": 610 }, { "KL/chosen_KL_mean": -175.27462768554688, "KL/mean": -229.12286376953125, "KL/rejected_KL_mean": -282.97113037109375, "KL/std": 114.62615203857422, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0038343756459653378, "fcm_dpo/delta": -0.027197841554880142, "fcm_dpo/margin": 107.69646453857422, "fcm_dpo/q_t": 0.4007180333137512, "grad_norm": 9.86976146697998, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.044619083404541, "logits/rejected": -1.6977062225341797, "logps/chosen": -262.0579833984375, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -372.8141784667969, "loss": 1.0674, "margin_dpo/margin_mean": 107.69645690917969, "margin_dpo/margin_std": 116.46023559570312, "step": 611 }, { "KL/chosen_KL_mean": -193.84970092773438, "KL/mean": -248.76661682128906, "KL/rejected_KL_mean": -303.6835632324219, "KL/std": 108.68096923828125, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.003591400571167469, "fcm_dpo/delta": 0.008999021723866463, "fcm_dpo/margin": 109.83390808105469, "fcm_dpo/q_t": 0.4073523283004761, "grad_norm": 8.541043281555176, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.123892068862915, "logits/rejected": -1.2865272760391235, "logps/chosen": -294.8698425292969, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -397.46661376953125, "loss": 1.0702, "margin_dpo/margin_mean": 109.83390045166016, "margin_dpo/margin_std": 114.20893859863281, "step": 612 }, { "KL/chosen_KL_mean": -154.19638061523438, "KL/mean": -216.7333526611328, "KL/rejected_KL_mean": -279.27032470703125, "KL/std": 110.97260284423828, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0033657357562333345, "fcm_dpo/delta": -0.039123985916376114, "fcm_dpo/margin": 125.07391357421875, "fcm_dpo/q_t": 0.3956943452358246, "grad_norm": 7.36722469329834, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.8887113332748413, "logits/rejected": -1.6909781694412231, "logps/chosen": -243.1850128173828, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -389.26580810546875, "loss": 1.0408, "margin_dpo/margin_mean": 125.07391357421875, "margin_dpo/margin_std": 108.45684814453125, "step": 613 }, { "KL/chosen_KL_mean": -194.4849853515625, "KL/mean": -245.27745056152344, "KL/rejected_KL_mean": -296.0699157714844, "KL/std": 105.448486328125, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0036228166427463293, "fcm_dpo/delta": 0.0505104660987854, "fcm_dpo/margin": 101.5849609375, "fcm_dpo/q_t": 0.4169062674045563, "grad_norm": 9.838240623474121, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.3196218013763428, "logits/rejected": -1.3128750324249268, "logps/chosen": -274.68505859375, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -405.93231201171875, "loss": 1.1065, "margin_dpo/margin_mean": 101.5849609375, "margin_dpo/margin_std": 129.9075469970703, "step": 614 }, { "KL/chosen_KL_mean": -180.88412475585938, "KL/mean": -222.33145141601562, "KL/rejected_KL_mean": -263.7787780761719, "KL/std": 106.7667236328125, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.00368721061386168, "fcm_dpo/delta": -0.011486921459436417, "fcm_dpo/margin": 82.89461517333984, "fcm_dpo/q_t": 0.42598122358322144, "grad_norm": 8.58275032043457, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.8514395952224731, "logits/rejected": -1.4578708410263062, "logps/chosen": -281.31939697265625, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -364.9588317871094, "loss": 1.1422, "margin_dpo/margin_mean": 82.89461517333984, "margin_dpo/margin_std": 102.32865142822266, "step": 615 }, { "KL/chosen_KL_mean": -183.52247619628906, "KL/mean": -240.36044311523438, "KL/rejected_KL_mean": -297.19842529296875, "KL/std": 106.22665405273438, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0036491299979388714, "fcm_dpo/delta": -0.02815670520067215, "fcm_dpo/margin": 113.6759262084961, "fcm_dpo/q_t": 0.3997732996940613, "grad_norm": 8.315059661865234, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.700222373008728, "logits/rejected": -1.1932988166809082, "logps/chosen": -276.0154113769531, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -379.2590637207031, "loss": 1.0663, "margin_dpo/margin_mean": 113.67591857910156, "margin_dpo/margin_std": 124.83656311035156, "step": 616 }, { "KL/chosen_KL_mean": -191.12066650390625, "KL/mean": -235.8386993408203, "KL/rejected_KL_mean": -280.5567626953125, "KL/std": 103.76573181152344, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0038889972493052483, "fcm_dpo/delta": 0.07889638841152191, "fcm_dpo/margin": 89.43608093261719, "fcm_dpo/q_t": 0.42325645685195923, "grad_norm": 9.403630256652832, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6301215887069702, "logits/rejected": -0.9936158061027527, "logps/chosen": -282.2176513671875, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -350.966796875, "loss": 1.1175, "margin_dpo/margin_mean": 89.43608093261719, "margin_dpo/margin_std": 115.92320251464844, "step": 617 }, { "KL/chosen_KL_mean": -192.927490234375, "KL/mean": -243.25135803222656, "KL/rejected_KL_mean": -293.5752258300781, "KL/std": 115.24060821533203, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.00397268682718277, "fcm_dpo/delta": -0.0003412282094359398, "fcm_dpo/margin": 100.64775085449219, "fcm_dpo/q_t": 0.40549546480178833, "grad_norm": 8.630677223205566, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.977842092514038, "logits/rejected": -1.611716389656067, "logps/chosen": -294.9480895996094, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -393.37640380859375, "loss": 1.088, "margin_dpo/margin_mean": 100.64775085449219, "margin_dpo/margin_std": 127.69452667236328, "step": 618 }, { "KL/chosen_KL_mean": -186.8675537109375, "KL/mean": -231.8062744140625, "KL/rejected_KL_mean": -276.7450256347656, "KL/std": 113.64497375488281, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004224720411002636, "fcm_dpo/delta": 0.03265788406133652, "fcm_dpo/margin": 89.87748718261719, "fcm_dpo/q_t": 0.4161718487739563, "grad_norm": 9.13076400756836, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.6555290222167969, "logits/rejected": -1.3010427951812744, "logps/chosen": -276.6089172363281, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -376.64642333984375, "loss": 1.1194, "margin_dpo/margin_mean": 89.87748718261719, "margin_dpo/margin_std": 136.66563415527344, "step": 619 }, { "KL/chosen_KL_mean": -178.8975830078125, "KL/mean": -227.89410400390625, "KL/rejected_KL_mean": -276.890625, "KL/std": 119.03985595703125, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0034739631228148937, "fcm_dpo/delta": -0.10706700384616852, "fcm_dpo/margin": 97.9930648803711, "fcm_dpo/q_t": 0.40776526927948, "grad_norm": 8.543606758117676, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.1572556495666504, "logits/rejected": -1.6279418468475342, "logps/chosen": -264.02191162109375, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -362.30316162109375, "loss": 1.1278, "margin_dpo/margin_mean": 97.9930648803711, "margin_dpo/margin_std": 127.8524169921875, "step": 620 }, { "KL/chosen_KL_mean": -182.1210479736328, "KL/mean": -230.4354248046875, "KL/rejected_KL_mean": -278.74981689453125, "KL/std": 114.39322662353516, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.003337735775858164, "fcm_dpo/delta": -0.040836963802576065, "fcm_dpo/margin": 96.62877655029297, "fcm_dpo/q_t": 0.4209139943122864, "grad_norm": 7.764411449432373, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.8189711570739746, "logits/rejected": -1.5511138439178467, "logps/chosen": -271.36944580078125, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -374.2144470214844, "loss": 1.1467, "margin_dpo/margin_mean": 96.62877655029297, "margin_dpo/margin_std": 133.39520263671875, "step": 621 }, { "KL/chosen_KL_mean": -180.35296630859375, "KL/mean": -231.60781860351562, "KL/rejected_KL_mean": -282.8627014160156, "KL/std": 102.44929504394531, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.003489724826067686, "fcm_dpo/delta": 0.06612639129161835, "fcm_dpo/margin": 102.50973510742188, "fcm_dpo/q_t": 0.4198324680328369, "grad_norm": 10.149873733520508, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.5453710556030273, "logits/rejected": -0.8962064981460571, "logps/chosen": -280.1470947265625, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -373.69091796875, "loss": 1.0987, "margin_dpo/margin_mean": 102.5097427368164, "margin_dpo/margin_std": 111.28376770019531, "step": 622 }, { "KL/chosen_KL_mean": -187.14022827148438, "KL/mean": -223.52069091796875, "KL/rejected_KL_mean": -259.901123046875, "KL/std": 111.1199951171875, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0043245647102594376, "fcm_dpo/delta": 0.12556609511375427, "fcm_dpo/margin": 72.76090240478516, "fcm_dpo/q_t": 0.43608760833740234, "grad_norm": 10.768909454345703, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.1749253273010254, "logits/rejected": -1.732398271560669, "logps/chosen": -277.59576416015625, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -351.223876953125, "loss": 1.1843, "margin_dpo/margin_mean": 72.76091003417969, "margin_dpo/margin_std": 136.98724365234375, "step": 623 }, { "KL/chosen_KL_mean": -193.1251983642578, "KL/mean": -251.42764282226562, "KL/rejected_KL_mean": -309.7301025390625, "KL/std": 109.90451049804688, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.003906491212546825, "fcm_dpo/delta": -0.10964608937501907, "fcm_dpo/margin": 116.60491943359375, "fcm_dpo/q_t": 0.3833053708076477, "grad_norm": 10.237390518188477, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.5014538764953613, "logits/rejected": -1.5833245515823364, "logps/chosen": -303.00042724609375, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -414.5033264160156, "loss": 1.0383, "margin_dpo/margin_mean": 116.60491180419922, "margin_dpo/margin_std": 125.76460266113281, "step": 624 }, { "KL/chosen_KL_mean": -201.10609436035156, "KL/mean": -242.6293182373047, "KL/rejected_KL_mean": -284.15252685546875, "KL/std": 109.1131362915039, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0042776307091116905, "fcm_dpo/delta": 0.06345093995332718, "fcm_dpo/margin": 83.04643249511719, "fcm_dpo/q_t": 0.42123591899871826, "grad_norm": 9.957439422607422, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.924459457397461, "logits/rejected": -1.562745451927185, "logps/chosen": -288.2742614746094, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -376.0140075683594, "loss": 1.1214, "margin_dpo/margin_mean": 83.04643249511719, "margin_dpo/margin_std": 116.76962280273438, "step": 625 }, { "KL/chosen_KL_mean": -189.80743408203125, "KL/mean": -231.91506958007812, "KL/rejected_KL_mean": -274.022705078125, "KL/std": 120.48194885253906, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.004563533701002598, "fcm_dpo/delta": 0.021859100088477135, "fcm_dpo/margin": 84.21527862548828, "fcm_dpo/q_t": 0.4141620397567749, "grad_norm": 12.42094612121582, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.2341217994689941, "logits/rejected": -0.7871816158294678, "logps/chosen": -294.0316467285156, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -378.2001953125, "loss": 1.1228, "margin_dpo/margin_mean": 84.21527862548828, "margin_dpo/margin_std": 131.31936645507812, "step": 626 }, { "KL/chosen_KL_mean": -167.6511993408203, "KL/mean": -220.68850708007812, "KL/rejected_KL_mean": -273.725830078125, "KL/std": 123.97897338867188, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004084354266524315, "fcm_dpo/delta": -0.05956938862800598, "fcm_dpo/margin": 106.07462310791016, "fcm_dpo/q_t": 0.39623594284057617, "grad_norm": 7.655340671539307, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.3955450057983398, "logits/rejected": -1.5500872135162354, "logps/chosen": -242.5858154296875, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -386.2987060546875, "loss": 1.0711, "margin_dpo/margin_mean": 106.07463073730469, "margin_dpo/margin_std": 137.12155151367188, "step": 627 }, { "KL/chosen_KL_mean": -177.60476684570312, "KL/mean": -231.82476806640625, "KL/rejected_KL_mean": -286.04473876953125, "KL/std": 111.46701049804688, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0038206344470381737, "fcm_dpo/delta": -0.024627530947327614, "fcm_dpo/margin": 108.43998718261719, "fcm_dpo/q_t": 0.40116366744041443, "grad_norm": 9.27859115600586, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.9642155170440674, "logits/rejected": -1.7030571699142456, "logps/chosen": -271.29583740234375, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -393.388671875, "loss": 1.0695, "margin_dpo/margin_mean": 108.43998718261719, "margin_dpo/margin_std": 125.644287109375, "step": 628 }, { "KL/chosen_KL_mean": -191.24415588378906, "KL/mean": -240.6210479736328, "KL/rejected_KL_mean": -289.9979248046875, "KL/std": 112.11629486083984, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0038484809920191765, "fcm_dpo/delta": 0.031160593032836914, "fcm_dpo/margin": 98.7537612915039, "fcm_dpo/q_t": 0.41235750913619995, "grad_norm": 9.491218566894531, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.9260261058807373, "logits/rejected": -1.480367660522461, "logps/chosen": -294.47454833984375, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -387.1663513183594, "loss": 1.0956, "margin_dpo/margin_mean": 98.75376892089844, "margin_dpo/margin_std": 119.63480377197266, "step": 629 }, { "KL/chosen_KL_mean": -173.53781127929688, "KL/mean": -205.4443359375, "KL/rejected_KL_mean": -237.350830078125, "KL/std": 104.29745483398438, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.004194583743810654, "fcm_dpo/delta": 0.04070357233285904, "fcm_dpo/margin": 63.813018798828125, "fcm_dpo/q_t": 0.4391627609729767, "grad_norm": 11.68310832977295, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.9379148483276367, "logits/rejected": -1.4072331190109253, "logps/chosen": -267.43536376953125, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -326.7252197265625, "loss": 1.1955, "margin_dpo/margin_mean": 63.81300735473633, "margin_dpo/margin_std": 117.75784301757812, "step": 630 }, { "KL/chosen_KL_mean": -171.4090576171875, "KL/mean": -219.42654418945312, "KL/rejected_KL_mean": -267.44403076171875, "KL/std": 104.20513916015625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.004232443869113922, "fcm_dpo/delta": -0.011658208444714546, "fcm_dpo/margin": 96.03498077392578, "fcm_dpo/q_t": 0.40369170904159546, "grad_norm": 9.161296844482422, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.361480712890625, "logits/rejected": -1.9244675636291504, "logps/chosen": -259.17987060546875, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -356.12646484375, "loss": 1.081, "margin_dpo/margin_mean": 96.03497314453125, "margin_dpo/margin_std": 118.9144515991211, "step": 631 }, { "KL/chosen_KL_mean": -168.761474609375, "KL/mean": -221.1422576904297, "KL/rejected_KL_mean": -273.52301025390625, "KL/std": 109.14592742919922, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.003883753903210163, "fcm_dpo/delta": -0.023460566997528076, "fcm_dpo/margin": 104.76156616210938, "fcm_dpo/q_t": 0.4025465250015259, "grad_norm": 11.344853401184082, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.059548854827881, "logits/rejected": -1.6356297731399536, "logps/chosen": -257.38800048828125, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -364.97393798828125, "loss": 1.0892, "margin_dpo/margin_mean": 104.76156616210938, "margin_dpo/margin_std": 134.80462646484375, "step": 632 }, { "KL/chosen_KL_mean": -209.85525512695312, "KL/mean": -240.19407653808594, "KL/rejected_KL_mean": -270.53289794921875, "KL/std": 110.52017211914062, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0041841198690235615, "fcm_dpo/delta": 0.027486614882946014, "fcm_dpo/margin": 60.67763137817383, "fcm_dpo/q_t": 0.4441055655479431, "grad_norm": 11.261579513549805, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.3442649841308594, "logits/rejected": -2.159053087234497, "logps/chosen": -291.22967529296875, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -369.15863037109375, "loss": 1.2386, "margin_dpo/margin_mean": 60.677635192871094, "margin_dpo/margin_std": 148.62936401367188, "step": 633 }, { "KL/chosen_KL_mean": -193.4292449951172, "KL/mean": -233.07876586914062, "KL/rejected_KL_mean": -272.7283020019531, "KL/std": 100.68098449707031, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.004114243667572737, "fcm_dpo/delta": -0.04496065899729729, "fcm_dpo/margin": 79.299072265625, "fcm_dpo/q_t": 0.42000672221183777, "grad_norm": 10.751233100891113, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.278041124343872, "logits/rejected": -1.1035919189453125, "logps/chosen": -289.5520935058594, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -385.57611083984375, "loss": 1.1389, "margin_dpo/margin_mean": 79.299072265625, "margin_dpo/margin_std": 103.21263122558594, "step": 634 }, { "KL/chosen_KL_mean": -203.36485290527344, "KL/mean": -233.70462036132812, "KL/rejected_KL_mean": -264.04437255859375, "KL/std": 109.88600158691406, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.004092206247150898, "fcm_dpo/delta": 0.03980815410614014, "fcm_dpo/margin": 60.67949676513672, "fcm_dpo/q_t": 0.4445072412490845, "grad_norm": 11.778306007385254, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.6583452224731445, "logits/rejected": -1.107499122619629, "logps/chosen": -302.0489501953125, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -353.94354248046875, "loss": 1.2219, "margin_dpo/margin_mean": 60.67949676513672, "margin_dpo/margin_std": 132.08792114257812, "step": 635 }, { "KL/chosen_KL_mean": -179.08676147460938, "KL/mean": -232.29171752929688, "KL/rejected_KL_mean": -285.4966735839844, "KL/std": 112.73333740234375, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004011464770883322, "fcm_dpo/delta": -0.0467059388756752, "fcm_dpo/margin": 106.4099349975586, "fcm_dpo/q_t": 0.39600124955177307, "grad_norm": 9.43135929107666, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.5647802352905273, "logits/rejected": -1.0869214534759521, "logps/chosen": -269.5047302246094, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -373.20355224609375, "loss": 1.0534, "margin_dpo/margin_mean": 106.4099349975586, "margin_dpo/margin_std": 112.21414184570312, "step": 636 }, { "KL/chosen_KL_mean": -179.98703002929688, "KL/mean": -236.8654022216797, "KL/rejected_KL_mean": -293.7437744140625, "KL/std": 125.1199951171875, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0037105996161699295, "fcm_dpo/delta": -0.03811420500278473, "fcm_dpo/margin": 113.75675964355469, "fcm_dpo/q_t": 0.3970419764518738, "grad_norm": 7.611179828643799, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.025611162185669, "logits/rejected": -1.9239931106567383, "logps/chosen": -262.4367370605469, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -397.77239990234375, "loss": 1.0562, "margin_dpo/margin_mean": 113.75675201416016, "margin_dpo/margin_std": 118.36184692382812, "step": 637 }, { "KL/chosen_KL_mean": -169.13177490234375, "KL/mean": -223.94229125976562, "KL/rejected_KL_mean": -278.7528076171875, "KL/std": 111.11815643310547, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0037070652469992638, "fcm_dpo/delta": -0.017498120665550232, "fcm_dpo/margin": 109.62104797363281, "fcm_dpo/q_t": 0.4012555480003357, "grad_norm": 7.861967086791992, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.4996144771575928, "logits/rejected": -0.8118008971214294, "logps/chosen": -261.0567626953125, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -365.03985595703125, "loss": 1.0609, "margin_dpo/margin_mean": 109.62104797363281, "margin_dpo/margin_std": 103.22618103027344, "step": 638 }, { "KL/chosen_KL_mean": -224.55215454101562, "KL/mean": -250.35830688476562, "KL/rejected_KL_mean": -276.1644287109375, "KL/std": 102.90052032470703, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0037945895455777645, "fcm_dpo/delta": 0.08638568222522736, "fcm_dpo/margin": 51.61226272583008, "fcm_dpo/q_t": 0.4569549560546875, "grad_norm": 8.540910720825195, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.500808596611023, "logits/rejected": -0.752601683139801, "logps/chosen": -329.0797119140625, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -357.644775390625, "loss": 1.2536, "margin_dpo/margin_mean": 51.612266540527344, "margin_dpo/margin_std": 127.99241638183594, "step": 639 }, { "KL/chosen_KL_mean": -190.81591796875, "KL/mean": -240.73876953125, "KL/rejected_KL_mean": -290.66162109375, "KL/std": 111.65750122070312, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.004237450193613768, "fcm_dpo/delta": -0.061611540615558624, "fcm_dpo/margin": 99.845703125, "fcm_dpo/q_t": 0.39364397525787354, "grad_norm": 13.039816856384277, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.939971685409546, "logits/rejected": -1.4322419166564941, "logps/chosen": -288.701171875, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -403.36663818359375, "loss": 1.0887, "margin_dpo/margin_mean": 99.845703125, "margin_dpo/margin_std": 111.98009490966797, "step": 640 }, { "KL/chosen_KL_mean": -174.44528198242188, "KL/mean": -226.30564880371094, "KL/rejected_KL_mean": -278.166015625, "KL/std": 129.78750610351562, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0037611054722219706, "fcm_dpo/delta": 0.015808213502168655, "fcm_dpo/margin": 103.72076416015625, "fcm_dpo/q_t": 0.41258272528648376, "grad_norm": 8.087779998779297, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.5708346366882324, "logits/rejected": -2.0594120025634766, "logps/chosen": -270.8909606933594, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -373.30169677734375, "loss": 1.1006, "margin_dpo/margin_mean": 103.72076416015625, "margin_dpo/margin_std": 144.43911743164062, "step": 641 }, { "KL/chosen_KL_mean": -194.87344360351562, "KL/mean": -242.7490234375, "KL/rejected_KL_mean": -290.62457275390625, "KL/std": 111.49491882324219, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.004008971154689789, "fcm_dpo/delta": 0.022690841928124428, "fcm_dpo/margin": 95.75111389160156, "fcm_dpo/q_t": 0.4139209985733032, "grad_norm": 11.340514183044434, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.6047385931015015, "logits/rejected": -1.276477336883545, "logps/chosen": -295.63330078125, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -405.33221435546875, "loss": 1.1113, "margin_dpo/margin_mean": 95.75111389160156, "margin_dpo/margin_std": 139.60003662109375, "step": 642 }, { "KL/chosen_KL_mean": -181.2464599609375, "KL/mean": -227.6654815673828, "KL/rejected_KL_mean": -274.08453369140625, "KL/std": 103.76243591308594, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.003944522235542536, "fcm_dpo/delta": 0.0452963225543499, "fcm_dpo/margin": 92.83805847167969, "fcm_dpo/q_t": 0.41609328985214233, "grad_norm": 11.660050392150879, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.178657054901123, "logits/rejected": -1.8052642345428467, "logps/chosen": -265.9901123046875, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -368.4029541015625, "loss": 1.119, "margin_dpo/margin_mean": 92.83806610107422, "margin_dpo/margin_std": 119.31580352783203, "step": 643 }, { "KL/chosen_KL_mean": -188.20547485351562, "KL/mean": -225.98110961914062, "KL/rejected_KL_mean": -263.7567443847656, "KL/std": 104.86798095703125, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004757949151098728, "fcm_dpo/delta": 0.06207982078194618, "fcm_dpo/margin": 75.55123138427734, "fcm_dpo/q_t": 0.42117783427238464, "grad_norm": 12.316163063049316, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.7614539861679077, "logits/rejected": -1.4923545122146606, "logps/chosen": -273.5282287597656, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -358.3653564453125, "loss": 1.1286, "margin_dpo/margin_mean": 75.55123138427734, "margin_dpo/margin_std": 114.39704132080078, "step": 644 }, { "KL/chosen_KL_mean": -199.39306640625, "KL/mean": -224.5082550048828, "KL/rejected_KL_mean": -249.6234588623047, "KL/std": 113.54199981689453, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.005164915230125189, "fcm_dpo/delta": 0.05558210611343384, "fcm_dpo/margin": 50.23040771484375, "fcm_dpo/q_t": 0.44426625967025757, "grad_norm": 12.803762435913086, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.2191295623779297, "logits/rejected": -1.0481048822402954, "logps/chosen": -281.9833068847656, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -352.627197265625, "loss": 1.2465, "margin_dpo/margin_mean": 50.23040771484375, "margin_dpo/margin_std": 128.9382781982422, "step": 645 }, { "KL/chosen_KL_mean": -176.78076171875, "KL/mean": -225.6451416015625, "KL/rejected_KL_mean": -274.509521484375, "KL/std": 98.8961181640625, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.004435589537024498, "fcm_dpo/delta": -0.08747022598981857, "fcm_dpo/margin": 97.728759765625, "fcm_dpo/q_t": 0.388482004404068, "grad_norm": 9.608895301818848, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.9299499988555908, "logits/rejected": -1.338326334953308, "logps/chosen": -267.9728088378906, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -358.287841796875, "loss": 1.0566, "margin_dpo/margin_mean": 97.728759765625, "margin_dpo/margin_std": 108.04932403564453, "step": 646 }, { "KL/chosen_KL_mean": -190.97116088867188, "KL/mean": -234.54177856445312, "KL/rejected_KL_mean": -278.1123962402344, "KL/std": 119.36813354492188, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004580165259540081, "fcm_dpo/delta": 0.0014598136767745018, "fcm_dpo/margin": 87.14126586914062, "fcm_dpo/q_t": 0.40976202487945557, "grad_norm": 11.136417388916016, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.5198596715927124, "logits/rejected": -1.3834283351898193, "logps/chosen": -279.81561279296875, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -377.6107177734375, "loss": 1.1118, "margin_dpo/margin_mean": 87.14126586914062, "margin_dpo/margin_std": 132.04574584960938, "step": 647 }, { "KL/chosen_KL_mean": -179.01278686523438, "KL/mean": -238.150390625, "KL/rejected_KL_mean": -297.2879943847656, "KL/std": 122.4760513305664, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.003922610078006983, "fcm_dpo/delta": -0.11907849460840225, "fcm_dpo/margin": 118.27520751953125, "fcm_dpo/q_t": 0.3814942240715027, "grad_norm": 9.543350219726562, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.8873274326324463, "logits/rejected": -1.815600037574768, "logps/chosen": -266.86090087890625, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -401.95806884765625, "loss": 1.0418, "margin_dpo/margin_mean": 118.27520751953125, "margin_dpo/margin_std": 130.57003784179688, "step": 648 }, { "KL/chosen_KL_mean": -163.5318145751953, "KL/mean": -213.79010009765625, "KL/rejected_KL_mean": -264.04840087890625, "KL/std": 106.80451965332031, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0037007341161370277, "fcm_dpo/delta": 0.04241820424795151, "fcm_dpo/margin": 100.5166015625, "fcm_dpo/q_t": 0.4146074950695038, "grad_norm": 9.996441841125488, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.484617233276367, "logits/rejected": -2.0876147747039795, "logps/chosen": -253.2243194580078, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -352.7550048828125, "loss": 1.0907, "margin_dpo/margin_mean": 100.5166015625, "margin_dpo/margin_std": 104.00312805175781, "step": 649 }, { "KL/chosen_KL_mean": -194.07156372070312, "KL/mean": -241.14962768554688, "KL/rejected_KL_mean": -288.2276916503906, "KL/std": 115.24767303466797, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004103176761418581, "fcm_dpo/delta": 0.021683285012841225, "fcm_dpo/margin": 94.1561279296875, "fcm_dpo/q_t": 0.41205382347106934, "grad_norm": 9.797582626342773, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.245701789855957, "logits/rejected": -1.0449899435043335, "logps/chosen": -284.0021667480469, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -390.84051513671875, "loss": 1.1094, "margin_dpo/margin_mean": 94.1561279296875, "margin_dpo/margin_std": 136.19216918945312, "step": 650 }, { "KL/chosen_KL_mean": -160.09527587890625, "KL/mean": -217.7837677001953, "KL/rejected_KL_mean": -275.47222900390625, "KL/std": 111.25703430175781, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.003727199276909232, "fcm_dpo/delta": -0.055380407720804214, "fcm_dpo/margin": 115.37696838378906, "fcm_dpo/q_t": 0.39239633083343506, "grad_norm": 8.388117790222168, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.081007957458496, "logits/rejected": -1.9054217338562012, "logps/chosen": -239.28260803222656, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -381.40557861328125, "loss": 1.0411, "margin_dpo/margin_mean": 115.37696838378906, "margin_dpo/margin_std": 108.61811828613281, "step": 651 }, { "KL/chosen_KL_mean": -167.25015258789062, "KL/mean": -215.64263916015625, "KL/rejected_KL_mean": -264.03509521484375, "KL/std": 103.54389953613281, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.003940294496715069, "fcm_dpo/delta": 0.02678321674466133, "fcm_dpo/margin": 96.78495788574219, "fcm_dpo/q_t": 0.4112705588340759, "grad_norm": 9.320079803466797, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.5432748794555664, "logits/rejected": -2.252794027328491, "logps/chosen": -269.0403747558594, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -380.35968017578125, "loss": 1.0773, "margin_dpo/margin_mean": 96.78495788574219, "margin_dpo/margin_std": 98.68894958496094, "step": 652 }, { "KL/chosen_KL_mean": -191.82504272460938, "KL/mean": -239.25421142578125, "KL/rejected_KL_mean": -286.683349609375, "KL/std": 113.43395233154297, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004075777716934681, "fcm_dpo/delta": 0.020319540053606033, "fcm_dpo/margin": 94.85832214355469, "fcm_dpo/q_t": 0.4093397259712219, "grad_norm": 9.756814002990723, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.130279064178467, "logits/rejected": -1.8762699365615845, "logps/chosen": -285.17864990234375, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -395.8066101074219, "loss": 1.0867, "margin_dpo/margin_mean": 94.85832977294922, "margin_dpo/margin_std": 111.10971069335938, "step": 653 }, { "KL/chosen_KL_mean": -192.541259765625, "KL/mean": -233.25634765625, "KL/rejected_KL_mean": -273.971435546875, "KL/std": 108.80006408691406, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.004438761621713638, "fcm_dpo/delta": 0.05952010303735733, "fcm_dpo/margin": 81.43017578125, "fcm_dpo/q_t": 0.42280304431915283, "grad_norm": 12.541304588317871, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.362118721008301, "logits/rejected": -2.183539390563965, "logps/chosen": -281.65679931640625, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -378.891357421875, "loss": 1.1439, "margin_dpo/margin_mean": 81.43017578125, "margin_dpo/margin_std": 137.55726623535156, "step": 654 }, { "KL/chosen_KL_mean": -176.20314025878906, "KL/mean": -224.53475952148438, "KL/rejected_KL_mean": -272.86639404296875, "KL/std": 118.22506713867188, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.004352295771241188, "fcm_dpo/delta": -0.03566489741206169, "fcm_dpo/margin": 96.66325378417969, "fcm_dpo/q_t": 0.3992576599121094, "grad_norm": 9.315327644348145, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.8777406215667725, "logits/rejected": -1.8827571868896484, "logps/chosen": -257.2392578125, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -375.6687316894531, "loss": 1.083, "margin_dpo/margin_mean": 96.66325378417969, "margin_dpo/margin_std": 126.80682373046875, "step": 655 }, { "KL/chosen_KL_mean": -178.04112243652344, "KL/mean": -239.47198486328125, "KL/rejected_KL_mean": -300.90289306640625, "KL/std": 118.55474853515625, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0036259321495890617, "fcm_dpo/delta": -0.08893150091171265, "fcm_dpo/margin": 122.86174011230469, "fcm_dpo/q_t": 0.38725727796554565, "grad_norm": 8.980226516723633, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.4840726852416992, "logits/rejected": -1.5549906492233276, "logps/chosen": -263.366455078125, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -419.24151611328125, "loss": 1.0356, "margin_dpo/margin_mean": 122.86174011230469, "margin_dpo/margin_std": 122.34724426269531, "step": 656 }, { "KL/chosen_KL_mean": -193.40625, "KL/mean": -239.70220947265625, "KL/rejected_KL_mean": -285.9981689453125, "KL/std": 110.19371032714844, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.003807145170867443, "fcm_dpo/delta": 0.07270748168230057, "fcm_dpo/margin": 92.59193420410156, "fcm_dpo/q_t": 0.4233360290527344, "grad_norm": 10.566231727600098, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9400272369384766, "logits/rejected": -1.7732443809509277, "logps/chosen": -273.6039733886719, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -388.5794677734375, "loss": 1.1311, "margin_dpo/margin_mean": 92.59193420410156, "margin_dpo/margin_std": 132.83250427246094, "step": 657 }, { "KL/chosen_KL_mean": -190.4767303466797, "KL/mean": -232.88429260253906, "KL/rejected_KL_mean": -275.2918701171875, "KL/std": 103.33901977539062, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.004455907270312309, "fcm_dpo/delta": 0.030175216495990753, "fcm_dpo/margin": 84.81514739990234, "fcm_dpo/q_t": 0.4136660695075989, "grad_norm": 9.574654579162598, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.9732880592346191, "logits/rejected": -1.5025033950805664, "logps/chosen": -285.11126708984375, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -376.9234924316406, "loss": 1.1029, "margin_dpo/margin_mean": 84.81513977050781, "margin_dpo/margin_std": 110.81301879882812, "step": 658 }, { "KL/chosen_KL_mean": -173.20248413085938, "KL/mean": -224.6419677734375, "KL/rejected_KL_mean": -276.08148193359375, "KL/std": 111.455078125, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.003984848968684673, "fcm_dpo/delta": -0.02642359957098961, "fcm_dpo/margin": 102.87895202636719, "fcm_dpo/q_t": 0.4023115038871765, "grad_norm": 8.573958396911621, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.152470827102661, "logits/rejected": -1.1904791593551636, "logps/chosen": -285.75836181640625, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -370.0030517578125, "loss": 1.1022, "margin_dpo/margin_mean": 102.87895965576172, "margin_dpo/margin_std": 144.26956176757812, "step": 659 }, { "KL/chosen_KL_mean": -182.75485229492188, "KL/mean": -235.36965942382812, "KL/rejected_KL_mean": -287.9844665527344, "KL/std": 108.21517944335938, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.003954469226300716, "fcm_dpo/delta": -0.027955632656812668, "fcm_dpo/margin": 105.2296142578125, "fcm_dpo/q_t": 0.4001622200012207, "grad_norm": 10.951987266540527, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.9371836185455322, "logits/rejected": -1.3201818466186523, "logps/chosen": -276.13226318359375, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -369.3792724609375, "loss": 1.0643, "margin_dpo/margin_mean": 105.22962188720703, "margin_dpo/margin_std": 118.40858459472656, "step": 660 }, { "KL/chosen_KL_mean": -179.39801025390625, "KL/mean": -238.43182373046875, "KL/rejected_KL_mean": -297.46563720703125, "KL/std": 121.52531433105469, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0036623452324420214, "fcm_dpo/delta": -0.05677337199449539, "fcm_dpo/margin": 118.0676498413086, "fcm_dpo/q_t": 0.394684761762619, "grad_norm": 8.693258285522461, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.8435614109039307, "logits/rejected": -1.3327176570892334, "logps/chosen": -267.253173828125, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -389.86895751953125, "loss": 1.0589, "margin_dpo/margin_mean": 118.06764221191406, "margin_dpo/margin_std": 135.48348999023438, "step": 661 }, { "KL/chosen_KL_mean": -175.073974609375, "KL/mean": -229.34359741210938, "KL/rejected_KL_mean": -283.61322021484375, "KL/std": 105.62258911132812, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0036186687648296356, "fcm_dpo/delta": 0.010547422803938389, "fcm_dpo/margin": 108.53923034667969, "fcm_dpo/q_t": 0.40749233961105347, "grad_norm": 9.345247268676758, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.6995309591293335, "logits/rejected": -1.1259841918945312, "logps/chosen": -277.853759765625, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -378.8385009765625, "loss": 1.0721, "margin_dpo/margin_mean": 108.53923034667969, "margin_dpo/margin_std": 111.81908416748047, "step": 662 }, { "KL/chosen_KL_mean": -177.70394897460938, "KL/mean": -226.72525024414062, "KL/rejected_KL_mean": -275.74652099609375, "KL/std": 109.47953796386719, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.003844293300062418, "fcm_dpo/delta": 0.034141410142183304, "fcm_dpo/margin": 98.04255676269531, "fcm_dpo/q_t": 0.4140503704547882, "grad_norm": 10.556899070739746, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.3128662109375, "logits/rejected": -2.101339817047119, "logps/chosen": -274.050537109375, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -396.2729797363281, "loss": 1.1026, "margin_dpo/margin_mean": 98.04255676269531, "margin_dpo/margin_std": 127.70820617675781, "step": 663 }, { "KL/chosen_KL_mean": -196.51882934570312, "KL/mean": -240.89715576171875, "KL/rejected_KL_mean": -285.2755126953125, "KL/std": 112.72706604003906, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.004183623008430004, "fcm_dpo/delta": 0.041208114475011826, "fcm_dpo/margin": 88.7567138671875, "fcm_dpo/q_t": 0.415682315826416, "grad_norm": 8.88394832611084, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.752948522567749, "logits/rejected": -1.583505392074585, "logps/chosen": -288.3612365722656, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -397.1122131347656, "loss": 1.1021, "margin_dpo/margin_mean": 88.7567138671875, "margin_dpo/margin_std": 110.88386535644531, "step": 664 }, { "KL/chosen_KL_mean": -168.92645263671875, "KL/mean": -217.39718627929688, "KL/rejected_KL_mean": -265.867919921875, "KL/std": 107.89190673828125, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.00418365653604269, "fcm_dpo/delta": -0.010507695376873016, "fcm_dpo/margin": 96.94146728515625, "fcm_dpo/q_t": 0.40335601568222046, "grad_norm": 8.283790588378906, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.4065263271331787, "logits/rejected": -1.9780369997024536, "logps/chosen": -257.1126403808594, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -357.7799377441406, "loss": 1.0749, "margin_dpo/margin_mean": 96.94146728515625, "margin_dpo/margin_std": 112.09625244140625, "step": 665 }, { "KL/chosen_KL_mean": -184.3155517578125, "KL/mean": -233.93145751953125, "KL/rejected_KL_mean": -283.5473937988281, "KL/std": 106.31968688964844, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.003983003087341785, "fcm_dpo/delta": 0.005934743210673332, "fcm_dpo/margin": 99.23184204101562, "fcm_dpo/q_t": 0.4084092974662781, "grad_norm": 10.487371444702148, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.7294948101043701, "logits/rejected": -1.3571486473083496, "logps/chosen": -285.2901611328125, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -384.79730224609375, "loss": 1.0935, "margin_dpo/margin_mean": 99.23184204101562, "margin_dpo/margin_std": 126.25028228759766, "step": 666 }, { "KL/chosen_KL_mean": -202.2776336669922, "KL/mean": -252.7703399658203, "KL/rejected_KL_mean": -303.2630615234375, "KL/std": 110.861328125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.004001175053417683, "fcm_dpo/delta": -0.007650203071534634, "fcm_dpo/margin": 100.98538208007812, "fcm_dpo/q_t": 0.40368151664733887, "grad_norm": 9.416135787963867, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.5772507190704346, "logits/rejected": -0.931223452091217, "logps/chosen": -299.84881591796875, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -396.8478088378906, "loss": 1.0665, "margin_dpo/margin_mean": 100.98538208007812, "margin_dpo/margin_std": 106.93710327148438, "step": 667 }, { "KL/chosen_KL_mean": -179.31602478027344, "KL/mean": -238.24148559570312, "KL/rejected_KL_mean": -297.1669616699219, "KL/std": 118.2341537475586, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.003698737360537052, "fcm_dpo/delta": -0.06390460580587387, "fcm_dpo/margin": 117.8509521484375, "fcm_dpo/q_t": 0.39344412088394165, "grad_norm": 9.20440673828125, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.237743616104126, "logits/rejected": -1.713510513305664, "logps/chosen": -281.8910827636719, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -405.9846496582031, "loss": 1.0509, "margin_dpo/margin_mean": 117.85094451904297, "margin_dpo/margin_std": 129.33010864257812, "step": 668 }, { "KL/chosen_KL_mean": -164.02557373046875, "KL/mean": -221.1015625, "KL/rejected_KL_mean": -278.17755126953125, "KL/std": 115.56050872802734, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.003540206467732787, "fcm_dpo/delta": -0.006924290210008621, "fcm_dpo/margin": 114.15196228027344, "fcm_dpo/q_t": 0.4040784239768982, "grad_norm": 8.825647354125977, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.642411231994629, "logits/rejected": -1.2440868616104126, "logps/chosen": -253.52035522460938, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -360.6970520019531, "loss": 1.0678, "margin_dpo/margin_mean": 114.15196228027344, "margin_dpo/margin_std": 123.40472412109375, "step": 669 }, { "KL/chosen_KL_mean": -163.63180541992188, "KL/mean": -226.52066040039062, "KL/rejected_KL_mean": -289.4095153808594, "KL/std": 117.39219665527344, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0033607278019189835, "fcm_dpo/delta": -0.039186082780361176, "fcm_dpo/margin": 125.77770233154297, "fcm_dpo/q_t": 0.3956890404224396, "grad_norm": 6.787951469421387, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.565321445465088, "logits/rejected": -1.0149550437927246, "logps/chosen": -259.0863952636719, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -390.94244384765625, "loss": 1.0497, "margin_dpo/margin_mean": 125.77770233154297, "margin_dpo/margin_std": 122.97918701171875, "step": 670 }, { "KL/chosen_KL_mean": -199.9897003173828, "KL/mean": -242.2918701171875, "KL/rejected_KL_mean": -284.59405517578125, "KL/std": 108.8224868774414, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.003316506277769804, "fcm_dpo/delta": 0.020676199346780777, "fcm_dpo/margin": 84.6043701171875, "fcm_dpo/q_t": 0.43457281589508057, "grad_norm": 8.23491096496582, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.5693416595458984, "logits/rejected": -1.3420217037200928, "logps/chosen": -282.1128234863281, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -374.81378173828125, "loss": 1.165, "margin_dpo/margin_mean": 84.6043701171875, "margin_dpo/margin_std": 121.10334777832031, "step": 671 }, { "KL/chosen_KL_mean": -199.75030517578125, "KL/mean": -236.72218322753906, "KL/rejected_KL_mean": -273.69403076171875, "KL/std": 104.5766830444336, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.004134608898311853, "fcm_dpo/delta": 0.13449162244796753, "fcm_dpo/margin": 73.94376373291016, "fcm_dpo/q_t": 0.4369118809700012, "grad_norm": 12.722521781921387, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.014944314956665, "logits/rejected": -1.6872189044952393, "logps/chosen": -289.81219482421875, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -374.14727783203125, "loss": 1.1624, "margin_dpo/margin_mean": 73.94375610351562, "margin_dpo/margin_std": 111.26020812988281, "step": 672 }, { "KL/chosen_KL_mean": -178.6166229248047, "KL/mean": -219.91123962402344, "KL/rejected_KL_mean": -261.2058410644531, "KL/std": 107.61972045898438, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004690292291343212, "fcm_dpo/delta": 0.017132729291915894, "fcm_dpo/margin": 82.5892333984375, "fcm_dpo/q_t": 0.41122865676879883, "grad_norm": 13.064983367919922, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.8018460273742676, "logits/rejected": -1.429476261138916, "logps/chosen": -257.8797607421875, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -352.55511474609375, "loss": 1.0994, "margin_dpo/margin_mean": 82.5892333984375, "margin_dpo/margin_std": 109.64390563964844, "step": 673 }, { "KL/chosen_KL_mean": -166.8162841796875, "KL/mean": -226.696533203125, "KL/rejected_KL_mean": -286.5767822265625, "KL/std": 107.868408203125, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.003801655489951372, "fcm_dpo/delta": -0.10994095355272293, "fcm_dpo/margin": 119.76048278808594, "fcm_dpo/q_t": 0.3832871913909912, "grad_norm": 7.571317672729492, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.4267520904541016, "logits/rejected": -1.0126957893371582, "logps/chosen": -242.27459716796875, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -362.7803955078125, "loss": 1.0294, "margin_dpo/margin_mean": 119.76048278808594, "margin_dpo/margin_std": 118.6670150756836, "step": 674 }, { "KL/chosen_KL_mean": -182.8865203857422, "KL/mean": -235.39389038085938, "KL/rejected_KL_mean": -287.90130615234375, "KL/std": 116.76221466064453, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0036114403046667576, "fcm_dpo/delta": 0.026208996772766113, "fcm_dpo/margin": 105.01478576660156, "fcm_dpo/q_t": 0.412223219871521, "grad_norm": 9.368300437927246, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.3153630495071411, "logits/rejected": -0.9642319679260254, "logps/chosen": -262.7960510253906, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -369.1195373535156, "loss": 1.1156, "margin_dpo/margin_mean": 105.0147705078125, "margin_dpo/margin_std": 143.08631896972656, "step": 675 }, { "KL/chosen_KL_mean": -197.23211669921875, "KL/mean": -248.78367614746094, "KL/rejected_KL_mean": -300.335205078125, "KL/std": 111.93342590332031, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.003833297174423933, "fcm_dpo/delta": 0.006847789511084557, "fcm_dpo/margin": 103.10308837890625, "fcm_dpo/q_t": 0.40744078159332275, "grad_norm": 8.313961029052734, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.8276386260986328, "logits/rejected": -1.2288813591003418, "logps/chosen": -295.40325927734375, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -395.35986328125, "loss": 1.0865, "margin_dpo/margin_mean": 103.10308837890625, "margin_dpo/margin_std": 125.90435791015625, "step": 676 }, { "KL/chosen_KL_mean": -176.2576904296875, "KL/mean": -228.17527770996094, "KL/rejected_KL_mean": -280.09283447265625, "KL/std": 110.50440979003906, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0038336405996233225, "fcm_dpo/delta": 0.0012787915766239166, "fcm_dpo/margin": 103.83514404296875, "fcm_dpo/q_t": 0.40587905049324036, "grad_norm": 8.118304252624512, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.6081550121307373, "logits/rejected": -0.9217997789382935, "logps/chosen": -267.636962890625, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -362.9706115722656, "loss": 1.0703, "margin_dpo/margin_mean": 103.83514404296875, "margin_dpo/margin_std": 108.56427764892578, "step": 677 }, { "KL/chosen_KL_mean": -166.8870849609375, "KL/mean": -228.26943969726562, "KL/rejected_KL_mean": -289.65179443359375, "KL/std": 118.30532836914062, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.00363665958866477, "fcm_dpo/delta": -0.08375559747219086, "fcm_dpo/margin": 122.76473999023438, "fcm_dpo/q_t": 0.38726821541786194, "grad_norm": 9.36252212524414, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.5841816663742065, "logits/rejected": -1.2538963556289673, "logps/chosen": -242.532958984375, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -376.617919921875, "loss": 1.0395, "margin_dpo/margin_mean": 122.76473999023438, "margin_dpo/margin_std": 120.33235168457031, "step": 678 }, { "KL/chosen_KL_mean": -194.30740356445312, "KL/mean": -244.59243774414062, "KL/rejected_KL_mean": -294.87744140625, "KL/std": 98.64520263671875, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0036171013489365578, "fcm_dpo/delta": 0.056782498955726624, "fcm_dpo/margin": 100.57003784179688, "fcm_dpo/q_t": 0.4181651473045349, "grad_norm": 8.145146369934082, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.1176650524139404, "logits/rejected": -1.5924410820007324, "logps/chosen": -275.08087158203125, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -377.7559509277344, "loss": 1.0898, "margin_dpo/margin_mean": 100.57003784179688, "margin_dpo/margin_std": 105.20979309082031, "step": 679 }, { "KL/chosen_KL_mean": -193.63201904296875, "KL/mean": -248.73953247070312, "KL/rejected_KL_mean": -303.8470458984375, "KL/std": 116.85638427734375, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.003798204706981778, "fcm_dpo/delta": -0.04228535294532776, "fcm_dpo/margin": 110.21501159667969, "fcm_dpo/q_t": 0.3982860743999481, "grad_norm": 7.169468402862549, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.8135547637939453, "logits/rejected": -1.4741668701171875, "logps/chosen": -301.31494140625, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -419.94189453125, "loss": 1.0768, "margin_dpo/margin_mean": 110.21501159667969, "margin_dpo/margin_std": 126.71295928955078, "step": 680 }, { "KL/chosen_KL_mean": -177.21469116210938, "KL/mean": -225.8577880859375, "KL/rejected_KL_mean": -274.5008850097656, "KL/std": 110.65364074707031, "epoch": 1.0, "fcm_dpo/beta": 0.003775274381041527, "fcm_dpo/delta": 0.04817819595336914, "fcm_dpo/margin": 97.28620910644531, "fcm_dpo/q_t": 0.41718482971191406, "grad_norm": 8.877148628234863, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.2065558433532715, "logits/rejected": -1.8909339904785156, "logps/chosen": -270.22576904296875, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -369.32305908203125, "loss": 1.1026, "margin_dpo/margin_mean": 97.28620910644531, "margin_dpo/margin_std": 121.05557250976562, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1187300043134087, "train_runtime": 1971.3015, "train_samples_per_second": 22.116, "train_steps_per_second": 0.345 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }