{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4998469352722168, "grad_norm": 3.8972043991088867, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3857, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.500311017036438, "grad_norm": 2.9605307579040527, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3875, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.060626983642578125, "KL/mean": 0.01270710676908493, "KL/rejected_KL_mean": -0.03520965576171875, "KL/std": 0.3118070960044861, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09583590924739838, "fcm_dpo/q_t": 0.4997604191303253, "grad_norm": 2.612171173095703, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4266408681869507, "logits/rejected": 0.6500343680381775, "logps/chosen": -93.75035858154297, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.26470947265625, "loss": 1.3853, "margin_dpo/margin_mean": 0.09583564102649689, "margin_dpo/margin_std": 0.4318687915802002, "step": 3 }, { "KL/chosen_KL_mean": -0.017520904541015625, "KL/mean": -0.008819274604320526, "KL/rejected_KL_mean": -0.000118255615234375, "KL/std": 0.34834155440330505, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.017399147152900696, "fcm_dpo/q_t": 0.500043511390686, "grad_norm": 3.2696681022644043, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.647315502166748, "logits/rejected": 0.7790212035179138, "logps/chosen": -87.30998992919922, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.82437133789062, "loss": 1.3865, "margin_dpo/margin_mean": -0.017399266362190247, "margin_dpo/margin_std": 0.4597775936126709, "step": 4 }, { "KL/chosen_KL_mean": -0.009525299072265625, "KL/mean": -0.022157929837703705, "KL/rejected_KL_mean": -0.034793853759765625, "KL/std": 0.29867932200431824, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02526743710041046, "fcm_dpo/q_t": 0.4999368488788605, "grad_norm": 3.516111373901367, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4831095337867737, "logits/rejected": 0.6967244148254395, "logps/chosen": -89.34628295898438, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.78262329101562, "loss": 1.386, "margin_dpo/margin_mean": 0.025267452001571655, "margin_dpo/margin_std": 0.42025989294052124, "step": 5 }, { "KL/chosen_KL_mean": -0.060009002685546875, "KL/mean": -0.040336593985557556, "KL/rejected_KL_mean": -0.020664215087890625, "KL/std": 0.31888359785079956, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0393487811088562, "fcm_dpo/q_t": 0.5000983476638794, "grad_norm": 3.6075973510742188, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.5734951496124268, "logits/rejected": 0.8959733247756958, "logps/chosen": -97.38148498535156, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.90411376953125, "loss": 1.3867, "margin_dpo/margin_mean": -0.039348989725112915, "margin_dpo/margin_std": 0.4025424122810364, "step": 6 }, { "KL/chosen_KL_mean": 0.000461578369140625, "KL/mean": -0.02657490223646164, "KL/rejected_KL_mean": -0.053615570068359375, "KL/std": 0.31015637516975403, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.054074302315711975, "fcm_dpo/q_t": 0.4998648166656494, "grad_norm": 3.828531503677368, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.49870598316192627, "logits/rejected": 0.5466470718383789, "logps/chosen": -86.6480712890625, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.6697998046875, "loss": 1.3858, "margin_dpo/margin_mean": 0.054072946310043335, "margin_dpo/margin_std": 0.43503981828689575, "step": 7 }, { "KL/chosen_KL_mean": -0.054386138916015625, "KL/mean": -0.0010024979710578918, "KL/rejected_KL_mean": 0.052387237548828125, "KL/std": 0.3091372847557068, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.10677047073841095, "fcm_dpo/q_t": 0.5002669095993042, "grad_norm": 3.2822604179382324, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.3903715908527374, "logits/rejected": 0.6992945671081543, "logps/chosen": -89.97807312011719, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.17565155029297, "loss": 1.3874, "margin_dpo/margin_mean": -0.1067705899477005, "margin_dpo/margin_std": 0.42788729071617126, "step": 8 }, { "KL/chosen_KL_mean": 0.0516510009765625, "KL/mean": 0.023035302758216858, "KL/rejected_KL_mean": -0.005573272705078125, "KL/std": 0.30714738368988037, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05721995234489441, "fcm_dpo/q_t": 0.49985694885253906, "grad_norm": 3.6161868572235107, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.6870657801628113, "logits/rejected": 0.8684152960777283, "logps/chosen": -103.80548095703125, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.32489013671875, "loss": 1.3857, "margin_dpo/margin_mean": 0.05721965432167053, "margin_dpo/margin_std": 0.4447464942932129, "step": 9 }, { "KL/chosen_KL_mean": 0.073089599609375, "KL/mean": 0.10309600085020065, "KL/rejected_KL_mean": 0.1331024169921875, "KL/std": 0.26194673776626587, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06001533567905426, "fcm_dpo/q_t": 0.5001499652862549, "grad_norm": 3.345100164413452, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5467858910560608, "logits/rejected": 0.761015772819519, "logps/chosen": -76.13127136230469, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.01899719238281, "loss": 1.3869, "margin_dpo/margin_mean": -0.06001552939414978, "margin_dpo/margin_std": 0.4085805416107178, "step": 10 }, { "KL/chosen_KL_mean": 0.00664520263671875, "KL/mean": 0.0015933215618133545, "KL/rejected_KL_mean": -0.003459930419921875, "KL/std": 0.31740203499794006, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.010099649429321289, "fcm_dpo/q_t": 0.49997472763061523, "grad_norm": 3.7119381427764893, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.3259147107601166, "logits/rejected": 0.5436388850212097, "logps/chosen": -82.35984802246094, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.26808166503906, "loss": 1.3862, "margin_dpo/margin_mean": 0.010100305080413818, "margin_dpo/margin_std": 0.46445393562316895, "step": 11 }, { "KL/chosen_KL_mean": 0.014369964599609375, "KL/mean": -0.013533636927604675, "KL/rejected_KL_mean": -0.04143524169921875, "KL/std": 0.33080875873565674, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.055797189474105835, "fcm_dpo/q_t": 0.4998604953289032, "grad_norm": 4.011943817138672, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.3617513179779053, "logits/rejected": 0.48908764123916626, "logps/chosen": -99.09112548828125, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.31285095214844, "loss": 1.3857, "margin_dpo/margin_mean": 0.055797427892684937, "margin_dpo/margin_std": 0.5010286569595337, "step": 12 }, { "KL/chosen_KL_mean": 0.00399017333984375, "KL/mean": -0.014975771307945251, "KL/rejected_KL_mean": -0.03394317626953125, "KL/std": 0.3364943265914917, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03793713450431824, "fcm_dpo/q_t": 0.49990513920783997, "grad_norm": 4.179642677307129, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.5760356187820435, "logits/rejected": 0.8394055366516113, "logps/chosen": -90.55574035644531, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.72503662109375, "loss": 1.3859, "margin_dpo/margin_mean": 0.03793707489967346, "margin_dpo/margin_std": 0.5292723178863525, "step": 13 }, { "KL/chosen_KL_mean": -0.06419754028320312, "KL/mean": -0.005205325782299042, "KL/rejected_KL_mean": 0.053783416748046875, "KL/std": 0.3167896866798401, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11798731982707977, "fcm_dpo/q_t": 0.5002949237823486, "grad_norm": 3.572885751724243, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.993935227394104, "logits/rejected": 1.0861668586730957, "logps/chosen": -99.89137268066406, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.88821411132812, "loss": 1.3875, "margin_dpo/margin_mean": -0.11798706650733948, "margin_dpo/margin_std": 0.4105429947376251, "step": 14 }, { "KL/chosen_KL_mean": 0.002269744873046875, "KL/mean": -0.022190526127815247, "KL/rejected_KL_mean": -0.04665374755859375, "KL/std": 0.33972322940826416, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04892250895500183, "fcm_dpo/q_t": 0.4998776912689209, "grad_norm": 3.1954455375671387, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.2503818869590759, "logits/rejected": 0.4099924564361572, "logps/chosen": -78.90769958496094, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.10899353027344, "loss": 1.3858, "margin_dpo/margin_mean": 0.048922598361968994, "margin_dpo/margin_std": 0.49507975578308105, "step": 15 }, { "KL/chosen_KL_mean": -0.042339324951171875, "KL/mean": -0.020286060869693756, "KL/rejected_KL_mean": 0.0017547607421875, "KL/std": 0.3447006046772003, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04409150779247284, "fcm_dpo/q_t": 0.5001102089881897, "grad_norm": 3.463486433029175, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.3753458261489868, "logits/rejected": 0.6139321327209473, "logps/chosen": -97.46562194824219, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.59770202636719, "loss": 1.3867, "margin_dpo/margin_mean": -0.044091492891311646, "margin_dpo/margin_std": 0.4672156274318695, "step": 16 }, { "KL/chosen_KL_mean": 0.020748138427734375, "KL/mean": -0.0027421042323112488, "KL/rejected_KL_mean": -0.02623748779296875, "KL/std": 0.30819302797317505, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.046993061900138855, "fcm_dpo/q_t": 0.4998824894428253, "grad_norm": 3.57242751121521, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.44832202792167664, "logits/rejected": 0.6737120151519775, "logps/chosen": -104.34355926513672, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.49395751953125, "loss": 1.3858, "margin_dpo/margin_mean": 0.04699251055717468, "margin_dpo/margin_std": 0.45155084133148193, "step": 17 }, { "KL/chosen_KL_mean": 0.0085601806640625, "KL/mean": 0.016665160655975342, "KL/rejected_KL_mean": 0.024768829345703125, "KL/std": 0.3104175329208374, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.016209125518798828, "fcm_dpo/q_t": 0.5000405311584473, "grad_norm": 4.181760787963867, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.21446029841899872, "logits/rejected": 0.5393970012664795, "logps/chosen": -87.0833969116211, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.8259506225586, "loss": 1.3865, "margin_dpo/margin_mean": -0.01620902121067047, "margin_dpo/margin_std": 0.4746027886867523, "step": 18 }, { "KL/chosen_KL_mean": -0.06595993041992188, "KL/mean": -0.05578558146953583, "KL/rejected_KL_mean": -0.045612335205078125, "KL/std": 0.3467903733253479, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.020346850156784058, "fcm_dpo/q_t": 0.5000509023666382, "grad_norm": 3.1384856700897217, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6972318887710571, "logits/rejected": 0.9823744297027588, "logps/chosen": -105.93949890136719, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.97584533691406, "loss": 1.3865, "margin_dpo/margin_mean": -0.020346760749816895, "margin_dpo/margin_std": 0.4981383681297302, "step": 19 }, { "KL/chosen_KL_mean": 0.025135040283203125, "KL/mean": 0.001785367727279663, "KL/rejected_KL_mean": -0.021572113037109375, "KL/std": 0.342011421918869, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.046715497970581055, "fcm_dpo/q_t": 0.4998832046985626, "grad_norm": 3.2411563396453857, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.27013835310935974, "logits/rejected": 0.5498103499412537, "logps/chosen": -90.73298645019531, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.93389129638672, "loss": 1.3858, "margin_dpo/margin_mean": 0.04671585559844971, "margin_dpo/margin_std": 0.4321746826171875, "step": 20 }, { "KL/chosen_KL_mean": 0.01403045654296875, "KL/mean": -0.020621351897716522, "KL/rejected_KL_mean": -0.055278778076171875, "KL/std": 0.2674625515937805, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06931330263614655, "fcm_dpo/q_t": 0.49982669949531555, "grad_norm": 3.167379379272461, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5158483982086182, "logits/rejected": 0.6921500563621521, "logps/chosen": -80.31942749023438, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.989013671875, "loss": 1.3856, "margin_dpo/margin_mean": 0.06931328773498535, "margin_dpo/margin_std": 0.3796931803226471, "step": 21 }, { "KL/chosen_KL_mean": 0.06995391845703125, "KL/mean": 0.0149935781955719, "KL/rejected_KL_mean": -0.039966583251953125, "KL/std": 0.3316918611526489, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10991716384887695, "fcm_dpo/q_t": 0.49972522258758545, "grad_norm": 4.277241230010986, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.757103681564331, "logits/rejected": 0.8835777640342712, "logps/chosen": -95.32534790039062, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.51348876953125, "loss": 1.3852, "margin_dpo/margin_mean": 0.10991713404655457, "margin_dpo/margin_std": 0.43410128355026245, "step": 22 }, { "KL/chosen_KL_mean": 0.018236160278320312, "KL/mean": 0.049478352069854736, "KL/rejected_KL_mean": 0.08072280883789062, "KL/std": 0.33478474617004395, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06248193979263306, "fcm_dpo/q_t": 0.5001561641693115, "grad_norm": 3.1362879276275635, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.9032010436058044, "logits/rejected": 1.0919687747955322, "logps/chosen": -90.61927795410156, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.51353454589844, "loss": 1.3869, "margin_dpo/margin_mean": -0.06248199939727783, "margin_dpo/margin_std": 0.41144853830337524, "step": 23 }, { "KL/chosen_KL_mean": 0.027345657348632812, "KL/mean": 0.01501135528087616, "KL/rejected_KL_mean": 0.002674102783203125, "KL/std": 0.3166768252849579, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.024668008089065552, "fcm_dpo/q_t": 0.49993833899497986, "grad_norm": 4.427740573883057, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.580396294593811, "logits/rejected": 0.6122506856918335, "logps/chosen": -69.88994598388672, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.60722351074219, "loss": 1.3861, "margin_dpo/margin_mean": 0.024668186902999878, "margin_dpo/margin_std": 0.4463178217411041, "step": 24 }, { "KL/chosen_KL_mean": -0.033657073974609375, "KL/mean": -0.05675327032804489, "KL/rejected_KL_mean": -0.0798492431640625, "KL/std": 0.2764991521835327, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.046184852719306946, "fcm_dpo/q_t": 0.4998845160007477, "grad_norm": 3.7257206439971924, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.318434476852417, "logits/rejected": 0.4787173271179199, "logps/chosen": -80.85914611816406, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.03695678710938, "loss": 1.3858, "margin_dpo/margin_mean": 0.04618537425994873, "margin_dpo/margin_std": 0.32028645277023315, "step": 25 }, { "KL/chosen_KL_mean": -0.0009002685546875, "KL/mean": -0.030067041516304016, "KL/rejected_KL_mean": -0.059230804443359375, "KL/std": 0.3645031452178955, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05833384394645691, "fcm_dpo/q_t": 0.4998541474342346, "grad_norm": 4.134904861450195, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.4308297038078308, "logits/rejected": 0.6430095434188843, "logps/chosen": -88.90206909179688, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.88742065429688, "loss": 1.3857, "margin_dpo/margin_mean": 0.05833446979522705, "margin_dpo/margin_std": 0.49705055356025696, "step": 26 }, { "KL/chosen_KL_mean": -0.011791229248046875, "KL/mean": -0.06920512020587921, "KL/rejected_KL_mean": -0.12662887573242188, "KL/std": 0.3557009696960449, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11483576893806458, "fcm_dpo/q_t": 0.49971291422843933, "grad_norm": 4.456466197967529, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.1622442603111267, "logits/rejected": 0.3499611020088196, "logps/chosen": -77.60780334472656, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.06513977050781, "loss": 1.3852, "margin_dpo/margin_mean": 0.11483535170555115, "margin_dpo/margin_std": 0.5264773368835449, "step": 27 }, { "KL/chosen_KL_mean": 0.13916015625, "KL/mean": 0.04853326082229614, "KL/rejected_KL_mean": -0.04209136962890625, "KL/std": 0.3585261106491089, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18125373125076294, "fcm_dpo/q_t": 0.4995468854904175, "grad_norm": 3.6978936195373535, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.4771879315376282, "logits/rejected": 0.6875078678131104, "logps/chosen": -102.08940887451172, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.0015869140625, "loss": 1.3845, "margin_dpo/margin_mean": 0.1812543272972107, "margin_dpo/margin_std": 0.44115889072418213, "step": 28 }, { "KL/chosen_KL_mean": -0.06749343872070312, "KL/mean": -0.10962507873773575, "KL/rejected_KL_mean": -0.15175247192382812, "KL/std": 0.3522123396396637, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08426420390605927, "fcm_dpo/q_t": 0.4997893273830414, "grad_norm": 4.262432098388672, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.4404758810997009, "logits/rejected": 0.7082964777946472, "logps/chosen": -88.71454620361328, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.11186218261719, "loss": 1.3855, "margin_dpo/margin_mean": 0.08426548540592194, "margin_dpo/margin_std": 0.4988255500793457, "step": 29 }, { "KL/chosen_KL_mean": 0.043041229248046875, "KL/mean": -0.011111453175544739, "KL/rejected_KL_mean": -0.06526947021484375, "KL/std": 0.3095162808895111, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10831108689308167, "fcm_dpo/q_t": 0.4997292160987854, "grad_norm": 3.937917947769165, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.5152514576911926, "logits/rejected": 0.6268141269683838, "logps/chosen": -88.34534454345703, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.38416290283203, "loss": 1.3852, "margin_dpo/margin_mean": 0.10831132531166077, "margin_dpo/margin_std": 0.4113122224807739, "step": 30 }, { "KL/chosen_KL_mean": -0.08150100708007812, "KL/mean": -0.07936333864927292, "KL/rejected_KL_mean": -0.07722854614257812, "KL/std": 0.34083086252212524, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.004279986023902893, "fcm_dpo/q_t": 0.5000107288360596, "grad_norm": 3.161220073699951, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.891455352306366, "logits/rejected": 1.1593793630599976, "logps/chosen": -101.20716094970703, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.93565368652344, "loss": 1.3863, "margin_dpo/margin_mean": -0.0042799413204193115, "margin_dpo/margin_std": 0.4104807376861572, "step": 31 }, { "KL/chosen_KL_mean": -0.05982208251953125, "KL/mean": -0.09715349972248077, "KL/rejected_KL_mean": -0.13448715209960938, "KL/std": 0.3144843280315399, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07466724514961243, "fcm_dpo/q_t": 0.4998133182525635, "grad_norm": 3.57645583152771, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.3012698292732239, "logits/rejected": 0.4341125786304474, "logps/chosen": -96.67684936523438, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.35885620117188, "loss": 1.3856, "margin_dpo/margin_mean": 0.07466785609722137, "margin_dpo/margin_std": 0.41121289134025574, "step": 32 }, { "KL/chosen_KL_mean": -0.0246429443359375, "KL/mean": -0.05574314296245575, "KL/rejected_KL_mean": -0.08684158325195312, "KL/std": 0.3581833839416504, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.062201112508773804, "fcm_dpo/q_t": 0.499844491481781, "grad_norm": 3.415233612060547, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6383552551269531, "logits/rejected": 0.8197961449623108, "logps/chosen": -81.545654296875, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.89279174804688, "loss": 1.3857, "margin_dpo/margin_mean": 0.06220182776451111, "margin_dpo/margin_std": 0.4963094890117645, "step": 33 }, { "KL/chosen_KL_mean": 0.013980865478515625, "KL/mean": -0.08518728613853455, "KL/rejected_KL_mean": -0.18435287475585938, "KL/std": 0.36769458651542664, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19833755493164062, "fcm_dpo/q_t": 0.4995041787624359, "grad_norm": 4.187440395355225, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.6751847267150879, "logits/rejected": 0.8481490015983582, "logps/chosen": -77.18805694580078, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.90312194824219, "loss": 1.3843, "margin_dpo/margin_mean": 0.19833725690841675, "margin_dpo/margin_std": 0.5220308303833008, "step": 34 }, { "KL/chosen_KL_mean": 0.004302978515625, "KL/mean": -0.09369708597660065, "KL/rejected_KL_mean": -0.19169998168945312, "KL/std": 0.3430505394935608, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1960069239139557, "fcm_dpo/q_t": 0.4995099902153015, "grad_norm": 4.337993144989014, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.23031751811504364, "logits/rejected": 0.3870696723461151, "logps/chosen": -77.56604766845703, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.3802490234375, "loss": 1.3843, "margin_dpo/margin_mean": 0.19600746035575867, "margin_dpo/margin_std": 0.480034738779068, "step": 35 }, { "KL/chosen_KL_mean": -0.0174407958984375, "KL/mean": -0.09184802323579788, "KL/rejected_KL_mean": -0.166259765625, "KL/std": 0.3608782887458801, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14881213009357452, "fcm_dpo/q_t": 0.4996279776096344, "grad_norm": 3.336711883544922, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.36793357133865356, "logits/rejected": 0.55533367395401, "logps/chosen": -83.30567932128906, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.92367553710938, "loss": 1.3848, "margin_dpo/margin_mean": 0.1488122045993805, "margin_dpo/margin_std": 0.4680359363555908, "step": 36 }, { "KL/chosen_KL_mean": -0.04923248291015625, "KL/mean": -0.1567155122756958, "KL/rejected_KL_mean": -0.26419830322265625, "KL/std": 0.38862401247024536, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21496252715587616, "fcm_dpo/q_t": 0.4994626045227051, "grad_norm": 3.7315804958343506, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.8063580989837646, "logits/rejected": 0.9789649248123169, "logps/chosen": -94.82032775878906, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -85.98146057128906, "loss": 1.3842, "margin_dpo/margin_mean": 0.2149631530046463, "margin_dpo/margin_std": 0.5128874778747559, "step": 37 }, { "KL/chosen_KL_mean": 0.07415390014648438, "KL/mean": -0.03252837806940079, "KL/rejected_KL_mean": -0.13920974731445312, "KL/std": 0.4372931122779846, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21336908638477325, "fcm_dpo/q_t": 0.49946659803390503, "grad_norm": 3.8419246673583984, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.42919063568115234, "logits/rejected": 0.567008376121521, "logps/chosen": -75.90435028076172, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.24322509765625, "loss": 1.3842, "margin_dpo/margin_mean": 0.21336936950683594, "margin_dpo/margin_std": 0.5725588798522949, "step": 38 }, { "KL/chosen_KL_mean": -0.06533050537109375, "KL/mean": -0.19702431559562683, "KL/rejected_KL_mean": -0.3287239074707031, "KL/std": 0.4287729859352112, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26339173316955566, "fcm_dpo/q_t": 0.49934154748916626, "grad_norm": 3.731046199798584, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.36311447620391846, "logits/rejected": 0.5430086255073547, "logps/chosen": -81.25111389160156, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.52464294433594, "loss": 1.3837, "margin_dpo/margin_mean": 0.26339179277420044, "margin_dpo/margin_std": 0.5903570652008057, "step": 39 }, { "KL/chosen_KL_mean": 0.0037384033203125, "KL/mean": -0.08793337643146515, "KL/rejected_KL_mean": -0.17960739135742188, "KL/std": 0.32459837198257446, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18334946036338806, "fcm_dpo/q_t": 0.49954164028167725, "grad_norm": 2.9677717685699463, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.7570903301239014, "logits/rejected": 0.9644452333450317, "logps/chosen": -83.3288345336914, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.43551635742188, "loss": 1.3845, "margin_dpo/margin_mean": 0.18334898352622986, "margin_dpo/margin_std": 0.41630327701568604, "step": 40 }, { "KL/chosen_KL_mean": -0.07762908935546875, "KL/mean": -0.17948748171329498, "KL/rejected_KL_mean": -0.281341552734375, "KL/std": 0.4152122139930725, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20371463894844055, "fcm_dpo/q_t": 0.49949073791503906, "grad_norm": 3.4317567348480225, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6156439781188965, "logits/rejected": 0.8021235466003418, "logps/chosen": -93.22628784179688, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.36054992675781, "loss": 1.3843, "margin_dpo/margin_mean": 0.2037147581577301, "margin_dpo/margin_std": 0.6016439199447632, "step": 41 }, { "KL/chosen_KL_mean": 0.0311737060546875, "KL/mean": -0.1622164249420166, "KL/rejected_KL_mean": -0.3556098937988281, "KL/std": 0.511090874671936, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.38678163290023804, "fcm_dpo/q_t": 0.49903303384780884, "grad_norm": 4.243740558624268, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.5114056468009949, "logits/rejected": 0.654529333114624, "logps/chosen": -90.67044830322266, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.22026062011719, "loss": 1.3824, "margin_dpo/margin_mean": 0.3867817521095276, "margin_dpo/margin_std": 0.5972121953964233, "step": 42 }, { "KL/chosen_KL_mean": 0.009227752685546875, "KL/mean": -0.21126289665699005, "KL/rejected_KL_mean": -0.4317626953125, "KL/std": 0.5266244411468506, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.44098547101020813, "fcm_dpo/q_t": 0.4988975524902344, "grad_norm": 3.976940393447876, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.5298972129821777, "logits/rejected": 0.7174561023712158, "logps/chosen": -89.63480377197266, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.38362121582031, "loss": 1.3819, "margin_dpo/margin_mean": 0.44098585844039917, "margin_dpo/margin_std": 0.759847104549408, "step": 43 }, { "KL/chosen_KL_mean": -0.096893310546875, "KL/mean": -0.32582932710647583, "KL/rejected_KL_mean": -0.5547637939453125, "KL/std": 0.5657384395599365, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.45787274837493896, "fcm_dpo/q_t": 0.498855322599411, "grad_norm": 3.57468843460083, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.7373632788658142, "logits/rejected": 0.8608587980270386, "logps/chosen": -81.97521209716797, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.97639465332031, "loss": 1.3817, "margin_dpo/margin_mean": 0.4578734338283539, "margin_dpo/margin_std": 0.645193338394165, "step": 44 }, { "KL/chosen_KL_mean": -0.11119461059570312, "KL/mean": -0.21965579688549042, "KL/rejected_KL_mean": -0.32811737060546875, "KL/std": 0.39221513271331787, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2169242799282074, "fcm_dpo/q_t": 0.4994576871395111, "grad_norm": 2.5110526084899902, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.6506820321083069, "logits/rejected": 0.7620583772659302, "logps/chosen": -77.45579528808594, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.57586669921875, "loss": 1.3841, "margin_dpo/margin_mean": 0.2169249951839447, "margin_dpo/margin_std": 0.5452648401260376, "step": 45 }, { "KL/chosen_KL_mean": -0.02693939208984375, "KL/mean": -0.24246153235435486, "KL/rejected_KL_mean": -0.457977294921875, "KL/std": 0.5581282377243042, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.43103957176208496, "fcm_dpo/q_t": 0.4989224076271057, "grad_norm": 3.4595718383789062, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.5100475549697876, "logits/rejected": 0.7528989315032959, "logps/chosen": -90.36776733398438, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.13449096679688, "loss": 1.382, "margin_dpo/margin_mean": 0.4310394525527954, "margin_dpo/margin_std": 0.5961135625839233, "step": 46 }, { "KL/chosen_KL_mean": -0.01157379150390625, "KL/mean": -0.24259042739868164, "KL/rejected_KL_mean": -0.4736061096191406, "KL/std": 0.5181233882904053, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4620252251625061, "fcm_dpo/q_t": 0.4988449513912201, "grad_norm": 3.530991315841675, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.43653130531311035, "logits/rejected": 0.6187427043914795, "logps/chosen": -104.4228744506836, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.72850036621094, "loss": 1.3817, "margin_dpo/margin_mean": 0.4620243012905121, "margin_dpo/margin_std": 0.7321393489837646, "step": 47 }, { "KL/chosen_KL_mean": 0.047882080078125, "KL/mean": -0.2590615451335907, "KL/rejected_KL_mean": -0.5660018920898438, "KL/std": 0.6804758310317993, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6138830184936523, "fcm_dpo/q_t": 0.49846532940864563, "grad_norm": 4.120362758636475, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.14840303361415863, "logits/rejected": 0.34348806738853455, "logps/chosen": -91.0523910522461, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.64657592773438, "loss": 1.3802, "margin_dpo/margin_mean": 0.6138839721679688, "margin_dpo/margin_std": 0.8362932205200195, "step": 48 }, { "KL/chosen_KL_mean": 0.03246307373046875, "KL/mean": -0.361356258392334, "KL/rejected_KL_mean": -0.7551803588867188, "KL/std": 0.7855098247528076, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7876377701759338, "fcm_dpo/q_t": 0.49803096055984497, "grad_norm": 4.344902515411377, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.005891956388950348, "logits/rejected": 0.4659579396247864, "logps/chosen": -92.97122192382812, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.50418090820312, "loss": 1.3785, "margin_dpo/margin_mean": 0.7876376509666443, "margin_dpo/margin_std": 0.9311748743057251, "step": 49 }, { "KL/chosen_KL_mean": -0.09606170654296875, "KL/mean": -0.4339733421802521, "KL/rejected_KL_mean": -0.7718772888183594, "KL/std": 0.7358265519142151, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6758142113685608, "fcm_dpo/q_t": 0.49831050634384155, "grad_norm": 3.542573928833008, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.4410429894924164, "logits/rejected": 0.608002245426178, "logps/chosen": -94.72286987304688, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.34623718261719, "loss": 1.3796, "margin_dpo/margin_mean": 0.6758142709732056, "margin_dpo/margin_std": 0.8745262026786804, "step": 50 }, { "KL/chosen_KL_mean": -0.0229339599609375, "KL/mean": -0.38021451234817505, "KL/rejected_KL_mean": -0.7374954223632812, "KL/std": 0.9364984631538391, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7145578861236572, "fcm_dpo/q_t": 0.4982137084007263, "grad_norm": 3.2724063396453857, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.4823940694332123, "logits/rejected": 0.6161498427391052, "logps/chosen": -87.53021240234375, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.20985412597656, "loss": 1.3792, "margin_dpo/margin_mean": 0.7145578265190125, "margin_dpo/margin_std": 1.2138879299163818, "step": 51 }, { "KL/chosen_KL_mean": -0.018474578857421875, "KL/mean": -0.4423646330833435, "KL/rejected_KL_mean": -0.8662605285644531, "KL/std": 0.7832540273666382, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8477820158004761, "fcm_dpo/q_t": 0.49788060784339905, "grad_norm": 3.4505019187927246, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.6609782576560974, "logits/rejected": 0.8892603516578674, "logps/chosen": -90.64874267578125, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.90531921386719, "loss": 1.3779, "margin_dpo/margin_mean": 0.847781777381897, "margin_dpo/margin_std": 0.9090744256973267, "step": 52 }, { "KL/chosen_KL_mean": -0.03527069091796875, "KL/mean": -0.5740271806716919, "KL/rejected_KL_mean": -1.1127853393554688, "KL/std": 1.0661849975585938, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0775216817855835, "fcm_dpo/q_t": 0.4973064363002777, "grad_norm": 4.4886932373046875, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.12041836977005005, "logits/rejected": 0.23411646485328674, "logps/chosen": -81.61834716796875, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.7743148803711, "loss": 1.3756, "margin_dpo/margin_mean": 1.0775214433670044, "margin_dpo/margin_std": 1.2855572700500488, "step": 53 }, { "KL/chosen_KL_mean": -0.055660247802734375, "KL/mean": -0.649177074432373, "KL/rejected_KL_mean": -1.2426948547363281, "KL/std": 1.3952505588531494, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1870307922363281, "fcm_dpo/q_t": 0.49703267216682434, "grad_norm": 4.4586358070373535, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.3733167052268982, "logits/rejected": 0.5522476434707642, "logps/chosen": -88.96582794189453, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.36024475097656, "loss": 1.3745, "margin_dpo/margin_mean": 1.1870307922363281, "margin_dpo/margin_std": 1.4224739074707031, "step": 54 }, { "KL/chosen_KL_mean": 0.13286590576171875, "KL/mean": -0.4376147985458374, "KL/rejected_KL_mean": -1.0080947875976562, "KL/std": 1.178413987159729, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1409590244293213, "fcm_dpo/q_t": 0.49714791774749756, "grad_norm": 4.032066345214844, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.026751890778541565, "logits/rejected": 0.3259810209274292, "logps/chosen": -92.32305908203125, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -93.97903442382812, "loss": 1.375, "margin_dpo/margin_mean": 1.1409589052200317, "margin_dpo/margin_std": 1.570730209350586, "step": 55 }, { "KL/chosen_KL_mean": -0.06808853149414062, "KL/mean": -0.6193695068359375, "KL/rejected_KL_mean": -1.1706504821777344, "KL/std": 1.19675874710083, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1025652885437012, "fcm_dpo/q_t": 0.49724388122558594, "grad_norm": 3.763031005859375, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.12196091562509537, "logits/rejected": 0.4548357129096985, "logps/chosen": -87.30474853515625, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.81619262695312, "loss": 1.3754, "margin_dpo/margin_mean": 1.1025644540786743, "margin_dpo/margin_std": 1.559999704360962, "step": 56 }, { "KL/chosen_KL_mean": -0.1284637451171875, "KL/mean": -0.6432129144668579, "KL/rejected_KL_mean": -1.1579704284667969, "KL/std": 1.1979900598526, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.029515027999878, "fcm_dpo/q_t": 0.49742645025253296, "grad_norm": 3.8324778079986572, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.28898710012435913, "logits/rejected": 0.5046127438545227, "logps/chosen": -98.27920532226562, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.21080780029297, "loss": 1.3761, "margin_dpo/margin_mean": 1.0295151472091675, "margin_dpo/margin_std": 1.5201349258422852, "step": 57 }, { "KL/chosen_KL_mean": 0.2729644775390625, "KL/mean": -0.5275334119796753, "KL/rejected_KL_mean": -1.3280296325683594, "KL/std": 1.476077914237976, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6009939908981323, "fcm_dpo/q_t": 0.4959982931613922, "grad_norm": 4.522995471954346, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.5334012508392334, "logits/rejected": 0.8932734131813049, "logps/chosen": -99.3080062866211, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.55030822753906, "loss": 1.3705, "margin_dpo/margin_mean": 1.6009936332702637, "margin_dpo/margin_std": 1.9369651079177856, "step": 58 }, { "KL/chosen_KL_mean": 0.054233551025390625, "KL/mean": -0.5713481307029724, "KL/rejected_KL_mean": -1.1969261169433594, "KL/std": 1.16945481300354, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2511570453643799, "fcm_dpo/q_t": 0.4968723654747009, "grad_norm": 3.867556571960449, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.02424241602420807, "logits/rejected": 0.15718887746334076, "logps/chosen": -89.74809265136719, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.47360229492188, "loss": 1.3739, "margin_dpo/margin_mean": 1.2511565685272217, "margin_dpo/margin_std": 1.3844916820526123, "step": 59 }, { "KL/chosen_KL_mean": -0.04192352294921875, "KL/mean": -0.6248098015785217, "KL/rejected_KL_mean": -1.20770263671875, "KL/std": 1.235998511314392, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1657778024673462, "fcm_dpo/q_t": 0.4970857501029968, "grad_norm": 3.186699628829956, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.44631266593933105, "logits/rejected": 0.7265136241912842, "logps/chosen": -95.1976318359375, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.58013916015625, "loss": 1.3747, "margin_dpo/margin_mean": 1.165778398513794, "margin_dpo/margin_std": 1.4111766815185547, "step": 60 }, { "KL/chosen_KL_mean": -0.2119293212890625, "KL/mean": -0.7931567430496216, "KL/rejected_KL_mean": -1.3743820190429688, "KL/std": 1.403143286705017, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1624557971954346, "fcm_dpo/q_t": 0.4970943331718445, "grad_norm": 3.3458242416381836, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.21277017891407013, "logits/rejected": 0.3528522253036499, "logps/chosen": -85.9442367553711, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.34014129638672, "loss": 1.3748, "margin_dpo/margin_mean": 1.1624560356140137, "margin_dpo/margin_std": 1.7646458148956299, "step": 61 }, { "KL/chosen_KL_mean": -0.1403656005859375, "KL/mean": -0.7419915795326233, "KL/rejected_KL_mean": -1.3436241149902344, "KL/std": 1.3611252307891846, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2032649517059326, "fcm_dpo/q_t": 0.49699217081069946, "grad_norm": 3.3833842277526855, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.3071219325065613, "logits/rejected": 0.5096158981323242, "logps/chosen": -81.77574920654297, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.3819351196289, "loss": 1.3744, "margin_dpo/margin_mean": 1.2032641172409058, "margin_dpo/margin_std": 1.786369800567627, "step": 62 }, { "KL/chosen_KL_mean": -0.094207763671875, "KL/mean": -0.87010657787323, "KL/rejected_KL_mean": -1.6460075378417969, "KL/std": 1.4670695066452026, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5518038272857666, "fcm_dpo/q_t": 0.4961209297180176, "grad_norm": 4.042943000793457, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.32477718591690063, "logits/rejected": 0.537598729133606, "logps/chosen": -103.71826171875, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -105.04904174804688, "loss": 1.3709, "margin_dpo/margin_mean": 1.5518033504486084, "margin_dpo/margin_std": 1.7345173358917236, "step": 63 }, { "KL/chosen_KL_mean": -0.08426284790039062, "KL/mean": -0.9460600018501282, "KL/rejected_KL_mean": -1.8078689575195312, "KL/std": 1.8018391132354736, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.723604679107666, "fcm_dpo/q_t": 0.4956918954849243, "grad_norm": 4.3125762939453125, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.49780601263046265, "logits/rejected": 0.7157726883888245, "logps/chosen": -87.08584594726562, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.39334106445312, "loss": 1.3693, "margin_dpo/margin_mean": 1.723604440689087, "margin_dpo/margin_std": 2.085357189178467, "step": 64 }, { "KL/chosen_KL_mean": -0.08560943603515625, "KL/mean": -0.9987833499908447, "KL/rejected_KL_mean": -1.911956787109375, "KL/std": 1.6989288330078125, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8263461589813232, "fcm_dpo/q_t": 0.4954349994659424, "grad_norm": 4.65037727355957, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.006236381828784943, "logits/rejected": 0.15087318420410156, "logps/chosen": -91.30752563476562, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.24748992919922, "loss": 1.3682, "margin_dpo/margin_mean": 1.8263461589813232, "margin_dpo/margin_std": 2.0813519954681396, "step": 65 }, { "KL/chosen_KL_mean": -0.37430572509765625, "KL/mean": -0.9875328540802002, "KL/rejected_KL_mean": -1.6007575988769531, "KL/std": 1.6093077659606934, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.226452112197876, "fcm_dpo/q_t": 0.4969343841075897, "grad_norm": 2.995617628097534, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.08493616431951523, "logits/rejected": 0.2860184907913208, "logps/chosen": -84.15853118896484, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.35285949707031, "loss": 1.3742, "margin_dpo/margin_mean": 1.2264525890350342, "margin_dpo/margin_std": 1.968803882598877, "step": 66 }, { "KL/chosen_KL_mean": -0.8975181579589844, "KL/mean": -1.455113410949707, "KL/rejected_KL_mean": -2.0127029418945312, "KL/std": 2.04282283782959, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1151870489120483, "fcm_dpo/q_t": 0.4972130060195923, "grad_norm": 2.800211191177368, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.5644781589508057, "logits/rejected": 0.8630934357643127, "logps/chosen": -88.57048034667969, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.09945678710938, "loss": 1.3754, "margin_dpo/margin_mean": 1.115187168121338, "margin_dpo/margin_std": 2.765322685241699, "step": 67 }, { "KL/chosen_KL_mean": -0.25252532958984375, "KL/mean": -1.3631634712219238, "KL/rejected_KL_mean": -2.473796844482422, "KL/std": 2.327113151550293, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.221273899078369, "fcm_dpo/q_t": 0.4944491982460022, "grad_norm": 3.67305064201355, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.01236661709845066, "logits/rejected": 0.2912748456001282, "logps/chosen": -97.33137512207031, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.592529296875, "loss": 1.3644, "margin_dpo/margin_mean": 2.22127366065979, "margin_dpo/margin_std": 2.911048173904419, "step": 68 }, { "KL/chosen_KL_mean": -0.3267555236816406, "KL/mean": -1.5004740953445435, "KL/rejected_KL_mean": -2.6741905212402344, "KL/std": 2.5831289291381836, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.3474366664886475, "fcm_dpo/q_t": 0.4941343367099762, "grad_norm": 4.144824981689453, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.23180922865867615, "logits/rejected": 0.34237736463546753, "logps/chosen": -86.04646301269531, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.1543960571289, "loss": 1.3632, "margin_dpo/margin_mean": 2.347437858581543, "margin_dpo/margin_std": 3.236788272857666, "step": 69 }, { "KL/chosen_KL_mean": -0.690399169921875, "KL/mean": -2.022078514099121, "KL/rejected_KL_mean": -3.3537521362304688, "KL/std": 2.877958297729492, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.66334867477417, "fcm_dpo/q_t": 0.49334579706192017, "grad_norm": 4.100100040435791, "learning_rate": 5e-07, "logits/chosen": 0.4959897994995117, "logits/rejected": 0.6912732124328613, "logps/chosen": -95.70034790039062, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.56647491455078, "loss": 1.3602, "margin_dpo/margin_mean": 2.6633481979370117, "margin_dpo/margin_std": 3.5159859657287598, "step": 70 }, { "KL/chosen_KL_mean": -0.043689727783203125, "KL/mean": -1.9659751653671265, "KL/rejected_KL_mean": -3.8882598876953125, "KL/std": 3.711668014526367, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.844571352005005, "fcm_dpo/q_t": 0.49039706587791443, "grad_norm": 5.126366138458252, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.0657881647348404, "logits/rejected": 0.31025803089141846, "logps/chosen": -91.02104187011719, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.47929382324219, "loss": 1.3487, "margin_dpo/margin_mean": 3.8445706367492676, "margin_dpo/margin_std": 4.241009712219238, "step": 71 }, { "KL/chosen_KL_mean": -0.6575431823730469, "KL/mean": -2.132457733154297, "KL/rejected_KL_mean": -3.6073760986328125, "KL/std": 3.0286974906921387, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.949838161468506, "fcm_dpo/q_t": 0.49262887239456177, "grad_norm": 4.038745403289795, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.2787877023220062, "logits/rejected": 0.04342615604400635, "logps/chosen": -98.549072265625, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.79908752441406, "loss": 1.3573, "margin_dpo/margin_mean": 2.9498376846313477, "margin_dpo/margin_std": 3.4324464797973633, "step": 72 }, { "KL/chosen_KL_mean": -0.9098396301269531, "KL/mean": -2.512251377105713, "KL/rejected_KL_mean": -4.114662170410156, "KL/std": 4.013312816619873, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.204820394515991, "fcm_dpo/q_t": 0.4919975996017456, "grad_norm": 4.525479316711426, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.17641638219356537, "logits/rejected": 0.1488437056541443, "logps/chosen": -96.67886352539062, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -100.04710388183594, "loss": 1.3552, "margin_dpo/margin_mean": 3.204820156097412, "margin_dpo/margin_std": 5.137542724609375, "step": 73 }, { "KL/chosen_KL_mean": -1.0602302551269531, "KL/mean": -2.8652851581573486, "KL/rejected_KL_mean": -4.670337677001953, "KL/std": 3.9167075157165527, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6101081371307373, "fcm_dpo/q_t": 0.4909824728965759, "grad_norm": 4.963025093078613, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.07164964079856873, "logits/rejected": 0.07294321805238724, "logps/chosen": -79.86862182617188, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.31710815429688, "loss": 1.3511, "margin_dpo/margin_mean": 3.6101083755493164, "margin_dpo/margin_std": 4.633073806762695, "step": 74 }, { "KL/chosen_KL_mean": -1.2048187255859375, "KL/mean": -3.571885585784912, "KL/rejected_KL_mean": -5.938945770263672, "KL/std": 4.859118461608887, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.734132766723633, "fcm_dpo/q_t": 0.4881826341152191, "grad_norm": 5.819903373718262, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.39377301931381226, "logits/rejected": -0.1896842122077942, "logps/chosen": -79.4866714477539, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.34205627441406, "loss": 1.3404, "margin_dpo/margin_mean": 4.734132766723633, "margin_dpo/margin_std": 5.7849016189575195, "step": 75 }, { "KL/chosen_KL_mean": -1.6757850646972656, "KL/mean": -3.219648838043213, "KL/rejected_KL_mean": -4.763519287109375, "KL/std": 3.973135471343994, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.087735414505005, "fcm_dpo/q_t": 0.49228888750076294, "grad_norm": 3.1143081188201904, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.5701890587806702, "logits/rejected": -0.17589715123176575, "logps/chosen": -89.55292510986328, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.48320007324219, "loss": 1.3562, "margin_dpo/margin_mean": 3.087735652923584, "margin_dpo/margin_std": 4.7748212814331055, "step": 76 }, { "KL/chosen_KL_mean": -1.7800140380859375, "KL/mean": -3.5272903442382812, "KL/rejected_KL_mean": -5.274570465087891, "KL/std": 4.715568542480469, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4945545196533203, "fcm_dpo/q_t": 0.49127188324928284, "grad_norm": 4.101231098175049, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.16603220999240875, "logits/rejected": 0.21527859568595886, "logps/chosen": -101.48036193847656, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.48033142089844, "loss": 1.3524, "margin_dpo/margin_mean": 3.4945549964904785, "margin_dpo/margin_std": 5.382877349853516, "step": 77 }, { "KL/chosen_KL_mean": -0.8605232238769531, "KL/mean": -2.731081008911133, "KL/rejected_KL_mean": -4.601634979248047, "KL/std": 3.879783868789673, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.741114616394043, "fcm_dpo/q_t": 0.4906550645828247, "grad_norm": 4.006865978240967, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.19206181168556213, "logits/rejected": 0.4419671893119812, "logps/chosen": -91.15723419189453, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.73936462402344, "loss": 1.3497, "margin_dpo/margin_mean": 3.741114377975464, "margin_dpo/margin_std": 4.52927303314209, "step": 78 }, { "KL/chosen_KL_mean": -1.1005134582519531, "KL/mean": -3.5610108375549316, "KL/rejected_KL_mean": -6.021511077880859, "KL/std": 5.683281898498535, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.921000957489014, "fcm_dpo/q_t": 0.4877166748046875, "grad_norm": 5.0408830642700195, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6842678785324097, "logits/rejected": -0.2966717481613159, "logps/chosen": -87.47883605957031, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -100.12928009033203, "loss": 1.3387, "margin_dpo/margin_mean": 4.921000957489014, "margin_dpo/margin_std": 6.309035301208496, "step": 79 }, { "KL/chosen_KL_mean": -1.8358879089355469, "KL/mean": -3.8267693519592285, "KL/rejected_KL_mean": -5.817653656005859, "KL/std": 5.191744804382324, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.981762170791626, "fcm_dpo/q_t": 0.49006152153015137, "grad_norm": 4.424638748168945, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.675911545753479, "logits/rejected": -0.3012281656265259, "logps/chosen": -95.80620574951172, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -98.39207458496094, "loss": 1.3477, "margin_dpo/margin_mean": 3.981761932373047, "margin_dpo/margin_std": 5.789755821228027, "step": 80 }, { "KL/chosen_KL_mean": -1.307525634765625, "KL/mean": -4.120117664337158, "KL/rejected_KL_mean": -6.932704925537109, "KL/std": 6.302674293518066, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.625184535980225, "fcm_dpo/q_t": 0.4859771728515625, "grad_norm": 5.7542643547058105, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5459493398666382, "logits/rejected": -0.10945140570402145, "logps/chosen": -101.13764953613281, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.60516357421875, "loss": 1.3323, "margin_dpo/margin_mean": 5.625185012817383, "margin_dpo/margin_std": 7.509007453918457, "step": 81 }, { "KL/chosen_KL_mean": -2.3378753662109375, "KL/mean": -4.278228759765625, "KL/rejected_KL_mean": -6.2185821533203125, "KL/std": 5.001008987426758, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.880702495574951, "fcm_dpo/q_t": 0.4903103709220886, "grad_norm": 4.099402904510498, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.47486573457717896, "logits/rejected": -0.2541789412498474, "logps/chosen": -85.38385772705078, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.74453735351562, "loss": 1.3487, "margin_dpo/margin_mean": 3.8807015419006348, "margin_dpo/margin_std": 5.492362976074219, "step": 82 }, { "KL/chosen_KL_mean": -2.8978538513183594, "KL/mean": -5.075529098510742, "KL/rejected_KL_mean": -7.253204345703125, "KL/std": 5.85251522064209, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.355353355407715, "fcm_dpo/q_t": 0.48913419246673584, "grad_norm": 4.316220283508301, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6124655604362488, "logits/rejected": -0.17772410809993744, "logps/chosen": -95.07406616210938, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.99784851074219, "loss": 1.3446, "margin_dpo/margin_mean": 4.355353832244873, "margin_dpo/margin_std": 7.551729202270508, "step": 83 }, { "KL/chosen_KL_mean": -3.6184730529785156, "KL/mean": -4.918801784515381, "KL/rejected_KL_mean": -6.219135284423828, "KL/std": 5.299543380737305, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.6006557941436768, "fcm_dpo/q_t": 0.4935084581375122, "grad_norm": 3.236095666885376, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.34674158692359924, "logits/rejected": 0.07833932340145111, "logps/chosen": -105.14496612548828, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.64521789550781, "loss": 1.3614, "margin_dpo/margin_mean": 2.600656032562256, "margin_dpo/margin_std": 5.66920280456543, "step": 84 }, { "KL/chosen_KL_mean": -3.2306556701660156, "KL/mean": -5.051130771636963, "KL/rejected_KL_mean": -6.871608734130859, "KL/std": 5.796492576599121, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6409552097320557, "fcm_dpo/q_t": 0.49091219902038574, "grad_norm": 3.128195285797119, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.4785706400871277, "logits/rejected": -0.13358959555625916, "logps/chosen": -99.3580322265625, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.92679595947266, "loss": 1.3515, "margin_dpo/margin_mean": 3.6409552097320557, "margin_dpo/margin_std": 7.0428876876831055, "step": 85 }, { "KL/chosen_KL_mean": -2.6890830993652344, "KL/mean": -5.054637908935547, "KL/rejected_KL_mean": -7.420200347900391, "KL/std": 6.70869255065918, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.731121063232422, "fcm_dpo/q_t": 0.48821350932121277, "grad_norm": 4.0614914894104, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.7449065446853638, "logits/rejected": -0.49459749460220337, "logps/chosen": -84.39334106445312, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -101.07569885253906, "loss": 1.3414, "margin_dpo/margin_mean": 4.731121063232422, "margin_dpo/margin_std": 8.502593040466309, "step": 86 }, { "KL/chosen_KL_mean": -3.240489959716797, "KL/mean": -5.815988540649414, "KL/rejected_KL_mean": -8.3914794921875, "KL/std": 7.529698371887207, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.1509881019592285, "fcm_dpo/q_t": 0.48714694380760193, "grad_norm": 4.2488226890563965, "learning_rate": 4.990486745229364e-07, "logits/chosen": -1.0236594676971436, "logits/rejected": -0.6372860670089722, "logps/chosen": -95.92645263671875, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -111.30966186523438, "loss": 1.3376, "margin_dpo/margin_mean": 5.15098762512207, "margin_dpo/margin_std": 9.021978378295898, "step": 87 }, { "KL/chosen_KL_mean": -3.7897186279296875, "KL/mean": -5.925696849822998, "KL/rejected_KL_mean": -8.06167221069336, "KL/std": 7.414109230041504, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.271944046020508, "fcm_dpo/q_t": 0.48934561014175415, "grad_norm": 3.9654178619384766, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.42611634731292725, "logits/rejected": -0.22464719414710999, "logps/chosen": -104.55270385742188, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -121.21204376220703, "loss": 1.3456, "margin_dpo/margin_mean": 4.271944046020508, "margin_dpo/margin_std": 7.840633392333984, "step": 88 }, { "KL/chosen_KL_mean": -3.0912399291992188, "KL/mean": -5.413605213165283, "KL/rejected_KL_mean": -7.735969543457031, "KL/std": 6.284661293029785, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.644725799560547, "fcm_dpo/q_t": 0.48842862248420715, "grad_norm": 4.301324367523193, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.9739890098571777, "logits/rejected": -0.631820797920227, "logps/chosen": -92.78231811523438, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -96.30430603027344, "loss": 1.3421, "margin_dpo/margin_mean": 4.644725799560547, "margin_dpo/margin_std": 8.163424491882324, "step": 89 }, { "KL/chosen_KL_mean": -2.9526100158691406, "KL/mean": -6.12854528427124, "KL/rejected_KL_mean": -9.30447769165039, "KL/std": 7.824243545532227, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.351874351501465, "fcm_dpo/q_t": 0.484183132648468, "grad_norm": 6.3789753913879395, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6813183426856995, "logits/rejected": -0.5565989017486572, "logps/chosen": -84.33516693115234, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -127.08161926269531, "loss": 1.3261, "margin_dpo/margin_mean": 6.351874351501465, "margin_dpo/margin_std": 9.545162200927734, "step": 90 }, { "KL/chosen_KL_mean": -3.5996170043945312, "KL/mean": -6.084131240844727, "KL/rejected_KL_mean": -8.568645477294922, "KL/std": 7.575854301452637, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.969034194946289, "fcm_dpo/q_t": 0.48762696981430054, "grad_norm": 3.7572855949401855, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6459864377975464, "logits/rejected": -0.20866045355796814, "logps/chosen": -100.82150268554688, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -101.54539489746094, "loss": 1.3394, "margin_dpo/margin_mean": 4.969033241271973, "margin_dpo/margin_std": 9.269001007080078, "step": 91 }, { "KL/chosen_KL_mean": -3.684173583984375, "KL/mean": -5.949444770812988, "KL/rejected_KL_mean": -8.214710235595703, "KL/std": 7.545066833496094, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.53053617477417, "fcm_dpo/q_t": 0.4887111485004425, "grad_norm": 4.094282627105713, "learning_rate": 4.984074589033043e-07, "logits/chosen": -1.015108346939087, "logits/rejected": -0.6290768980979919, "logps/chosen": -88.21444702148438, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -92.71603393554688, "loss": 1.3432, "margin_dpo/margin_mean": 4.530535697937012, "margin_dpo/margin_std": 8.246196746826172, "step": 92 }, { "KL/chosen_KL_mean": -4.436882019042969, "KL/mean": -6.444909572601318, "KL/rejected_KL_mean": -8.452938079833984, "KL/std": 7.253467082977295, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.016058921813965, "fcm_dpo/q_t": 0.4899873733520508, "grad_norm": 3.305091142654419, "learning_rate": 4.982595640958425e-07, "logits/chosen": -1.2338569164276123, "logits/rejected": -0.6369494199752808, "logps/chosen": -94.68731689453125, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -92.54714965820312, "loss": 1.3485, "margin_dpo/margin_mean": 4.016058921813965, "margin_dpo/margin_std": 8.843799591064453, "step": 93 }, { "KL/chosen_KL_mean": -4.851509094238281, "KL/mean": -8.259111404418945, "KL/rejected_KL_mean": -11.66671371459961, "KL/std": 8.952959060668945, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.8152079582214355, "fcm_dpo/q_t": 0.48302096128463745, "grad_norm": 5.590983867645264, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.9338712096214294, "logits/rejected": -0.6044620275497437, "logps/chosen": -97.30572509765625, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -116.91400146484375, "loss": 1.3221, "margin_dpo/margin_mean": 6.815207004547119, "margin_dpo/margin_std": 10.361207962036133, "step": 94 }, { "KL/chosen_KL_mean": -3.9511985778808594, "KL/mean": -6.591326713562012, "KL/rejected_KL_mean": -9.231452941894531, "KL/std": 7.576604843139648, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.2802605628967285, "fcm_dpo/q_t": 0.48685526847839355, "grad_norm": 3.9137790203094482, "learning_rate": 4.979441529392784e-07, "logits/chosen": -1.3029019832611084, "logits/rejected": -0.9494895935058594, "logps/chosen": -82.82490539550781, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -92.82266235351562, "loss": 1.3365, "margin_dpo/margin_mean": 5.28026008605957, "margin_dpo/margin_std": 9.47518253326416, "step": 95 }, { "KL/chosen_KL_mean": -4.138420104980469, "KL/mean": -7.605649948120117, "KL/rejected_KL_mean": -11.0728759765625, "KL/std": 9.30474853515625, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.934466361999512, "fcm_dpo/q_t": 0.4827427864074707, "grad_norm": 4.6927642822265625, "learning_rate": 4.977766449015534e-07, "logits/chosen": -1.2680134773254395, "logits/rejected": -0.7970230579376221, "logps/chosen": -110.73058319091797, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -112.84090423583984, "loss": 1.321, "margin_dpo/margin_mean": 6.93446683883667, "margin_dpo/margin_std": 10.548545837402344, "step": 96 }, { "KL/chosen_KL_mean": -5.848964691162109, "KL/mean": -8.807226181030273, "KL/rejected_KL_mean": -11.765483856201172, "KL/std": 8.714092254638672, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.916518211364746, "fcm_dpo/q_t": 0.48525339365005493, "grad_norm": 4.363819122314453, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.9856381416320801, "logits/rejected": -0.5741415619850159, "logps/chosen": -94.52884674072266, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -96.57777404785156, "loss": 1.3305, "margin_dpo/margin_mean": 5.91651725769043, "margin_dpo/margin_std": 9.581679344177246, "step": 97 }, { "KL/chosen_KL_mean": -5.538139343261719, "KL/mean": -9.199954986572266, "KL/rejected_KL_mean": -12.861774444580078, "KL/std": 8.616477966308594, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.32363224029541, "fcm_dpo/q_t": 0.481766015291214, "grad_norm": 4.541478633880615, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.9994081854820251, "logits/rejected": -0.7022345662117004, "logps/chosen": -97.7806396484375, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -114.38125610351562, "loss": 1.317, "margin_dpo/margin_mean": 7.32363224029541, "margin_dpo/margin_std": 9.873504638671875, "step": 98 }, { "KL/chosen_KL_mean": -4.145988464355469, "KL/mean": -8.628467559814453, "KL/rejected_KL_mean": -13.110950469970703, "KL/std": 10.724343299865723, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.964962005615234, "fcm_dpo/q_t": 0.4777979254722595, "grad_norm": 5.380415916442871, "learning_rate": 4.972349644343108e-07, "logits/chosen": -1.1264692544937134, "logits/rejected": -0.9258356094360352, "logps/chosen": -76.33064270019531, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -104.99227142333984, "loss": 1.3035, "margin_dpo/margin_mean": 8.964961051940918, "margin_dpo/margin_std": 14.017382621765137, "step": 99 }, { "KL/chosen_KL_mean": -5.477771759033203, "KL/mean": -8.168551445007324, "KL/rejected_KL_mean": -10.859333038330078, "KL/std": 8.560338973999023, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.381558895111084, "fcm_dpo/q_t": 0.48659729957580566, "grad_norm": 3.312379837036133, "learning_rate": 4.970413680203148e-07, "logits/chosen": -1.2460837364196777, "logits/rejected": -0.7035455703735352, "logps/chosen": -94.9916000366211, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -92.07646179199219, "loss": 1.3355, "margin_dpo/margin_mean": 5.381559371948242, "margin_dpo/margin_std": 9.596693992614746, "step": 100 }, { "KL/chosen_KL_mean": -8.034740447998047, "KL/mean": -10.663750648498535, "KL/rejected_KL_mean": -13.29275894165039, "KL/std": 12.035667419433594, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.258004665374756, "fcm_dpo/q_t": 0.4868850111961365, "grad_norm": 3.910324811935425, "learning_rate": 4.968412618365215e-07, "logits/chosen": -1.3099093437194824, "logits/rejected": -0.9940763711929321, "logps/chosen": -102.43793487548828, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -103.20277404785156, "loss": 1.3388, "margin_dpo/margin_mean": 5.258005142211914, "margin_dpo/margin_std": 12.93232250213623, "step": 101 }, { "KL/chosen_KL_mean": -9.223350524902344, "KL/mean": -11.285970687866211, "KL/rejected_KL_mean": -13.348590850830078, "KL/std": 12.11407470703125, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.125253677368164, "fcm_dpo/q_t": 0.48978811502456665, "grad_norm": 3.8780553340911865, "learning_rate": 4.966346511559149e-07, "logits/chosen": -1.5820343494415283, "logits/rejected": -1.1506949663162231, "logps/chosen": -100.83277893066406, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -89.11750793457031, "loss": 1.3512, "margin_dpo/margin_mean": 4.125253677368164, "margin_dpo/margin_std": 14.684951782226562, "step": 102 }, { "KL/chosen_KL_mean": -8.201019287109375, "KL/mean": -12.196084976196289, "KL/rejected_KL_mean": -16.19115447998047, "KL/std": 12.6515474319458, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.990135192871094, "fcm_dpo/q_t": 0.4801647663116455, "grad_norm": 4.072584629058838, "learning_rate": 4.964215414228785e-07, "logits/chosen": -1.3708975315093994, "logits/rejected": -0.9027442336082458, "logps/chosen": -105.60340881347656, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -115.17283630371094, "loss": 1.3137, "margin_dpo/margin_mean": 7.990135192871094, "margin_dpo/margin_std": 14.88979721069336, "step": 103 }, { "KL/chosen_KL_mean": -7.4892425537109375, "KL/mean": -11.769170761108398, "KL/rejected_KL_mean": -16.049102783203125, "KL/std": 13.109146118164062, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.559856414794922, "fcm_dpo/q_t": 0.4788459837436676, "grad_norm": 3.9202091693878174, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.8855440616607666, "logits/rejected": -1.4627628326416016, "logps/chosen": -102.3456039428711, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -113.07579803466797, "loss": 1.3094, "margin_dpo/margin_mean": 8.559856414794922, "margin_dpo/margin_std": 16.49858856201172, "step": 104 }, { "KL/chosen_KL_mean": -9.175403594970703, "KL/mean": -13.801017761230469, "KL/rejected_KL_mean": -18.4266357421875, "KL/std": 12.005559921264648, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.251230239868164, "fcm_dpo/q_t": 0.4770241677761078, "grad_norm": 4.768394947052002, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.905242919921875, "logits/rejected": -1.387826919555664, "logps/chosen": -105.93330383300781, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -120.0041275024414, "loss": 1.3014, "margin_dpo/margin_mean": 9.251230239868164, "margin_dpo/margin_std": 14.707832336425781, "step": 105 }, { "KL/chosen_KL_mean": -9.23647689819336, "KL/mean": -13.072514533996582, "KL/rejected_KL_mean": -16.908550262451172, "KL/std": 12.319353103637695, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.672076225280762, "fcm_dpo/q_t": 0.48102515935897827, "grad_norm": 3.959104537963867, "learning_rate": 4.957432749209755e-07, "logits/chosen": -1.3598814010620117, "logits/rejected": -0.8790042400360107, "logps/chosen": -101.67438507080078, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -107.91825103759766, "loss": 1.318, "margin_dpo/margin_mean": 7.672076225280762, "margin_dpo/margin_std": 16.527145385742188, "step": 106 }, { "KL/chosen_KL_mean": -8.4798583984375, "KL/mean": -13.308723449707031, "KL/rejected_KL_mean": -18.137596130371094, "KL/std": 15.852505683898926, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.657736778259277, "fcm_dpo/q_t": 0.47654756903648376, "grad_norm": 4.3160600662231445, "learning_rate": 4.955042268449307e-07, "logits/chosen": -1.454092025756836, "logits/rejected": -0.8274974822998047, "logps/chosen": -113.60486602783203, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -119.32009887695312, "loss": 1.3023, "margin_dpo/margin_mean": 9.657737731933594, "margin_dpo/margin_std": 19.12749481201172, "step": 107 }, { "KL/chosen_KL_mean": -9.763282775878906, "KL/mean": -15.02826976776123, "KL/rejected_KL_mean": -20.293254852294922, "KL/std": 17.15796661376953, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.529966354370117, "fcm_dpo/q_t": 0.4741785526275635, "grad_norm": 5.147180557250977, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.8017053604125977, "logits/rejected": -1.3483786582946777, "logps/chosen": -98.97264099121094, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -122.131103515625, "loss": 1.294, "margin_dpo/margin_mean": 10.529967308044434, "margin_dpo/margin_std": 20.094833374023438, "step": 108 }, { "KL/chosen_KL_mean": -8.822704315185547, "KL/mean": -14.600711822509766, "KL/rejected_KL_mean": -20.37873077392578, "KL/std": 14.947261810302734, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.556026458740234, "fcm_dpo/q_t": 0.47153112292289734, "grad_norm": 4.397295951843262, "learning_rate": 4.95006729368358e-07, "logits/chosen": -1.3030229806900024, "logits/rejected": -1.0237191915512085, "logps/chosen": -109.00979614257812, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -123.91612243652344, "loss": 1.2841, "margin_dpo/margin_mean": 11.556026458740234, "margin_dpo/margin_std": 20.050033569335938, "step": 109 }, { "KL/chosen_KL_mean": -11.244876861572266, "KL/mean": -13.942317962646484, "KL/rejected_KL_mean": -16.639766693115234, "KL/std": 14.593774795532227, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.394887447357178, "fcm_dpo/q_t": 0.4866444766521454, "grad_norm": 4.799775123596191, "learning_rate": 4.947482930773511e-07, "logits/chosen": -1.1008121967315674, "logits/rejected": -0.5783033967018127, "logps/chosen": -118.14059448242188, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -102.45097351074219, "loss": 1.3405, "margin_dpo/margin_mean": 5.394888401031494, "margin_dpo/margin_std": 16.578697204589844, "step": 110 }, { "KL/chosen_KL_mean": -9.616039276123047, "KL/mean": -16.42453384399414, "KL/rejected_KL_mean": -23.233028411865234, "KL/std": 17.029062271118164, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.61699104309082, "fcm_dpo/q_t": 0.46635541319847107, "grad_norm": 4.519567012786865, "learning_rate": 4.944834074412042e-07, "logits/chosen": -1.837559700012207, "logits/rejected": -1.5204964876174927, "logps/chosen": -102.94540405273438, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -127.41051483154297, "loss": 1.2662, "margin_dpo/margin_mean": 13.616990089416504, "margin_dpo/margin_std": 21.270469665527344, "step": 111 }, { "KL/chosen_KL_mean": -9.74685287475586, "KL/mean": -12.787385940551758, "KL/rejected_KL_mean": -15.827919006347656, "KL/std": 10.627685546875, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.0810699462890625, "fcm_dpo/q_t": 0.4849131107330322, "grad_norm": 3.6866533756256104, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.9612890481948853, "logits/rejected": -1.3536148071289062, "logps/chosen": -92.493896484375, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -87.38636779785156, "loss": 1.3318, "margin_dpo/margin_mean": 6.081070423126221, "margin_dpo/margin_std": 14.733182907104492, "step": 112 }, { "KL/chosen_KL_mean": -12.506614685058594, "KL/mean": -15.923879623413086, "KL/rejected_KL_mean": -19.341144561767578, "KL/std": 15.009801864624023, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.834536552429199, "fcm_dpo/q_t": 0.4831368327140808, "grad_norm": 4.734821796417236, "learning_rate": 4.939343162231841e-07, "logits/chosen": -1.8280048370361328, "logits/rejected": -1.2222900390625, "logps/chosen": -110.40611267089844, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -103.7607421875, "loss": 1.3255, "margin_dpo/margin_mean": 6.834536552429199, "margin_dpo/margin_std": 15.690375328063965, "step": 113 }, { "KL/chosen_KL_mean": -12.311607360839844, "KL/mean": -17.343799591064453, "KL/rejected_KL_mean": -22.375991821289062, "KL/std": 15.594621658325195, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.064388275146484, "fcm_dpo/q_t": 0.47525012493133545, "grad_norm": 4.536149024963379, "learning_rate": 4.936501251103751e-07, "logits/chosen": -2.0100290775299072, "logits/rejected": -1.413813829421997, "logps/chosen": -108.80672454833984, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -114.391357421875, "loss": 1.2995, "margin_dpo/margin_mean": 10.064388275146484, "margin_dpo/margin_std": 21.149805068969727, "step": 114 }, { "KL/chosen_KL_mean": -13.5145263671875, "KL/mean": -18.722551345825195, "KL/rejected_KL_mean": -23.93056869506836, "KL/std": 18.716575622558594, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.416046142578125, "fcm_dpo/q_t": 0.4744112491607666, "grad_norm": 4.465400218963623, "learning_rate": 4.933595135901732e-07, "logits/chosen": -1.698897123336792, "logits/rejected": -1.2478255033493042, "logps/chosen": -116.69465637207031, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -129.6465301513672, "loss": 1.297, "margin_dpo/margin_mean": 10.416046142578125, "margin_dpo/margin_std": 21.574260711669922, "step": 115 }, { "KL/chosen_KL_mean": -11.00699234008789, "KL/mean": -15.604896545410156, "KL/rejected_KL_mean": -20.202808380126953, "KL/std": 14.46591567993164, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.19581413269043, "fcm_dpo/q_t": 0.47727298736572266, "grad_norm": 4.040833473205566, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.9338021278381348, "logits/rejected": -1.5297337770462036, "logps/chosen": -92.16536712646484, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -106.5086669921875, "loss": 1.3033, "margin_dpo/margin_mean": 9.19581413269043, "margin_dpo/margin_std": 15.961064338684082, "step": 116 }, { "KL/chosen_KL_mean": -13.49578857421875, "KL/mean": -15.721643447875977, "KL/rejected_KL_mean": -17.947494506835938, "KL/std": 14.654977798461914, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.451704978942871, "fcm_dpo/q_t": 0.4889603853225708, "grad_norm": 5.193813323974609, "learning_rate": 4.927590601281083e-07, "logits/chosen": -1.3423629999160767, "logits/rejected": -0.7927243709564209, "logps/chosen": -110.57674407958984, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -93.08740997314453, "loss": 1.3524, "margin_dpo/margin_mean": 4.451704978942871, "margin_dpo/margin_std": 20.149227142333984, "step": 117 }, { "KL/chosen_KL_mean": -10.407882690429688, "KL/mean": -15.332218170166016, "KL/rejected_KL_mean": -20.256549835205078, "KL/std": 15.275903701782227, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.848665237426758, "fcm_dpo/q_t": 0.475645512342453, "grad_norm": 3.989236354827881, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.8827531337738037, "logits/rejected": -1.5232150554656982, "logps/chosen": -96.94412231445312, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -103.19219207763672, "loss": 1.2975, "margin_dpo/margin_mean": 9.848665237426758, "margin_dpo/margin_std": 17.061870574951172, "step": 118 }, { "KL/chosen_KL_mean": -13.97714614868164, "KL/mean": -19.694637298583984, "KL/rejected_KL_mean": -25.412128448486328, "KL/std": 18.110122680664062, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.434971809387207, "fcm_dpo/q_t": 0.4718303084373474, "grad_norm": 4.654934883117676, "learning_rate": 4.92133019126601e-07, "logits/chosen": -2.2220773696899414, "logits/rejected": -1.8641358613967896, "logps/chosen": -118.58622741699219, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -134.5699462890625, "loss": 1.2887, "margin_dpo/margin_mean": 11.434972763061523, "margin_dpo/margin_std": 23.396766662597656, "step": 119 }, { "KL/chosen_KL_mean": -14.429119110107422, "KL/mean": -20.570026397705078, "KL/rejected_KL_mean": -26.71092987060547, "KL/std": 20.479721069335938, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.281810760498047, "fcm_dpo/q_t": 0.4699445366859436, "grad_norm": 5.2194318771362305, "learning_rate": 4.918104238142103e-07, "logits/chosen": -2.157106876373291, "logits/rejected": -1.5463621616363525, "logps/chosen": -129.11654663085938, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -137.89678955078125, "loss": 1.2847, "margin_dpo/margin_mean": 12.281808853149414, "margin_dpo/margin_std": 26.596324920654297, "step": 120 }, { "KL/chosen_KL_mean": -11.793159484863281, "KL/mean": -18.972925186157227, "KL/rejected_KL_mean": -26.152694702148438, "KL/std": 17.301753997802734, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.359539985656738, "fcm_dpo/q_t": 0.4645058512687683, "grad_norm": 5.625041484832764, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.9931845664978027, "logits/rejected": -1.6752583980560303, "logps/chosen": -103.90982055664062, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -141.12559509277344, "loss": 1.2574, "margin_dpo/margin_mean": 14.359539985656738, "margin_dpo/margin_std": 19.54151153564453, "step": 121 }, { "KL/chosen_KL_mean": -11.211135864257812, "KL/mean": -19.405075073242188, "KL/rejected_KL_mean": -27.599014282226562, "KL/std": 17.524368286132812, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.387876510620117, "fcm_dpo/q_t": 0.45965027809143066, "grad_norm": 5.365407943725586, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.5984790325164795, "logits/rejected": -1.5931143760681152, "logps/chosen": -96.08870697021484, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -138.6820068359375, "loss": 1.241, "margin_dpo/margin_mean": 16.387874603271484, "margin_dpo/margin_std": 21.540977478027344, "step": 122 }, { "KL/chosen_KL_mean": -13.262821197509766, "KL/mean": -18.574787139892578, "KL/rejected_KL_mean": -23.886760711669922, "KL/std": 16.135135650634766, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.623932838439941, "fcm_dpo/q_t": 0.4737852215766907, "grad_norm": 4.273542404174805, "learning_rate": 4.908044411417711e-07, "logits/chosen": -2.0809192657470703, "logits/rejected": -1.7527475357055664, "logps/chosen": -111.74156951904297, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -117.74093627929688, "loss": 1.2936, "margin_dpo/margin_mean": 10.623932838439941, "margin_dpo/margin_std": 20.432016372680664, "step": 123 }, { "KL/chosen_KL_mean": -14.17916488647461, "KL/mean": -21.159587860107422, "KL/rejected_KL_mean": -28.14000701904297, "KL/std": 20.843353271484375, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.960832595825195, "fcm_dpo/q_t": 0.46610793471336365, "grad_norm": 4.826685905456543, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.8583539724349976, "logits/rejected": -1.6234264373779297, "logps/chosen": -116.94206237792969, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -151.1456298828125, "loss": 1.2708, "margin_dpo/margin_mean": 13.960832595825195, "margin_dpo/margin_std": 27.38876724243164, "step": 124 }, { "KL/chosen_KL_mean": -13.807785034179688, "KL/mean": -18.04694366455078, "KL/rejected_KL_mean": -22.286102294921875, "KL/std": 16.836322784423828, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.478311538696289, "fcm_dpo/q_t": 0.47910600900650024, "grad_norm": 4.202136039733887, "learning_rate": 4.90102044194588e-07, "logits/chosen": -1.7084598541259766, "logits/rejected": -1.45163893699646, "logps/chosen": -104.23406219482422, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -111.53511810302734, "loss": 1.3149, "margin_dpo/margin_mean": 8.478311538696289, "margin_dpo/margin_std": 21.41693687438965, "step": 125 }, { "KL/chosen_KL_mean": -13.46868896484375, "KL/mean": -20.062969207763672, "KL/rejected_KL_mean": -26.65725326538086, "KL/std": 19.818378448486328, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.188570976257324, "fcm_dpo/q_t": 0.4675453305244446, "grad_norm": 4.5179443359375, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.4186737537384033, "logits/rejected": -1.1245858669281006, "logps/chosen": -107.89157104492188, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -130.538818359375, "loss": 1.2717, "margin_dpo/margin_mean": 13.188570976257324, "margin_dpo/margin_std": 22.842647552490234, "step": 126 }, { "KL/chosen_KL_mean": -13.122596740722656, "KL/mean": -17.383634567260742, "KL/rejected_KL_mean": -21.644676208496094, "KL/std": 16.07625961303711, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.522075653076172, "fcm_dpo/q_t": 0.4790321886539459, "grad_norm": 4.36868953704834, "learning_rate": 4.89374339765481e-07, "logits/chosen": -1.5369030237197876, "logits/rejected": -1.1407138109207153, "logps/chosen": -97.87364959716797, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -105.18991088867188, "loss": 1.3131, "margin_dpo/margin_mean": 8.522075653076172, "margin_dpo/margin_std": 20.263111114501953, "step": 127 }, { "KL/chosen_KL_mean": -13.434154510498047, "KL/mean": -18.11103057861328, "KL/rejected_KL_mean": -22.787900924682617, "KL/std": 19.6984806060791, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.353751182556152, "fcm_dpo/q_t": 0.47717300057411194, "grad_norm": 4.761756896972656, "learning_rate": 4.890010211106795e-07, "logits/chosen": -1.5687198638916016, "logits/rejected": -1.0241076946258545, "logps/chosen": -104.05380249023438, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -105.89137268066406, "loss": 1.3121, "margin_dpo/margin_mean": 9.353750228881836, "margin_dpo/margin_std": 26.291912078857422, "step": 128 }, { "KL/chosen_KL_mean": -14.25033950805664, "KL/mean": -19.694782257080078, "KL/rejected_KL_mean": -25.13922882080078, "KL/std": 18.880294799804688, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.888891220092773, "fcm_dpo/q_t": 0.4738141596317291, "grad_norm": 4.24107551574707, "learning_rate": 4.88621404556699e-07, "logits/chosen": -1.6655076742172241, "logits/rejected": -1.3444623947143555, "logps/chosen": -113.16905212402344, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -128.42343139648438, "loss": 1.2987, "margin_dpo/margin_mean": 10.888891220092773, "margin_dpo/margin_std": 27.003910064697266, "step": 129 }, { "KL/chosen_KL_mean": -11.294525146484375, "KL/mean": -18.074857711791992, "KL/rejected_KL_mean": -24.85519027709961, "KL/std": 16.855791091918945, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.560664176940918, "fcm_dpo/q_t": 0.46662935614585876, "grad_norm": 4.419915199279785, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.9005060195922852, "logits/rejected": -1.7033705711364746, "logps/chosen": -84.47821044921875, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -112.8767318725586, "loss": 1.2677, "margin_dpo/margin_mean": 13.560665130615234, "margin_dpo/margin_std": 21.716400146484375, "step": 130 }, { "KL/chosen_KL_mean": -10.320560455322266, "KL/mean": -17.93349838256836, "KL/rejected_KL_mean": -25.546432495117188, "KL/std": 18.844680786132812, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.225872039794922, "fcm_dpo/q_t": 0.46253088116645813, "grad_norm": 5.320019721984863, "learning_rate": 4.878433179298909e-07, "logits/chosen": -2.065002679824829, "logits/rejected": -1.9137024879455566, "logps/chosen": -84.27700805664062, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -120.00225830078125, "loss": 1.2521, "margin_dpo/margin_mean": 15.225871086120605, "margin_dpo/margin_std": 21.907752990722656, "step": 131 }, { "KL/chosen_KL_mean": -13.953727722167969, "KL/mean": -19.925594329833984, "KL/rejected_KL_mean": -25.89745330810547, "KL/std": 19.79730987548828, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.943717002868652, "fcm_dpo/q_t": 0.4708418846130371, "grad_norm": 4.646473407745361, "learning_rate": 4.874448683603694e-07, "logits/chosen": -2.162374496459961, "logits/rejected": -1.7399506568908691, "logps/chosen": -106.3932113647461, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -120.02996826171875, "loss": 1.2879, "margin_dpo/margin_mean": 11.943716049194336, "margin_dpo/margin_std": 26.509193420410156, "step": 132 }, { "KL/chosen_KL_mean": -13.466751098632812, "KL/mean": -19.94275665283203, "KL/rejected_KL_mean": -26.418766021728516, "KL/std": 18.24095916748047, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.952018737792969, "fcm_dpo/q_t": 0.46815139055252075, "grad_norm": 4.544924259185791, "learning_rate": 4.870401618977415e-07, "logits/chosen": -2.3544094562530518, "logits/rejected": -1.9635837078094482, "logps/chosen": -106.58341217041016, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -128.513671875, "loss": 1.272, "margin_dpo/margin_mean": 12.952018737792969, "margin_dpo/margin_std": 21.06024169921875, "step": 133 }, { "KL/chosen_KL_mean": -13.033435821533203, "KL/mean": -20.096099853515625, "KL/rejected_KL_mean": -27.158763885498047, "KL/std": 17.378429412841797, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.12532901763916, "fcm_dpo/q_t": 0.46520036458969116, "grad_norm": 4.832637310028076, "learning_rate": 4.866292092063986e-07, "logits/chosen": -2.3934192657470703, "logits/rejected": -2.074002265930176, "logps/chosen": -96.58355712890625, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -121.04032897949219, "loss": 1.2605, "margin_dpo/margin_mean": 14.12532901763916, "margin_dpo/margin_std": 19.75324821472168, "step": 134 }, { "KL/chosen_KL_mean": -11.793933868408203, "KL/mean": -20.817569732666016, "KL/rejected_KL_mean": -29.841209411621094, "KL/std": 21.06104278564453, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 18.047258377075195, "fcm_dpo/q_t": 0.4562903046607971, "grad_norm": 6.105976581573486, "learning_rate": 4.862120211153265e-07, "logits/chosen": -2.0459752082824707, "logits/rejected": -2.04719877243042, "logps/chosen": -89.37655639648438, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -150.89768981933594, "loss": 1.2325, "margin_dpo/margin_mean": 18.047260284423828, "margin_dpo/margin_std": 26.235458374023438, "step": 135 }, { "KL/chosen_KL_mean": -15.605792999267578, "KL/mean": -22.385486602783203, "KL/rejected_KL_mean": -29.16518783569336, "KL/std": 23.626855850219727, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.009999999776482582, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.559389114379883, "fcm_dpo/q_t": 0.46754294633865356, "grad_norm": 5.022453784942627, "learning_rate": 4.857886086178193e-07, "logits/chosen": -2.088524341583252, "logits/rejected": -1.7058684825897217, "logps/chosen": -109.77852630615234, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -132.42947387695312, "loss": 1.2749, "margin_dpo/margin_mean": 13.55938720703125, "margin_dpo/margin_std": 28.631629943847656, "step": 136 }, { "KL/chosen_KL_mean": -11.945697784423828, "KL/mean": -24.10503387451172, "KL/rejected_KL_mean": -36.264366149902344, "KL/std": 27.348052978515625, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.010227885097265244, "fcm_dpo/delta": 0.15522728860378265, "fcm_dpo/margin": 24.318674087524414, "fcm_dpo/q_t": 0.44159436225891113, "grad_norm": 5.515376567840576, "learning_rate": 4.853589828711902e-07, "logits/chosen": -2.3214468955993652, "logits/rejected": -2.2008535861968994, "logps/chosen": -90.06443786621094, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -154.18673706054688, "loss": 1.1804, "margin_dpo/margin_mean": 24.318675994873047, "margin_dpo/margin_std": 32.59141540527344, "step": 137 }, { "KL/chosen_KL_mean": -16.007213592529297, "KL/mean": -22.633390426635742, "KL/rejected_KL_mean": -29.259571075439453, "KL/std": 19.728729248046875, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.252360343933105, "fcm_dpo/q_t": 0.4664079546928406, "grad_norm": 4.831506729125977, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.6070399284362793, "logits/rejected": -1.30340576171875, "logps/chosen": -108.837158203125, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -127.90797424316406, "loss": 1.2681, "margin_dpo/margin_mean": 13.252360343933105, "margin_dpo/margin_std": 22.882272720336914, "step": 138 }, { "KL/chosen_KL_mean": -11.518566131591797, "KL/mean": -19.26103973388672, "KL/rejected_KL_mean": -27.00351333618164, "KL/std": 18.675678253173828, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.484947204589844, "fcm_dpo/q_t": 0.4606620669364929, "grad_norm": 4.712732315063477, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.9638400077819824, "logits/rejected": -1.6488885879516602, "logps/chosen": -90.52157592773438, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -112.83084106445312, "loss": 1.2485, "margin_dpo/margin_mean": 15.484947204589844, "margin_dpo/margin_std": 24.23459815979004, "step": 139 }, { "KL/chosen_KL_mean": -13.871238708496094, "KL/mean": -20.5947208404541, "KL/rejected_KL_mean": -27.31821060180664, "KL/std": 18.34686279296875, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.446975708007812, "fcm_dpo/q_t": 0.46586087346076965, "grad_norm": 4.753627777099609, "learning_rate": 4.840329401637809e-07, "logits/chosen": -2.3088369369506836, "logits/rejected": -1.905047059059143, "logps/chosen": -102.22368621826172, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -117.89298248291016, "loss": 1.2653, "margin_dpo/margin_mean": 13.446976661682129, "margin_dpo/margin_std": 22.154033660888672, "step": 140 }, { "KL/chosen_KL_mean": -19.097705841064453, "KL/mean": -25.781448364257812, "KL/rejected_KL_mean": -32.46519088745117, "KL/std": 27.119401931762695, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.367481231689453, "fcm_dpo/q_t": 0.46580761671066284, "grad_norm": 5.317202091217041, "learning_rate": 4.83578576263792e-07, "logits/chosen": -2.256279230117798, "logits/rejected": -1.8157012462615967, "logps/chosen": -126.31787109375, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -137.17742919921875, "loss": 1.2748, "margin_dpo/margin_mean": 13.367481231689453, "margin_dpo/margin_std": 28.643779754638672, "step": 141 }, { "KL/chosen_KL_mean": -15.686511993408203, "KL/mean": -22.92099952697754, "KL/rejected_KL_mean": -30.15549087524414, "KL/std": 22.597476959228516, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.468978881835938, "fcm_dpo/q_t": 0.46368053555488586, "grad_norm": 4.813089847564697, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.9123042821884155, "logits/rejected": -1.6630275249481201, "logps/chosen": -104.41545867919922, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -129.88587951660156, "loss": 1.2586, "margin_dpo/margin_mean": 14.468977928161621, "margin_dpo/margin_std": 24.35361099243164, "step": 142 }, { "KL/chosen_KL_mean": -14.960289001464844, "KL/mean": -20.12690544128418, "KL/rejected_KL_mean": -25.29351806640625, "KL/std": 18.0384464263916, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.333226203918457, "fcm_dpo/q_t": 0.4737424850463867, "grad_norm": 5.141211032867432, "learning_rate": 4.826513955607734e-07, "logits/chosen": -2.2664759159088135, "logits/rejected": -1.8361139297485352, "logps/chosen": -100.4648666381836, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -111.09918212890625, "loss": 1.2928, "margin_dpo/margin_mean": 10.333226203918457, "margin_dpo/margin_std": 19.352519989013672, "step": 143 }, { "KL/chosen_KL_mean": -19.058509826660156, "KL/mean": -23.19949722290039, "KL/rejected_KL_mean": -27.34048843383789, "KL/std": 20.207927703857422, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.281972885131836, "fcm_dpo/q_t": 0.4788573086261749, "grad_norm": 6.380902290344238, "learning_rate": 4.821786031898176e-07, "logits/chosen": -2.2588913440704346, "logits/rejected": -1.6646013259887695, "logps/chosen": -123.74560546875, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -115.0595703125, "loss": 1.3196, "margin_dpo/margin_mean": 8.281972885131836, "margin_dpo/margin_std": 24.94827651977539, "step": 144 }, { "KL/chosen_KL_mean": -14.070140838623047, "KL/mean": -20.415206909179688, "KL/rejected_KL_mean": -26.760276794433594, "KL/std": 18.05329704284668, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.690128326416016, "fcm_dpo/q_t": 0.46778449416160583, "grad_norm": 5.348937511444092, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.6592739820480347, "logits/rejected": -1.0994408130645752, "logps/chosen": -102.59002685546875, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -110.52542114257812, "loss": 1.2713, "margin_dpo/margin_mean": 12.690128326416016, "margin_dpo/margin_std": 20.78740119934082, "step": 145 }, { "KL/chosen_KL_mean": -15.837150573730469, "KL/mean": -22.180126190185547, "KL/rejected_KL_mean": -28.523101806640625, "KL/std": 19.36863136291504, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.685955047607422, "fcm_dpo/q_t": 0.4677453339099884, "grad_norm": 5.804346084594727, "learning_rate": 4.812146767012779e-07, "logits/chosen": -2.1731395721435547, "logits/rejected": -1.4527220726013184, "logps/chosen": -118.74301147460938, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -117.47246551513672, "loss": 1.2747, "margin_dpo/margin_mean": 12.685956001281738, "margin_dpo/margin_std": 23.663103103637695, "step": 146 }, { "KL/chosen_KL_mean": -14.558509826660156, "KL/mean": -21.202747344970703, "KL/rejected_KL_mean": -27.846988677978516, "KL/std": 17.663497924804688, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.010315323248505592, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.288483619689941, "fcm_dpo/q_t": 0.46636486053466797, "grad_norm": 5.360836029052734, "learning_rate": 4.807235679840536e-07, "logits/chosen": -2.0523457527160645, "logits/rejected": -1.4634252786636353, "logps/chosen": -103.82069396972656, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -104.34710693359375, "loss": 1.265, "margin_dpo/margin_mean": 13.288483619689941, "margin_dpo/margin_std": 20.52991485595703, "step": 147 }, { "KL/chosen_KL_mean": -12.84716796875, "KL/mean": -20.85495376586914, "KL/rejected_KL_mean": -28.86273956298828, "KL/std": 21.408485412597656, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.010396521538496017, "fcm_dpo/delta": 0.07810341566801071, "fcm_dpo/margin": 16.01557159423828, "fcm_dpo/q_t": 0.46016862988471985, "grad_norm": 4.9590253829956055, "learning_rate": 4.802263794862384e-07, "logits/chosen": -2.1854052543640137, "logits/rejected": -1.8153210878372192, "logps/chosen": -104.61004638671875, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -139.24826049804688, "loss": 1.2468, "margin_dpo/margin_mean": 16.01557159423828, "margin_dpo/margin_std": 25.303375244140625, "step": 148 }, { "KL/chosen_KL_mean": -16.810348510742188, "KL/mean": -23.670978546142578, "KL/rejected_KL_mean": -30.531604766845703, "KL/std": 22.748842239379883, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.010477720759809017, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.721254348754883, "fcm_dpo/q_t": 0.4657023549079895, "grad_norm": 5.090395927429199, "learning_rate": 4.797231243092118e-07, "logits/chosen": -2.079342842102051, "logits/rejected": -1.711609959602356, "logps/chosen": -110.29436492919922, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -136.17530822753906, "loss": 1.2743, "margin_dpo/margin_mean": 13.721254348754883, "margin_dpo/margin_std": 30.55900001525879, "step": 149 }, { "KL/chosen_KL_mean": -13.610740661621094, "KL/mean": -20.800823211669922, "KL/rejected_KL_mean": -27.99090576171875, "KL/std": 22.339248657226562, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.010477720759809017, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.380167007446289, "fcm_dpo/q_t": 0.4633002281188965, "grad_norm": 4.784445285797119, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.9591243267059326, "logits/rejected": -1.7481008768081665, "logps/chosen": -89.3894271850586, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -118.01766967773438, "loss": 1.2586, "margin_dpo/margin_mean": 14.380166053771973, "margin_dpo/margin_std": 25.275150299072266, "step": 150 }, { "KL/chosen_KL_mean": -17.141075134277344, "KL/mean": -24.229610443115234, "KL/rejected_KL_mean": -31.318138122558594, "KL/std": 22.053836822509766, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.010477720759809017, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.17706298828125, "fcm_dpo/q_t": 0.46391913294792175, "grad_norm": 5.613164901733398, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.6637558937072754, "logits/rejected": -2.0851352214813232, "logps/chosen": -119.87529754638672, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -137.35595703125, "loss": 1.263, "margin_dpo/margin_mean": 14.177062034606934, "margin_dpo/margin_std": 26.962430953979492, "step": 151 }, { "KL/chosen_KL_mean": -14.400520324707031, "KL/mean": -23.314071655273438, "KL/rejected_KL_mean": -32.22761535644531, "KL/std": 22.00121307373047, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.010477720759809017, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.82708740234375, "fcm_dpo/q_t": 0.45450079441070557, "grad_norm": 5.169353008270264, "learning_rate": 4.78177092112495e-07, "logits/chosen": -2.0961546897888184, "logits/rejected": -1.6915212869644165, "logps/chosen": -111.99126434326172, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -142.72674560546875, "loss": 1.2304, "margin_dpo/margin_mean": 17.82708740234375, "margin_dpo/margin_std": 28.0440673828125, "step": 152 }, { "KL/chosen_KL_mean": -15.235832214355469, "KL/mean": -21.334306716918945, "KL/rejected_KL_mean": -27.432785034179688, "KL/std": 21.2337589263916, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.010477720759809017, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.1969633102417, "fcm_dpo/q_t": 0.46887972950935364, "grad_norm": 5.190633773803711, "learning_rate": 4.776497044244016e-07, "logits/chosen": -2.425900936126709, "logits/rejected": -2.1150050163269043, "logps/chosen": -102.26142883300781, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -118.50132751464844, "loss": 1.2824, "margin_dpo/margin_mean": 12.196962356567383, "margin_dpo/margin_std": 26.90621566772461, "step": 153 }, { "KL/chosen_KL_mean": -14.308284759521484, "KL/mean": -24.374820709228516, "KL/rejected_KL_mean": -34.44136047363281, "KL/std": 22.47675323486328, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.010553887113928795, "fcm_dpo/delta": 0.07217022031545639, "fcm_dpo/margin": 20.133092880249023, "fcm_dpo/q_t": 0.448677122592926, "grad_norm": 5.4404120445251465, "learning_rate": 4.771163179548808e-07, "logits/chosen": -2.145632743835449, "logits/rejected": -1.8567649126052856, "logps/chosen": -105.41395568847656, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -145.81375122070312, "loss": 1.2073, "margin_dpo/margin_mean": 20.13309097290039, "margin_dpo/margin_std": 28.40338134765625, "step": 154 }, { "KL/chosen_KL_mean": -16.572925567626953, "KL/mean": -22.798816680908203, "KL/rejected_KL_mean": -29.024700164794922, "KL/std": 22.30667495727539, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.010630053468048573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.451776504516602, "fcm_dpo/q_t": 0.46701568365097046, "grad_norm": 5.636878967285156, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -2.1570992469787598, "logits/rejected": -1.7269625663757324, "logps/chosen": -111.13389587402344, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -113.79829406738281, "loss": 1.2832, "margin_dpo/margin_mean": 12.451776504516602, "margin_dpo/margin_std": 29.974637985229492, "step": 155 }, { "KL/chosen_KL_mean": -18.98804473876953, "KL/mean": -24.65170669555664, "KL/rejected_KL_mean": -30.31536102294922, "KL/std": 21.2137451171875, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.010630053468048573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.32731819152832, "fcm_dpo/q_t": 0.47122183442115784, "grad_norm": 5.707695484161377, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -2.1856603622436523, "logits/rejected": -1.6934802532196045, "logps/chosen": -113.6434326171875, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -114.73470306396484, "loss": 1.2939, "margin_dpo/margin_mean": 11.327317237854004, "margin_dpo/margin_std": 28.57672119140625, "step": 156 }, { "KL/chosen_KL_mean": -16.922870635986328, "KL/mean": -27.436203002929688, "KL/rejected_KL_mean": -37.949527740478516, "KL/std": 24.338939666748047, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.010918639600276947, "fcm_dpo/delta": 0.17428763210773468, "fcm_dpo/margin": 21.026653289794922, "fcm_dpo/q_t": 0.44512778520584106, "grad_norm": 5.779012680053711, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.514077663421631, "logits/rejected": -2.0966784954071045, "logps/chosen": -113.93525695800781, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -149.65087890625, "loss": 1.1941, "margin_dpo/margin_mean": 21.026653289794922, "margin_dpo/margin_std": 28.262229919433594, "step": 157 }, { "KL/chosen_KL_mean": -16.10519027709961, "KL/mean": -26.264909744262695, "KL/rejected_KL_mean": -36.42462921142578, "KL/std": 26.171859741210938, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.011306250467896461, "fcm_dpo/delta": 0.17412203550338745, "fcm_dpo/margin": 20.319440841674805, "fcm_dpo/q_t": 0.4461653232574463, "grad_norm": 5.876125335693359, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.9129363298416138, "logits/rejected": -1.7055774927139282, "logps/chosen": -110.81815338134766, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -157.36196899414062, "loss": 1.1996, "margin_dpo/margin_mean": 20.319438934326172, "margin_dpo/margin_std": 31.336315155029297, "step": 158 }, { "KL/chosen_KL_mean": -17.41028594970703, "KL/mean": -25.169261932373047, "KL/rejected_KL_mean": -32.92824172973633, "KL/std": 24.364898681640625, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.011397194117307663, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 15.517956733703613, "fcm_dpo/q_t": 0.4572281241416931, "grad_norm": 6.080030918121338, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.6008901596069336, "logits/rejected": -2.0952985286712646, "logps/chosen": -125.62852478027344, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -127.76726531982422, "loss": 1.2427, "margin_dpo/margin_mean": 15.51795768737793, "margin_dpo/margin_std": 28.3680419921875, "step": 159 }, { "KL/chosen_KL_mean": -14.577560424804688, "KL/mean": -22.859004974365234, "KL/rejected_KL_mean": -31.14044952392578, "KL/std": 19.859323501586914, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.011604917235672474, "fcm_dpo/delta": 0.09030850976705551, "fcm_dpo/margin": 16.56287956237793, "fcm_dpo/q_t": 0.4534924626350403, "grad_norm": 5.596906661987305, "learning_rate": 4.737908228387656e-07, "logits/chosen": -2.335120677947998, "logits/rejected": -1.847808837890625, "logps/chosen": -112.740966796875, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -133.4214324951172, "loss": 1.221, "margin_dpo/margin_mean": 16.562881469726562, "margin_dpo/margin_std": 22.968341827392578, "step": 160 }, { "KL/chosen_KL_mean": -15.27872085571289, "KL/mean": -22.161968231201172, "KL/rejected_KL_mean": -29.045211791992188, "KL/std": 18.21184539794922, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.011604917235672474, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.766493797302246, "fcm_dpo/q_t": 0.4611618220806122, "grad_norm": 5.7197418212890625, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -2.273632049560547, "logits/rejected": -1.7997932434082031, "logps/chosen": -101.87059020996094, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -118.0616455078125, "loss": 1.2557, "margin_dpo/margin_mean": 13.766494750976562, "margin_dpo/margin_std": 25.65907096862793, "step": 161 }, { "KL/chosen_KL_mean": -14.74441146850586, "KL/mean": -22.964149475097656, "KL/rejected_KL_mean": -31.183883666992188, "KL/std": 22.499574661254883, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.0117735480889678, "fcm_dpo/delta": 0.07213227450847626, "fcm_dpo/margin": 16.43947410583496, "fcm_dpo/q_t": 0.4535808861255646, "grad_norm": 6.077866554260254, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.8551239967346191, "logits/rejected": -1.449777364730835, "logps/chosen": -112.14729309082031, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -127.05311584472656, "loss": 1.2265, "margin_dpo/margin_mean": 16.439476013183594, "margin_dpo/margin_std": 26.34282684326172, "step": 162 }, { "KL/chosen_KL_mean": -13.091423034667969, "KL/mean": -22.161598205566406, "KL/rejected_KL_mean": -31.231773376464844, "KL/std": 22.77787971496582, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.011946926824748516, "fcm_dpo/delta": 0.07309354841709137, "fcm_dpo/margin": 18.140352249145508, "fcm_dpo/q_t": 0.44838595390319824, "grad_norm": 5.629769802093506, "learning_rate": 4.720482655449212e-07, "logits/chosen": -2.0775585174560547, "logits/rejected": -1.6824802160263062, "logps/chosen": -95.26681518554688, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -114.09651184082031, "loss": 1.2062, "margin_dpo/margin_mean": 18.140352249145508, "margin_dpo/margin_std": 26.733760833740234, "step": 163 }, { "KL/chosen_KL_mean": -17.281421661376953, "KL/mean": -24.069910049438477, "KL/rejected_KL_mean": -30.85839080810547, "KL/std": 18.620283126831055, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.012158727273344994, "fcm_dpo/delta": 0.0878656804561615, "fcm_dpo/margin": 13.576963424682617, "fcm_dpo/q_t": 0.4602315127849579, "grad_norm": 6.342441558837891, "learning_rate": 4.714556901942599e-07, "logits/chosen": -2.060515880584717, "logits/rejected": -1.596917748451233, "logps/chosen": -107.75569152832031, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -117.88648223876953, "loss": 1.2502, "margin_dpo/margin_mean": 13.57696533203125, "margin_dpo/margin_std": 24.09943389892578, "step": 164 }, { "KL/chosen_KL_mean": -12.480415344238281, "KL/mean": -21.161205291748047, "KL/rejected_KL_mean": -29.841999053955078, "KL/std": 19.48574447631836, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.012372849509119987, "fcm_dpo/delta": 0.08728645741939545, "fcm_dpo/margin": 17.36157989501953, "fcm_dpo/q_t": 0.44894033670425415, "grad_norm": 5.659285545349121, "learning_rate": 4.708572792802069e-07, "logits/chosen": -2.2916436195373535, "logits/rejected": -1.836799144744873, "logps/chosen": -99.79364013671875, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -111.74458312988281, "loss": 1.2062, "margin_dpo/margin_mean": 17.36157989501953, "margin_dpo/margin_std": 24.590404510498047, "step": 165 }, { "KL/chosen_KL_mean": -16.520648956298828, "KL/mean": -27.144981384277344, "KL/rejected_KL_mean": -37.769309997558594, "KL/std": 26.39926528930664, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.01262185163795948, "fcm_dpo/delta": 0.1354774534702301, "fcm_dpo/margin": 21.248655319213867, "fcm_dpo/q_t": 0.4389752149581909, "grad_norm": 6.446889400482178, "learning_rate": 4.702530485714461e-07, "logits/chosen": -2.0672483444213867, "logits/rejected": -1.8985710144042969, "logps/chosen": -98.10671997070312, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -140.93447875976562, "loss": 1.1784, "margin_dpo/margin_mean": 21.2486572265625, "margin_dpo/margin_std": 33.91762924194336, "step": 166 }, { "KL/chosen_KL_mean": -15.84384536743164, "KL/mean": -26.15508460998535, "KL/rejected_KL_mean": -36.46632766723633, "KL/std": 22.715862274169922, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.01281344797462225, "fcm_dpo/delta": 0.039476338773965836, "fcm_dpo/margin": 20.622478485107422, "fcm_dpo/q_t": 0.43691909313201904, "grad_norm": 6.174230575561523, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.627135992050171, "logits/rejected": -1.4863570928573608, "logps/chosen": -99.405517578125, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -135.51937866210938, "loss": 1.1679, "margin_dpo/margin_mean": 20.622478485107422, "margin_dpo/margin_std": 26.534364700317383, "step": 167 }, { "KL/chosen_KL_mean": -14.314018249511719, "KL/mean": -23.096111297607422, "KL/rejected_KL_mean": -31.87820816040039, "KL/std": 20.628662109375, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.01315833069384098, "fcm_dpo/delta": 0.17274583876132965, "fcm_dpo/margin": 17.564186096191406, "fcm_dpo/q_t": 0.4451214671134949, "grad_norm": 6.0556511878967285, "learning_rate": 4.690271916109034e-07, "logits/chosen": -2.1242005825042725, "logits/rejected": -1.7847068309783936, "logps/chosen": -94.24525451660156, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -111.98037719726562, "loss": 1.1927, "margin_dpo/margin_mean": 17.564186096191406, "margin_dpo/margin_std": 23.987106323242188, "step": 168 }, { "KL/chosen_KL_mean": -12.374855041503906, "KL/mean": -22.888263702392578, "KL/rejected_KL_mean": -33.40167236328125, "KL/std": 23.44970703125, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.01347257662564516, "fcm_dpo/delta": 0.12028877437114716, "fcm_dpo/margin": 21.026809692382812, "fcm_dpo/q_t": 0.4331369996070862, "grad_norm": 5.855408668518066, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -2.0839014053344727, "logits/rejected": -1.8767797946929932, "logps/chosen": -95.278564453125, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -123.5235824584961, "loss": 1.154, "margin_dpo/margin_mean": 21.026809692382812, "margin_dpo/margin_std": 26.248252868652344, "step": 169 }, { "KL/chosen_KL_mean": -13.467720031738281, "KL/mean": -21.924579620361328, "KL/rejected_KL_mean": -30.381450653076172, "KL/std": 18.7718448638916, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.013726375997066498, "fcm_dpo/delta": 0.051085058599710464, "fcm_dpo/margin": 16.913728713989258, "fcm_dpo/q_t": 0.44384893774986267, "grad_norm": 6.870417594909668, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.8733181953430176, "logits/rejected": -1.6833090782165527, "logps/chosen": -88.44969177246094, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -112.54705810546875, "loss": 1.188, "margin_dpo/margin_mean": 16.913728713989258, "margin_dpo/margin_std": 20.49029541015625, "step": 170 }, { "KL/chosen_KL_mean": -21.33121109008789, "KL/mean": -30.184993743896484, "KL/rejected_KL_mean": -39.03879165649414, "KL/std": 31.521778106689453, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.014068006537854671, "fcm_dpo/delta": 0.1544691026210785, "fcm_dpo/margin": 17.70758056640625, "fcm_dpo/q_t": 0.44547730684280396, "grad_norm": 9.687833786010742, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -2.4707581996917725, "logits/rejected": -1.9285330772399902, "logps/chosen": -140.3097381591797, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -154.80810546875, "loss": 1.2299, "margin_dpo/margin_mean": 17.70758056640625, "margin_dpo/margin_std": 41.68859100341797, "step": 171 }, { "KL/chosen_KL_mean": -20.819087982177734, "KL/mean": -27.89054298400879, "KL/rejected_KL_mean": -34.961997985839844, "KL/std": 20.982099533081055, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.014418627135455608, "fcm_dpo/delta": 0.09153927117586136, "fcm_dpo/margin": 14.142910957336426, "fcm_dpo/q_t": 0.4508058428764343, "grad_norm": 7.979934215545654, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.8934574127197266, "logits/rejected": -2.0934500694274902, "logps/chosen": -121.9728775024414, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -125.96768188476562, "loss": 1.2116, "margin_dpo/margin_mean": 14.142910957336426, "margin_dpo/margin_std": 19.069021224975586, "step": 172 }, { "KL/chosen_KL_mean": -16.284358978271484, "KL/mean": -23.25412368774414, "KL/rejected_KL_mean": -30.223888397216797, "KL/std": 19.511213302612305, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.0145356310531497, "fcm_dpo/delta": 0.0804961770772934, "fcm_dpo/margin": 13.939532279968262, "fcm_dpo/q_t": 0.4512094259262085, "grad_norm": 6.909392833709717, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -2.2185277938842773, "logits/rejected": -1.7112140655517578, "logps/chosen": -115.34104919433594, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -114.07461547851562, "loss": 1.2152, "margin_dpo/margin_mean": 13.939532279968262, "margin_dpo/margin_std": 20.292552947998047, "step": 173 }, { "KL/chosen_KL_mean": -16.220869064331055, "KL/mean": -27.566532135009766, "KL/rejected_KL_mean": -38.912200927734375, "KL/std": 28.592411041259766, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.01483137533068657, "fcm_dpo/delta": 0.06539694964885712, "fcm_dpo/margin": 22.691328048706055, "fcm_dpo/q_t": 0.42303580045700073, "grad_norm": 7.1526994705200195, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.9584803581237793, "logits/rejected": -1.9635305404663086, "logps/chosen": -96.62977600097656, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -147.11392211914062, "loss": 1.1313, "margin_dpo/margin_mean": 22.691326141357422, "margin_dpo/margin_std": 32.59711456298828, "step": 174 }, { "KL/chosen_KL_mean": -19.236377716064453, "KL/mean": -29.37299346923828, "KL/rejected_KL_mean": -39.509620666503906, "KL/std": 23.96701431274414, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.015071099624037743, "fcm_dpo/delta": 0.09737876057624817, "fcm_dpo/margin": 20.273239135742188, "fcm_dpo/q_t": 0.42707228660583496, "grad_norm": 8.050992012023926, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.5847499370574951, "logits/rejected": -1.307321548461914, "logps/chosen": -101.95286560058594, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -136.1767578125, "loss": 1.1403, "margin_dpo/margin_mean": 20.273239135742188, "margin_dpo/margin_std": 25.314807891845703, "step": 175 }, { "KL/chosen_KL_mean": -15.757843017578125, "KL/mean": -27.28162384033203, "KL/rejected_KL_mean": -38.80540084838867, "KL/std": 25.286073684692383, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.015256403014063835, "fcm_dpo/delta": 0.05012023448944092, "fcm_dpo/margin": 23.04755973815918, "fcm_dpo/q_t": 0.41844916343688965, "grad_norm": 7.943886756896973, "learning_rate": 4.638942309888058e-07, "logits/chosen": -2.201439619064331, "logits/rejected": -2.0829544067382812, "logps/chosen": -95.38714599609375, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -141.49398803710938, "loss": 1.1172, "margin_dpo/margin_mean": 23.04755973815918, "margin_dpo/margin_std": 30.871421813964844, "step": 176 }, { "KL/chosen_KL_mean": -19.17926788330078, "KL/mean": -31.370956420898438, "KL/rejected_KL_mean": -43.56264877319336, "KL/std": 29.176395416259766, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.015390634536743164, "fcm_dpo/delta": 0.025462238118052483, "fcm_dpo/margin": 24.383377075195312, "fcm_dpo/q_t": 0.4144711494445801, "grad_norm": 7.732017993927002, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -2.001267433166504, "logits/rejected": -1.6747379302978516, "logps/chosen": -111.74037170410156, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -146.90103149414062, "loss": 1.1013, "margin_dpo/margin_mean": 24.383378982543945, "margin_dpo/margin_std": 32.05248260498047, "step": 177 }, { "KL/chosen_KL_mean": -17.328685760498047, "KL/mean": -32.49454879760742, "KL/rejected_KL_mean": -47.66040802001953, "KL/std": 28.86639404296875, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.015210134908556938, "fcm_dpo/delta": -0.06437690556049347, "fcm_dpo/margin": 30.33172607421875, "fcm_dpo/q_t": 0.39307671785354614, "grad_norm": 8.103397369384766, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -2.06088924407959, "logits/rejected": -1.8953661918640137, "logps/chosen": -101.594482421875, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -158.4577178955078, "loss": 1.029, "margin_dpo/margin_mean": 30.331722259521484, "margin_dpo/margin_std": 31.507015228271484, "step": 178 }, { "KL/chosen_KL_mean": -22.87517547607422, "KL/mean": -31.03832244873047, "KL/rejected_KL_mean": -39.20146179199219, "KL/std": 26.659008026123047, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.01531896274536848, "fcm_dpo/delta": 0.0483752116560936, "fcm_dpo/margin": 16.32628631591797, "fcm_dpo/q_t": 0.4433177709579468, "grad_norm": 9.356552124023438, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.9116811752319336, "logits/rejected": -1.5053656101226807, "logps/chosen": -116.32315063476562, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -133.45687866210938, "loss": 1.2167, "margin_dpo/margin_mean": 16.32628631591797, "margin_dpo/margin_std": 34.10221862792969, "step": 179 }, { "KL/chosen_KL_mean": -17.767791748046875, "KL/mean": -30.52987289428711, "KL/rejected_KL_mean": -43.291954040527344, "KL/std": 24.810630798339844, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.015322528779506683, "fcm_dpo/delta": 0.009251517243683338, "fcm_dpo/margin": 25.52416229248047, "fcm_dpo/q_t": 0.40902501344680786, "grad_norm": 8.983099937438965, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.8629512786865234, "logits/rejected": -1.663163423538208, "logps/chosen": -106.60333251953125, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -147.19891357421875, "loss": 1.073, "margin_dpo/margin_mean": 25.52416229248047, "margin_dpo/margin_std": 27.524240493774414, "step": 180 }, { "KL/chosen_KL_mean": -17.022998809814453, "KL/mean": -24.74761199951172, "KL/rejected_KL_mean": -32.472225189208984, "KL/std": 23.084880828857422, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.015738680958747864, "fcm_dpo/delta": 0.1605420857667923, "fcm_dpo/margin": 15.449227333068848, "fcm_dpo/q_t": 0.4441846013069153, "grad_norm": 7.9050493240356445, "learning_rate": 4.605024008834863e-07, "logits/chosen": -2.230503559112549, "logits/rejected": -1.7836577892303467, "logps/chosen": -96.2569580078125, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -101.64665222167969, "loss": 1.2001, "margin_dpo/margin_mean": 15.449227333068848, "margin_dpo/margin_std": 26.963146209716797, "step": 181 }, { "KL/chosen_KL_mean": -19.75183868408203, "KL/mean": -32.27485656738281, "KL/rejected_KL_mean": -44.797874450683594, "KL/std": 27.880767822265625, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.015842024236917496, "fcm_dpo/delta": 0.0033403472043573856, "fcm_dpo/margin": 25.04602813720703, "fcm_dpo/q_t": 0.40880951285362244, "grad_norm": 8.327579498291016, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.4302818775177002, "logits/rejected": -1.2526754140853882, "logps/chosen": -95.83402252197266, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -139.7664794921875, "loss": 1.087, "margin_dpo/margin_mean": 25.04602813720703, "margin_dpo/margin_std": 31.580062866210938, "step": 182 }, { "KL/chosen_KL_mean": -24.43130874633789, "KL/mean": -33.79136657714844, "KL/rejected_KL_mean": -43.15143585205078, "KL/std": 24.186115264892578, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.01587144285440445, "fcm_dpo/delta": 0.004016148392111063, "fcm_dpo/margin": 18.720117568969727, "fcm_dpo/q_t": 0.42975932359695435, "grad_norm": 9.648603439331055, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.9849950075149536, "logits/rejected": -1.3848059177398682, "logps/chosen": -118.33921813964844, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -126.71205139160156, "loss": 1.1531, "margin_dpo/margin_mean": 18.72011947631836, "margin_dpo/margin_std": 25.527969360351562, "step": 183 }, { "KL/chosen_KL_mean": -21.663288116455078, "KL/mean": -28.37442398071289, "KL/rejected_KL_mean": -35.0855598449707, "KL/std": 24.478296279907227, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.015987733379006386, "fcm_dpo/delta": 0.07273852080106735, "fcm_dpo/margin": 13.422273635864258, "fcm_dpo/q_t": 0.4490703344345093, "grad_norm": 8.421429634094238, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.9063217639923096, "logits/rejected": -1.6032493114471436, "logps/chosen": -108.53119659423828, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -119.3701171875, "loss": 1.2311, "margin_dpo/margin_mean": 13.422273635864258, "margin_dpo/margin_std": 27.896526336669922, "step": 184 }, { "KL/chosen_KL_mean": -22.14864730834961, "KL/mean": -29.5606689453125, "KL/rejected_KL_mean": -36.972694396972656, "KL/std": 23.954322814941406, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.016215771436691284, "fcm_dpo/delta": 0.06891398876905441, "fcm_dpo/margin": 14.824050903320312, "fcm_dpo/q_t": 0.44434428215026855, "grad_norm": 9.427929878234863, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.4545114040374756, "logits/rejected": -1.8272576332092285, "logps/chosen": -116.17007446289062, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -118.42937469482422, "loss": 1.2106, "margin_dpo/margin_mean": 14.824051856994629, "margin_dpo/margin_std": 28.342208862304688, "step": 185 }, { "KL/chosen_KL_mean": -29.471981048583984, "KL/mean": -37.76907730102539, "KL/rejected_KL_mean": -46.066162109375, "KL/std": 24.864803314208984, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.016600944101810455, "fcm_dpo/delta": 0.1282443106174469, "fcm_dpo/margin": 16.594175338745117, "fcm_dpo/q_t": 0.43593889474868774, "grad_norm": 11.484820365905762, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.5599873065948486, "logits/rejected": -1.603615403175354, "logps/chosen": -143.66366577148438, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -139.40528869628906, "loss": 1.1874, "margin_dpo/margin_mean": 16.594175338745117, "margin_dpo/margin_std": 29.353029251098633, "step": 186 }, { "KL/chosen_KL_mean": -21.679534912109375, "KL/mean": -31.454111099243164, "KL/rejected_KL_mean": -41.22868347167969, "KL/std": 26.028564453125, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.016981925815343857, "fcm_dpo/delta": 0.06991731375455856, "fcm_dpo/margin": 19.549148559570312, "fcm_dpo/q_t": 0.42360594868659973, "grad_norm": 8.60029125213623, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -2.0268306732177734, "logits/rejected": -1.802510380744934, "logps/chosen": -98.54908752441406, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -129.3936309814453, "loss": 1.1399, "margin_dpo/margin_mean": 19.549148559570312, "margin_dpo/margin_std": 29.345401763916016, "step": 187 }, { "KL/chosen_KL_mean": -18.187660217285156, "KL/mean": -28.987823486328125, "KL/rejected_KL_mean": -39.787994384765625, "KL/std": 25.639720916748047, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.017024017870426178, "fcm_dpo/delta": 0.033437229692935944, "fcm_dpo/margin": 21.600326538085938, "fcm_dpo/q_t": 0.4129903316497803, "grad_norm": 8.757622718811035, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.242361307144165, "logits/rejected": -2.0096662044525146, "logps/chosen": -97.38246154785156, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -136.61688232421875, "loss": 1.102, "margin_dpo/margin_mean": 21.600324630737305, "margin_dpo/margin_std": 25.959243774414062, "step": 188 }, { "KL/chosen_KL_mean": -22.786869049072266, "KL/mean": -32.364715576171875, "KL/rejected_KL_mean": -41.942562103271484, "KL/std": 24.538928985595703, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.017350463196635246, "fcm_dpo/delta": 0.06933768093585968, "fcm_dpo/margin": 19.15569496154785, "fcm_dpo/q_t": 0.42485880851745605, "grad_norm": 9.162005424499512, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -2.023186206817627, "logits/rejected": -1.369532823562622, "logps/chosen": -106.67967224121094, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -121.40846252441406, "loss": 1.1412, "margin_dpo/margin_mean": 19.15569496154785, "margin_dpo/margin_std": 29.334354400634766, "step": 189 }, { "KL/chosen_KL_mean": -20.05907440185547, "KL/mean": -33.100990295410156, "KL/rejected_KL_mean": -46.142921447753906, "KL/std": 28.73394012451172, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.017248596996068954, "fcm_dpo/delta": -0.05225612595677376, "fcm_dpo/margin": 26.083843231201172, "fcm_dpo/q_t": 0.396478533744812, "grad_norm": 9.155226707458496, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -2.1661195755004883, "logits/rejected": -1.581076979637146, "logps/chosen": -123.50054168701172, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -155.35794067382812, "loss": 1.0457, "margin_dpo/margin_mean": 26.083843231201172, "margin_dpo/margin_std": 29.458080291748047, "step": 190 }, { "KL/chosen_KL_mean": -19.078758239746094, "KL/mean": -34.923248291015625, "KL/rejected_KL_mean": -50.767738342285156, "KL/std": 32.010887145996094, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.016823021695017815, "fcm_dpo/delta": -0.1404791623353958, "fcm_dpo/margin": 31.68897819519043, "fcm_dpo/q_t": 0.3797943592071533, "grad_norm": 9.150219917297363, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.8687258958816528, "logits/rejected": -1.573800802230835, "logps/chosen": -118.86427307128906, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -166.7680206298828, "loss": 0.9968, "margin_dpo/margin_mean": 31.688976287841797, "margin_dpo/margin_std": 34.668861389160156, "step": 191 }, { "KL/chosen_KL_mean": -27.272037506103516, "KL/mean": -38.11181640625, "KL/rejected_KL_mean": -48.95158767700195, "KL/std": 25.05974578857422, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.016792040318250656, "fcm_dpo/delta": 0.037267740815877914, "fcm_dpo/margin": 21.67954444885254, "fcm_dpo/q_t": 0.4168747663497925, "grad_norm": 9.52097225189209, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.8875877857208252, "logits/rejected": -1.648698329925537, "logps/chosen": -128.13235473632812, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -158.39659118652344, "loss": 1.1144, "margin_dpo/margin_mean": 21.679546356201172, "margin_dpo/margin_std": 30.27237319946289, "step": 192 }, { "KL/chosen_KL_mean": -23.521041870117188, "KL/mean": -31.185344696044922, "KL/rejected_KL_mean": -38.849647521972656, "KL/std": 26.877199172973633, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.01688910275697708, "fcm_dpo/delta": 0.04711629077792168, "fcm_dpo/margin": 15.32861328125, "fcm_dpo/q_t": 0.4406198561191559, "grad_norm": 10.282949447631836, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.3613088130950928, "logits/rejected": -1.775997519493103, "logps/chosen": -116.45935821533203, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -121.95883178710938, "loss": 1.2188, "margin_dpo/margin_mean": 15.32861328125, "margin_dpo/margin_std": 33.33778381347656, "step": 193 }, { "KL/chosen_KL_mean": -20.09249496459961, "KL/mean": -29.859996795654297, "KL/rejected_KL_mean": -39.62749481201172, "KL/std": 23.718292236328125, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.01717711240053177, "fcm_dpo/delta": 0.06642194837331772, "fcm_dpo/margin": 19.535003662109375, "fcm_dpo/q_t": 0.4220060408115387, "grad_norm": 8.41250228881836, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.61328125, "logits/rejected": -2.226808786392212, "logps/chosen": -83.88406372070312, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -108.3541488647461, "loss": 1.1233, "margin_dpo/margin_mean": 19.535003662109375, "margin_dpo/margin_std": 25.31645393371582, "step": 194 }, { "KL/chosen_KL_mean": -21.718467712402344, "KL/mean": -34.85675048828125, "KL/rejected_KL_mean": -47.99503707885742, "KL/std": 26.282644271850586, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.01708962395787239, "fcm_dpo/delta": -0.051331739872694016, "fcm_dpo/margin": 26.276569366455078, "fcm_dpo/q_t": 0.39610666036605835, "grad_norm": 8.504365921020508, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.5736221075057983, "logits/rejected": -1.4837653636932373, "logps/chosen": -105.60392761230469, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -151.9962921142578, "loss": 1.0482, "margin_dpo/margin_mean": 26.276569366455078, "margin_dpo/margin_std": 30.085912704467773, "step": 195 }, { "KL/chosen_KL_mean": -24.905841827392578, "KL/mean": -36.63442611694336, "KL/rejected_KL_mean": -48.36301040649414, "KL/std": 27.632535934448242, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.017011653631925583, "fcm_dpo/delta": 0.0009867791086435318, "fcm_dpo/margin": 23.457168579101562, "fcm_dpo/q_t": 0.40587007999420166, "grad_norm": 8.727965354919434, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.6957306861877441, "logits/rejected": -1.2538894414901733, "logps/chosen": -114.30720520019531, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -131.71035766601562, "loss": 1.0807, "margin_dpo/margin_mean": 23.457168579101562, "margin_dpo/margin_std": 28.04401397705078, "step": 196 }, { "KL/chosen_KL_mean": -24.708362579345703, "KL/mean": -31.423316955566406, "KL/rejected_KL_mean": -38.13827133178711, "KL/std": 20.61960220336914, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.017469994723796844, "fcm_dpo/delta": 0.1692475974559784, "fcm_dpo/margin": 13.429903030395508, "fcm_dpo/q_t": 0.44468483328819275, "grad_norm": 9.150611877441406, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.796918511390686, "logits/rejected": -1.4673511981964111, "logps/chosen": -111.549072265625, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -120.93116760253906, "loss": 1.205, "margin_dpo/margin_mean": 13.429903030395508, "margin_dpo/margin_std": 23.14694595336914, "step": 197 }, { "KL/chosen_KL_mean": -27.77411651611328, "KL/mean": -40.47635269165039, "KL/rejected_KL_mean": -53.17859649658203, "KL/std": 27.400245666503906, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.017489265650510788, "fcm_dpo/delta": -0.046363409608602524, "fcm_dpo/margin": 25.404481887817383, "fcm_dpo/q_t": 0.3969578146934509, "grad_norm": 9.66495418548584, "learning_rate": 4.479470611971645e-07, "logits/chosen": -2.085875988006592, "logits/rejected": -1.9388020038604736, "logps/chosen": -112.82235717773438, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -157.28225708007812, "loss": 1.0415, "margin_dpo/margin_mean": 25.40447998046875, "margin_dpo/margin_std": 27.213031768798828, "step": 198 }, { "KL/chosen_KL_mean": -26.51614761352539, "KL/mean": -36.821311950683594, "KL/rejected_KL_mean": -47.12649154663086, "KL/std": 24.95035743713379, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.017553452402353287, "fcm_dpo/delta": 0.03964417427778244, "fcm_dpo/margin": 20.61034393310547, "fcm_dpo/q_t": 0.4161791205406189, "grad_norm": 8.823667526245117, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.9238916635513306, "logits/rejected": -1.671958088874817, "logps/chosen": -108.30284118652344, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -137.91726684570312, "loss": 1.1142, "margin_dpo/margin_mean": 20.61034393310547, "margin_dpo/margin_std": 28.11554718017578, "step": 199 }, { "KL/chosen_KL_mean": -25.263015747070312, "KL/mean": -36.977294921875, "KL/rejected_KL_mean": -48.69157409667969, "KL/std": 26.513355255126953, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.01749037578701973, "fcm_dpo/delta": -0.010539419949054718, "fcm_dpo/margin": 23.42855453491211, "fcm_dpo/q_t": 0.4065903425216675, "grad_norm": 9.214178085327148, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -2.120130777359009, "logits/rejected": -1.5873587131500244, "logps/chosen": -109.53111267089844, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -135.65943908691406, "loss": 1.0941, "margin_dpo/margin_mean": 23.428552627563477, "margin_dpo/margin_std": 32.451751708984375, "step": 200 }, { "KL/chosen_KL_mean": -29.78036117553711, "KL/mean": -41.33283996582031, "KL/rejected_KL_mean": -52.88531494140625, "KL/std": 31.11529541015625, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.017637979239225388, "fcm_dpo/delta": -0.008608940988779068, "fcm_dpo/margin": 23.10496711730957, "fcm_dpo/q_t": 0.4081525504589081, "grad_norm": 9.595117568969727, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.5936212539672852, "logits/rejected": -1.2256463766098022, "logps/chosen": -137.5528564453125, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -172.67779541015625, "loss": 1.1046, "margin_dpo/margin_mean": 23.104969024658203, "margin_dpo/margin_std": 34.17507553100586, "step": 201 }, { "KL/chosen_KL_mean": -24.94731903076172, "KL/mean": -35.4049072265625, "KL/rejected_KL_mean": -45.86250305175781, "KL/std": 24.300334930419922, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.017618495970964432, "fcm_dpo/delta": 0.0326709970831871, "fcm_dpo/margin": 20.915180206298828, "fcm_dpo/q_t": 0.4163872003555298, "grad_norm": 8.935579299926758, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.9219000339508057, "logits/rejected": -1.8084826469421387, "logps/chosen": -100.91976928710938, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -140.32247924804688, "loss": 1.1175, "margin_dpo/margin_mean": 20.915180206298828, "margin_dpo/margin_std": 29.48224449157715, "step": 202 }, { "KL/chosen_KL_mean": -23.97006607055664, "KL/mean": -34.6287841796875, "KL/rejected_KL_mean": -45.287506103515625, "KL/std": 24.272319793701172, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.017713358625769615, "fcm_dpo/delta": 0.02322772517800331, "fcm_dpo/margin": 21.317440032958984, "fcm_dpo/q_t": 0.4123542308807373, "grad_norm": 9.287670135498047, "learning_rate": 4.439630306414758e-07, "logits/chosen": -2.1872100830078125, "logits/rejected": -1.7428826093673706, "logps/chosen": -118.93722534179688, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -138.1751708984375, "loss": 1.0989, "margin_dpo/margin_mean": 21.317440032958984, "margin_dpo/margin_std": 27.405914306640625, "step": 203 }, { "KL/chosen_KL_mean": -29.77129364013672, "KL/mean": -39.09203338623047, "KL/rejected_KL_mean": -48.412776947021484, "KL/std": 26.413461685180664, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.017913253977894783, "fcm_dpo/delta": 0.06827335804700851, "fcm_dpo/margin": 18.64148712158203, "fcm_dpo/q_t": 0.4241909980773926, "grad_norm": 10.967944145202637, "learning_rate": 4.431508065452897e-07, "logits/chosen": -2.1346569061279297, "logits/rejected": -1.469660997390747, "logps/chosen": -146.12847900390625, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -141.81036376953125, "loss": 1.1587, "margin_dpo/margin_mean": 18.64148712158203, "margin_dpo/margin_std": 31.378631591796875, "step": 204 }, { "KL/chosen_KL_mean": -26.42535400390625, "KL/mean": -39.396942138671875, "KL/rejected_KL_mean": -52.36852264404297, "KL/std": 27.76251983642578, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.017725832760334015, "fcm_dpo/delta": -0.06325547397136688, "fcm_dpo/margin": 25.94317626953125, "fcm_dpo/q_t": 0.3913671374320984, "grad_norm": 8.770440101623535, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.5739998817443848, "logits/rejected": -2.146010160446167, "logps/chosen": -115.28469848632812, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -144.2230224609375, "loss": 1.0218, "margin_dpo/margin_mean": 25.94317626953125, "margin_dpo/margin_std": 24.040271759033203, "step": 205 }, { "KL/chosen_KL_mean": -24.985347747802734, "KL/mean": -37.6280403137207, "KL/rejected_KL_mean": -50.27073287963867, "KL/std": 25.333263397216797, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.017554234713315964, "fcm_dpo/delta": -0.04627152532339096, "fcm_dpo/margin": 25.28538703918457, "fcm_dpo/q_t": 0.3940865993499756, "grad_norm": 9.081438064575195, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.886652946472168, "logits/rejected": -1.753605842590332, "logps/chosen": -101.53169250488281, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -153.223876953125, "loss": 1.0282, "margin_dpo/margin_mean": 25.285388946533203, "margin_dpo/margin_std": 22.291887283325195, "step": 206 }, { "KL/chosen_KL_mean": -28.43319320678711, "KL/mean": -41.75574493408203, "KL/rejected_KL_mean": -55.078311920166016, "KL/std": 33.581573486328125, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.017405685037374496, "fcm_dpo/delta": -0.06684759259223938, "fcm_dpo/margin": 26.645109176635742, "fcm_dpo/q_t": 0.3991878628730774, "grad_norm": 10.24443531036377, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.313065767288208, "logits/rejected": -1.8947019577026367, "logps/chosen": -114.66484069824219, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -145.73342895507812, "loss": 1.0784, "margin_dpo/margin_mean": 26.645109176635742, "margin_dpo/margin_std": 39.75422286987305, "step": 207 }, { "KL/chosen_KL_mean": -29.381633758544922, "KL/mean": -43.0147819519043, "KL/rejected_KL_mean": -56.64793395996094, "KL/std": 28.169836044311523, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.01719864085316658, "fcm_dpo/delta": -0.07229123264551163, "fcm_dpo/margin": 27.266292572021484, "fcm_dpo/q_t": 0.3894117772579193, "grad_norm": 9.6349458694458, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.873849868774414, "logits/rejected": -1.5258948802947998, "logps/chosen": -123.54118347167969, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -157.61026000976562, "loss": 1.0207, "margin_dpo/margin_mean": 27.266292572021484, "margin_dpo/margin_std": 26.43233299255371, "step": 208 }, { "KL/chosen_KL_mean": -25.148887634277344, "KL/mean": -36.304039001464844, "KL/rejected_KL_mean": -47.45919418334961, "KL/std": 25.465927124023438, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.01708749681711197, "fcm_dpo/delta": 0.019400183111429214, "fcm_dpo/margin": 22.310302734375, "fcm_dpo/q_t": 0.41238778829574585, "grad_norm": 9.375060081481934, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.4571962356567383, "logits/rejected": -1.9883074760437012, "logps/chosen": -109.3194580078125, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -135.0787353515625, "loss": 1.1048, "margin_dpo/margin_mean": 22.310302734375, "margin_dpo/margin_std": 30.07697868347168, "step": 209 }, { "KL/chosen_KL_mean": -26.901321411132812, "KL/mean": -38.547264099121094, "KL/rejected_KL_mean": -50.19320297241211, "KL/std": 26.310874938964844, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.01717599481344223, "fcm_dpo/delta": -9.740982204675674e-05, "fcm_dpo/margin": 23.291885375976562, "fcm_dpo/q_t": 0.40764176845550537, "grad_norm": 8.423360824584961, "learning_rate": 4.381713366536311e-07, "logits/chosen": -2.33752703666687, "logits/rejected": -1.965346336364746, "logps/chosen": -108.07249450683594, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -134.36798095703125, "loss": 1.0806, "margin_dpo/margin_mean": 23.291887283325195, "margin_dpo/margin_std": 28.31826400756836, "step": 210 }, { "KL/chosen_KL_mean": -33.176021575927734, "KL/mean": -43.6624641418457, "KL/rejected_KL_mean": -54.14890670776367, "KL/std": 28.498428344726562, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.017306886613368988, "fcm_dpo/delta": 0.03808807581663132, "fcm_dpo/margin": 20.972881317138672, "fcm_dpo/q_t": 0.4174515902996063, "grad_norm": 10.620423316955566, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.3510818481445312, "logits/rejected": -1.7273385524749756, "logps/chosen": -141.88873291015625, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -147.70455932617188, "loss": 1.1208, "margin_dpo/margin_mean": 20.972881317138672, "margin_dpo/margin_std": 30.49523162841797, "step": 211 }, { "KL/chosen_KL_mean": -22.752887725830078, "KL/mean": -39.854400634765625, "KL/rejected_KL_mean": -56.95591735839844, "KL/std": 34.515403747558594, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.016757890582084656, "fcm_dpo/delta": -0.18462014198303223, "fcm_dpo/margin": 34.203025817871094, "fcm_dpo/q_t": 0.37160539627075195, "grad_norm": 8.50251579284668, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.938838243484497, "logits/rejected": -1.5229421854019165, "logps/chosen": -121.11483764648438, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -166.84591674804688, "loss": 0.9782, "margin_dpo/margin_mean": 34.203025817871094, "margin_dpo/margin_std": 37.13557052612305, "step": 212 }, { "KL/chosen_KL_mean": -26.234722137451172, "KL/mean": -38.72975540161133, "KL/rejected_KL_mean": -51.22478485107422, "KL/std": 29.904008865356445, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.01658651977777481, "fcm_dpo/delta": -0.015242788940668106, "fcm_dpo/margin": 24.99005889892578, "fcm_dpo/q_t": 0.40395012497901917, "grad_norm": 10.706770896911621, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -2.194777011871338, "logits/rejected": -1.4027342796325684, "logps/chosen": -134.2900390625, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -151.36892700195312, "loss": 1.0876, "margin_dpo/margin_mean": 24.99005889892578, "margin_dpo/margin_std": 33.95264434814453, "step": 213 }, { "KL/chosen_KL_mean": -27.298892974853516, "KL/mean": -38.716007232666016, "KL/rejected_KL_mean": -50.13312530517578, "KL/std": 31.12869644165039, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.016669873148202896, "fcm_dpo/delta": 0.0200960636138916, "fcm_dpo/margin": 22.8342342376709, "fcm_dpo/q_t": 0.414794921875, "grad_norm": 8.502174377441406, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -2.0881295204162598, "logits/rejected": -1.965996265411377, "logps/chosen": -105.10363006591797, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -134.40890502929688, "loss": 1.1208, "margin_dpo/margin_mean": 22.83423614501953, "margin_dpo/margin_std": 35.53411865234375, "step": 214 }, { "KL/chosen_KL_mean": -23.402442932128906, "KL/mean": -38.997703552246094, "KL/rejected_KL_mean": -54.592960357666016, "KL/std": 30.682048797607422, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.01638612523674965, "fcm_dpo/delta": -0.1170823723077774, "fcm_dpo/margin": 31.190513610839844, "fcm_dpo/q_t": 0.3822213411331177, "grad_norm": 8.394270896911621, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -2.0505785942077637, "logits/rejected": -1.5892293453216553, "logps/chosen": -108.51634216308594, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -151.4544677734375, "loss": 0.9973, "margin_dpo/margin_mean": 31.190513610839844, "margin_dpo/margin_std": 31.179595947265625, "step": 215 }, { "KL/chosen_KL_mean": -25.889972686767578, "KL/mean": -38.56311798095703, "KL/rejected_KL_mean": -51.23626708984375, "KL/std": 26.026596069335938, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.016317319124937057, "fcm_dpo/delta": -0.014423396438360214, "fcm_dpo/margin": 25.346296310424805, "fcm_dpo/q_t": 0.40359801054000854, "grad_norm": 8.231871604919434, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.6618878841400146, "logits/rejected": -1.2215979099273682, "logps/chosen": -106.4822769165039, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -132.65609741210938, "loss": 1.0674, "margin_dpo/margin_mean": 25.346298217773438, "margin_dpo/margin_std": 28.58609390258789, "step": 216 }, { "KL/chosen_KL_mean": -32.19425964355469, "KL/mean": -43.11487579345703, "KL/rejected_KL_mean": -54.03549575805664, "KL/std": 29.280494689941406, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.016359776258468628, "fcm_dpo/delta": 0.044226787984371185, "fcm_dpo/margin": 21.84123992919922, "fcm_dpo/q_t": 0.4202713966369629, "grad_norm": 10.815791130065918, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.279926061630249, "logits/rejected": -1.974079966545105, "logps/chosen": -125.66683959960938, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -157.52401733398438, "loss": 1.1436, "margin_dpo/margin_mean": 21.841238021850586, "margin_dpo/margin_std": 36.76457595825195, "step": 217 }, { "KL/chosen_KL_mean": -30.197803497314453, "KL/mean": -43.41337585449219, "KL/rejected_KL_mean": -56.62895202636719, "KL/std": 29.887683868408203, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.016277320683002472, "fcm_dpo/delta": -0.03172491118311882, "fcm_dpo/margin": 26.43113899230957, "fcm_dpo/q_t": 0.40346798300743103, "grad_norm": 9.106010437011719, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.8466994762420654, "logits/rejected": -1.4801476001739502, "logps/chosen": -119.25663757324219, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -150.93576049804688, "loss": 1.0892, "margin_dpo/margin_mean": 26.431137084960938, "margin_dpo/margin_std": 38.04444885253906, "step": 218 }, { "KL/chosen_KL_mean": -28.42550277709961, "KL/mean": -39.77777099609375, "KL/rejected_KL_mean": -51.13003921508789, "KL/std": 33.96781539916992, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.016473740339279175, "fcm_dpo/delta": 0.025521598756313324, "fcm_dpo/margin": 22.70452880859375, "fcm_dpo/q_t": 0.4151086211204529, "grad_norm": 10.738029479980469, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.9513671398162842, "logits/rejected": -1.4867024421691895, "logps/chosen": -129.43284606933594, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -143.5979766845703, "loss": 1.1422, "margin_dpo/margin_mean": 22.704530715942383, "margin_dpo/margin_std": 38.13640594482422, "step": 219 }, { "KL/chosen_KL_mean": -28.592758178710938, "KL/mean": -39.36803436279297, "KL/rejected_KL_mean": -50.143310546875, "KL/std": 26.446380615234375, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.016496650874614716, "fcm_dpo/delta": 0.046052463352680206, "fcm_dpo/margin": 21.550561904907227, "fcm_dpo/q_t": 0.41648364067077637, "grad_norm": 9.55201244354248, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5094305276870728, "logits/rejected": -1.158909797668457, "logps/chosen": -124.12997436523438, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -144.45034790039062, "loss": 1.1095, "margin_dpo/margin_mean": 21.55056381225586, "margin_dpo/margin_std": 26.793121337890625, "step": 220 }, { "KL/chosen_KL_mean": -25.947792053222656, "KL/mean": -43.942298889160156, "KL/rejected_KL_mean": -61.93679428100586, "KL/std": 34.90479278564453, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.01614297740161419, "fcm_dpo/delta": -0.19169840216636658, "fcm_dpo/margin": 35.98900604248047, "fcm_dpo/q_t": 0.3672521114349365, "grad_norm": 9.276050567626953, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.348720669746399, "logits/rejected": -1.218428134918213, "logps/chosen": -108.79265594482422, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -172.74859619140625, "loss": 0.9548, "margin_dpo/margin_mean": 35.989009857177734, "margin_dpo/margin_std": 33.244483947753906, "step": 221 }, { "KL/chosen_KL_mean": -33.69205093383789, "KL/mean": -47.56593704223633, "KL/rejected_KL_mean": -61.4398193359375, "KL/std": 33.31191635131836, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.015864061191678047, "fcm_dpo/delta": -0.04223465919494629, "fcm_dpo/margin": 27.74776840209961, "fcm_dpo/q_t": 0.4003182053565979, "grad_norm": 10.087751388549805, "learning_rate": 4.276818137766118e-07, "logits/chosen": -2.220752477645874, "logits/rejected": -1.7969143390655518, "logps/chosen": -128.8340301513672, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -168.24423217773438, "loss": 1.0937, "margin_dpo/margin_mean": 27.74776840209961, "margin_dpo/margin_std": 41.15275573730469, "step": 222 }, { "KL/chosen_KL_mean": -31.778278350830078, "KL/mean": -41.96113967895508, "KL/rejected_KL_mean": -52.14400863647461, "KL/std": 27.973480224609375, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.015912488102912903, "fcm_dpo/delta": 0.07854747772216797, "fcm_dpo/margin": 20.36573028564453, "fcm_dpo/q_t": 0.42583921551704407, "grad_norm": 9.76085376739502, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.415830612182617, "logits/rejected": -2.0337021350860596, "logps/chosen": -117.3533935546875, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -138.5963897705078, "loss": 1.15, "margin_dpo/margin_mean": 20.36573028564453, "margin_dpo/margin_std": 32.15615463256836, "step": 223 }, { "KL/chosen_KL_mean": -30.003841400146484, "KL/mean": -43.716217041015625, "KL/rejected_KL_mean": -57.428592681884766, "KL/std": 31.491840362548828, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.015905670821666718, "fcm_dpo/delta": -0.0379190668463707, "fcm_dpo/margin": 27.424747467041016, "fcm_dpo/q_t": 0.4008179306983948, "grad_norm": 9.272350311279297, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.9724560976028442, "logits/rejected": -1.5177730321884155, "logps/chosen": -112.72764587402344, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -140.0239715576172, "loss": 1.0662, "margin_dpo/margin_mean": 27.424747467041016, "margin_dpo/margin_std": 34.76381301879883, "step": 224 }, { "KL/chosen_KL_mean": -34.10304641723633, "KL/mean": -49.334754943847656, "KL/rejected_KL_mean": -64.56646728515625, "KL/std": 33.443199157714844, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.015757910907268524, "fcm_dpo/delta": -0.08407371491193771, "fcm_dpo/margin": 30.463424682617188, "fcm_dpo/q_t": 0.39220595359802246, "grad_norm": 10.175551414489746, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.215939521789551, "logits/rejected": -1.7437918186187744, "logps/chosen": -129.78073120117188, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -169.66334533691406, "loss": 1.0491, "margin_dpo/margin_mean": 30.463424682617188, "margin_dpo/margin_std": 39.77890396118164, "step": 225 }, { "KL/chosen_KL_mean": -28.513259887695312, "KL/mean": -42.17583084106445, "KL/rejected_KL_mean": -55.838401794433594, "KL/std": 31.58776092529297, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.015480760484933853, "fcm_dpo/delta": -0.024820774793624878, "fcm_dpo/margin": 27.325145721435547, "fcm_dpo/q_t": 0.40160930156707764, "grad_norm": 8.474968910217285, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.830456256866455, "logits/rejected": -1.2791507244110107, "logps/chosen": -121.97418212890625, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -142.5401611328125, "loss": 1.0576, "margin_dpo/margin_mean": 27.325145721435547, "margin_dpo/margin_std": 29.010629653930664, "step": 226 }, { "KL/chosen_KL_mean": -38.45386505126953, "KL/mean": -52.95548629760742, "KL/rejected_KL_mean": -67.45710754394531, "KL/std": 32.051517486572266, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.015430268831551075, "fcm_dpo/delta": -0.04976673051714897, "fcm_dpo/margin": 29.003246307373047, "fcm_dpo/q_t": 0.3989933729171753, "grad_norm": 9.103921890258789, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.7413547039031982, "logits/rejected": -1.399796962738037, "logps/chosen": -131.26812744140625, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -172.19403076171875, "loss": 1.07, "margin_dpo/margin_mean": 29.003246307373047, "margin_dpo/margin_std": 38.73999786376953, "step": 227 }, { "KL/chosen_KL_mean": -35.93561553955078, "KL/mean": -47.07569122314453, "KL/rejected_KL_mean": -58.21577072143555, "KL/std": 29.864261627197266, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.015531130135059357, "fcm_dpo/delta": 0.05586491525173187, "fcm_dpo/margin": 22.280149459838867, "fcm_dpo/q_t": 0.419780433177948, "grad_norm": 9.322175979614258, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.8605533838272095, "logits/rejected": -1.4225263595581055, "logps/chosen": -129.97274780273438, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -154.23728942871094, "loss": 1.1313, "margin_dpo/margin_mean": 22.2801513671875, "margin_dpo/margin_std": 32.78272247314453, "step": 228 }, { "KL/chosen_KL_mean": -32.390480041503906, "KL/mean": -49.98347854614258, "KL/rejected_KL_mean": -67.57647705078125, "KL/std": 36.191810607910156, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.015227205120027065, "fcm_dpo/delta": -0.14359506964683533, "fcm_dpo/margin": 35.18598937988281, "fcm_dpo/q_t": 0.3800284266471863, "grad_norm": 9.111041069030762, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.942089557647705, "logits/rejected": -1.3045574426651, "logps/chosen": -127.96814727783203, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -156.75027465820312, "loss": 1.0036, "margin_dpo/margin_mean": 35.18598937988281, "margin_dpo/margin_std": 40.58952331542969, "step": 229 }, { "KL/chosen_KL_mean": -31.653038024902344, "KL/mean": -50.54296875, "KL/rejected_KL_mean": -69.43289184570312, "KL/std": 33.91448974609375, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.014761574566364288, "fcm_dpo/delta": -0.16695085167884827, "fcm_dpo/margin": 37.77986145019531, "fcm_dpo/q_t": 0.370009183883667, "grad_norm": 9.242754936218262, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8968827724456787, "logits/rejected": -1.8432289361953735, "logps/chosen": -95.36666870117188, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -161.34161376953125, "loss": 0.9603, "margin_dpo/margin_mean": 37.77985382080078, "margin_dpo/margin_std": 33.667198181152344, "step": 230 }, { "KL/chosen_KL_mean": -36.92931365966797, "KL/mean": -51.35365295410156, "KL/rejected_KL_mean": -65.77799224853516, "KL/std": 32.0013542175293, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.014533329755067825, "fcm_dpo/delta": -0.020446501672267914, "fcm_dpo/margin": 28.848684310913086, "fcm_dpo/q_t": 0.4015029966831207, "grad_norm": 8.962400436401367, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -2.034181594848633, "logits/rejected": -1.477712631225586, "logps/chosen": -132.38497924804688, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -146.73367309570312, "loss": 1.0538, "margin_dpo/margin_mean": 28.848684310913086, "margin_dpo/margin_std": 29.264429092407227, "step": 231 }, { "KL/chosen_KL_mean": -38.16975402832031, "KL/mean": -51.125511169433594, "KL/rejected_KL_mean": -64.08126831054688, "KL/std": 30.500152587890625, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.014597484841942787, "fcm_dpo/delta": 0.022600093856453896, "fcm_dpo/margin": 25.911521911621094, "fcm_dpo/q_t": 0.41191238164901733, "grad_norm": 8.652594566345215, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.5768864154815674, "logits/rejected": -1.089969515800476, "logps/chosen": -135.0670166015625, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -153.84588623046875, "loss": 1.0949, "margin_dpo/margin_mean": 25.911521911621094, "margin_dpo/margin_std": 31.672883987426758, "step": 232 }, { "KL/chosen_KL_mean": -39.25528335571289, "KL/mean": -57.939453125, "KL/rejected_KL_mean": -76.62361907958984, "KL/std": 39.38589859008789, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.01433255709707737, "fcm_dpo/delta": -0.14321675896644592, "fcm_dpo/margin": 37.36833953857422, "fcm_dpo/q_t": 0.3773807883262634, "grad_norm": 9.161920547485352, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.8010529279708862, "logits/rejected": -1.6099367141723633, "logps/chosen": -128.30560302734375, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -189.38278198242188, "loss": 0.9962, "margin_dpo/margin_mean": 37.36833953857422, "margin_dpo/margin_std": 40.20466613769531, "step": 233 }, { "KL/chosen_KL_mean": -37.29386520385742, "KL/mean": -56.80888366699219, "KL/rejected_KL_mean": -76.32390594482422, "KL/std": 37.52827453613281, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.013798411935567856, "fcm_dpo/delta": -0.14902280271053314, "fcm_dpo/margin": 39.03003692626953, "fcm_dpo/q_t": 0.37817296385765076, "grad_norm": 8.631239891052246, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.3795958757400513, "logits/rejected": -1.317468523979187, "logps/chosen": -111.61222839355469, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -182.71148681640625, "loss": 1.002, "margin_dpo/margin_mean": 39.03003692626953, "margin_dpo/margin_std": 42.974639892578125, "step": 234 }, { "KL/chosen_KL_mean": -45.769187927246094, "KL/mean": -60.254547119140625, "KL/rejected_KL_mean": -74.73991394042969, "KL/std": 33.46485900878906, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.013814171776175499, "fcm_dpo/delta": -0.00022443151101469994, "fcm_dpo/margin": 28.970733642578125, "fcm_dpo/q_t": 0.40687045454978943, "grad_norm": 8.71873664855957, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.7581822872161865, "logits/rejected": -1.2036162614822388, "logps/chosen": -143.98623657226562, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -171.9866943359375, "loss": 1.0776, "margin_dpo/margin_mean": 28.970733642578125, "margin_dpo/margin_std": 34.12029266357422, "step": 235 }, { "KL/chosen_KL_mean": -36.23759460449219, "KL/mean": -55.996788024902344, "KL/rejected_KL_mean": -75.7559814453125, "KL/std": 43.22687530517578, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.013549413532018661, "fcm_dpo/delta": -0.14288415014743805, "fcm_dpo/margin": 39.51837921142578, "fcm_dpo/q_t": 0.38449084758758545, "grad_norm": 8.847535133361816, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.5576131343841553, "logits/rejected": -1.5691535472869873, "logps/chosen": -115.07532501220703, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -184.8194122314453, "loss": 1.0104, "margin_dpo/margin_mean": 39.518375396728516, "margin_dpo/margin_std": 48.84423828125, "step": 236 }, { "KL/chosen_KL_mean": -43.16505813598633, "KL/mean": -57.514320373535156, "KL/rejected_KL_mean": -71.86357879638672, "KL/std": 37.10009002685547, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.013454707339406013, "fcm_dpo/delta": 0.014404786750674248, "fcm_dpo/margin": 28.698505401611328, "fcm_dpo/q_t": 0.41587740182876587, "grad_norm": 8.519947052001953, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.7905049324035645, "logits/rejected": -1.447874903678894, "logps/chosen": -128.3763427734375, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -171.77357482910156, "loss": 1.1346, "margin_dpo/margin_mean": 28.698505401611328, "margin_dpo/margin_std": 49.22452163696289, "step": 237 }, { "KL/chosen_KL_mean": -46.77350616455078, "KL/mean": -62.90650177001953, "KL/rejected_KL_mean": -79.03948974609375, "KL/std": 37.97175598144531, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.01335754245519638, "fcm_dpo/delta": -0.0325838178396225, "fcm_dpo/margin": 32.2659912109375, "fcm_dpo/q_t": 0.4019027352333069, "grad_norm": 8.209688186645508, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.4632775783538818, "logits/rejected": -1.315436840057373, "logps/chosen": -125.50474548339844, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -174.4578857421875, "loss": 1.0688, "margin_dpo/margin_mean": 32.2659912109375, "margin_dpo/margin_std": 40.838706970214844, "step": 238 }, { "KL/chosen_KL_mean": -43.33333206176758, "KL/mean": -58.01361083984375, "KL/rejected_KL_mean": -72.69389343261719, "KL/std": 32.67463302612305, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.013320745900273323, "fcm_dpo/delta": 0.008829150348901749, "fcm_dpo/margin": 29.360557556152344, "fcm_dpo/q_t": 0.4094046354293823, "grad_norm": 8.689838409423828, "learning_rate": 4.116778689174514e-07, "logits/chosen": -2.1153571605682373, "logits/rejected": -1.6535060405731201, "logps/chosen": -135.93426513671875, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -173.21157836914062, "loss": 1.0941, "margin_dpo/margin_mean": 29.36055564880371, "margin_dpo/margin_std": 37.31492614746094, "step": 239 }, { "KL/chosen_KL_mean": -44.414981842041016, "KL/mean": -57.108177185058594, "KL/rejected_KL_mean": -69.80137634277344, "KL/std": 33.59710693359375, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.013474713079631329, "fcm_dpo/delta": 0.05999016761779785, "fcm_dpo/margin": 25.38640594482422, "fcm_dpo/q_t": 0.42082566022872925, "grad_norm": 10.12807559967041, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.2856268882751465, "logits/rejected": -1.9244661331176758, "logps/chosen": -130.5747528076172, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -150.25704956054688, "loss": 1.1318, "margin_dpo/margin_mean": 25.38640594482422, "margin_dpo/margin_std": 36.51958084106445, "step": 240 }, { "KL/chosen_KL_mean": -40.523014068603516, "KL/mean": -63.76449966430664, "KL/rejected_KL_mean": -87.0059814453125, "KL/std": 39.155677795410156, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.013172032311558723, "fcm_dpo/delta": -0.2255103439092636, "fcm_dpo/margin": 46.48295593261719, "fcm_dpo/q_t": 0.3610575795173645, "grad_norm": 9.034218788146973, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.7049872875213623, "logits/rejected": -1.0001187324523926, "logps/chosen": -141.56895446777344, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -181.05532836914062, "loss": 0.9417, "margin_dpo/margin_mean": 46.48295593261719, "margin_dpo/margin_std": 44.19648361206055, "step": 241 }, { "KL/chosen_KL_mean": -47.686641693115234, "KL/mean": -64.19827270507812, "KL/rejected_KL_mean": -80.70990753173828, "KL/std": 34.94841766357422, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.012959599494934082, "fcm_dpo/delta": -0.029541093856096268, "fcm_dpo/margin": 33.02326965332031, "fcm_dpo/q_t": 0.39976340532302856, "grad_norm": 8.978475570678711, "learning_rate": 4.087222918524807e-07, "logits/chosen": -2.062251329421997, "logits/rejected": -1.3969969749450684, "logps/chosen": -143.35931396484375, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -171.3644561767578, "loss": 1.0647, "margin_dpo/margin_mean": 33.02326965332031, "margin_dpo/margin_std": 38.674556732177734, "step": 242 }, { "KL/chosen_KL_mean": -46.53954315185547, "KL/mean": -63.015777587890625, "KL/rejected_KL_mean": -79.49201202392578, "KL/std": 36.648963928222656, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.01292099617421627, "fcm_dpo/delta": -0.027841519564390182, "fcm_dpo/margin": 32.95246887207031, "fcm_dpo/q_t": 0.40235626697540283, "grad_norm": 8.784867286682129, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.9720041751861572, "logits/rejected": -1.1311167478561401, "logps/chosen": -144.57095336914062, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -162.68008422851562, "loss": 1.0818, "margin_dpo/margin_mean": 32.95246887207031, "margin_dpo/margin_std": 42.897010803222656, "step": 243 }, { "KL/chosen_KL_mean": -47.65789794921875, "KL/mean": -65.68315124511719, "KL/rejected_KL_mean": -83.70840454101562, "KL/std": 41.41551971435547, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.012669552117586136, "fcm_dpo/delta": -0.05964727699756622, "fcm_dpo/margin": 36.05051040649414, "fcm_dpo/q_t": 0.3941190838813782, "grad_norm": 8.55615520477295, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.1039748191833496, "logits/rejected": -1.7686223983764648, "logps/chosen": -136.55181884765625, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -186.28118896484375, "loss": 1.0383, "margin_dpo/margin_mean": 36.05051040649414, "margin_dpo/margin_std": 39.575828552246094, "step": 244 }, { "KL/chosen_KL_mean": -42.22422409057617, "KL/mean": -59.96978759765625, "KL/rejected_KL_mean": -77.7153549194336, "KL/std": 36.44927978515625, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.012573031708598137, "fcm_dpo/delta": -0.04836679622530937, "fcm_dpo/margin": 35.49113845825195, "fcm_dpo/q_t": 0.39646679162979126, "grad_norm": 8.119379997253418, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.3652031421661377, "logits/rejected": -1.5181705951690674, "logps/chosen": -146.418212890625, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -170.37181091308594, "loss": 1.0441, "margin_dpo/margin_mean": 35.49113464355469, "margin_dpo/margin_std": 38.718421936035156, "step": 245 }, { "KL/chosen_KL_mean": -53.294517517089844, "KL/mean": -67.98114013671875, "KL/rejected_KL_mean": -82.66775512695312, "KL/std": 36.6029052734375, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.012552263215184212, "fcm_dpo/delta": 0.03238074108958244, "fcm_dpo/margin": 29.37323570251465, "fcm_dpo/q_t": 0.4149385094642639, "grad_norm": 8.362699508666992, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.95768141746521, "logits/rejected": -1.4092146158218384, "logps/chosen": -156.51356506347656, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -173.65997314453125, "loss": 1.1103, "margin_dpo/margin_mean": 29.37323570251465, "margin_dpo/margin_std": 39.6980094909668, "step": 246 }, { "KL/chosen_KL_mean": -47.18656921386719, "KL/mean": -71.58561706542969, "KL/rejected_KL_mean": -95.98464965820312, "KL/std": 42.870521545410156, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.012181243859231472, "fcm_dpo/delta": -0.20791128277778625, "fcm_dpo/margin": 48.79808044433594, "fcm_dpo/q_t": 0.36552467942237854, "grad_norm": 8.605443000793457, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.3975281715393066, "logits/rejected": -2.228116035461426, "logps/chosen": -134.18093872070312, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -207.32266235351562, "loss": 0.9536, "margin_dpo/margin_mean": 48.79808044433594, "margin_dpo/margin_std": 47.528289794921875, "step": 247 }, { "KL/chosen_KL_mean": -54.21684646606445, "KL/mean": -70.67132568359375, "KL/rejected_KL_mean": -87.12580871582031, "KL/std": 36.2819938659668, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.012127427384257317, "fcm_dpo/delta": 0.0008472912013530731, "fcm_dpo/margin": 32.908958435058594, "fcm_dpo/q_t": 0.4066827595233917, "grad_norm": 8.731497764587402, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.871152400970459, "logits/rejected": -1.8521233797073364, "logps/chosen": -129.0023651123047, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -185.4027099609375, "loss": 1.0751, "margin_dpo/margin_mean": 32.908958435058594, "margin_dpo/margin_std": 37.336761474609375, "step": 248 }, { "KL/chosen_KL_mean": -61.01692581176758, "KL/mean": -74.92108917236328, "KL/rejected_KL_mean": -88.82525634765625, "KL/std": 39.88531494140625, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.012173151597380638, "fcm_dpo/delta": 0.06359433382749557, "fcm_dpo/margin": 27.808330535888672, "fcm_dpo/q_t": 0.4226782023906708, "grad_norm": 9.450992584228516, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.4192285537719727, "logits/rejected": -1.886960506439209, "logps/chosen": -156.72071838378906, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -175.9717254638672, "loss": 1.1484, "margin_dpo/margin_mean": 27.808330535888672, "margin_dpo/margin_std": 45.1094970703125, "step": 249 }, { "KL/chosen_KL_mean": -55.28938674926758, "KL/mean": -70.495849609375, "KL/rejected_KL_mean": -85.70230102539062, "KL/std": 37.98749542236328, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.012255854904651642, "fcm_dpo/delta": 0.02805461175739765, "fcm_dpo/margin": 30.412919998168945, "fcm_dpo/q_t": 0.4126467704772949, "grad_norm": 9.292584419250488, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.1410789489746094, "logits/rejected": -1.2116292715072632, "logps/chosen": -169.3415985107422, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -166.78997802734375, "loss": 1.1146, "margin_dpo/margin_mean": 30.412919998168945, "margin_dpo/margin_std": 41.89973449707031, "step": 250 }, { "KL/chosen_KL_mean": -54.71305847167969, "KL/mean": -68.89373779296875, "KL/rejected_KL_mean": -83.07443237304688, "KL/std": 35.78728485107422, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.012357255443930626, "fcm_dpo/delta": 0.05104972422122955, "fcm_dpo/margin": 28.361370086669922, "fcm_dpo/q_t": 0.42028236389160156, "grad_norm": 9.521484375, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.8973891735076904, "logits/rejected": -1.1809018850326538, "logps/chosen": -152.42434692382812, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -166.6018524169922, "loss": 1.1375, "margin_dpo/margin_mean": 28.361370086669922, "margin_dpo/margin_std": 43.66529083251953, "step": 251 }, { "KL/chosen_KL_mean": -55.585365295410156, "KL/mean": -70.29450988769531, "KL/rejected_KL_mean": -85.003662109375, "KL/std": 36.89997863769531, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.012531939893960953, "fcm_dpo/delta": 0.032440055161714554, "fcm_dpo/margin": 29.418283462524414, "fcm_dpo/q_t": 0.41412076354026794, "grad_norm": 9.815316200256348, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.1424832344055176, "logits/rejected": -1.5915915966033936, "logps/chosen": -164.57147216796875, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -176.56790161132812, "loss": 1.1103, "margin_dpo/margin_mean": 29.418285369873047, "margin_dpo/margin_std": 39.59583282470703, "step": 252 }, { "KL/chosen_KL_mean": -60.83100128173828, "KL/mean": -74.92231750488281, "KL/rejected_KL_mean": -89.01364135742188, "KL/std": 40.093467712402344, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.012698895297944546, "fcm_dpo/delta": 0.04247502610087395, "fcm_dpo/margin": 28.182632446289062, "fcm_dpo/q_t": 0.41598159074783325, "grad_norm": 9.582547187805176, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.120654821395874, "logits/rejected": -1.9146623611450195, "logps/chosen": -161.04730224609375, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -194.69033813476562, "loss": 1.1149, "margin_dpo/margin_mean": 28.182634353637695, "margin_dpo/margin_std": 36.35641860961914, "step": 253 }, { "KL/chosen_KL_mean": -59.673030853271484, "KL/mean": -73.04058837890625, "KL/rejected_KL_mean": -86.40814971923828, "KL/std": 38.27387237548828, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.01270817220211029, "fcm_dpo/delta": 0.06228437274694443, "fcm_dpo/margin": 26.73511505126953, "fcm_dpo/q_t": 0.42076778411865234, "grad_norm": 9.319372177124023, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.3528027534484863, "logits/rejected": -1.8330146074295044, "logps/chosen": -158.40821838378906, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -180.14639282226562, "loss": 1.1383, "margin_dpo/margin_mean": 26.735116958618164, "margin_dpo/margin_std": 39.85004425048828, "step": 254 }, { "KL/chosen_KL_mean": -61.31330490112305, "KL/mean": -81.67536926269531, "KL/rejected_KL_mean": -102.03741455078125, "KL/std": 42.34226989746094, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.012583614327013493, "fcm_dpo/delta": -0.11847008764743805, "fcm_dpo/margin": 40.7241096496582, "fcm_dpo/q_t": 0.38156333565711975, "grad_norm": 9.832037925720215, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.048488140106201, "logits/rejected": -1.7065234184265137, "logps/chosen": -151.6958770751953, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -199.11366271972656, "loss": 1.0082, "margin_dpo/margin_mean": 40.7241096496582, "margin_dpo/margin_std": 43.84959411621094, "step": 255 }, { "KL/chosen_KL_mean": -57.9482536315918, "KL/mean": -76.62069702148438, "KL/rejected_KL_mean": -95.29313659667969, "KL/std": 44.662967681884766, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.012408062815666199, "fcm_dpo/delta": -0.06641862541437149, "fcm_dpo/margin": 37.344886779785156, "fcm_dpo/q_t": 0.3952305316925049, "grad_norm": 9.085689544677734, "learning_rate": 3.944434578520628e-07, "logits/chosen": -2.1287851333618164, "logits/rejected": -1.7968530654907227, "logps/chosen": -146.70106506347656, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -193.78697204589844, "loss": 1.0721, "margin_dpo/margin_mean": 37.344886779785156, "margin_dpo/margin_std": 52.2286262512207, "step": 256 }, { "KL/chosen_KL_mean": -60.037818908691406, "KL/mean": -78.71626281738281, "KL/rejected_KL_mean": -97.39469909667969, "KL/std": 45.956581115722656, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.012136580422520638, "fcm_dpo/delta": -0.05756368115544319, "fcm_dpo/margin": 37.35689163208008, "fcm_dpo/q_t": 0.3966999351978302, "grad_norm": 8.956185340881348, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.0078070163726807, "logits/rejected": -1.602112054824829, "logps/chosen": -142.84133911132812, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -183.2624969482422, "loss": 1.0618, "margin_dpo/margin_mean": 37.35689163208008, "margin_dpo/margin_std": 46.09087371826172, "step": 257 }, { "KL/chosen_KL_mean": -62.33671569824219, "KL/mean": -79.77466583251953, "KL/rejected_KL_mean": -97.2126235961914, "KL/std": 42.82039260864258, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.012208860367536545, "fcm_dpo/delta": -0.02736835926771164, "fcm_dpo/margin": 34.87590789794922, "fcm_dpo/q_t": 0.40020641684532166, "grad_norm": 9.280653953552246, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.466237783432007, "logits/rejected": -2.1252899169921875, "logps/chosen": -152.5244598388672, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -200.68331909179688, "loss": 1.0696, "margin_dpo/margin_mean": 34.87590789794922, "margin_dpo/margin_std": 41.70686340332031, "step": 258 }, { "KL/chosen_KL_mean": -68.26213836669922, "KL/mean": -83.51335144042969, "KL/rejected_KL_mean": -98.76455688476562, "KL/std": 43.780364990234375, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.012117289006710052, "fcm_dpo/delta": 0.031116724014282227, "fcm_dpo/margin": 30.502416610717773, "fcm_dpo/q_t": 0.4149153530597687, "grad_norm": 9.3866605758667, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.514399528503418, "logits/rejected": -1.2635788917541504, "logps/chosen": -159.03468322753906, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -193.35272216796875, "loss": 1.137, "margin_dpo/margin_mean": 30.502416610717773, "margin_dpo/margin_std": 49.53575897216797, "step": 259 }, { "KL/chosen_KL_mean": -69.18380737304688, "KL/mean": -82.53229522705078, "KL/rejected_KL_mean": -95.88078308105469, "KL/std": 42.835960388183594, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.012340893968939781, "fcm_dpo/delta": 0.07290994375944138, "fcm_dpo/margin": 26.696977615356445, "fcm_dpo/q_t": 0.4247264862060547, "grad_norm": 10.635502815246582, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.8554587364196777, "logits/rejected": -1.883858561515808, "logps/chosen": -144.77650451660156, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -196.726318359375, "loss": 1.1733, "margin_dpo/margin_mean": 26.696975708007812, "margin_dpo/margin_std": 49.96846008300781, "step": 260 }, { "KL/chosen_KL_mean": -54.995731353759766, "KL/mean": -76.44453430175781, "KL/rejected_KL_mean": -97.8933334350586, "KL/std": 44.764652252197266, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.012087922543287277, "fcm_dpo/delta": -0.12626712024211884, "fcm_dpo/margin": 42.897613525390625, "fcm_dpo/q_t": 0.3814753293991089, "grad_norm": 9.348803520202637, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.35200834274292, "logits/rejected": -1.995513677597046, "logps/chosen": -149.25064086914062, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -191.55032348632812, "loss": 1.003, "margin_dpo/margin_mean": 42.897613525390625, "margin_dpo/margin_std": 45.74822998046875, "step": 261 }, { "KL/chosen_KL_mean": -58.700660705566406, "KL/mean": -78.9190673828125, "KL/rejected_KL_mean": -99.13748168945312, "KL/std": 45.787357330322266, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.011897856369614601, "fcm_dpo/delta": -0.08545419573783875, "fcm_dpo/margin": 40.43681335449219, "fcm_dpo/q_t": 0.38921335339546204, "grad_norm": 8.728130340576172, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.7637333869934082, "logits/rejected": -1.2180311679840088, "logps/chosen": -143.96795654296875, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -189.96356201171875, "loss": 1.0281, "margin_dpo/margin_mean": 40.43681335449219, "margin_dpo/margin_std": 45.22480010986328, "step": 262 }, { "KL/chosen_KL_mean": -48.023353576660156, "KL/mean": -74.70487976074219, "KL/rejected_KL_mean": -101.38641357421875, "KL/std": 51.94933319091797, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.011477399617433548, "fcm_dpo/delta": -0.2263338267803192, "fcm_dpo/margin": 53.36305236816406, "fcm_dpo/q_t": 0.3607654869556427, "grad_norm": 8.27263355255127, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.3429393768310547, "logits/rejected": -1.8116565942764282, "logps/chosen": -143.78125, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -203.22018432617188, "loss": 0.9557, "margin_dpo/margin_mean": 53.36305236816406, "margin_dpo/margin_std": 55.583412170410156, "step": 263 }, { "KL/chosen_KL_mean": -54.96421813964844, "KL/mean": -77.7083740234375, "KL/rejected_KL_mean": -100.45252990722656, "KL/std": 47.737709045410156, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.011140530928969383, "fcm_dpo/delta": -0.11264801770448685, "fcm_dpo/margin": 45.48832702636719, "fcm_dpo/q_t": 0.382796049118042, "grad_norm": 8.28264045715332, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.2537662982940674, "logits/rejected": -1.975048303604126, "logps/chosen": -145.58688354492188, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -200.778076171875, "loss": 1.0117, "margin_dpo/margin_mean": 45.48832702636719, "margin_dpo/margin_std": 48.94868087768555, "step": 264 }, { "KL/chosen_KL_mean": -58.163726806640625, "KL/mean": -78.68659973144531, "KL/rejected_KL_mean": -99.20947265625, "KL/std": 48.115394592285156, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.010960336774587631, "fcm_dpo/delta": -0.05276045948266983, "fcm_dpo/margin": 41.045753479003906, "fcm_dpo/q_t": 0.3979823589324951, "grad_norm": 8.163066864013672, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.167593002319336, "logits/rejected": -1.7623989582061768, "logps/chosen": -150.5360565185547, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -193.83705139160156, "loss": 1.0656, "margin_dpo/margin_mean": 41.045753479003906, "margin_dpo/margin_std": 53.08613204956055, "step": 265 }, { "KL/chosen_KL_mean": -67.99655151367188, "KL/mean": -85.7921142578125, "KL/rejected_KL_mean": -103.58767700195312, "KL/std": 47.691524505615234, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.010968511924147606, "fcm_dpo/delta": 0.009945307858288288, "fcm_dpo/margin": 35.591129302978516, "fcm_dpo/q_t": 0.41062456369400024, "grad_norm": 9.756659507751465, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.513324499130249, "logits/rejected": -1.705636739730835, "logps/chosen": -170.1965789794922, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -177.9541015625, "loss": 1.105, "margin_dpo/margin_mean": 35.591129302978516, "margin_dpo/margin_std": 49.34724044799805, "step": 266 }, { "KL/chosen_KL_mean": -58.2864875793457, "KL/mean": -79.94828796386719, "KL/rejected_KL_mean": -101.61006927490234, "KL/std": 47.29210662841797, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.010831637308001518, "fcm_dpo/delta": -0.07333911210298538, "fcm_dpo/margin": 43.32357406616211, "fcm_dpo/q_t": 0.39144057035446167, "grad_norm": 8.762503623962402, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.8738054037094116, "logits/rejected": -1.2419886589050293, "logps/chosen": -162.57247924804688, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -197.59725952148438, "loss": 1.0295, "margin_dpo/margin_mean": 43.32357406616211, "margin_dpo/margin_std": 45.70309066772461, "step": 267 }, { "KL/chosen_KL_mean": -57.017433166503906, "KL/mean": -81.46762084960938, "KL/rejected_KL_mean": -105.91781616210938, "KL/std": 46.895904541015625, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.010600624606013298, "fcm_dpo/delta": -0.12549108266830444, "fcm_dpo/margin": 48.90038299560547, "fcm_dpo/q_t": 0.37868785858154297, "grad_norm": 9.320385932922363, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.888838768005371, "logits/rejected": -1.7399232387542725, "logps/chosen": -143.71365356445312, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -215.10964965820312, "loss": 0.9926, "margin_dpo/margin_mean": 48.90038299560547, "margin_dpo/margin_std": 47.2347297668457, "step": 268 }, { "KL/chosen_KL_mean": -61.533931732177734, "KL/mean": -86.1348876953125, "KL/rejected_KL_mean": -110.73583984375, "KL/std": 51.69200134277344, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.010412340052425861, "fcm_dpo/delta": -0.11820147931575775, "fcm_dpo/margin": 49.201908111572266, "fcm_dpo/q_t": 0.38135969638824463, "grad_norm": 9.154202461242676, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.8302907943725586, "logits/rejected": -1.365046739578247, "logps/chosen": -166.5057373046875, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -223.21231079101562, "loss": 1.0019, "margin_dpo/margin_mean": 49.201908111572266, "margin_dpo/margin_std": 50.1748046875, "step": 269 }, { "KL/chosen_KL_mean": -65.81491088867188, "KL/mean": -87.59219360351562, "KL/rejected_KL_mean": -109.36947631835938, "KL/std": 51.7882194519043, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.010277560912072659, "fcm_dpo/delta": -0.049918632954359055, "fcm_dpo/margin": 43.554561614990234, "fcm_dpo/q_t": 0.3980548679828644, "grad_norm": 8.784394264221191, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.650465965270996, "logits/rejected": -2.246046781539917, "logps/chosen": -166.8887481689453, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -227.12237548828125, "loss": 1.0775, "margin_dpo/margin_mean": 43.5545654296875, "margin_dpo/margin_std": 59.9603271484375, "step": 270 }, { "KL/chosen_KL_mean": -52.35586166381836, "KL/mean": -79.21272277832031, "KL/rejected_KL_mean": -106.06957244873047, "KL/std": 48.555877685546875, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.009992143139243126, "fcm_dpo/delta": -0.14461538195610046, "fcm_dpo/margin": 53.713706970214844, "fcm_dpo/q_t": 0.3741515874862671, "grad_norm": 8.869087219238281, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.6483261585235596, "logits/rejected": -1.1975396871566772, "logps/chosen": -149.08045959472656, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -204.593994140625, "loss": 0.9892, "margin_dpo/margin_mean": 53.713706970214844, "margin_dpo/margin_std": 53.93389892578125, "step": 271 }, { "KL/chosen_KL_mean": -46.10403823852539, "KL/mean": -70.56965637207031, "KL/rejected_KL_mean": -95.03527069091797, "KL/std": 52.984893798828125, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.009795510210096836, "fcm_dpo/delta": -0.08339318633079529, "fcm_dpo/margin": 48.931236267089844, "fcm_dpo/q_t": 0.39013439416885376, "grad_norm": 9.086271286010742, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.662524700164795, "logits/rejected": -1.554401159286499, "logps/chosen": -127.08125305175781, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -203.59063720703125, "loss": 1.0369, "margin_dpo/margin_mean": 48.931236267089844, "margin_dpo/margin_std": 57.55793762207031, "step": 272 }, { "KL/chosen_KL_mean": -62.92029571533203, "KL/mean": -88.55552673339844, "KL/rejected_KL_mean": -114.19078063964844, "KL/std": 54.27576446533203, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.00954618863761425, "fcm_dpo/delta": -0.09617571532726288, "fcm_dpo/margin": 51.270477294921875, "fcm_dpo/q_t": 0.3871602416038513, "grad_norm": 8.507856369018555, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.8342642784118652, "logits/rejected": -1.4296852350234985, "logps/chosen": -155.1448974609375, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -211.5538330078125, "loss": 1.0378, "margin_dpo/margin_mean": 51.270477294921875, "margin_dpo/margin_std": 60.37902069091797, "step": 273 }, { "KL/chosen_KL_mean": -63.861148834228516, "KL/mean": -82.91728210449219, "KL/rejected_KL_mean": -101.97341918945312, "KL/std": 54.0129280090332, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.009611910209059715, "fcm_dpo/delta": 0.034953825175762177, "fcm_dpo/margin": 38.11227798461914, "fcm_dpo/q_t": 0.4146655201911926, "grad_norm": 11.220690727233887, "learning_rate": 3.75e-07, "logits/chosen": -2.153634548187256, "logits/rejected": -1.7627897262573242, "logps/chosen": -146.12203979492188, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -185.8504180908203, "loss": 1.1052, "margin_dpo/margin_mean": 38.11227798461914, "margin_dpo/margin_std": 48.65495681762695, "step": 274 }, { "KL/chosen_KL_mean": -53.924381256103516, "KL/mean": -75.63973236083984, "KL/rejected_KL_mean": -97.35507202148438, "KL/std": 52.042388916015625, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.009667777456343174, "fcm_dpo/delta": -0.021275632083415985, "fcm_dpo/margin": 43.430694580078125, "fcm_dpo/q_t": 0.4026668667793274, "grad_norm": 8.835113525390625, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.2808096408843994, "logits/rejected": -1.9565479755401611, "logps/chosen": -133.61134338378906, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -196.10604858398438, "loss": 1.0931, "margin_dpo/margin_mean": 43.430694580078125, "margin_dpo/margin_std": 60.18267822265625, "step": 275 }, { "KL/chosen_KL_mean": -64.31425476074219, "KL/mean": -87.07012176513672, "KL/rejected_KL_mean": -109.82598114013672, "KL/std": 53.86808776855469, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.009487581439316273, "fcm_dpo/delta": -0.03446374088525772, "fcm_dpo/margin": 45.5117301940918, "fcm_dpo/q_t": 0.4017857313156128, "grad_norm": 10.774099349975586, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.3474433422088623, "logits/rejected": -1.7884430885314941, "logps/chosen": -150.85397338867188, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -187.679931640625, "loss": 1.0811, "margin_dpo/margin_mean": 45.5117301940918, "margin_dpo/margin_std": 60.91232681274414, "step": 276 }, { "KL/chosen_KL_mean": -69.51609802246094, "KL/mean": -87.15257263183594, "KL/rejected_KL_mean": -104.7890625, "KL/std": 51.432010650634766, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.009598245844244957, "fcm_dpo/delta": 0.0635959729552269, "fcm_dpo/margin": 35.272972106933594, "fcm_dpo/q_t": 0.42142653465270996, "grad_norm": 9.2035493850708, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.334512233734131, "logits/rejected": -1.9248089790344238, "logps/chosen": -153.76022338867188, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -203.62327575683594, "loss": 1.1354, "margin_dpo/margin_mean": 35.272972106933594, "margin_dpo/margin_std": 51.49891662597656, "step": 277 }, { "KL/chosen_KL_mean": -66.83633422851562, "KL/mean": -88.02229309082031, "KL/rejected_KL_mean": -109.20826721191406, "KL/std": 54.609867095947266, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.009615002200007439, "fcm_dpo/delta": -0.00798194482922554, "fcm_dpo/margin": 42.37193298339844, "fcm_dpo/q_t": 0.4049244821071625, "grad_norm": 10.54725170135498, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.2496023178100586, "logits/rejected": -2.032130718231201, "logps/chosen": -149.26821899414062, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -207.065185546875, "loss": 1.0913, "margin_dpo/margin_mean": 42.37193298339844, "margin_dpo/margin_std": 56.2381591796875, "step": 278 }, { "KL/chosen_KL_mean": -69.5898666381836, "KL/mean": -95.86959838867188, "KL/rejected_KL_mean": -122.14934539794922, "KL/std": 59.020835876464844, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.00945627223700285, "fcm_dpo/delta": -0.1029166430234909, "fcm_dpo/margin": 52.55947494506836, "fcm_dpo/q_t": 0.38634830713272095, "grad_norm": 12.870723724365234, "learning_rate": 3.6940245560867e-07, "logits/chosen": -2.068427324295044, "logits/rejected": -1.862480878829956, "logps/chosen": -154.75787353515625, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -216.27598571777344, "loss": 1.0234, "margin_dpo/margin_mean": 52.55947494506836, "margin_dpo/margin_std": 59.82342529296875, "step": 279 }, { "KL/chosen_KL_mean": -67.66307067871094, "KL/mean": -95.85108184814453, "KL/rejected_KL_mean": -124.03909301757812, "KL/std": 55.23677444458008, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.009279094636440277, "fcm_dpo/delta": -0.1297796219587326, "fcm_dpo/margin": 56.37602233886719, "fcm_dpo/q_t": 0.3780668079853058, "grad_norm": 9.198201179504395, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.9606478214263916, "logits/rejected": -1.7617120742797852, "logps/chosen": -153.5194854736328, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -228.15768432617188, "loss": 0.9964, "margin_dpo/margin_mean": 56.37602233886719, "margin_dpo/margin_std": 56.99542236328125, "step": 280 }, { "KL/chosen_KL_mean": -75.49485778808594, "KL/mean": -104.29064178466797, "KL/rejected_KL_mean": -133.08642578125, "KL/std": 63.72148513793945, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.009017860516905785, "fcm_dpo/delta": -0.12618514895439148, "fcm_dpo/margin": 57.5915641784668, "fcm_dpo/q_t": 0.3818970024585724, "grad_norm": 7.7242960929870605, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.202223777770996, "logits/rejected": -1.4587171077728271, "logps/chosen": -162.04568481445312, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -221.715087890625, "loss": 1.0123, "margin_dpo/margin_mean": 57.59156036376953, "margin_dpo/margin_std": 65.92282104492188, "step": 281 }, { "KL/chosen_KL_mean": -73.2401123046875, "KL/mean": -109.09881591796875, "KL/rejected_KL_mean": -144.95751953125, "KL/std": 63.48696517944336, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.00865398719906807, "fcm_dpo/delta": -0.23566648364067078, "fcm_dpo/margin": 71.71742248535156, "fcm_dpo/q_t": 0.3572981357574463, "grad_norm": 7.348165988922119, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.8948813676834106, "logits/rejected": -1.5178947448730469, "logps/chosen": -178.34521484375, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -247.81088256835938, "loss": 0.9287, "margin_dpo/margin_mean": 71.71742248535156, "margin_dpo/margin_std": 64.93849182128906, "step": 282 }, { "KL/chosen_KL_mean": -90.18972778320312, "KL/mean": -114.05986022949219, "KL/rejected_KL_mean": -137.9299774169922, "KL/std": 62.16899871826172, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.008550295606255531, "fcm_dpo/delta": -0.008543584495782852, "fcm_dpo/margin": 47.740238189697266, "fcm_dpo/q_t": 0.4064582586288452, "grad_norm": 10.282188415527344, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.3904569149017334, "logits/rejected": -2.0626134872436523, "logps/chosen": -185.24234008789062, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -241.47451782226562, "loss": 1.1002, "margin_dpo/margin_mean": 47.740238189697266, "margin_dpo/margin_std": 68.55302429199219, "step": 283 }, { "KL/chosen_KL_mean": -79.76167297363281, "KL/mean": -112.1309814453125, "KL/rejected_KL_mean": -144.5002899169922, "KL/std": 63.768280029296875, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.008334267884492874, "fcm_dpo/delta": -0.14821693301200867, "fcm_dpo/margin": 64.73861694335938, "fcm_dpo/q_t": 0.37382328510284424, "grad_norm": 8.829617500305176, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.9759297370910645, "logits/rejected": -1.6459615230560303, "logps/chosen": -167.4281463623047, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -243.25132751464844, "loss": 0.9714, "margin_dpo/margin_mean": 64.73861694335938, "margin_dpo/margin_std": 58.17889404296875, "step": 284 }, { "KL/chosen_KL_mean": -92.73271942138672, "KL/mean": -114.74484252929688, "KL/rejected_KL_mean": -136.7569580078125, "KL/std": 56.658447265625, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.008357180282473564, "fcm_dpo/delta": 0.033142201602458954, "fcm_dpo/margin": 44.02423858642578, "fcm_dpo/q_t": 0.413303017616272, "grad_norm": 11.129773139953613, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.5493059158325195, "logits/rejected": -2.2726621627807617, "logps/chosen": -187.16575622558594, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -240.82891845703125, "loss": 1.1, "margin_dpo/margin_mean": 44.02423858642578, "margin_dpo/margin_std": 53.51174545288086, "step": 285 }, { "KL/chosen_KL_mean": -99.92044067382812, "KL/mean": -130.57144165039062, "KL/rejected_KL_mean": -161.22244262695312, "KL/std": 62.01511001586914, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.008209067396819592, "fcm_dpo/delta": -0.1088700070977211, "fcm_dpo/margin": 61.3020133972168, "fcm_dpo/q_t": 0.38278040289878845, "grad_norm": 8.652714729309082, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.146725654602051, "logits/rejected": -1.6866660118103027, "logps/chosen": -203.64083862304688, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -257.480224609375, "loss": 1.0104, "margin_dpo/margin_mean": 61.30200958251953, "margin_dpo/margin_std": 63.83982467651367, "step": 286 }, { "KL/chosen_KL_mean": -116.81156158447266, "KL/mean": -134.49107360839844, "KL/rejected_KL_mean": -152.17059326171875, "KL/std": 55.566307067871094, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.008180721662938595, "fcm_dpo/delta": 0.008885523304343224, "fcm_dpo/margin": 35.35902404785156, "fcm_dpo/q_t": 0.43394941091537476, "grad_norm": 8.412991523742676, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.530669927597046, "logits/rejected": -1.9513375759124756, "logps/chosen": -210.70144653320312, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -235.50424194335938, "loss": 1.1853, "margin_dpo/margin_mean": 35.35902404785156, "margin_dpo/margin_std": 63.763790130615234, "step": 287 }, { "KL/chosen_KL_mean": -113.35980224609375, "KL/mean": -144.95272827148438, "KL/rejected_KL_mean": -176.545654296875, "KL/std": 62.226043701171875, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.008040027692914009, "fcm_dpo/delta": -0.11397960036993027, "fcm_dpo/margin": 63.18585968017578, "fcm_dpo/q_t": 0.38355880975723267, "grad_norm": 8.010588645935059, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.895991563796997, "logits/rejected": -1.4773237705230713, "logps/chosen": -201.51580810546875, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -269.8276062011719, "loss": 1.0091, "margin_dpo/margin_mean": 63.185855865478516, "margin_dpo/margin_std": 67.9988021850586, "step": 288 }, { "KL/chosen_KL_mean": -111.06316375732422, "KL/mean": -143.11058044433594, "KL/rejected_KL_mean": -175.15798950195312, "KL/std": 65.94894409179688, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.007887642830610275, "fcm_dpo/delta": -0.11104057729244232, "fcm_dpo/margin": 64.0948257446289, "fcm_dpo/q_t": 0.3870524764060974, "grad_norm": 7.423426151275635, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.1484756469726562, "logits/rejected": -1.946473240852356, "logps/chosen": -186.45608520507812, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -268.312255859375, "loss": 1.027, "margin_dpo/margin_mean": 64.0948257446289, "margin_dpo/margin_std": 77.39784240722656, "step": 289 }, { "KL/chosen_KL_mean": -121.11951446533203, "KL/mean": -154.60501098632812, "KL/rejected_KL_mean": -188.09048461914062, "KL/std": 71.20973205566406, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.007734727114439011, "fcm_dpo/delta": -0.12450462579727173, "fcm_dpo/margin": 66.9709701538086, "fcm_dpo/q_t": 0.3831160366535187, "grad_norm": 7.971763610839844, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.423424482345581, "logits/rejected": -1.4723480939865112, "logps/chosen": -209.1614990234375, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -311.3026428222656, "loss": 1.0116, "margin_dpo/margin_mean": 66.9709701538086, "margin_dpo/margin_std": 74.78608703613281, "step": 290 }, { "KL/chosen_KL_mean": -116.64927673339844, "KL/mean": -143.7423095703125, "KL/rejected_KL_mean": -170.8353271484375, "KL/std": 65.04029846191406, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.00763185229152441, "fcm_dpo/delta": -0.014157518744468689, "fcm_dpo/margin": 54.18605041503906, "fcm_dpo/q_t": 0.4030645787715912, "grad_norm": 9.332014083862305, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.4590530395507812, "logits/rejected": -1.9975454807281494, "logps/chosen": -210.74452209472656, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -266.8853759765625, "loss": 1.0745, "margin_dpo/margin_mean": 54.18605041503906, "margin_dpo/margin_std": 65.6194839477539, "step": 291 }, { "KL/chosen_KL_mean": -118.75586700439453, "KL/mean": -148.55972290039062, "KL/rejected_KL_mean": -178.36361694335938, "KL/std": 63.7007942199707, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.007549101021140814, "fcm_dpo/delta": -0.05238755792379379, "fcm_dpo/margin": 59.607757568359375, "fcm_dpo/q_t": 0.3942567706108093, "grad_norm": 8.402032852172852, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -1.997333288192749, "logits/rejected": -1.5479841232299805, "logps/chosen": -207.00628662109375, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -274.78125, "loss": 1.0318, "margin_dpo/margin_mean": 59.607757568359375, "margin_dpo/margin_std": 58.811317443847656, "step": 292 }, { "KL/chosen_KL_mean": -120.85020446777344, "KL/mean": -145.15524291992188, "KL/rejected_KL_mean": -169.46029663085938, "KL/std": 67.47537231445312, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.007600134238600731, "fcm_dpo/delta": 0.03126843273639679, "fcm_dpo/margin": 48.61009979248047, "fcm_dpo/q_t": 0.41595351696014404, "grad_norm": 7.724319934844971, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.5667004585266113, "logits/rejected": -2.0035290718078613, "logps/chosen": -208.22674560546875, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -255.2161102294922, "loss": 1.1197, "margin_dpo/margin_mean": 48.61009979248047, "margin_dpo/margin_std": 71.07196044921875, "step": 293 }, { "KL/chosen_KL_mean": -123.35670471191406, "KL/mean": -143.08340454101562, "KL/rejected_KL_mean": -162.81011962890625, "KL/std": 62.65153503417969, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.007713350001722574, "fcm_dpo/delta": 0.09829793870449066, "fcm_dpo/margin": 39.453399658203125, "fcm_dpo/q_t": 0.43074169754981995, "grad_norm": 8.450900077819824, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.6437623500823975, "logits/rejected": -1.4731113910675049, "logps/chosen": -196.86465454101562, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -250.89889526367188, "loss": 1.1825, "margin_dpo/margin_mean": 39.453392028808594, "margin_dpo/margin_std": 73.74832153320312, "step": 294 }, { "KL/chosen_KL_mean": -120.57691955566406, "KL/mean": -149.28170776367188, "KL/rejected_KL_mean": -177.98654174804688, "KL/std": 64.97038269042969, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.007648756727576256, "fcm_dpo/delta": -0.041542332619428635, "fcm_dpo/margin": 57.40959930419922, "fcm_dpo/q_t": 0.3991323709487915, "grad_norm": 8.54820442199707, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.7784404754638672, "logits/rejected": -1.6270692348480225, "logps/chosen": -198.3424072265625, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -272.2337951660156, "loss": 1.0635, "margin_dpo/margin_mean": 57.409603118896484, "margin_dpo/margin_std": 70.80119323730469, "step": 295 }, { "KL/chosen_KL_mean": -129.88304138183594, "KL/mean": -160.40492248535156, "KL/rejected_KL_mean": -190.9268035888672, "KL/std": 64.90210723876953, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.00757030863314867, "fcm_dpo/delta": -0.06552498042583466, "fcm_dpo/margin": 61.04376983642578, "fcm_dpo/q_t": 0.3926340341567993, "grad_norm": 11.619428634643555, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.6971914768218994, "logits/rejected": -2.2201807498931885, "logps/chosen": -227.7394561767578, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -291.74310302734375, "loss": 1.0379, "margin_dpo/margin_mean": 61.04376983642578, "margin_dpo/margin_std": 67.56732177734375, "step": 296 }, { "KL/chosen_KL_mean": -129.77468872070312, "KL/mean": -156.49513244628906, "KL/rejected_KL_mean": -183.215576171875, "KL/std": 65.45933532714844, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.007566890213638544, "fcm_dpo/delta": -0.004568807780742645, "fcm_dpo/margin": 53.44087219238281, "fcm_dpo/q_t": 0.4079003930091858, "grad_norm": 10.643759727478027, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.9542795419692993, "logits/rejected": -0.6251869201660156, "logps/chosen": -218.3405303955078, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -284.77215576171875, "loss": 1.0894, "margin_dpo/margin_mean": 53.44087219238281, "margin_dpo/margin_std": 71.53192138671875, "step": 297 }, { "KL/chosen_KL_mean": -111.24129486083984, "KL/mean": -144.49105834960938, "KL/rejected_KL_mean": -177.74085998535156, "KL/std": 72.44493865966797, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.007466558832675219, "fcm_dpo/delta": -0.10144035518169403, "fcm_dpo/margin": 66.49954986572266, "fcm_dpo/q_t": 0.38751447200775146, "grad_norm": 11.947007179260254, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.1249849796295166, "logits/rejected": -2.102611541748047, "logps/chosen": -206.12173461914062, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -300.0518798828125, "loss": 1.0204, "margin_dpo/margin_mean": 66.49954223632812, "margin_dpo/margin_std": 74.64753723144531, "step": 298 }, { "KL/chosen_KL_mean": -114.72303009033203, "KL/mean": -149.54359436035156, "KL/rejected_KL_mean": -184.36416625976562, "KL/std": 69.05644226074219, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.007277102209627628, "fcm_dpo/delta": -0.11266354471445084, "fcm_dpo/margin": 69.64112854003906, "fcm_dpo/q_t": 0.3831230401992798, "grad_norm": 9.74075698852539, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.2006276845932007, "logits/rejected": -0.9797923564910889, "logps/chosen": -195.13137817382812, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -273.90130615234375, "loss": 1.0082, "margin_dpo/margin_mean": 69.64112854003906, "margin_dpo/margin_std": 73.63960266113281, "step": 299 }, { "KL/chosen_KL_mean": -122.71484375, "KL/mean": -151.17672729492188, "KL/rejected_KL_mean": -179.63861083984375, "KL/std": 73.74125671386719, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0072031207382678986, "fcm_dpo/delta": -0.010844860225915909, "fcm_dpo/margin": 56.92377853393555, "fcm_dpo/q_t": 0.405791312456131, "grad_norm": 9.206564903259277, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.7652575969696045, "logits/rejected": -1.5570580959320068, "logps/chosen": -210.87374877929688, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -280.57781982421875, "loss": 1.0942, "margin_dpo/margin_mean": 56.92377471923828, "margin_dpo/margin_std": 78.5833969116211, "step": 300 }, { "KL/chosen_KL_mean": -111.0382080078125, "KL/mean": -148.02447509765625, "KL/rejected_KL_mean": -185.0107421875, "KL/std": 73.10354614257812, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.007042955607175827, "fcm_dpo/delta": -0.12945589423179626, "fcm_dpo/margin": 73.97254943847656, "fcm_dpo/q_t": 0.3806875944137573, "grad_norm": 9.115862846374512, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.8022961616516113, "logits/rejected": -1.3716025352478027, "logps/chosen": -196.54312133789062, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -283.9276123046875, "loss": 1.0091, "margin_dpo/margin_mean": 73.97254180908203, "margin_dpo/margin_std": 81.28394317626953, "step": 301 }, { "KL/chosen_KL_mean": -107.6657485961914, "KL/mean": -138.69863891601562, "KL/rejected_KL_mean": -169.73155212402344, "KL/std": 75.36006164550781, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0069905854761600494, "fcm_dpo/delta": -0.03560216352343559, "fcm_dpo/margin": 62.06580352783203, "fcm_dpo/q_t": 0.39924106001853943, "grad_norm": 8.750239372253418, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.5051417350769043, "logits/rejected": -1.332211971282959, "logps/chosen": -187.11614990234375, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -272.7906494140625, "loss": 1.0635, "margin_dpo/margin_mean": 62.0657958984375, "margin_dpo/margin_std": 74.9919662475586, "step": 302 }, { "KL/chosen_KL_mean": -99.88517761230469, "KL/mean": -134.6578826904297, "KL/rejected_KL_mean": -169.4305877685547, "KL/std": 70.67665100097656, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.006923149339854717, "fcm_dpo/delta": -0.08550924062728882, "fcm_dpo/margin": 69.54541015625, "fcm_dpo/q_t": 0.38780587911605835, "grad_norm": 8.039948463439941, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -2.007098913192749, "logits/rejected": -1.5541331768035889, "logps/chosen": -195.4834747314453, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -261.49517822265625, "loss": 1.0231, "margin_dpo/margin_mean": 69.54541778564453, "margin_dpo/margin_std": 73.43028259277344, "step": 303 }, { "KL/chosen_KL_mean": -100.0020523071289, "KL/mean": -135.5366668701172, "KL/rejected_KL_mean": -171.0712890625, "KL/std": 73.16687774658203, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0067902072332799435, "fcm_dpo/delta": -0.0867278128862381, "fcm_dpo/margin": 71.0692138671875, "fcm_dpo/q_t": 0.38736873865127563, "grad_norm": 8.281644821166992, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.4791178703308105, "logits/rejected": -2.07542085647583, "logps/chosen": -199.4552001953125, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -269.153564453125, "loss": 1.0218, "margin_dpo/margin_mean": 71.0692138671875, "margin_dpo/margin_std": 75.09916687011719, "step": 304 }, { "KL/chosen_KL_mean": -106.36341094970703, "KL/mean": -140.8560791015625, "KL/rejected_KL_mean": -175.34872436523438, "KL/std": 75.91716003417969, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.006657380145043135, "fcm_dpo/delta": -0.06307508796453476, "fcm_dpo/margin": 68.98531341552734, "fcm_dpo/q_t": 0.39316385984420776, "grad_norm": 7.695662975311279, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.2690589427948, "logits/rejected": -1.7815592288970947, "logps/chosen": -209.62680053710938, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -273.1777038574219, "loss": 1.0491, "margin_dpo/margin_mean": 68.98531341552734, "margin_dpo/margin_std": 80.1463851928711, "step": 305 }, { "KL/chosen_KL_mean": -95.14529418945312, "KL/mean": -135.94577026367188, "KL/rejected_KL_mean": -176.74624633789062, "KL/std": 71.80307006835938, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.006518645212054253, "fcm_dpo/delta": -0.13982662558555603, "fcm_dpo/margin": 81.60094451904297, "fcm_dpo/q_t": 0.37376928329467773, "grad_norm": 8.77502155303955, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.4153013229370117, "logits/rejected": -1.8680719137191772, "logps/chosen": -195.44912719726562, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -268.97711181640625, "loss": 0.9671, "margin_dpo/margin_mean": 81.60094451904297, "margin_dpo/margin_std": 65.51980590820312, "step": 306 }, { "KL/chosen_KL_mean": -103.97486877441406, "KL/mean": -133.5328369140625, "KL/rejected_KL_mean": -163.0908203125, "KL/std": 73.75865173339844, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.006478797644376755, "fcm_dpo/delta": 0.017407860606908798, "fcm_dpo/margin": 59.115928649902344, "fcm_dpo/q_t": 0.411965548992157, "grad_norm": 7.9007391929626465, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.9843719005584717, "logits/rejected": -1.6293281316757202, "logps/chosen": -200.6246337890625, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -256.4394226074219, "loss": 1.1163, "margin_dpo/margin_mean": 59.115928649902344, "margin_dpo/margin_std": 87.17727661132812, "step": 307 }, { "KL/chosen_KL_mean": -108.65632629394531, "KL/mean": -137.90328979492188, "KL/rejected_KL_mean": -167.15023803710938, "KL/std": 73.87652587890625, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.006493743509054184, "fcm_dpo/delta": 0.020401351153850555, "fcm_dpo/margin": 58.49391174316406, "fcm_dpo/q_t": 0.41323065757751465, "grad_norm": 8.71318244934082, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.8327324390411377, "logits/rejected": -1.5532386302947998, "logps/chosen": -194.38101196289062, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -253.57342529296875, "loss": 1.1177, "margin_dpo/margin_mean": 58.49391174316406, "margin_dpo/margin_std": 84.89556884765625, "step": 308 }, { "KL/chosen_KL_mean": -118.33631134033203, "KL/mean": -146.01339721679688, "KL/rejected_KL_mean": -173.69049072265625, "KL/std": 79.33246612548828, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0065799979493021965, "fcm_dpo/delta": 0.03706915304064751, "fcm_dpo/margin": 55.35419464111328, "fcm_dpo/q_t": 0.415650337934494, "grad_norm": 11.278367042541504, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -2.0757246017456055, "logits/rejected": -1.6598957777023315, "logps/chosen": -201.48776245117188, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -260.2765197753906, "loss": 1.12, "margin_dpo/margin_mean": 55.35419464111328, "margin_dpo/margin_std": 78.12393188476562, "step": 309 }, { "KL/chosen_KL_mean": -106.81036376953125, "KL/mean": -137.88821411132812, "KL/rejected_KL_mean": -168.966064453125, "KL/std": 72.21171569824219, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006571663077920675, "fcm_dpo/delta": -0.008851460181176662, "fcm_dpo/margin": 62.15570068359375, "fcm_dpo/q_t": 0.40482378005981445, "grad_norm": 8.55673885345459, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.584489345550537, "logits/rejected": -2.1880922317504883, "logps/chosen": -196.54835510253906, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -256.0368957519531, "loss": 1.0778, "margin_dpo/margin_mean": 62.15570068359375, "margin_dpo/margin_std": 75.9321060180664, "step": 310 }, { "KL/chosen_KL_mean": -126.54210662841797, "KL/mean": -165.02098083496094, "KL/rejected_KL_mean": -203.4998779296875, "KL/std": 74.2469482421875, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.006476958282291889, "fcm_dpo/delta": -0.10357700288295746, "fcm_dpo/margin": 76.95777130126953, "fcm_dpo/q_t": 0.38404130935668945, "grad_norm": 8.585000038146973, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.3930598497390747, "logits/rejected": -0.6190176010131836, "logps/chosen": -215.06854248046875, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -285.7659606933594, "loss": 1.002, "margin_dpo/margin_mean": 76.95777130126953, "margin_dpo/margin_std": 74.63330841064453, "step": 311 }, { "KL/chosen_KL_mean": -122.20855712890625, "KL/mean": -155.1330108642578, "KL/rejected_KL_mean": -188.05746459960938, "KL/std": 82.42958068847656, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.00643577054142952, "fcm_dpo/delta": -0.02495800144970417, "fcm_dpo/margin": 65.84890747070312, "fcm_dpo/q_t": 0.40160131454467773, "grad_norm": 9.593628883361816, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.556093454360962, "logits/rejected": -1.014587640762329, "logps/chosen": -202.6092529296875, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -270.5046081542969, "loss": 1.0729, "margin_dpo/margin_mean": 65.84890747070312, "margin_dpo/margin_std": 81.86070251464844, "step": 312 }, { "KL/chosen_KL_mean": -117.53175354003906, "KL/mean": -148.73483276367188, "KL/rejected_KL_mean": -179.93792724609375, "KL/std": 80.07640075683594, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0064286962151527405, "fcm_dpo/delta": -0.0014817193150520325, "fcm_dpo/margin": 62.406185150146484, "fcm_dpo/q_t": 0.4073820114135742, "grad_norm": 9.254664421081543, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.523591995239258, "logits/rejected": -2.1188266277313232, "logps/chosen": -201.67364501953125, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -273.36474609375, "loss": 1.0954, "margin_dpo/margin_mean": 62.406185150146484, "margin_dpo/margin_std": 84.77334594726562, "step": 313 }, { "KL/chosen_KL_mean": -118.94080352783203, "KL/mean": -149.29037475585938, "KL/rejected_KL_mean": -179.63992309570312, "KL/std": 80.14220428466797, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.006397377699613571, "fcm_dpo/delta": 0.011995725333690643, "fcm_dpo/margin": 60.69911575317383, "fcm_dpo/q_t": 0.4110183119773865, "grad_norm": 9.230607032775879, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -2.232675313949585, "logits/rejected": -2.0229547023773193, "logps/chosen": -207.75051879882812, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -283.798828125, "loss": 1.1019, "margin_dpo/margin_mean": 60.699119567871094, "margin_dpo/margin_std": 82.67333984375, "step": 314 }, { "KL/chosen_KL_mean": -115.27936553955078, "KL/mean": -148.19140625, "KL/rejected_KL_mean": -181.10345458984375, "KL/std": 83.26962280273438, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.006390688940882683, "fcm_dpo/delta": -0.021650895476341248, "fcm_dpo/margin": 65.82408142089844, "fcm_dpo/q_t": 0.40416979789733887, "grad_norm": 8.843500137329102, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.535307765007019, "logits/rejected": -1.3604837656021118, "logps/chosen": -196.07305908203125, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -273.2405700683594, "loss": 1.0944, "margin_dpo/margin_mean": 65.82408142089844, "margin_dpo/margin_std": 94.52423095703125, "step": 315 }, { "KL/chosen_KL_mean": -122.24072265625, "KL/mean": -155.95314025878906, "KL/rejected_KL_mean": -189.6655731201172, "KL/std": 83.48139190673828, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.006371030583977699, "fcm_dpo/delta": -0.030883239582180977, "fcm_dpo/margin": 67.42485046386719, "fcm_dpo/q_t": 0.4023538827896118, "grad_norm": 8.660035133361816, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.906862735748291, "logits/rejected": -1.9115962982177734, "logps/chosen": -206.4249725341797, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -295.35638427734375, "loss": 1.0757, "margin_dpo/margin_mean": 67.42485046386719, "margin_dpo/margin_std": 89.07405090332031, "step": 316 }, { "KL/chosen_KL_mean": -124.50723266601562, "KL/mean": -166.64405822753906, "KL/rejected_KL_mean": -208.7808837890625, "KL/std": 91.66809844970703, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.006248924881219864, "fcm_dpo/delta": -0.13344231247901917, "fcm_dpo/margin": 84.27367401123047, "fcm_dpo/q_t": 0.3806785047054291, "grad_norm": 10.419573783874512, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -2.0108745098114014, "logits/rejected": -1.5357221364974976, "logps/chosen": -212.38072204589844, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -317.04608154296875, "loss": 1.0156, "margin_dpo/margin_mean": 84.27367401123047, "margin_dpo/margin_std": 98.39994812011719, "step": 317 }, { "KL/chosen_KL_mean": -122.97928619384766, "KL/mean": -160.24440002441406, "KL/rejected_KL_mean": -197.509521484375, "KL/std": 76.91849517822266, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.006125482264906168, "fcm_dpo/delta": -0.059331394731998444, "fcm_dpo/margin": 74.53024291992188, "fcm_dpo/q_t": 0.39375123381614685, "grad_norm": 10.163827896118164, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.2619564533233643, "logits/rejected": -1.5394580364227295, "logps/chosen": -221.184814453125, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -285.64581298828125, "loss": 1.0427, "margin_dpo/margin_mean": 74.53024291992188, "margin_dpo/margin_std": 84.05496215820312, "step": 318 }, { "KL/chosen_KL_mean": -104.52880859375, "KL/mean": -145.6531982421875, "KL/rejected_KL_mean": -186.777587890625, "KL/std": 79.65054321289062, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.006009139120578766, "fcm_dpo/delta": -0.09951581060886383, "fcm_dpo/margin": 82.248779296875, "fcm_dpo/q_t": 0.38576892018318176, "grad_norm": 9.248260498046875, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.8935785293579102, "logits/rejected": -1.829909086227417, "logps/chosen": -187.0321807861328, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -305.42828369140625, "loss": 1.02, "margin_dpo/margin_mean": 82.24878692626953, "margin_dpo/margin_std": 90.25145721435547, "step": 319 }, { "KL/chosen_KL_mean": -107.59471130371094, "KL/mean": -141.98947143554688, "KL/rejected_KL_mean": -176.38424682617188, "KL/std": 81.87004089355469, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005971145816147327, "fcm_dpo/delta": -0.011319499462842941, "fcm_dpo/margin": 68.78953552246094, "fcm_dpo/q_t": 0.4041629731655121, "grad_norm": 8.581494331359863, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.0385143756866455, "logits/rejected": -1.8494200706481934, "logps/chosen": -194.73193359375, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -279.6083679199219, "loss": 1.0706, "margin_dpo/margin_mean": 68.78953552246094, "margin_dpo/margin_std": 79.9781494140625, "step": 320 }, { "KL/chosen_KL_mean": -121.21678924560547, "KL/mean": -160.185791015625, "KL/rejected_KL_mean": -199.15481567382812, "KL/std": 83.05851745605469, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.005903697572648525, "fcm_dpo/delta": -0.06346674263477325, "fcm_dpo/margin": 77.93801879882812, "fcm_dpo/q_t": 0.3914802670478821, "grad_norm": 8.569597244262695, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -2.0273637771606445, "logits/rejected": -1.6354691982269287, "logps/chosen": -214.3214569091797, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -296.6229248046875, "loss": 1.0341, "margin_dpo/margin_mean": 77.93801879882812, "margin_dpo/margin_std": 81.77682495117188, "step": 321 }, { "KL/chosen_KL_mean": -114.07395935058594, "KL/mean": -157.701416015625, "KL/rejected_KL_mean": -201.32887268066406, "KL/std": 83.89845275878906, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.005771083291620016, "fcm_dpo/delta": -0.11058944463729858, "fcm_dpo/margin": 87.25491333007812, "fcm_dpo/q_t": 0.38283178210258484, "grad_norm": 10.080855369567871, "learning_rate": 3.184157475180207e-07, "logits/chosen": -2.0215957164764404, "logits/rejected": -1.6714890003204346, "logps/chosen": -206.26687622070312, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -302.22210693359375, "loss": 1.0107, "margin_dpo/margin_mean": 87.25491333007812, "margin_dpo/margin_std": 89.95396423339844, "step": 322 }, { "KL/chosen_KL_mean": -132.57562255859375, "KL/mean": -167.7520751953125, "KL/rejected_KL_mean": -202.92852783203125, "KL/std": 84.56470489501953, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.005759330932050943, "fcm_dpo/delta": -0.005491688847541809, "fcm_dpo/margin": 70.35289764404297, "fcm_dpo/q_t": 0.40673696994781494, "grad_norm": 8.757376670837402, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.1040544509887695, "logits/rejected": -1.723374605178833, "logps/chosen": -212.84771728515625, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -283.23187255859375, "loss": 1.0819, "margin_dpo/margin_mean": 70.35289764404297, "margin_dpo/margin_std": 87.58911895751953, "step": 323 }, { "KL/chosen_KL_mean": -132.39979553222656, "KL/mean": -169.245361328125, "KL/rejected_KL_mean": -206.09091186523438, "KL/std": 83.37654876708984, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.0057556889951229095, "fcm_dpo/delta": -0.025219524279236794, "fcm_dpo/margin": 73.69113159179688, "fcm_dpo/q_t": 0.39998936653137207, "grad_norm": 8.402904510498047, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -2.001081943511963, "logits/rejected": -1.3680001497268677, "logps/chosen": -228.129150390625, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -300.2601623535156, "loss": 1.0567, "margin_dpo/margin_mean": 73.69112396240234, "margin_dpo/margin_std": 79.77305603027344, "step": 324 }, { "KL/chosen_KL_mean": -122.16981506347656, "KL/mean": -161.32615661621094, "KL/rejected_KL_mean": -200.48248291015625, "KL/std": 80.2048568725586, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.005700238049030304, "fcm_dpo/delta": -0.04855629429221153, "fcm_dpo/margin": 78.31268310546875, "fcm_dpo/q_t": 0.3951469659805298, "grad_norm": 7.731611728668213, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.886455774307251, "logits/rejected": -1.1478431224822998, "logps/chosen": -220.87667846679688, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -276.8905944824219, "loss": 1.0426, "margin_dpo/margin_mean": 78.31268310546875, "margin_dpo/margin_std": 83.01351928710938, "step": 325 }, { "KL/chosen_KL_mean": -134.50033569335938, "KL/mean": -170.6287841796875, "KL/rejected_KL_mean": -206.7572021484375, "KL/std": 80.01649475097656, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.005670288577675819, "fcm_dpo/delta": -0.010161316022276878, "fcm_dpo/margin": 72.25686645507812, "fcm_dpo/q_t": 0.40363484621047974, "grad_norm": 7.200769424438477, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.670785903930664, "logits/rejected": -2.0097928047180176, "logps/chosen": -239.16165161132812, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -293.58258056640625, "loss": 1.0732, "margin_dpo/margin_mean": 72.25686645507812, "margin_dpo/margin_std": 84.6978759765625, "step": 326 }, { "KL/chosen_KL_mean": -129.2296142578125, "KL/mean": -164.94448852539062, "KL/rejected_KL_mean": -200.65936279296875, "KL/std": 74.24826049804688, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0056545548141002655, "fcm_dpo/delta": -0.004216842353343964, "fcm_dpo/margin": 71.42976379394531, "fcm_dpo/q_t": 0.4062386155128479, "grad_norm": 7.839048385620117, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.851598858833313, "logits/rejected": -1.2604849338531494, "logps/chosen": -236.6698455810547, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -317.71258544921875, "loss": 1.0753, "margin_dpo/margin_mean": 71.42976379394531, "margin_dpo/margin_std": 83.78807830810547, "step": 327 }, { "KL/chosen_KL_mean": -128.00503540039062, "KL/mean": -173.21142578125, "KL/rejected_KL_mean": -218.41781616210938, "KL/std": 87.85050964355469, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.005575449205935001, "fcm_dpo/delta": -0.10963813960552216, "fcm_dpo/margin": 90.41278076171875, "fcm_dpo/q_t": 0.38227465748786926, "grad_norm": 9.374929428100586, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6768348217010498, "logits/rejected": -1.375054121017456, "logps/chosen": -214.61572265625, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -315.8197021484375, "loss": 1.0009, "margin_dpo/margin_mean": 90.41278076171875, "margin_dpo/margin_std": 88.63432312011719, "step": 328 }, { "KL/chosen_KL_mean": -133.95986938476562, "KL/mean": -168.681884765625, "KL/rejected_KL_mean": -203.40390014648438, "KL/std": 82.48472595214844, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0055352747440338135, "fcm_dpo/delta": 0.01594502106308937, "fcm_dpo/margin": 69.44401550292969, "fcm_dpo/q_t": 0.4109278917312622, "grad_norm": 9.737687110900879, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.9152681827545166, "logits/rejected": -1.4291472434997559, "logps/chosen": -234.5729522705078, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -308.35760498046875, "loss": 1.099, "margin_dpo/margin_mean": 69.44401550292969, "margin_dpo/margin_std": 90.26435852050781, "step": 329 }, { "KL/chosen_KL_mean": -113.37420654296875, "KL/mean": -151.68533325195312, "KL/rejected_KL_mean": -189.99644470214844, "KL/std": 81.62052917480469, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005531922448426485, "fcm_dpo/delta": -0.02504011243581772, "fcm_dpo/margin": 76.62223815917969, "fcm_dpo/q_t": 0.39872580766677856, "grad_norm": 7.780351161956787, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.9576584100723267, "logits/rejected": -1.8802752494812012, "logps/chosen": -190.3326416015625, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -282.8114013671875, "loss": 1.0451, "margin_dpo/margin_mean": 76.62223815917969, "margin_dpo/margin_std": 71.32508850097656, "step": 330 }, { "KL/chosen_KL_mean": -122.11412811279297, "KL/mean": -164.014404296875, "KL/rejected_KL_mean": -205.91468811035156, "KL/std": 79.3971939086914, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.005498077254742384, "fcm_dpo/delta": -0.06365593522787094, "fcm_dpo/margin": 83.80056762695312, "fcm_dpo/q_t": 0.39148789644241333, "grad_norm": 8.933655738830566, "learning_rate": 3.072376374875335e-07, "logits/chosen": -2.0149483680725098, "logits/rejected": -1.7135872840881348, "logps/chosen": -205.17355346679688, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -300.04583740234375, "loss": 1.0232, "margin_dpo/margin_mean": 83.80056762695312, "margin_dpo/margin_std": 79.47540283203125, "step": 331 }, { "KL/chosen_KL_mean": -129.74179077148438, "KL/mean": -159.22467041015625, "KL/rejected_KL_mean": -188.70755004882812, "KL/std": 88.24655151367188, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005533217918127775, "fcm_dpo/delta": 0.0761144831776619, "fcm_dpo/margin": 58.96575927734375, "fcm_dpo/q_t": 0.423473060131073, "grad_norm": 7.477311611175537, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.0068295001983643, "logits/rejected": -1.562392234802246, "logps/chosen": -211.6726837158203, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -271.553466796875, "loss": 1.1395, "margin_dpo/margin_mean": 58.96575927734375, "margin_dpo/margin_std": 85.17802429199219, "step": 332 }, { "KL/chosen_KL_mean": -148.9146728515625, "KL/mean": -189.62020874023438, "KL/rejected_KL_mean": -230.32574462890625, "KL/std": 92.93616485595703, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.005479893647134304, "fcm_dpo/delta": -0.049086473882198334, "fcm_dpo/margin": 81.41110229492188, "fcm_dpo/q_t": 0.3970828056335449, "grad_norm": 8.315781593322754, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.7137458324432373, "logits/rejected": -1.3754115104675293, "logps/chosen": -237.44725036621094, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -332.791748046875, "loss": 1.0607, "margin_dpo/margin_mean": 81.41109466552734, "margin_dpo/margin_std": 99.22920989990234, "step": 333 }, { "KL/chosen_KL_mean": -137.14443969726562, "KL/mean": -173.208984375, "KL/rejected_KL_mean": -209.27354431152344, "KL/std": 87.6807632446289, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0054707275703549385, "fcm_dpo/delta": 0.005111131817102432, "fcm_dpo/margin": 72.12909698486328, "fcm_dpo/q_t": 0.4079854488372803, "grad_norm": 9.063214302062988, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.7211403846740723, "logits/rejected": -1.4371792078018188, "logps/chosen": -234.9022216796875, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -309.27935791015625, "loss": 1.1039, "margin_dpo/margin_mean": 72.12908935546875, "margin_dpo/margin_std": 100.05352783203125, "step": 334 }, { "KL/chosen_KL_mean": -135.29861450195312, "KL/mean": -168.27935791015625, "KL/rejected_KL_mean": -201.26010131835938, "KL/std": 86.92887115478516, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005529351532459259, "fcm_dpo/delta": 0.03661692142486572, "fcm_dpo/margin": 65.96151733398438, "fcm_dpo/q_t": 0.415336936712265, "grad_norm": 9.989916801452637, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.2843353748321533, "logits/rejected": -1.8450608253479004, "logps/chosen": -231.67465209960938, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -296.1841125488281, "loss": 1.1201, "margin_dpo/margin_mean": 65.96151733398438, "margin_dpo/margin_std": 94.99966430664062, "step": 335 }, { "KL/chosen_KL_mean": -136.21908569335938, "KL/mean": -178.16624450683594, "KL/rejected_KL_mean": -220.1134033203125, "KL/std": 89.85641479492188, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005501416511833668, "fcm_dpo/delta": -0.06448090821504593, "fcm_dpo/margin": 83.89433288574219, "fcm_dpo/q_t": 0.39262908697128296, "grad_norm": 10.329455375671387, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.6425557136535645, "logits/rejected": -1.489349365234375, "logps/chosen": -224.74859619140625, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -329.669189453125, "loss": 1.0363, "margin_dpo/margin_mean": 83.89433288574219, "margin_dpo/margin_std": 92.44712829589844, "step": 336 }, { "KL/chosen_KL_mean": -138.4334716796875, "KL/mean": -176.56419372558594, "KL/rejected_KL_mean": -214.69493103027344, "KL/std": 90.89530944824219, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.005465905647724867, "fcm_dpo/delta": -0.01761242188513279, "fcm_dpo/margin": 76.261474609375, "fcm_dpo/q_t": 0.40467244386672974, "grad_norm": 8.800559997558594, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.5268702507019043, "logits/rejected": -2.248110294342041, "logps/chosen": -235.745849609375, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -324.8262939453125, "loss": 1.0932, "margin_dpo/margin_mean": 76.261474609375, "margin_dpo/margin_std": 107.51373291015625, "step": 337 }, { "KL/chosen_KL_mean": -142.8409423828125, "KL/mean": -179.2299346923828, "KL/rejected_KL_mean": -215.61891174316406, "KL/std": 90.62361907958984, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.005448544397950172, "fcm_dpo/delta": 0.0035962210968136787, "fcm_dpo/margin": 72.77796936035156, "fcm_dpo/q_t": 0.40651267766952515, "grad_norm": 8.304313659667969, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7956328392028809, "logits/rejected": -1.6426862478256226, "logps/chosen": -226.4710693359375, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -309.311279296875, "loss": 1.0815, "margin_dpo/margin_mean": 72.77796936035156, "margin_dpo/margin_std": 86.35737609863281, "step": 338 }, { "KL/chosen_KL_mean": -151.6717987060547, "KL/mean": -190.01519775390625, "KL/rejected_KL_mean": -228.35861206054688, "KL/std": 90.26435852050781, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005470922216773033, "fcm_dpo/delta": -0.021044213324785233, "fcm_dpo/margin": 76.68679809570312, "fcm_dpo/q_t": 0.4029090702533722, "grad_norm": 7.918790340423584, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.974137783050537, "logits/rejected": -2.5411906242370605, "logps/chosen": -251.52362060546875, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -337.4329833984375, "loss": 1.083, "margin_dpo/margin_mean": 76.6867904663086, "margin_dpo/margin_std": 99.385498046875, "step": 339 }, { "KL/chosen_KL_mean": -163.9344482421875, "KL/mean": -198.1103515625, "KL/rejected_KL_mean": -232.28622436523438, "KL/std": 83.49043273925781, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.00545636098831892, "fcm_dpo/delta": 0.028081975877285004, "fcm_dpo/margin": 68.35176849365234, "fcm_dpo/q_t": 0.41289421916007996, "grad_norm": 8.38933277130127, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.566434621810913, "logits/rejected": -1.0863769054412842, "logps/chosen": -248.90841674804688, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -311.0865478515625, "loss": 1.1076, "margin_dpo/margin_mean": 68.35176849365234, "margin_dpo/margin_std": 93.10060119628906, "step": 340 }, { "KL/chosen_KL_mean": -172.1874542236328, "KL/mean": -205.05572509765625, "KL/rejected_KL_mean": -237.92396545410156, "KL/std": 87.1167221069336, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0054856156930327415, "fcm_dpo/delta": 0.04087613895535469, "fcm_dpo/margin": 65.73651123046875, "fcm_dpo/q_t": 0.4172903597354889, "grad_norm": 10.664201736450195, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.5940091609954834, "logits/rejected": -1.3462456464767456, "logps/chosen": -257.4514465332031, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -344.0099182128906, "loss": 1.1281, "margin_dpo/margin_mean": 65.73651123046875, "margin_dpo/margin_std": 98.62066650390625, "step": 341 }, { "KL/chosen_KL_mean": -136.70199584960938, "KL/mean": -185.01284790039062, "KL/rejected_KL_mean": -233.32369995117188, "KL/std": 87.99820709228516, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.005373295396566391, "fcm_dpo/delta": -0.12708258628845215, "fcm_dpo/margin": 96.62168884277344, "fcm_dpo/q_t": 0.3772847652435303, "grad_norm": 7.556240081787109, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.441150188446045, "logits/rejected": -0.6723727583885193, "logps/chosen": -237.7406005859375, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -332.22222900390625, "loss": 0.9971, "margin_dpo/margin_mean": 96.62168884277344, "margin_dpo/margin_std": 93.32367706298828, "step": 342 }, { "KL/chosen_KL_mean": -149.5662841796875, "KL/mean": -188.89089965820312, "KL/rejected_KL_mean": -228.21551513671875, "KL/std": 90.73258972167969, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.005381445400416851, "fcm_dpo/delta": -0.024871759116649628, "fcm_dpo/margin": 78.64924621582031, "fcm_dpo/q_t": 0.40158963203430176, "grad_norm": 9.025823593139648, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1432039737701416, "logits/rejected": -2.033994674682617, "logps/chosen": -233.31365966796875, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -344.79461669921875, "loss": 1.0722, "margin_dpo/margin_mean": 78.64924621582031, "margin_dpo/margin_std": 94.7224349975586, "step": 343 }, { "KL/chosen_KL_mean": -150.06822204589844, "KL/mean": -193.07989501953125, "KL/rejected_KL_mean": -236.091552734375, "KL/std": 97.09487915039062, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.005322422832250595, "fcm_dpo/delta": -0.06088802218437195, "fcm_dpo/margin": 86.02333068847656, "fcm_dpo/q_t": 0.39479249715805054, "grad_norm": 8.927608489990234, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.6667547225952148, "logits/rejected": -1.4194403886795044, "logps/chosen": -228.3221893310547, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -331.520263671875, "loss": 1.0518, "margin_dpo/margin_mean": 86.02334594726562, "margin_dpo/margin_std": 102.62258911132812, "step": 344 }, { "KL/chosen_KL_mean": -152.90719604492188, "KL/mean": -197.71490478515625, "KL/rejected_KL_mean": -242.5226287841797, "KL/std": 98.58704376220703, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.005212096497416496, "fcm_dpo/delta": -0.0705864354968071, "fcm_dpo/margin": 89.61544799804688, "fcm_dpo/q_t": 0.39214351773262024, "grad_norm": 8.821065902709961, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.324622392654419, "logits/rejected": -1.0158771276474, "logps/chosen": -250.9866943359375, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -346.5574035644531, "loss": 1.0368, "margin_dpo/margin_mean": 89.61544799804688, "margin_dpo/margin_std": 100.80644226074219, "step": 345 }, { "KL/chosen_KL_mean": -171.75332641601562, "KL/mean": -206.89913940429688, "KL/rejected_KL_mean": -242.04490661621094, "KL/std": 91.3446044921875, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.005229143425822258, "fcm_dpo/delta": 0.033675748854875565, "fcm_dpo/margin": 70.29158020019531, "fcm_dpo/q_t": 0.41544109582901, "grad_norm": 8.38217830657959, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.8644046783447266, "logits/rejected": -1.5510649681091309, "logps/chosen": -252.43682861328125, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -327.9375, "loss": 1.1219, "margin_dpo/margin_mean": 70.29158020019531, "margin_dpo/margin_std": 104.17990112304688, "step": 346 }, { "KL/chosen_KL_mean": -161.090576171875, "KL/mean": -195.0328826904297, "KL/rejected_KL_mean": -228.97518920898438, "KL/std": 90.44676208496094, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.00528854550793767, "fcm_dpo/delta": 0.04217086359858513, "fcm_dpo/margin": 67.88462829589844, "fcm_dpo/q_t": 0.4173012673854828, "grad_norm": 8.082019805908203, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.8615248203277588, "logits/rejected": -1.5334457159042358, "logps/chosen": -254.7699432373047, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -326.5177001953125, "loss": 1.1242, "margin_dpo/margin_mean": 67.8846206665039, "margin_dpo/margin_std": 98.63946533203125, "step": 347 }, { "KL/chosen_KL_mean": -140.37342834472656, "KL/mean": -178.1192626953125, "KL/rejected_KL_mean": -215.86509704589844, "KL/std": 85.59811401367188, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.0052436888217926025, "fcm_dpo/delta": 0.0030514225363731384, "fcm_dpo/margin": 75.49166870117188, "fcm_dpo/q_t": 0.40680891275405884, "grad_norm": 7.615900993347168, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.1755945682525635, "logits/rejected": -1.906559944152832, "logps/chosen": -230.62567138671875, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -311.6353759765625, "loss": 1.0836, "margin_dpo/margin_mean": 75.49166107177734, "margin_dpo/margin_std": 85.67681884765625, "step": 348 }, { "KL/chosen_KL_mean": -152.44927978515625, "KL/mean": -187.54908752441406, "KL/rejected_KL_mean": -222.6488800048828, "KL/std": 93.9215087890625, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.00532772159203887, "fcm_dpo/delta": 0.02659936062991619, "fcm_dpo/margin": 70.19960021972656, "fcm_dpo/q_t": 0.41398370265960693, "grad_norm": 9.72806167602539, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.1878151893615723, "logits/rejected": -1.964611291885376, "logps/chosen": -244.50738525390625, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -326.6141357421875, "loss": 1.1129, "margin_dpo/margin_mean": 70.19960021972656, "margin_dpo/margin_std": 98.30331420898438, "step": 349 }, { "KL/chosen_KL_mean": -161.92453002929688, "KL/mean": -190.67897033691406, "KL/rejected_KL_mean": -219.43341064453125, "KL/std": 85.01283264160156, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005382169969379902, "fcm_dpo/delta": 0.09345383197069168, "fcm_dpo/margin": 57.508888244628906, "fcm_dpo/q_t": 0.42798513174057007, "grad_norm": 11.079474449157715, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8963922262191772, "logits/rejected": -1.6701393127441406, "logps/chosen": -248.97872924804688, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -315.15533447265625, "loss": 1.16, "margin_dpo/margin_mean": 57.50888442993164, "margin_dpo/margin_std": 92.15670013427734, "step": 350 }, { "KL/chosen_KL_mean": -158.1461181640625, "KL/mean": -201.73284912109375, "KL/rejected_KL_mean": -245.319580078125, "KL/std": 96.9740219116211, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.005366134457290173, "fcm_dpo/delta": -0.07104984670877457, "fcm_dpo/margin": 87.17343139648438, "fcm_dpo/q_t": 0.3914119005203247, "grad_norm": 7.967398643493652, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.3186531066894531, "logits/rejected": -1.0006015300750732, "logps/chosen": -247.31515502929688, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -353.61761474609375, "loss": 1.0428, "margin_dpo/margin_mean": 87.17343139648438, "margin_dpo/margin_std": 102.06587982177734, "step": 351 }, { "KL/chosen_KL_mean": -147.1536865234375, "KL/mean": -186.831787109375, "KL/rejected_KL_mean": -226.50991821289062, "KL/std": 95.82431030273438, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.005316582508385181, "fcm_dpo/delta": -0.022892624139785767, "fcm_dpo/margin": 79.35623931884766, "fcm_dpo/q_t": 0.4040263593196869, "grad_norm": 8.046867370605469, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.788487434387207, "logits/rejected": -1.2211673259735107, "logps/chosen": -243.96514892578125, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -310.91473388671875, "loss": 1.0801, "margin_dpo/margin_mean": 79.35623931884766, "margin_dpo/margin_std": 103.37153625488281, "step": 352 }, { "KL/chosen_KL_mean": -166.29953002929688, "KL/mean": -192.50955200195312, "KL/rejected_KL_mean": -218.71957397460938, "KL/std": 85.62002563476562, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.005393030121922493, "fcm_dpo/delta": 0.12088752537965775, "fcm_dpo/margin": 52.42003631591797, "fcm_dpo/q_t": 0.43419569730758667, "grad_norm": 9.481779098510742, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.2872443199157715, "logits/rejected": -1.9025554656982422, "logps/chosen": -251.66201782226562, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -306.0868835449219, "loss": 1.1869, "margin_dpo/margin_mean": 52.42003631591797, "margin_dpo/margin_std": 93.78532409667969, "step": 353 }, { "KL/chosen_KL_mean": -145.4825897216797, "KL/mean": -185.92393493652344, "KL/rejected_KL_mean": -226.36529541015625, "KL/std": 85.61680603027344, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.005425235256552696, "fcm_dpo/delta": -0.04062160104513168, "fcm_dpo/margin": 80.8826904296875, "fcm_dpo/q_t": 0.39690786600112915, "grad_norm": 8.143423080444336, "learning_rate": 2.781732916288303e-07, "logits/chosen": -2.056746006011963, "logits/rejected": -1.589310884475708, "logps/chosen": -241.28375244140625, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -321.2314453125, "loss": 1.0387, "margin_dpo/margin_mean": 80.8826904296875, "margin_dpo/margin_std": 79.38536071777344, "step": 354 }, { "KL/chosen_KL_mean": -142.357666015625, "KL/mean": -179.97850036621094, "KL/rejected_KL_mean": -217.59933471679688, "KL/std": 87.13818359375, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.005375292152166367, "fcm_dpo/delta": -0.004951227456331253, "fcm_dpo/margin": 75.24168395996094, "fcm_dpo/q_t": 0.405413955450058, "grad_norm": 8.98030948638916, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4683008193969727, "logits/rejected": -1.977631688117981, "logps/chosen": -228.543212890625, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -306.0609130859375, "loss": 1.0828, "margin_dpo/margin_mean": 75.24168395996094, "margin_dpo/margin_std": 90.99357604980469, "step": 355 }, { "KL/chosen_KL_mean": -168.6410369873047, "KL/mean": -196.64903259277344, "KL/rejected_KL_mean": -224.6570281982422, "KL/std": 85.98258972167969, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005486851558089256, "fcm_dpo/delta": 0.09516627341508865, "fcm_dpo/margin": 56.01601791381836, "fcm_dpo/q_t": 0.4288730025291443, "grad_norm": 9.846342086791992, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -2.014887571334839, "logits/rejected": -1.551250696182251, "logps/chosen": -259.4591369628906, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -306.78436279296875, "loss": 1.1732, "margin_dpo/margin_mean": 56.016014099121094, "margin_dpo/margin_std": 99.12171936035156, "step": 356 }, { "KL/chosen_KL_mean": -139.6346435546875, "KL/mean": -185.7269287109375, "KL/rejected_KL_mean": -231.8192138671875, "KL/std": 93.44927978515625, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.005379359703511, "fcm_dpo/delta": -0.10246876627206802, "fcm_dpo/margin": 92.18455505371094, "fcm_dpo/q_t": 0.38309624791145325, "grad_norm": 8.775167465209961, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -2.094320774078369, "logits/rejected": -1.727975606918335, "logps/chosen": -227.139404296875, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -324.0685729980469, "loss": 1.0025, "margin_dpo/margin_mean": 92.1845474243164, "margin_dpo/margin_std": 84.30912780761719, "step": 357 }, { "KL/chosen_KL_mean": -150.62889099121094, "KL/mean": -185.00140380859375, "KL/rejected_KL_mean": -219.37393188476562, "KL/std": 88.681884765625, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.00540516059845686, "fcm_dpo/delta": 0.029528388753533363, "fcm_dpo/margin": 68.74504089355469, "fcm_dpo/q_t": 0.415380597114563, "grad_norm": 8.465203285217285, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.658797025680542, "logits/rejected": -1.1850144863128662, "logps/chosen": -245.68435668945312, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -315.85003662109375, "loss": 1.1098, "margin_dpo/margin_mean": 68.74503326416016, "margin_dpo/margin_std": 94.95555877685547, "step": 358 }, { "KL/chosen_KL_mean": -166.68185424804688, "KL/mean": -199.73629760742188, "KL/rejected_KL_mean": -232.79074096679688, "KL/std": 83.94450378417969, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.005421824753284454, "fcm_dpo/delta": 0.042679473757743835, "fcm_dpo/margin": 66.10885620117188, "fcm_dpo/q_t": 0.41805434226989746, "grad_norm": 9.061989784240723, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.5053596496582031, "logits/rejected": -0.9163897037506104, "logps/chosen": -254.05519104003906, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -321.5626220703125, "loss": 1.1297, "margin_dpo/margin_mean": 66.10885620117188, "margin_dpo/margin_std": 98.10028076171875, "step": 359 }, { "KL/chosen_KL_mean": -154.70034790039062, "KL/mean": -194.94888305664062, "KL/rejected_KL_mean": -235.19741821289062, "KL/std": 83.0208969116211, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.005429339595139027, "fcm_dpo/delta": -0.03878094628453255, "fcm_dpo/margin": 80.49706268310547, "fcm_dpo/q_t": 0.3969472646713257, "grad_norm": 11.305315017700195, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.818256139755249, "logits/rejected": -1.480759859085083, "logps/chosen": -229.92379760742188, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -311.2914123535156, "loss": 1.0482, "margin_dpo/margin_mean": 80.49706268310547, "margin_dpo/margin_std": 86.21923065185547, "step": 360 }, { "KL/chosen_KL_mean": -157.9833984375, "KL/mean": -194.11631774902344, "KL/rejected_KL_mean": -230.24923706054688, "KL/std": 94.6265869140625, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005435779690742493, "fcm_dpo/delta": 0.007428489625453949, "fcm_dpo/margin": 72.26587677001953, "fcm_dpo/q_t": 0.4099791646003723, "grad_norm": 10.343612670898438, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.8339624404907227, "logits/rejected": -1.5910390615463257, "logps/chosen": -239.9367218017578, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -326.9284362792969, "loss": 1.0989, "margin_dpo/margin_mean": 72.26588439941406, "margin_dpo/margin_std": 98.6583023071289, "step": 361 }, { "KL/chosen_KL_mean": -161.15264892578125, "KL/mean": -199.39358520507812, "KL/rejected_KL_mean": -237.634521484375, "KL/std": 90.35491943359375, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.005407636985182762, "fcm_dpo/delta": -0.014297381974756718, "fcm_dpo/margin": 76.48185729980469, "fcm_dpo/q_t": 0.40349018573760986, "grad_norm": 9.412640571594238, "learning_rate": 2.679511629948319e-07, "logits/chosen": -2.1061692237854004, "logits/rejected": -1.9120397567749023, "logps/chosen": -250.49237060546875, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -348.136474609375, "loss": 1.0782, "margin_dpo/margin_mean": 76.48185729980469, "margin_dpo/margin_std": 94.35972595214844, "step": 362 }, { "KL/chosen_KL_mean": -150.69700622558594, "KL/mean": -200.70445251464844, "KL/rejected_KL_mean": -250.71188354492188, "KL/std": 95.1234359741211, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.005325894802808762, "fcm_dpo/delta": -0.13990448415279388, "fcm_dpo/margin": 100.014892578125, "fcm_dpo/q_t": 0.37722668051719666, "grad_norm": 9.762939453125, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.8377395868301392, "logits/rejected": -1.3983542919158936, "logps/chosen": -231.4720001220703, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -330.1445617675781, "loss": 0.991, "margin_dpo/margin_mean": 100.01488494873047, "margin_dpo/margin_std": 101.22518920898438, "step": 363 }, { "KL/chosen_KL_mean": -147.01959228515625, "KL/mean": -187.73715209960938, "KL/rejected_KL_mean": -228.4547119140625, "KL/std": 90.11917114257812, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.005271662957966328, "fcm_dpo/delta": -0.03109993413090706, "fcm_dpo/margin": 81.43513488769531, "fcm_dpo/q_t": 0.3990965485572815, "grad_norm": 9.738204002380371, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.1416172981262207, "logits/rejected": -1.9996737241744995, "logps/chosen": -223.5491943359375, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -326.0164489746094, "loss": 1.054, "margin_dpo/margin_mean": 81.43513488769531, "margin_dpo/margin_std": 85.69392395019531, "step": 364 }, { "KL/chosen_KL_mean": -143.95553588867188, "KL/mean": -188.41455078125, "KL/rejected_KL_mean": -232.87356567382812, "KL/std": 95.78887939453125, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.005203197710216045, "fcm_dpo/delta": -0.06576266139745712, "fcm_dpo/margin": 88.91804504394531, "fcm_dpo/q_t": 0.39465266466140747, "grad_norm": 8.756814002990723, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7383065223693848, "logits/rejected": -1.616851568222046, "logps/chosen": -224.50042724609375, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -332.19671630859375, "loss": 1.0501, "margin_dpo/margin_mean": 88.91803741455078, "margin_dpo/margin_std": 107.74969482421875, "step": 365 }, { "KL/chosen_KL_mean": -147.58839416503906, "KL/mean": -183.84132385253906, "KL/rejected_KL_mean": -220.09423828125, "KL/std": 98.51033020019531, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.005182204768061638, "fcm_dpo/delta": 0.025213927030563354, "fcm_dpo/margin": 72.50584411621094, "fcm_dpo/q_t": 0.41465622186660767, "grad_norm": 7.991236209869385, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.8776271343231201, "logits/rejected": -1.9153807163238525, "logps/chosen": -219.82647705078125, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -311.76605224609375, "loss": 1.103, "margin_dpo/margin_mean": 72.50584411621094, "margin_dpo/margin_std": 96.99908447265625, "step": 366 }, { "KL/chosen_KL_mean": -134.3475341796875, "KL/mean": -176.96646118164062, "KL/rejected_KL_mean": -219.58535766601562, "KL/std": 86.25660705566406, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.005162697285413742, "fcm_dpo/delta": -0.04188086837530136, "fcm_dpo/margin": 85.23782348632812, "fcm_dpo/q_t": 0.3969029188156128, "grad_norm": 9.104174613952637, "learning_rate": 2.615458646614349e-07, "logits/chosen": -2.0649051666259766, "logits/rejected": -1.7248246669769287, "logps/chosen": -219.76954650878906, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -303.1031494140625, "loss": 1.0426, "margin_dpo/margin_mean": 85.23782348632812, "margin_dpo/margin_std": 88.7197265625, "step": 367 }, { "KL/chosen_KL_mean": -126.20015716552734, "KL/mean": -174.28509521484375, "KL/rejected_KL_mean": -222.3700408935547, "KL/std": 94.73281860351562, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.005095783621072769, "fcm_dpo/delta": -0.09464528411626816, "fcm_dpo/margin": 96.16989135742188, "fcm_dpo/q_t": 0.38474807143211365, "grad_norm": 8.314908027648926, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.5860313177108765, "logits/rejected": -1.5578964948654175, "logps/chosen": -200.04129028320312, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -327.0653076171875, "loss": 1.0134, "margin_dpo/margin_mean": 96.16989135742188, "margin_dpo/margin_std": 95.0321044921875, "step": 368 }, { "KL/chosen_KL_mean": -141.8168487548828, "KL/mean": -185.76425170898438, "KL/rejected_KL_mean": -229.71163940429688, "KL/std": 87.55709838867188, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.005026928149163723, "fcm_dpo/delta": -0.043759785592556, "fcm_dpo/margin": 87.89480590820312, "fcm_dpo/q_t": 0.3963191509246826, "grad_norm": 8.04337215423584, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.7968530654907227, "logits/rejected": -1.4356284141540527, "logps/chosen": -240.27880859375, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -328.86834716796875, "loss": 1.0379, "margin_dpo/margin_mean": 87.89480590820312, "margin_dpo/margin_std": 87.50447082519531, "step": 369 }, { "KL/chosen_KL_mean": -157.39646911621094, "KL/mean": -187.2655487060547, "KL/rejected_KL_mean": -217.13461303710938, "KL/std": 83.47927856445312, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.005087011959403753, "fcm_dpo/delta": 0.09899773448705673, "fcm_dpo/margin": 59.73814392089844, "fcm_dpo/q_t": 0.4286961555480957, "grad_norm": 8.693794250488281, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -2.229179620742798, "logits/rejected": -2.0390121936798096, "logps/chosen": -254.17221069335938, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -319.87872314453125, "loss": 1.1644, "margin_dpo/margin_mean": 59.73814392089844, "margin_dpo/margin_std": 97.18853759765625, "step": 370 }, { "KL/chosen_KL_mean": -150.46633911132812, "KL/mean": -183.99876403808594, "KL/rejected_KL_mean": -217.53118896484375, "KL/std": 84.90621948242188, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005163062363862991, "fcm_dpo/delta": 0.05538462847471237, "fcm_dpo/margin": 67.06483459472656, "fcm_dpo/q_t": 0.41865336894989014, "grad_norm": 10.725748062133789, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.7539564371109009, "logits/rejected": -1.3721293210983276, "logps/chosen": -253.45654296875, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -322.8818054199219, "loss": 1.1252, "margin_dpo/margin_mean": 67.06483459472656, "margin_dpo/margin_std": 92.47673034667969, "step": 371 }, { "KL/chosen_KL_mean": -130.90386962890625, "KL/mean": -180.30844116210938, "KL/rejected_KL_mean": -229.7130126953125, "KL/std": 98.82281494140625, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.005087047815322876, "fcm_dpo/delta": -0.1079922616481781, "fcm_dpo/margin": 98.80916595458984, "fcm_dpo/q_t": 0.38361629843711853, "grad_norm": 7.496952533721924, "learning_rate": 2.551329606220976e-07, "logits/chosen": -2.1031112670898438, "logits/rejected": -1.4824285507202148, "logps/chosen": -221.9300994873047, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -315.3232421875, "loss": 1.0067, "margin_dpo/margin_mean": 98.80915832519531, "margin_dpo/margin_std": 100.46953582763672, "step": 372 }, { "KL/chosen_KL_mean": -149.92190551757812, "KL/mean": -187.0259246826172, "KL/rejected_KL_mean": -224.12994384765625, "KL/std": 95.12792205810547, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.005068117752671242, "fcm_dpo/delta": 0.024845201522111893, "fcm_dpo/margin": 74.20803833007812, "fcm_dpo/q_t": 0.4126342535018921, "grad_norm": 11.673218727111816, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.250734806060791, "logits/rejected": -1.7334730625152588, "logps/chosen": -249.9949951171875, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -315.24627685546875, "loss": 1.1078, "margin_dpo/margin_mean": 74.20803833007812, "margin_dpo/margin_std": 101.13754272460938, "step": 373 }, { "KL/chosen_KL_mean": -130.8233184814453, "KL/mean": -172.83274841308594, "KL/rejected_KL_mean": -214.84219360351562, "KL/std": 90.33714294433594, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.005047214217483997, "fcm_dpo/delta": -0.025318361818790436, "fcm_dpo/margin": 84.01887512207031, "fcm_dpo/q_t": 0.40069830417633057, "grad_norm": 8.113106727600098, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.390965461730957, "logits/rejected": -2.007880210876465, "logps/chosen": -228.83566284179688, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -315.59442138671875, "loss": 1.053, "margin_dpo/margin_mean": 84.01887512207031, "margin_dpo/margin_std": 87.80509948730469, "step": 374 }, { "KL/chosen_KL_mean": -134.156494140625, "KL/mean": -180.2216796875, "KL/rejected_KL_mean": -226.28689575195312, "KL/std": 86.51730346679688, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0049772970378398895, "fcm_dpo/delta": -0.06244664639234543, "fcm_dpo/margin": 92.13040161132812, "fcm_dpo/q_t": 0.391700804233551, "grad_norm": 8.560799598693848, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.8873395919799805, "logits/rejected": -1.6612257957458496, "logps/chosen": -218.55276489257812, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -322.0057373046875, "loss": 1.0315, "margin_dpo/margin_mean": 92.13040161132812, "margin_dpo/margin_std": 91.73725891113281, "step": 375 }, { "KL/chosen_KL_mean": -126.76222229003906, "KL/mean": -178.85284423828125, "KL/rejected_KL_mean": -230.9434814453125, "KL/std": 100.60958862304688, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004900106228888035, "fcm_dpo/delta": -0.11657736450433731, "fcm_dpo/margin": 104.18125915527344, "fcm_dpo/q_t": 0.38141870498657227, "grad_norm": 8.838798522949219, "learning_rate": 2.5e-07, "logits/chosen": -1.7525087594985962, "logits/rejected": -1.4346784353256226, "logps/chosen": -222.4999237060547, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -342.91680908203125, "loss": 1.0028, "margin_dpo/margin_mean": 104.1812515258789, "margin_dpo/margin_std": 106.73007202148438, "step": 376 }, { "KL/chosen_KL_mean": -138.8360137939453, "KL/mean": -192.2803192138672, "KL/rejected_KL_mean": -245.724609375, "KL/std": 102.01841735839844, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004774793051183224, "fcm_dpo/delta": -0.11696825176477432, "fcm_dpo/margin": 106.88860321044922, "fcm_dpo/q_t": 0.3800773024559021, "grad_norm": 8.10577392578125, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.6642727851867676, "logits/rejected": -1.5481679439544678, "logps/chosen": -217.76296997070312, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -351.62353515625, "loss": 0.9943, "margin_dpo/margin_mean": 106.88859558105469, "margin_dpo/margin_std": 100.59651947021484, "step": 377 }, { "KL/chosen_KL_mean": -133.3759765625, "KL/mean": -176.62396240234375, "KL/rejected_KL_mean": -219.87191772460938, "KL/std": 96.11386108398438, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004774591885507107, "fcm_dpo/delta": -0.013896126300096512, "fcm_dpo/margin": 86.49594116210938, "fcm_dpo/q_t": 0.40213143825531006, "grad_norm": 9.039429664611816, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.9743247032165527, "logits/rejected": -1.6904141902923584, "logps/chosen": -208.74838256835938, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -313.5760192871094, "loss": 1.0697, "margin_dpo/margin_mean": 86.49594116210938, "margin_dpo/margin_std": 97.98692321777344, "step": 378 }, { "KL/chosen_KL_mean": -157.49053955078125, "KL/mean": -195.89984130859375, "KL/rejected_KL_mean": -234.30911254882812, "KL/std": 99.6199951171875, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004770830273628235, "fcm_dpo/delta": 0.03480100259184837, "fcm_dpo/margin": 76.81858825683594, "fcm_dpo/q_t": 0.4159185588359833, "grad_norm": 8.880955696105957, "learning_rate": 2.461501611777483e-07, "logits/chosen": -2.036651849746704, "logits/rejected": -1.964677333831787, "logps/chosen": -246.34622192382812, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -355.69024658203125, "loss": 1.1215, "margin_dpo/margin_mean": 76.81858825683594, "margin_dpo/margin_std": 112.8366470336914, "step": 379 }, { "KL/chosen_KL_mean": -171.22727966308594, "KL/mean": -219.68356323242188, "KL/rejected_KL_mean": -268.1398620605469, "KL/std": 102.53096008300781, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004737130831927061, "fcm_dpo/delta": -0.06195715814828873, "fcm_dpo/margin": 96.91256713867188, "fcm_dpo/q_t": 0.394929975271225, "grad_norm": 10.908204078674316, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.2247734069824219, "logits/rejected": -1.2011778354644775, "logps/chosen": -251.06048583984375, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -378.8846130371094, "loss": 1.0476, "margin_dpo/margin_mean": 96.91255950927734, "margin_dpo/margin_std": 116.21565246582031, "step": 380 }, { "KL/chosen_KL_mean": -152.4805450439453, "KL/mean": -192.46453857421875, "KL/rejected_KL_mean": -232.44854736328125, "KL/std": 99.30533599853516, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004742340184748173, "fcm_dpo/delta": 0.021558944135904312, "fcm_dpo/margin": 79.96800231933594, "fcm_dpo/q_t": 0.4111793637275696, "grad_norm": 8.339578628540039, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.9136242866516113, "logits/rejected": -1.3661218881607056, "logps/chosen": -239.04736328125, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -316.86529541015625, "loss": 1.1032, "margin_dpo/margin_mean": 79.96800231933594, "margin_dpo/margin_std": 106.66758728027344, "step": 381 }, { "KL/chosen_KL_mean": -150.27328491210938, "KL/mean": -196.13119506835938, "KL/rejected_KL_mean": -241.9890899658203, "KL/std": 95.32405090332031, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004695836454629898, "fcm_dpo/delta": -0.032834455370903015, "fcm_dpo/margin": 91.71578216552734, "fcm_dpo/q_t": 0.3987146019935608, "grad_norm": 8.380996704101562, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.7391798496246338, "logits/rejected": -1.4679232835769653, "logps/chosen": -254.73509216308594, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -356.59130859375, "loss": 1.053, "margin_dpo/margin_mean": 91.71577453613281, "margin_dpo/margin_std": 97.11197662353516, "step": 382 }, { "KL/chosen_KL_mean": -164.20101928710938, "KL/mean": -203.4794158935547, "KL/rejected_KL_mean": -242.75782775878906, "KL/std": 99.30279541015625, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.004725228063762188, "fcm_dpo/delta": 0.02989334426820278, "fcm_dpo/margin": 78.55680847167969, "fcm_dpo/q_t": 0.4133113622665405, "grad_norm": 9.14516830444336, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.7964081764221191, "logits/rejected": -1.6766221523284912, "logps/chosen": -246.31707763671875, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -341.53497314453125, "loss": 1.1042, "margin_dpo/margin_mean": 78.55680847167969, "margin_dpo/margin_std": 99.65657806396484, "step": 383 }, { "KL/chosen_KL_mean": -179.0583953857422, "KL/mean": -215.59246826171875, "KL/rejected_KL_mean": -252.12655639648438, "KL/std": 101.03443908691406, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.00480548944324255, "fcm_dpo/delta": 0.049621324986219406, "fcm_dpo/margin": 73.06816864013672, "fcm_dpo/q_t": 0.4178934693336487, "grad_norm": 10.472575187683105, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.9691556692123413, "logits/rejected": -1.749802827835083, "logps/chosen": -259.6436462402344, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -343.574462890625, "loss": 1.1399, "margin_dpo/margin_mean": 73.06816864013672, "margin_dpo/margin_std": 111.94465637207031, "step": 384 }, { "KL/chosen_KL_mean": -152.22396850585938, "KL/mean": -199.386962890625, "KL/rejected_KL_mean": -246.54994201660156, "KL/std": 90.49244689941406, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.00472596287727356, "fcm_dpo/delta": -0.048928648233413696, "fcm_dpo/margin": 94.32596588134766, "fcm_dpo/q_t": 0.39638376235961914, "grad_norm": 12.332011222839355, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.3004939556121826, "logits/rejected": -1.692992925643921, "logps/chosen": -244.984130859375, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -331.0099182128906, "loss": 1.0518, "margin_dpo/margin_mean": 94.32597351074219, "margin_dpo/margin_std": 106.3654556274414, "step": 385 }, { "KL/chosen_KL_mean": -158.19839477539062, "KL/mean": -204.92355346679688, "KL/rejected_KL_mean": -251.64874267578125, "KL/std": 92.99520874023438, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0047127865254879, "fcm_dpo/delta": -0.04227038100361824, "fcm_dpo/margin": 93.4503402709961, "fcm_dpo/q_t": 0.3953969478607178, "grad_norm": 7.631380081176758, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.8083001375198364, "logits/rejected": -1.524813175201416, "logps/chosen": -243.55734252929688, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -348.639404296875, "loss": 1.047, "margin_dpo/margin_mean": 93.4503402709961, "margin_dpo/margin_std": 99.73554992675781, "step": 386 }, { "KL/chosen_KL_mean": -183.40557861328125, "KL/mean": -223.63211059570312, "KL/rejected_KL_mean": -263.858642578125, "KL/std": 89.46792602539062, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004723359830677509, "fcm_dpo/delta": 0.02071293629705906, "fcm_dpo/margin": 80.45304870605469, "fcm_dpo/q_t": 0.41075658798217773, "grad_norm": 10.297039985656738, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.4340267181396484, "logits/rejected": -1.7421795129776, "logps/chosen": -286.0937194824219, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -363.2872314453125, "loss": 1.0886, "margin_dpo/margin_mean": 80.45305633544922, "margin_dpo/margin_std": 93.80541229248047, "step": 387 }, { "KL/chosen_KL_mean": -183.0073699951172, "KL/mean": -229.48019409179688, "KL/rejected_KL_mean": -275.9530029296875, "KL/std": 101.52085876464844, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.004694625269621611, "fcm_dpo/delta": -0.0380076989531517, "fcm_dpo/margin": 92.94563293457031, "fcm_dpo/q_t": 0.39773136377334595, "grad_norm": 8.645927429199219, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.8446111679077148, "logits/rejected": -1.7280243635177612, "logps/chosen": -266.59259033203125, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -386.0533752441406, "loss": 1.0539, "margin_dpo/margin_mean": 92.94562530517578, "margin_dpo/margin_std": 104.73236083984375, "step": 388 }, { "KL/chosen_KL_mean": -188.9106903076172, "KL/mean": -227.70822143554688, "KL/rejected_KL_mean": -266.5057373046875, "KL/std": 103.4793701171875, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004706930834800005, "fcm_dpo/delta": 0.03609923645853996, "fcm_dpo/margin": 77.59506225585938, "fcm_dpo/q_t": 0.4155357778072357, "grad_norm": 11.514424324035645, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.306762218475342, "logits/rejected": -1.8424410820007324, "logps/chosen": -278.73297119140625, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -351.3594970703125, "loss": 1.1194, "margin_dpo/margin_mean": 77.59505462646484, "margin_dpo/margin_std": 111.89553833007812, "step": 389 }, { "KL/chosen_KL_mean": -185.99693298339844, "KL/mean": -225.59730529785156, "KL/rejected_KL_mean": -265.19769287109375, "KL/std": 100.81751251220703, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004715648479759693, "fcm_dpo/delta": 0.02717376872897148, "fcm_dpo/margin": 79.20074462890625, "fcm_dpo/q_t": 0.4155096411705017, "grad_norm": 7.947685718536377, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.7250564098358154, "logits/rejected": -1.5200068950653076, "logps/chosen": -263.52935791015625, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -357.7816162109375, "loss": 1.1195, "margin_dpo/margin_mean": 79.20074462890625, "margin_dpo/margin_std": 117.11166381835938, "step": 390 }, { "KL/chosen_KL_mean": -169.03126525878906, "KL/mean": -210.40890502929688, "KL/rejected_KL_mean": -251.78656005859375, "KL/std": 96.36602783203125, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.00473739393055439, "fcm_dpo/delta": 0.008124185726046562, "fcm_dpo/margin": 82.75531768798828, "fcm_dpo/q_t": 0.40846309065818787, "grad_norm": 8.434361457824707, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.02353572845459, "logits/rejected": -1.5689680576324463, "logps/chosen": -257.2641906738281, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -344.79412841796875, "loss": 1.09, "margin_dpo/margin_mean": 82.75531768798828, "margin_dpo/margin_std": 102.68330383300781, "step": 391 }, { "KL/chosen_KL_mean": -169.90374755859375, "KL/mean": -221.17782592773438, "KL/rejected_KL_mean": -272.45196533203125, "KL/std": 108.45538330078125, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004706883803009987, "fcm_dpo/delta": -0.08680158108472824, "fcm_dpo/margin": 102.54820251464844, "fcm_dpo/q_t": 0.38790321350097656, "grad_norm": 9.269989967346191, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.609855055809021, "logits/rejected": -1.4940932989120483, "logps/chosen": -257.6481628417969, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -381.02264404296875, "loss": 1.0319, "margin_dpo/margin_mean": 102.54820251464844, "margin_dpo/margin_std": 116.597900390625, "step": 392 }, { "KL/chosen_KL_mean": -152.86354064941406, "KL/mean": -197.545654296875, "KL/rejected_KL_mean": -242.22775268554688, "KL/std": 101.83715057373047, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004660004749894142, "fcm_dpo/delta": -0.017150957137346268, "fcm_dpo/margin": 89.36422729492188, "fcm_dpo/q_t": 0.4024710953235626, "grad_norm": 8.467018127441406, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.9537690877914429, "logits/rejected": -1.6402723789215088, "logps/chosen": -229.42733764648438, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -321.9539489746094, "loss": 1.0633, "margin_dpo/margin_mean": 89.3642349243164, "margin_dpo/margin_std": 99.51588439941406, "step": 393 }, { "KL/chosen_KL_mean": -159.79202270507812, "KL/mean": -209.72756958007812, "KL/rejected_KL_mean": -259.66314697265625, "KL/std": 98.24201202392578, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004610900767147541, "fcm_dpo/delta": -0.06343074142932892, "fcm_dpo/margin": 99.87110137939453, "fcm_dpo/q_t": 0.39122474193573, "grad_norm": 8.039046287536621, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6830354928970337, "logits/rejected": -1.4530532360076904, "logps/chosen": -244.49069213867188, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -357.12091064453125, "loss": 1.0243, "margin_dpo/margin_mean": 99.87110137939453, "margin_dpo/margin_std": 95.92214965820312, "step": 394 }, { "KL/chosen_KL_mean": -165.34446716308594, "KL/mean": -203.86370849609375, "KL/rejected_KL_mean": -242.38294982910156, "KL/std": 101.82427978515625, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004608167801052332, "fcm_dpo/delta": 0.0464453399181366, "fcm_dpo/margin": 77.03848266601562, "fcm_dpo/q_t": 0.4201025664806366, "grad_norm": 8.337104797363281, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.6379506587982178, "logits/rejected": -2.236152172088623, "logps/chosen": -254.23211669921875, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -333.428466796875, "loss": 1.1579, "margin_dpo/margin_mean": 77.03848266601562, "margin_dpo/margin_std": 137.00987243652344, "step": 395 }, { "KL/chosen_KL_mean": -174.05441284179688, "KL/mean": -207.705810546875, "KL/rejected_KL_mean": -241.35723876953125, "KL/std": 89.62427520751953, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004699273966252804, "fcm_dpo/delta": 0.0864752009510994, "fcm_dpo/margin": 67.30282592773438, "fcm_dpo/q_t": 0.42649829387664795, "grad_norm": 11.247823715209961, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -2.039097547531128, "logits/rejected": -1.5504066944122314, "logps/chosen": -276.5257568359375, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -334.5920104980469, "loss": 1.1549, "margin_dpo/margin_mean": 67.30282592773438, "margin_dpo/margin_std": 106.7750244140625, "step": 396 }, { "KL/chosen_KL_mean": -154.9403839111328, "KL/mean": -215.87596130371094, "KL/rejected_KL_mean": -276.8115539550781, "KL/std": 105.98936462402344, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.00459482753649354, "fcm_dpo/delta": -0.1698470115661621, "fcm_dpo/margin": 121.87115478515625, "fcm_dpo/q_t": 0.37086302042007446, "grad_norm": 8.95605754852295, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.363847255706787, "logits/rejected": -1.7636604309082031, "logps/chosen": -255.98480224609375, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -385.6627197265625, "loss": 0.9816, "margin_dpo/margin_mean": 121.87115478515625, "margin_dpo/margin_std": 127.12533569335938, "step": 397 }, { "KL/chosen_KL_mean": -164.25010681152344, "KL/mean": -208.0979766845703, "KL/rejected_KL_mean": -251.9458770751953, "KL/std": 102.14480590820312, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004560126923024654, "fcm_dpo/delta": 9.118020534515381e-05, "fcm_dpo/margin": 87.69577026367188, "fcm_dpo/q_t": 0.4079226851463318, "grad_norm": 8.589481353759766, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.287520408630371, "logits/rejected": -2.013606309890747, "logps/chosen": -259.29931640625, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -365.9515075683594, "loss": 1.0986, "margin_dpo/margin_mean": 87.69577026367188, "margin_dpo/margin_std": 122.64488220214844, "step": 398 }, { "KL/chosen_KL_mean": -141.67657470703125, "KL/mean": -197.89175415039062, "KL/rejected_KL_mean": -254.10690307617188, "KL/std": 93.44123840332031, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.004496928304433823, "fcm_dpo/delta": -0.11107830703258514, "fcm_dpo/margin": 112.43032836914062, "fcm_dpo/q_t": 0.38102051615715027, "grad_norm": 8.359976768493652, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.131807327270508, "logits/rejected": -1.3167277574539185, "logps/chosen": -238.31512451171875, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -337.64337158203125, "loss": 0.9924, "margin_dpo/margin_mean": 112.43032836914062, "margin_dpo/margin_std": 101.65180969238281, "step": 399 }, { "KL/chosen_KL_mean": -147.84014892578125, "KL/mean": -203.76736450195312, "KL/rejected_KL_mean": -259.6946105957031, "KL/std": 102.62130737304688, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.004401477053761482, "fcm_dpo/delta": -0.09705789387226105, "fcm_dpo/margin": 111.85443115234375, "fcm_dpo/q_t": 0.3855084776878357, "grad_norm": 12.065513610839844, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.374598979949951, "logits/rejected": -1.8996994495391846, "logps/chosen": -253.143310546875, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -368.35015869140625, "loss": 1.0209, "margin_dpo/margin_mean": 111.85443115234375, "margin_dpo/margin_std": 120.0859146118164, "step": 400 }, { "KL/chosen_KL_mean": -163.86187744140625, "KL/mean": -198.77532958984375, "KL/rejected_KL_mean": -233.6887969970703, "KL/std": 104.78170776367188, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0044249361380934715, "fcm_dpo/delta": 0.09401515126228333, "fcm_dpo/margin": 69.826904296875, "fcm_dpo/q_t": 0.4312235713005066, "grad_norm": 10.356164932250977, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.6853857040405273, "logits/rejected": -1.4425228834152222, "logps/chosen": -248.1510467529297, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -324.63189697265625, "loss": 1.1758, "margin_dpo/margin_mean": 69.826904296875, "margin_dpo/margin_std": 128.068359375, "step": 401 }, { "KL/chosen_KL_mean": -142.47824096679688, "KL/mean": -209.77536010742188, "KL/rejected_KL_mean": -277.0724792480469, "KL/std": 113.08943176269531, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.00433918833732605, "fcm_dpo/delta": -0.19520936906337738, "fcm_dpo/margin": 134.59423828125, "fcm_dpo/q_t": 0.3625495433807373, "grad_norm": 8.505099296569824, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -2.0210964679718018, "logits/rejected": -1.5511560440063477, "logps/chosen": -226.0713653564453, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -364.88275146484375, "loss": 0.9408, "margin_dpo/margin_mean": 134.59423828125, "margin_dpo/margin_std": 111.78361511230469, "step": 402 }, { "KL/chosen_KL_mean": -146.95806884765625, "KL/mean": -196.53427124023438, "KL/rejected_KL_mean": -246.1104736328125, "KL/std": 107.58206176757812, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004264670889824629, "fcm_dpo/delta": -0.024014826864004135, "fcm_dpo/margin": 99.15240478515625, "fcm_dpo/q_t": 0.401262104511261, "grad_norm": 8.628694534301758, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.423916816711426, "logits/rejected": -2.0247087478637695, "logps/chosen": -220.71115112304688, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -330.03057861328125, "loss": 1.066, "margin_dpo/margin_mean": 99.15240478515625, "margin_dpo/margin_std": 116.03781127929688, "step": 403 }, { "KL/chosen_KL_mean": -173.4261474609375, "KL/mean": -217.1373748779297, "KL/rejected_KL_mean": -260.8486328125, "KL/std": 104.57717895507812, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.00427491869777441, "fcm_dpo/delta": 0.027224814519286156, "fcm_dpo/margin": 87.42247009277344, "fcm_dpo/q_t": 0.41368013620376587, "grad_norm": 11.648098945617676, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.1785948276519775, "logits/rejected": -1.8436429500579834, "logps/chosen": -253.10232543945312, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -345.129150390625, "loss": 1.1175, "margin_dpo/margin_mean": 87.42247009277344, "margin_dpo/margin_std": 127.5324478149414, "step": 404 }, { "KL/chosen_KL_mean": -177.88137817382812, "KL/mean": -221.41656494140625, "KL/rejected_KL_mean": -264.9517517089844, "KL/std": 97.75669860839844, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004325050860643387, "fcm_dpo/delta": 0.023899344727396965, "fcm_dpo/margin": 87.07038879394531, "fcm_dpo/q_t": 0.41207510232925415, "grad_norm": 8.232120513916016, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.760652780532837, "logits/rejected": -1.288390874862671, "logps/chosen": -274.34332275390625, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -357.8224792480469, "loss": 1.0935, "margin_dpo/margin_mean": 87.07038879394531, "margin_dpo/margin_std": 103.81724548339844, "step": 405 }, { "KL/chosen_KL_mean": -169.34246826171875, "KL/mean": -207.43161010742188, "KL/rejected_KL_mean": -245.520751953125, "KL/std": 103.5272216796875, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004358366131782532, "fcm_dpo/delta": 0.07032008469104767, "fcm_dpo/margin": 76.17831420898438, "fcm_dpo/q_t": 0.4229024052619934, "grad_norm": 13.215198516845703, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -2.055619478225708, "logits/rejected": -1.8901665210723877, "logps/chosen": -247.47642517089844, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -343.8043518066406, "loss": 1.1471, "margin_dpo/margin_mean": 76.17831420898438, "margin_dpo/margin_std": 120.0980453491211, "step": 406 }, { "KL/chosen_KL_mean": -174.6946563720703, "KL/mean": -208.89413452148438, "KL/rejected_KL_mean": -243.09359741210938, "KL/std": 102.3582763671875, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.004441599361598492, "fcm_dpo/delta": 0.09915069490671158, "fcm_dpo/margin": 68.3989486694336, "fcm_dpo/q_t": 0.4291294813156128, "grad_norm": 9.100320816040039, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.340054512023926, "logits/rejected": -1.824233055114746, "logps/chosen": -258.7074890136719, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -325.8746337890625, "loss": 1.1617, "margin_dpo/margin_mean": 68.39894104003906, "margin_dpo/margin_std": 109.83917999267578, "step": 407 }, { "KL/chosen_KL_mean": -168.3236083984375, "KL/mean": -202.4035186767578, "KL/rejected_KL_mean": -236.48342895507812, "KL/std": 98.47137451171875, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.004518316127359867, "fcm_dpo/delta": 0.09505480527877808, "fcm_dpo/margin": 68.15982055664062, "fcm_dpo/q_t": 0.42945224046707153, "grad_norm": 9.530012130737305, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.3856680393218994, "logits/rejected": -1.7627966403961182, "logps/chosen": -272.7853698730469, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -332.8556213378906, "loss": 1.1745, "margin_dpo/margin_mean": 68.1598129272461, "margin_dpo/margin_std": 122.35933685302734, "step": 408 }, { "KL/chosen_KL_mean": -180.0064697265625, "KL/mean": -214.9791259765625, "KL/rejected_KL_mean": -249.95181274414062, "KL/std": 99.62580871582031, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.004602537490427494, "fcm_dpo/delta": 0.08066413551568985, "fcm_dpo/margin": 69.94534301757812, "fcm_dpo/q_t": 0.42387086153030396, "grad_norm": 8.765697479248047, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.6009554862976074, "logits/rejected": -1.1614078283309937, "logps/chosen": -277.6747741699219, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -339.9976501464844, "loss": 1.155, "margin_dpo/margin_mean": 69.94534301757812, "margin_dpo/margin_std": 111.42561340332031, "step": 409 }, { "KL/chosen_KL_mean": -173.85867309570312, "KL/mean": -217.03939819335938, "KL/rejected_KL_mean": -260.22015380859375, "KL/std": 105.9308090209961, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0046184733510017395, "fcm_dpo/delta": 0.0011793216690421104, "fcm_dpo/margin": 86.36146545410156, "fcm_dpo/q_t": 0.40578579902648926, "grad_norm": 9.086349487304688, "learning_rate": 2.065879555832674e-07, "logits/chosen": -2.0635623931884766, "logits/rejected": -1.9388694763183594, "logps/chosen": -250.32791137695312, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -348.86077880859375, "loss": 1.083, "margin_dpo/margin_mean": 86.36146545410156, "margin_dpo/margin_std": 105.41732788085938, "step": 410 }, { "KL/chosen_KL_mean": -185.66275024414062, "KL/mean": -225.71131896972656, "KL/rejected_KL_mean": -265.7598876953125, "KL/std": 98.36116027832031, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004629787057638168, "fcm_dpo/delta": 0.03021075576543808, "fcm_dpo/margin": 80.09715270996094, "fcm_dpo/q_t": 0.41438037157058716, "grad_norm": 8.501738548278809, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.525538444519043, "logits/rejected": -2.128232955932617, "logps/chosen": -272.82904052734375, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -352.8559265136719, "loss": 1.108, "margin_dpo/margin_mean": 80.09715270996094, "margin_dpo/margin_std": 106.49466705322266, "step": 411 }, { "KL/chosen_KL_mean": -175.7507781982422, "KL/mean": -215.649658203125, "KL/rejected_KL_mean": -255.5485382080078, "KL/std": 99.66338348388672, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004658141639083624, "fcm_dpo/delta": 0.029316982254385948, "fcm_dpo/margin": 79.79776000976562, "fcm_dpo/q_t": 0.4158071279525757, "grad_norm": 10.53446102142334, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.71474027633667, "logits/rejected": -2.504957675933838, "logps/chosen": -254.6981201171875, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -361.6540832519531, "loss": 1.1339, "margin_dpo/margin_mean": 79.79776000976562, "margin_dpo/margin_std": 128.89404296875, "step": 412 }, { "KL/chosen_KL_mean": -189.5083770751953, "KL/mean": -234.19699096679688, "KL/rejected_KL_mean": -278.88555908203125, "KL/std": 106.91339111328125, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004651394672691822, "fcm_dpo/delta": -0.016660500317811966, "fcm_dpo/margin": 89.377197265625, "fcm_dpo/q_t": 0.40299007296562195, "grad_norm": 9.851682662963867, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.397071123123169, "logits/rejected": -1.1470975875854492, "logps/chosen": -285.2030944824219, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -386.846435546875, "loss": 1.0954, "margin_dpo/margin_mean": 89.377197265625, "margin_dpo/margin_std": 127.5826187133789, "step": 413 }, { "KL/chosen_KL_mean": -173.26870727539062, "KL/mean": -215.07135009765625, "KL/rejected_KL_mean": -256.8739929199219, "KL/std": 101.4033432006836, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.004675209987908602, "fcm_dpo/delta": 0.009442977607250214, "fcm_dpo/margin": 83.60529327392578, "fcm_dpo/q_t": 0.41080552339553833, "grad_norm": 12.022055625915527, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.825345516204834, "logits/rejected": -1.4784469604492188, "logps/chosen": -261.5453796386719, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -349.7440185546875, "loss": 1.1185, "margin_dpo/margin_mean": 83.60529327392578, "margin_dpo/margin_std": 129.45286560058594, "step": 414 }, { "KL/chosen_KL_mean": -190.52410888671875, "KL/mean": -225.69129943847656, "KL/rejected_KL_mean": -260.85845947265625, "KL/std": 93.8363265991211, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.00472237030044198, "fcm_dpo/delta": 0.07014335691928864, "fcm_dpo/margin": 70.33436584472656, "fcm_dpo/q_t": 0.42361748218536377, "grad_norm": 10.141068458557129, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.87493896484375, "logits/rejected": -1.501633644104004, "logps/chosen": -299.44000244140625, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -368.329833984375, "loss": 1.1504, "margin_dpo/margin_mean": 70.33436584472656, "margin_dpo/margin_std": 113.63925170898438, "step": 415 }, { "KL/chosen_KL_mean": -188.630859375, "KL/mean": -238.57974243164062, "KL/rejected_KL_mean": -288.52862548828125, "KL/std": 109.4114990234375, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004716603085398674, "fcm_dpo/delta": -0.07514870166778564, "fcm_dpo/margin": 99.89778137207031, "fcm_dpo/q_t": 0.3913673758506775, "grad_norm": 9.047630310058594, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8241703510284424, "logits/rejected": -1.312799334526062, "logps/chosen": -282.02972412109375, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -386.20159912109375, "loss": 1.0413, "margin_dpo/margin_mean": 99.89778137207031, "margin_dpo/margin_std": 113.03175354003906, "step": 416 }, { "KL/chosen_KL_mean": -176.7530517578125, "KL/mean": -222.2763671875, "KL/rejected_KL_mean": -267.79962158203125, "KL/std": 100.5050048828125, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0046548256650567055, "fcm_dpo/delta": -0.024861598387360573, "fcm_dpo/margin": 91.04660034179688, "fcm_dpo/q_t": 0.40077486634254456, "grad_norm": 8.948161125183105, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.424156665802002, "logits/rejected": -1.5217547416687012, "logps/chosen": -260.28839111328125, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -342.2414855957031, "loss": 1.0609, "margin_dpo/margin_mean": 91.04660034179688, "margin_dpo/margin_std": 102.41136932373047, "step": 417 }, { "KL/chosen_KL_mean": -169.13375854492188, "KL/mean": -226.98208618164062, "KL/rejected_KL_mean": -284.8304138183594, "KL/std": 106.58732604980469, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004591111093759537, "fcm_dpo/delta": -0.1390218287706375, "fcm_dpo/margin": 115.6966552734375, "fcm_dpo/q_t": 0.3765522241592407, "grad_norm": 11.547738075256348, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.1451568603515625, "logits/rejected": -1.474290370941162, "logps/chosen": -277.35528564453125, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -396.695068359375, "loss": 0.9855, "margin_dpo/margin_mean": 115.69666290283203, "margin_dpo/margin_std": 103.82411193847656, "step": 418 }, { "KL/chosen_KL_mean": -176.81838989257812, "KL/mean": -226.86956787109375, "KL/rejected_KL_mean": -276.9207458496094, "KL/std": 96.21568298339844, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.00449893344193697, "fcm_dpo/delta": -0.05286148935556412, "fcm_dpo/margin": 100.10234069824219, "fcm_dpo/q_t": 0.39485862851142883, "grad_norm": 11.03249454498291, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.336810827255249, "logits/rejected": -0.8732119798660278, "logps/chosen": -274.0016784667969, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -375.1060791015625, "loss": 1.0435, "margin_dpo/margin_mean": 100.10234069824219, "margin_dpo/margin_std": 108.45382690429688, "step": 419 }, { "KL/chosen_KL_mean": -181.53018188476562, "KL/mean": -226.72451782226562, "KL/rejected_KL_mean": -271.91888427734375, "KL/std": 101.53173065185547, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.004478839226067066, "fcm_dpo/delta": -0.005229433998465538, "fcm_dpo/margin": 90.38867950439453, "fcm_dpo/q_t": 0.40624892711639404, "grad_norm": 14.480390548706055, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.290055513381958, "logits/rejected": -1.1773285865783691, "logps/chosen": -295.83868408203125, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -347.6024475097656, "loss": 1.0868, "margin_dpo/margin_mean": 90.38868713378906, "margin_dpo/margin_std": 116.51036071777344, "step": 420 }, { "KL/chosen_KL_mean": -173.6748046875, "KL/mean": -223.16195678710938, "KL/rejected_KL_mean": -272.64910888671875, "KL/std": 104.17076110839844, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.00443071685731411, "fcm_dpo/delta": -0.04038522392511368, "fcm_dpo/margin": 98.9742660522461, "fcm_dpo/q_t": 0.3973526954650879, "grad_norm": 7.907308101654053, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.9838862419128418, "logits/rejected": -1.6159873008728027, "logps/chosen": -259.5546569824219, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -368.985595703125, "loss": 1.0565, "margin_dpo/margin_mean": 98.97427368164062, "margin_dpo/margin_std": 114.27567291259766, "step": 421 }, { "KL/chosen_KL_mean": -171.12313842773438, "KL/mean": -218.5703125, "KL/rejected_KL_mean": -266.01751708984375, "KL/std": 96.28176879882812, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004418655298650265, "fcm_dpo/delta": -0.02015058323740959, "fcm_dpo/margin": 94.89436340332031, "fcm_dpo/q_t": 0.4021940231323242, "grad_norm": 11.9756441116333, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.750154972076416, "logits/rejected": -2.404712438583374, "logps/chosen": -271.603759765625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -360.4257507324219, "loss": 1.0632, "margin_dpo/margin_mean": 94.89436340332031, "margin_dpo/margin_std": 107.58831787109375, "step": 422 }, { "KL/chosen_KL_mean": -158.46307373046875, "KL/mean": -207.01426696777344, "KL/rejected_KL_mean": -255.56546020507812, "KL/std": 116.15084075927734, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.00440713856369257, "fcm_dpo/delta": -0.029373470693826675, "fcm_dpo/margin": 97.10240173339844, "fcm_dpo/q_t": 0.402626633644104, "grad_norm": 9.354714393615723, "learning_rate": 1.902669377503756e-07, "logits/chosen": -2.0675859451293945, "logits/rejected": -1.9523124694824219, "logps/chosen": -236.9130096435547, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -347.61199951171875, "loss": 1.0832, "margin_dpo/margin_mean": 97.1023941040039, "margin_dpo/margin_std": 131.8529052734375, "step": 423 }, { "KL/chosen_KL_mean": -179.23037719726562, "KL/mean": -226.16944885253906, "KL/rejected_KL_mean": -273.1085205078125, "KL/std": 103.74246215820312, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.004373928532004356, "fcm_dpo/delta": -0.01108574029058218, "fcm_dpo/margin": 93.87814331054688, "fcm_dpo/q_t": 0.40330469608306885, "grad_norm": 9.277356147766113, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7981752157211304, "logits/rejected": -1.1994078159332275, "logps/chosen": -266.87274169921875, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -345.47418212890625, "loss": 1.0726, "margin_dpo/margin_mean": 93.8781509399414, "margin_dpo/margin_std": 109.91558074951172, "step": 424 }, { "KL/chosen_KL_mean": -184.39340209960938, "KL/mean": -217.05435180664062, "KL/rejected_KL_mean": -249.7152862548828, "KL/std": 107.04127502441406, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004438747186213732, "fcm_dpo/delta": 0.1134929433465004, "fcm_dpo/margin": 65.32186889648438, "fcm_dpo/q_t": 0.4354708790779114, "grad_norm": 10.538732528686523, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.7752799987792969, "logits/rejected": -1.713599681854248, "logps/chosen": -262.9132080078125, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -352.46392822265625, "loss": 1.1967, "margin_dpo/margin_mean": 65.32186889648438, "margin_dpo/margin_std": 130.77420043945312, "step": 425 }, { "KL/chosen_KL_mean": -160.472412109375, "KL/mean": -205.99484252929688, "KL/rejected_KL_mean": -251.5172576904297, "KL/std": 97.511474609375, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.00448061665520072, "fcm_dpo/delta": -0.008389696478843689, "fcm_dpo/margin": 91.04483795166016, "fcm_dpo/q_t": 0.40505120158195496, "grad_norm": 10.979317665100098, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.5496630668640137, "logits/rejected": -0.7743158936500549, "logps/chosen": -268.9782409667969, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -339.81732177734375, "loss": 1.0825, "margin_dpo/margin_mean": 91.04483795166016, "margin_dpo/margin_std": 114.3433837890625, "step": 426 }, { "KL/chosen_KL_mean": -130.78012084960938, "KL/mean": -173.346923828125, "KL/rejected_KL_mean": -215.9137420654297, "KL/std": 91.707275390625, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.004483185708522797, "fcm_dpo/delta": 0.019031396135687828, "fcm_dpo/margin": 85.13362121582031, "fcm_dpo/q_t": 0.41005170345306396, "grad_norm": 8.843528747558594, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.6542632579803467, "logits/rejected": -2.0402464866638184, "logps/chosen": -229.90057373046875, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -301.63836669921875, "loss": 1.0865, "margin_dpo/margin_mean": 85.13361358642578, "margin_dpo/margin_std": 97.78932189941406, "step": 427 }, { "KL/chosen_KL_mean": -189.37771606445312, "KL/mean": -225.734375, "KL/rejected_KL_mean": -262.09100341796875, "KL/std": 104.67610931396484, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0045131719671189785, "fcm_dpo/delta": 0.07417532801628113, "fcm_dpo/margin": 72.71328735351562, "fcm_dpo/q_t": 0.4243544936180115, "grad_norm": 9.241060256958008, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.9201531410217285, "logits/rejected": -1.6695094108581543, "logps/chosen": -295.34698486328125, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -371.19317626953125, "loss": 1.1523, "margin_dpo/margin_mean": 72.7132797241211, "margin_dpo/margin_std": 116.04498291015625, "step": 428 }, { "KL/chosen_KL_mean": -172.68801879882812, "KL/mean": -220.7059326171875, "KL/rejected_KL_mean": -268.7238464355469, "KL/std": 110.73622131347656, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0045561715960502625, "fcm_dpo/delta": -0.040000397711992264, "fcm_dpo/margin": 96.03580474853516, "fcm_dpo/q_t": 0.400225967168808, "grad_norm": 8.547696113586426, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9638710021972656, "logits/rejected": -1.24202561378479, "logps/chosen": -286.23291015625, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -366.9658508300781, "loss": 1.0779, "margin_dpo/margin_mean": 96.03580474853516, "margin_dpo/margin_std": 126.40265655517578, "step": 429 }, { "KL/chosen_KL_mean": -164.82662963867188, "KL/mean": -201.3826141357422, "KL/rejected_KL_mean": -237.9385986328125, "KL/std": 101.1016845703125, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.004572734236717224, "fcm_dpo/delta": 0.06762906908988953, "fcm_dpo/margin": 73.11199951171875, "fcm_dpo/q_t": 0.42331764101982117, "grad_norm": 9.119863510131836, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.47965669631958, "logits/rejected": -1.4472768306732178, "logps/chosen": -256.14599609375, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -348.0482482910156, "loss": 1.1485, "margin_dpo/margin_mean": 73.11199188232422, "margin_dpo/margin_std": 117.68975067138672, "step": 430 }, { "KL/chosen_KL_mean": -166.22142028808594, "KL/mean": -209.8226318359375, "KL/rejected_KL_mean": -253.423828125, "KL/std": 105.61477661132812, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004571778699755669, "fcm_dpo/delta": 0.0013569127768278122, "fcm_dpo/margin": 87.20242309570312, "fcm_dpo/q_t": 0.40780091285705566, "grad_norm": 10.647443771362305, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -2.0501842498779297, "logits/rejected": -1.6701467037200928, "logps/chosen": -259.40264892578125, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -351.55609130859375, "loss": 1.0989, "margin_dpo/margin_mean": 87.20242309570312, "margin_dpo/margin_std": 120.67742919921875, "step": 431 }, { "KL/chosen_KL_mean": -145.41818237304688, "KL/mean": -197.73910522460938, "KL/rejected_KL_mean": -250.05999755859375, "KL/std": 107.7015609741211, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.004539668560028076, "fcm_dpo/delta": -0.07874863594770432, "fcm_dpo/margin": 104.64180755615234, "fcm_dpo/q_t": 0.3887287974357605, "grad_norm": 8.719236373901367, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.481508493423462, "logits/rejected": -0.9180983304977417, "logps/chosen": -249.85296630859375, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -355.1495361328125, "loss": 1.0233, "margin_dpo/margin_mean": 104.64179992675781, "margin_dpo/margin_std": 105.14724731445312, "step": 432 }, { "KL/chosen_KL_mean": -174.86404418945312, "KL/mean": -216.2481689453125, "KL/rejected_KL_mean": -257.6322937011719, "KL/std": 102.30848693847656, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004542004317045212, "fcm_dpo/delta": 0.02452782541513443, "fcm_dpo/margin": 82.76824951171875, "fcm_dpo/q_t": 0.41315585374832153, "grad_norm": 9.17801570892334, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.8819167613983154, "logits/rejected": -1.569873332977295, "logps/chosen": -264.707275390625, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -359.3657531738281, "loss": 1.1131, "margin_dpo/margin_mean": 82.76824188232422, "margin_dpo/margin_std": 116.37222290039062, "step": 433 }, { "KL/chosen_KL_mean": -149.63015747070312, "KL/mean": -196.08721923828125, "KL/rejected_KL_mean": -242.54428100585938, "KL/std": 114.81493377685547, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004516917280852795, "fcm_dpo/delta": -0.020542694255709648, "fcm_dpo/margin": 92.91413879394531, "fcm_dpo/q_t": 0.4063739776611328, "grad_norm": 11.395620346069336, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.452796459197998, "logits/rejected": -1.7687480449676514, "logps/chosen": -247.32269287109375, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -326.6355895996094, "loss": 1.0912, "margin_dpo/margin_mean": 92.91413879394531, "margin_dpo/margin_std": 133.8496551513672, "step": 434 }, { "KL/chosen_KL_mean": -148.46206665039062, "KL/mean": -196.32749938964844, "KL/rejected_KL_mean": -244.19293212890625, "KL/std": 105.72044372558594, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.004494057968258858, "fcm_dpo/delta": -0.031570665538311005, "fcm_dpo/margin": 95.73086547851562, "fcm_dpo/q_t": 0.4012710452079773, "grad_norm": 9.527572631835938, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.474997043609619, "logits/rejected": -2.0265324115753174, "logps/chosen": -234.6339874267578, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -337.94464111328125, "loss": 1.0672, "margin_dpo/margin_mean": 95.73086547851562, "margin_dpo/margin_std": 118.62521362304688, "step": 435 }, { "KL/chosen_KL_mean": -123.94236755371094, "KL/mean": -174.63905334472656, "KL/rejected_KL_mean": -225.33572387695312, "KL/std": 89.7096939086914, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004441497381776571, "fcm_dpo/delta": -0.05279136821627617, "fcm_dpo/margin": 101.39334869384766, "fcm_dpo/q_t": 0.39310699701309204, "grad_norm": 9.700955390930176, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.8920304775238037, "logits/rejected": -1.0338414907455444, "logps/chosen": -229.72946166992188, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -313.96044921875, "loss": 1.0259, "margin_dpo/margin_mean": 101.39335632324219, "margin_dpo/margin_std": 92.38886260986328, "step": 436 }, { "KL/chosen_KL_mean": -144.56968688964844, "KL/mean": -184.92701721191406, "KL/rejected_KL_mean": -225.2843475341797, "KL/std": 95.94768524169922, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004464837722480297, "fcm_dpo/delta": 0.04106065630912781, "fcm_dpo/margin": 80.71465301513672, "fcm_dpo/q_t": 0.41409242153167725, "grad_norm": 10.165648460388184, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9713184833526611, "logits/rejected": -1.304339051246643, "logps/chosen": -240.63174438476562, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -314.29656982421875, "loss": 1.1025, "margin_dpo/margin_mean": 80.71465301513672, "margin_dpo/margin_std": 96.02731323242188, "step": 437 }, { "KL/chosen_KL_mean": -146.62527465820312, "KL/mean": -189.02610778808594, "KL/rejected_KL_mean": -231.4269256591797, "KL/std": 96.93711853027344, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.004467032849788666, "fcm_dpo/delta": 0.02176925726234913, "fcm_dpo/margin": 84.80162048339844, "fcm_dpo/q_t": 0.4110042452812195, "grad_norm": 10.265804290771484, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1290650367736816, "logits/rejected": -1.3002126216888428, "logps/chosen": -248.4806671142578, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -320.87457275390625, "loss": 1.0963, "margin_dpo/margin_mean": 84.80162811279297, "margin_dpo/margin_std": 103.88420104980469, "step": 438 }, { "KL/chosen_KL_mean": -159.9168243408203, "KL/mean": -201.1876678466797, "KL/rejected_KL_mean": -242.45849609375, "KL/std": 105.08197021484375, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004482748918235302, "fcm_dpo/delta": 0.030530009418725967, "fcm_dpo/margin": 82.54167175292969, "fcm_dpo/q_t": 0.41383278369903564, "grad_norm": 10.158411026000977, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.42020845413208, "logits/rejected": -1.1777371168136597, "logps/chosen": -241.67245483398438, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -333.04486083984375, "loss": 1.104, "margin_dpo/margin_mean": 82.54167175292969, "margin_dpo/margin_std": 102.21713256835938, "step": 439 }, { "KL/chosen_KL_mean": -143.21456909179688, "KL/mean": -194.09478759765625, "KL/rejected_KL_mean": -244.97500610351562, "KL/std": 106.45553588867188, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.004457351751625538, "fcm_dpo/delta": -0.056924235075712204, "fcm_dpo/margin": 101.76043701171875, "fcm_dpo/q_t": 0.3952861726284027, "grad_norm": 7.442152976989746, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.55845308303833, "logits/rejected": -2.0679030418395996, "logps/chosen": -248.85565185546875, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -348.37603759765625, "loss": 1.0495, "margin_dpo/margin_mean": 101.76043701171875, "margin_dpo/margin_std": 117.95341491699219, "step": 440 }, { "KL/chosen_KL_mean": -184.5184326171875, "KL/mean": -227.35459899902344, "KL/rejected_KL_mean": -270.1907653808594, "KL/std": 101.30107116699219, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.004473375156521797, "fcm_dpo/delta": 0.017417727038264275, "fcm_dpo/margin": 85.67237091064453, "fcm_dpo/q_t": 0.4122787117958069, "grad_norm": 8.727927207946777, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.6932966709136963, "logits/rejected": -1.327399730682373, "logps/chosen": -276.0479736328125, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -373.8101806640625, "loss": 1.0978, "margin_dpo/margin_mean": 85.67237091064453, "margin_dpo/margin_std": 113.0551986694336, "step": 441 }, { "KL/chosen_KL_mean": -178.54055786132812, "KL/mean": -232.89930725097656, "KL/rejected_KL_mean": -287.258056640625, "KL/std": 106.43873596191406, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0044275675900280476, "fcm_dpo/delta": -0.0854353979229927, "fcm_dpo/margin": 108.71748352050781, "fcm_dpo/q_t": 0.3873278796672821, "grad_norm": 11.545042991638184, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.612520456314087, "logits/rejected": -1.1302735805511475, "logps/chosen": -263.318115234375, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -371.08221435546875, "loss": 1.0172, "margin_dpo/margin_mean": 108.71748352050781, "margin_dpo/margin_std": 110.00927734375, "step": 442 }, { "KL/chosen_KL_mean": -174.53916931152344, "KL/mean": -208.8023681640625, "KL/rejected_KL_mean": -243.06556701660156, "KL/std": 104.12619018554688, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004454616457223892, "fcm_dpo/delta": 0.09780648350715637, "fcm_dpo/margin": 68.52639770507812, "fcm_dpo/q_t": 0.4287182092666626, "grad_norm": 10.27910327911377, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.9686793088912964, "logits/rejected": -1.4127156734466553, "logps/chosen": -277.1884460449219, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -336.1036376953125, "loss": 1.1542, "margin_dpo/margin_mean": 68.52639770507812, "margin_dpo/margin_std": 100.19795989990234, "step": 443 }, { "KL/chosen_KL_mean": -178.61236572265625, "KL/mean": -218.8331298828125, "KL/rejected_KL_mean": -259.05389404296875, "KL/std": 101.67933654785156, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0045218211598694324, "fcm_dpo/delta": 0.03758939355611801, "fcm_dpo/margin": 80.4415054321289, "fcm_dpo/q_t": 0.4161168932914734, "grad_norm": 11.025041580200195, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.7572442293167114, "logits/rejected": -1.5911502838134766, "logps/chosen": -266.5321044921875, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -362.3773193359375, "loss": 1.1228, "margin_dpo/margin_mean": 80.4415054321289, "margin_dpo/margin_std": 118.25935363769531, "step": 444 }, { "KL/chosen_KL_mean": -158.14138793945312, "KL/mean": -204.52484130859375, "KL/rejected_KL_mean": -250.9082794189453, "KL/std": 98.76119995117188, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0044996244832873344, "fcm_dpo/delta": -0.018469806760549545, "fcm_dpo/margin": 92.76690673828125, "fcm_dpo/q_t": 0.40219563245773315, "grad_norm": 9.163178443908691, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -2.079244613647461, "logits/rejected": -1.6684294939041138, "logps/chosen": -259.542236328125, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -349.9461669921875, "loss": 1.0652, "margin_dpo/margin_mean": 92.76690673828125, "margin_dpo/margin_std": 104.87440490722656, "step": 445 }, { "KL/chosen_KL_mean": -149.6294403076172, "KL/mean": -192.09832763671875, "KL/rejected_KL_mean": -234.56723022460938, "KL/std": 103.85661315917969, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.004529435187578201, "fcm_dpo/delta": 0.015794720500707626, "fcm_dpo/margin": 84.93778991699219, "fcm_dpo/q_t": 0.4098289906978607, "grad_norm": 12.737281799316406, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9022663831710815, "logits/rejected": -1.7596321105957031, "logps/chosen": -237.05177307128906, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -341.26800537109375, "loss": 1.0853, "margin_dpo/margin_mean": 84.93778991699219, "margin_dpo/margin_std": 98.15299987792969, "step": 446 }, { "KL/chosen_KL_mean": -149.81430053710938, "KL/mean": -205.07766723632812, "KL/rejected_KL_mean": -260.3410339355469, "KL/std": 108.79704284667969, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004458588548004627, "fcm_dpo/delta": -0.09762756526470184, "fcm_dpo/margin": 110.52674865722656, "fcm_dpo/q_t": 0.38602977991104126, "grad_norm": 8.59799575805664, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.209089756011963, "logits/rejected": -2.166133403778076, "logps/chosen": -233.4295654296875, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -365.25341796875, "loss": 1.0246, "margin_dpo/margin_mean": 110.52674865722656, "margin_dpo/margin_std": 123.33868408203125, "step": 447 }, { "KL/chosen_KL_mean": -159.804443359375, "KL/mean": -213.8458251953125, "KL/rejected_KL_mean": -267.88720703125, "KL/std": 113.37985229492188, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0044057914055883884, "fcm_dpo/delta": -0.08008042722940445, "fcm_dpo/margin": 108.08274841308594, "fcm_dpo/q_t": 0.39073750376701355, "grad_norm": 10.501388549804688, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.785628318786621, "logits/rejected": -1.5159730911254883, "logps/chosen": -252.38021850585938, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -375.57696533203125, "loss": 1.0468, "margin_dpo/margin_mean": 108.08274841308594, "margin_dpo/margin_std": 131.22662353515625, "step": 448 }, { "KL/chosen_KL_mean": -157.43699645996094, "KL/mean": -212.52853393554688, "KL/rejected_KL_mean": -267.62005615234375, "KL/std": 105.81903839111328, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.004315992351621389, "fcm_dpo/delta": -0.07931534200906754, "fcm_dpo/margin": 110.18311309814453, "fcm_dpo/q_t": 0.3891426920890808, "grad_norm": 8.778791427612305, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.5970497131347656, "logits/rejected": -1.1563621759414673, "logps/chosen": -259.8359375, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -362.7689514160156, "loss": 1.0302, "margin_dpo/margin_mean": 110.18312072753906, "margin_dpo/margin_std": 121.09993743896484, "step": 449 }, { "KL/chosen_KL_mean": -141.85342407226562, "KL/mean": -192.81353759765625, "KL/rejected_KL_mean": -243.77365112304688, "KL/std": 106.97483825683594, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0042708683758974075, "fcm_dpo/delta": -0.03694060444831848, "fcm_dpo/margin": 101.92021179199219, "fcm_dpo/q_t": 0.39787057042121887, "grad_norm": 9.022513389587402, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.4238805770874023, "logits/rejected": -2.2327139377593994, "logps/chosen": -228.84628295898438, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -352.3056945800781, "loss": 1.0556, "margin_dpo/margin_mean": 101.92021942138672, "margin_dpo/margin_std": 115.9273681640625, "step": 450 }, { "KL/chosen_KL_mean": -160.0928497314453, "KL/mean": -200.32745361328125, "KL/rejected_KL_mean": -240.56204223632812, "KL/std": 106.91096496582031, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.004309053998440504, "fcm_dpo/delta": 0.055011067539453506, "fcm_dpo/margin": 80.46920013427734, "fcm_dpo/q_t": 0.4207785129547119, "grad_norm": 14.08891487121582, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.5344135761260986, "logits/rejected": -2.0309348106384277, "logps/chosen": -246.9041290283203, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -320.4176330566406, "loss": 1.1478, "margin_dpo/margin_mean": 80.46920013427734, "margin_dpo/margin_std": 133.5021209716797, "step": 451 }, { "KL/chosen_KL_mean": -139.61328125, "KL/mean": -185.15553283691406, "KL/rejected_KL_mean": -230.6977996826172, "KL/std": 100.57363891601562, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004290041048079729, "fcm_dpo/delta": 0.008729591965675354, "fcm_dpo/margin": 91.08453369140625, "fcm_dpo/q_t": 0.40765416622161865, "grad_norm": 8.893556594848633, "learning_rate": 1.549222776991186e-07, "logits/chosen": -2.0020949840545654, "logits/rejected": -1.9463545083999634, "logps/chosen": -218.992919921875, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -334.4132080078125, "loss": 1.0771, "margin_dpo/margin_mean": 91.08453369140625, "margin_dpo/margin_std": 93.48591613769531, "step": 452 }, { "KL/chosen_KL_mean": -168.19985961914062, "KL/mean": -224.22207641601562, "KL/rejected_KL_mean": -280.2442626953125, "KL/std": 103.28233337402344, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.00427229143679142, "fcm_dpo/delta": -0.08258512616157532, "fcm_dpo/margin": 112.04440307617188, "fcm_dpo/q_t": 0.3884204924106598, "grad_norm": 12.891401290893555, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.9265592098236084, "logits/rejected": -1.4515278339385986, "logps/chosen": -255.89505004882812, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -370.302490234375, "loss": 1.0164, "margin_dpo/margin_mean": 112.0444107055664, "margin_dpo/margin_std": 110.38692474365234, "step": 453 }, { "KL/chosen_KL_mean": -177.1742706298828, "KL/mean": -222.58316040039062, "KL/rejected_KL_mean": -267.99200439453125, "KL/std": 105.75848388671875, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004277661442756653, "fcm_dpo/delta": 0.011479493230581284, "fcm_dpo/margin": 90.8177490234375, "fcm_dpo/q_t": 0.4108721613883972, "grad_norm": 9.441713333129883, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.133726119995117, "logits/rejected": -1.6900935173034668, "logps/chosen": -266.7405090332031, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -360.9130859375, "loss": 1.1171, "margin_dpo/margin_mean": 90.8177490234375, "margin_dpo/margin_std": 136.17330932617188, "step": 454 }, { "KL/chosen_KL_mean": -167.4620361328125, "KL/mean": -210.90606689453125, "KL/rejected_KL_mean": -254.35009765625, "KL/std": 105.5683364868164, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0042924778535962105, "fcm_dpo/delta": 0.027750549837946892, "fcm_dpo/margin": 86.88804626464844, "fcm_dpo/q_t": 0.4134191572666168, "grad_norm": 9.235121726989746, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8998043537139893, "logits/rejected": -1.8630216121673584, "logps/chosen": -245.09202575683594, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -373.32806396484375, "loss": 1.1123, "margin_dpo/margin_mean": 86.88804626464844, "margin_dpo/margin_std": 119.96641540527344, "step": 455 }, { "KL/chosen_KL_mean": -168.03346252441406, "KL/mean": -211.6600341796875, "KL/rejected_KL_mean": -255.28660583496094, "KL/std": 106.16474151611328, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.004308540374040604, "fcm_dpo/delta": 0.024872276932001114, "fcm_dpo/margin": 87.25312805175781, "fcm_dpo/q_t": 0.41188162565231323, "grad_norm": 11.717841148376465, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.3382792472839355, "logits/rejected": -2.2141289710998535, "logps/chosen": -257.65032958984375, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -364.8463134765625, "loss": 1.0955, "margin_dpo/margin_mean": 87.25312805175781, "margin_dpo/margin_std": 106.40306091308594, "step": 456 }, { "KL/chosen_KL_mean": -139.21568298339844, "KL/mean": -195.07440185546875, "KL/rejected_KL_mean": -250.93310546875, "KL/std": 106.17615509033203, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.004222998861223459, "fcm_dpo/delta": -0.07719205319881439, "fcm_dpo/margin": 111.71745300292969, "fcm_dpo/q_t": 0.3878215253353119, "grad_norm": 9.364872932434082, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.2088406085968018, "logits/rejected": -1.8950560092926025, "logps/chosen": -226.53736877441406, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -341.69970703125, "loss": 1.0106, "margin_dpo/margin_mean": 111.71745300292969, "margin_dpo/margin_std": 92.85293579101562, "step": 457 }, { "KL/chosen_KL_mean": -158.85406494140625, "KL/mean": -200.800537109375, "KL/rejected_KL_mean": -242.74700927734375, "KL/std": 106.13194274902344, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004275122657418251, "fcm_dpo/delta": 0.04271306097507477, "fcm_dpo/margin": 83.89295959472656, "fcm_dpo/q_t": 0.41820457577705383, "grad_norm": 9.407254219055176, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.7150627374649048, "logits/rejected": -1.4792304039001465, "logps/chosen": -252.37451171875, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -346.1159973144531, "loss": 1.1264, "margin_dpo/margin_mean": 83.89295196533203, "margin_dpo/margin_std": 125.17750549316406, "step": 458 }, { "KL/chosen_KL_mean": -152.1630859375, "KL/mean": -204.81871032714844, "KL/rejected_KL_mean": -257.4743347167969, "KL/std": 110.78013610839844, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0042257653549313545, "fcm_dpo/delta": -0.04777819663286209, "fcm_dpo/margin": 105.31124877929688, "fcm_dpo/q_t": 0.39695000648498535, "grad_norm": 7.0585036277771, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6379880905151367, "logits/rejected": -1.3867108821868896, "logps/chosen": -227.85128784179688, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -349.64483642578125, "loss": 1.0508, "margin_dpo/margin_mean": 105.31124877929688, "margin_dpo/margin_std": 119.13032531738281, "step": 459 }, { "KL/chosen_KL_mean": -157.10691833496094, "KL/mean": -204.5196533203125, "KL/rejected_KL_mean": -251.93240356445312, "KL/std": 107.03904724121094, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.004220187664031982, "fcm_dpo/delta": -0.0004334300756454468, "fcm_dpo/margin": 94.82549285888672, "fcm_dpo/q_t": 0.40700647234916687, "grad_norm": 11.014700889587402, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.172398090362549, "logits/rejected": -2.200178623199463, "logps/chosen": -238.2248077392578, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -362.24481201171875, "loss": 1.0893, "margin_dpo/margin_mean": 94.82548522949219, "margin_dpo/margin_std": 121.73226165771484, "step": 460 }, { "KL/chosen_KL_mean": -193.9816131591797, "KL/mean": -232.7714385986328, "KL/rejected_KL_mean": -271.5612487792969, "KL/std": 94.15699768066406, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004270569421350956, "fcm_dpo/delta": 0.07110296934843063, "fcm_dpo/margin": 77.57962799072266, "fcm_dpo/q_t": 0.423362672328949, "grad_norm": 8.84291934967041, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.8049015998840332, "logits/rejected": -1.6211200952529907, "logps/chosen": -275.56512451171875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -365.4383544921875, "loss": 1.1389, "margin_dpo/margin_mean": 77.57962799072266, "margin_dpo/margin_std": 113.31498718261719, "step": 461 }, { "KL/chosen_KL_mean": -146.3834686279297, "KL/mean": -196.44126892089844, "KL/rejected_KL_mean": -246.49908447265625, "KL/std": 101.21432495117188, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004281737841665745, "fcm_dpo/delta": -0.029948072507977486, "fcm_dpo/margin": 100.11558532714844, "fcm_dpo/q_t": 0.3997390568256378, "grad_norm": 8.168227195739746, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.4433646202087402, "logits/rejected": -1.9482452869415283, "logps/chosen": -240.58203125, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -332.1307067871094, "loss": 1.0535, "margin_dpo/margin_mean": 100.11559295654297, "margin_dpo/margin_std": 108.28557586669922, "step": 462 }, { "KL/chosen_KL_mean": -141.6971893310547, "KL/mean": -201.1674346923828, "KL/rejected_KL_mean": -260.6376953125, "KL/std": 109.64218139648438, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.004228126723319292, "fcm_dpo/delta": -0.10847333818674088, "fcm_dpo/margin": 118.94051361083984, "fcm_dpo/q_t": 0.38182759284973145, "grad_norm": 13.650558471679688, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.7737629413604736, "logits/rejected": -1.3401349782943726, "logps/chosen": -227.62193298339844, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -357.53955078125, "loss": 1.0084, "margin_dpo/margin_mean": 118.94052124023438, "margin_dpo/margin_std": 115.78070068359375, "step": 463 }, { "KL/chosen_KL_mean": -176.37635803222656, "KL/mean": -224.69273376464844, "KL/rejected_KL_mean": -273.0091247558594, "KL/std": 108.70243835449219, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004167080856859684, "fcm_dpo/delta": -0.002893369644880295, "fcm_dpo/margin": 96.63275146484375, "fcm_dpo/q_t": 0.4065708518028259, "grad_norm": 12.395657539367676, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.848388910293579, "logits/rejected": -1.9338229894638062, "logps/chosen": -256.0655517578125, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -380.30145263671875, "loss": 1.0877, "margin_dpo/margin_mean": 96.63275146484375, "margin_dpo/margin_std": 125.45909118652344, "step": 464 }, { "KL/chosen_KL_mean": -168.21897888183594, "KL/mean": -224.91424560546875, "KL/rejected_KL_mean": -281.6094970703125, "KL/std": 117.95230102539062, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.00412693340331316, "fcm_dpo/delta": -0.07139503955841064, "fcm_dpo/margin": 113.3905258178711, "fcm_dpo/q_t": 0.3898018002510071, "grad_norm": 9.160012245178223, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6664458513259888, "logits/rejected": -1.368147373199463, "logps/chosen": -260.0791931152344, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -400.31951904296875, "loss": 1.0216, "margin_dpo/margin_mean": 113.39053344726562, "margin_dpo/margin_std": 111.22815704345703, "step": 465 }, { "KL/chosen_KL_mean": -148.26141357421875, "KL/mean": -206.6683807373047, "KL/rejected_KL_mean": -265.07537841796875, "KL/std": 112.72630310058594, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0040648621506989, "fcm_dpo/delta": -0.0786508172750473, "fcm_dpo/margin": 116.81394958496094, "fcm_dpo/q_t": 0.3886815309524536, "grad_norm": 7.19753360748291, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.7005661725997925, "logits/rejected": -1.385525107383728, "logps/chosen": -232.9628143310547, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -361.126220703125, "loss": 1.0207, "margin_dpo/margin_mean": 116.81394958496094, "margin_dpo/margin_std": 117.99612426757812, "step": 466 }, { "KL/chosen_KL_mean": -192.64035034179688, "KL/mean": -240.55868530273438, "KL/rejected_KL_mean": -288.47698974609375, "KL/std": 117.74639892578125, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.004040064290165901, "fcm_dpo/delta": 0.012898050248622894, "fcm_dpo/margin": 95.83665466308594, "fcm_dpo/q_t": 0.4099825620651245, "grad_norm": 12.350601196289062, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.318258285522461, "logits/rejected": -1.7305917739868164, "logps/chosen": -301.93865966796875, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -397.3206787109375, "loss": 1.1052, "margin_dpo/margin_mean": 95.83665466308594, "margin_dpo/margin_std": 131.17947387695312, "step": 467 }, { "KL/chosen_KL_mean": -150.63729858398438, "KL/mean": -194.8858184814453, "KL/rejected_KL_mean": -239.1343536376953, "KL/std": 115.32505798339844, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0040825773030519485, "fcm_dpo/delta": 0.040174275636672974, "fcm_dpo/margin": 88.49705505371094, "fcm_dpo/q_t": 0.417630672454834, "grad_norm": 9.064801216125488, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.9644601345062256, "logits/rejected": -1.7594530582427979, "logps/chosen": -248.95895385742188, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -345.8148193359375, "loss": 1.1282, "margin_dpo/margin_mean": 88.49705505371094, "margin_dpo/margin_std": 134.62513732910156, "step": 468 }, { "KL/chosen_KL_mean": -182.2525177001953, "KL/mean": -238.32608032226562, "KL/rejected_KL_mean": -294.399658203125, "KL/std": 114.46591186523438, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.004040595144033432, "fcm_dpo/delta": -0.05647577345371246, "fcm_dpo/margin": 112.14710998535156, "fcm_dpo/q_t": 0.3942406177520752, "grad_norm": 10.027302742004395, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.350381851196289, "logits/rejected": -1.9336928129196167, "logps/chosen": -279.0167236328125, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -403.99462890625, "loss": 1.0432, "margin_dpo/margin_mean": 112.14710998535156, "margin_dpo/margin_std": 122.72891235351562, "step": 469 }, { "KL/chosen_KL_mean": -168.68856811523438, "KL/mean": -219.22811889648438, "KL/rejected_KL_mean": -269.7676696777344, "KL/std": 106.21034240722656, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.004042068962007761, "fcm_dpo/delta": -0.008942849934101105, "fcm_dpo/margin": 101.07913208007812, "fcm_dpo/q_t": 0.403676301240921, "grad_norm": 9.867769241333008, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.6487462520599365, "logits/rejected": -1.224298119544983, "logps/chosen": -252.18521118164062, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -358.25347900390625, "loss": 1.0636, "margin_dpo/margin_mean": 101.07913208007812, "margin_dpo/margin_std": 107.30561828613281, "step": 470 }, { "KL/chosen_KL_mean": -193.64801025390625, "KL/mean": -241.1115264892578, "KL/rejected_KL_mean": -288.5750427246094, "KL/std": 105.53306579589844, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004051464609801769, "fcm_dpo/delta": 0.016024025157094002, "fcm_dpo/margin": 94.92703247070312, "fcm_dpo/q_t": 0.40953636169433594, "grad_norm": 8.751537322998047, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.5959956645965576, "logits/rejected": -1.1718157529830933, "logps/chosen": -282.122314453125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -379.0567626953125, "loss": 1.0833, "margin_dpo/margin_mean": 94.92703247070312, "margin_dpo/margin_std": 107.78265380859375, "step": 471 }, { "KL/chosen_KL_mean": -181.3482208251953, "KL/mean": -237.97482299804688, "KL/rejected_KL_mean": -294.60137939453125, "KL/std": 106.03761291503906, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.004011916927993298, "fcm_dpo/delta": -0.057175736874341965, "fcm_dpo/margin": 113.2531509399414, "fcm_dpo/q_t": 0.3939012885093689, "grad_norm": 8.340659141540527, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.2897253036499023, "logits/rejected": -1.5998258590698242, "logps/chosen": -306.5819091796875, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -415.6549072265625, "loss": 1.0323, "margin_dpo/margin_mean": 113.25315856933594, "margin_dpo/margin_std": 113.64892578125, "step": 472 }, { "KL/chosen_KL_mean": -207.94384765625, "KL/mean": -242.455078125, "KL/rejected_KL_mean": -276.9663391113281, "KL/std": 108.63106536865234, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0040893517434597015, "fcm_dpo/delta": 0.12088309228420258, "fcm_dpo/margin": 69.02249145507812, "fcm_dpo/q_t": 0.4358173608779907, "grad_norm": 9.765830039978027, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -2.0205934047698975, "logits/rejected": -1.5394573211669922, "logps/chosen": -303.55523681640625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -365.11749267578125, "loss": 1.1867, "margin_dpo/margin_mean": 69.02249145507812, "margin_dpo/margin_std": 126.51701354980469, "step": 473 }, { "KL/chosen_KL_mean": -193.3594970703125, "KL/mean": -228.83523559570312, "KL/rejected_KL_mean": -264.31097412109375, "KL/std": 101.944580078125, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.004103817977011204, "fcm_dpo/delta": -0.008100904524326324, "fcm_dpo/margin": 70.95144653320312, "fcm_dpo/q_t": 0.432258665561676, "grad_norm": 13.180788040161133, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.499945878982544, "logits/rejected": -1.3067643642425537, "logps/chosen": -274.8392333984375, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -360.776611328125, "loss": 1.186, "margin_dpo/margin_mean": 70.95144653320312, "margin_dpo/margin_std": 126.04278564453125, "step": 474 }, { "KL/chosen_KL_mean": -145.73016357421875, "KL/mean": -201.44158935546875, "KL/rejected_KL_mean": -257.15301513671875, "KL/std": 109.03304290771484, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.004069517366588116, "fcm_dpo/delta": -0.055945247411727905, "fcm_dpo/margin": 111.42285919189453, "fcm_dpo/q_t": 0.3950110375881195, "grad_norm": 8.846735954284668, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.335242748260498, "logits/rejected": -2.2856156826019287, "logps/chosen": -219.9261474609375, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -351.8454284667969, "loss": 1.0443, "margin_dpo/margin_mean": 111.4228515625, "margin_dpo/margin_std": 125.65812683105469, "step": 475 }, { "KL/chosen_KL_mean": -170.17361450195312, "KL/mean": -219.81106567382812, "KL/rejected_KL_mean": -269.44854736328125, "KL/std": 111.3665771484375, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.004044758155941963, "fcm_dpo/delta": -0.0017142165452241898, "fcm_dpo/margin": 99.27491760253906, "fcm_dpo/q_t": 0.4071701467037201, "grad_norm": 9.653604507446289, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.0450503826141357, "logits/rejected": -1.6669292449951172, "logps/chosen": -242.14471435546875, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -349.7107849121094, "loss": 1.0804, "margin_dpo/margin_mean": 99.27491760253906, "margin_dpo/margin_std": 120.42430114746094, "step": 476 }, { "KL/chosen_KL_mean": -169.2335662841797, "KL/mean": -218.6215057373047, "KL/rejected_KL_mean": -268.00946044921875, "KL/std": 98.36669921875, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004055965691804886, "fcm_dpo/delta": -0.0006678062491118908, "fcm_dpo/margin": 98.77587127685547, "fcm_dpo/q_t": 0.4059140682220459, "grad_norm": 8.61225414276123, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.211313009262085, "logits/rejected": -1.2809290885925293, "logps/chosen": -274.2391357421875, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -349.88787841796875, "loss": 1.0853, "margin_dpo/margin_mean": 98.77587890625, "margin_dpo/margin_std": 123.81683349609375, "step": 477 }, { "KL/chosen_KL_mean": -164.13108825683594, "KL/mean": -215.61093139648438, "KL/rejected_KL_mean": -267.0907897949219, "KL/std": 105.70332336425781, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.004041607491672039, "fcm_dpo/delta": -0.016829386353492737, "fcm_dpo/margin": 102.9596939086914, "fcm_dpo/q_t": 0.40234002470970154, "grad_norm": 8.961562156677246, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.7456817626953125, "logits/rejected": -1.5721545219421387, "logps/chosen": -240.91929626464844, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -357.53070068359375, "loss": 1.0657, "margin_dpo/margin_mean": 102.95968627929688, "margin_dpo/margin_std": 116.734375, "step": 478 }, { "KL/chosen_KL_mean": -166.88809204101562, "KL/mean": -218.86416625976562, "KL/rejected_KL_mean": -270.8402404785156, "KL/std": 107.75678253173828, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004039865452796221, "fcm_dpo/delta": -0.020974837243556976, "fcm_dpo/margin": 103.95211029052734, "fcm_dpo/q_t": 0.40091264247894287, "grad_norm": 11.548354148864746, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.5958247184753418, "logits/rejected": -1.4554626941680908, "logps/chosen": -248.2504119873047, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -371.9313659667969, "loss": 1.0598, "margin_dpo/margin_mean": 103.95211029052734, "margin_dpo/margin_std": 111.98036193847656, "step": 479 }, { "KL/chosen_KL_mean": -194.1207275390625, "KL/mean": -235.00701904296875, "KL/rejected_KL_mean": -275.8932800292969, "KL/std": 102.23271179199219, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004067365545779467, "fcm_dpo/delta": 0.06965919584035873, "fcm_dpo/margin": 81.77256774902344, "fcm_dpo/q_t": 0.4220374822616577, "grad_norm": 9.688615798950195, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7350095510482788, "logits/rejected": -1.0408560037612915, "logps/chosen": -290.8946533203125, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -362.29803466796875, "loss": 1.131, "margin_dpo/margin_mean": 81.77256774902344, "margin_dpo/margin_std": 112.06007385253906, "step": 480 }, { "KL/chosen_KL_mean": -189.3058624267578, "KL/mean": -236.63197326660156, "KL/rejected_KL_mean": -283.95806884765625, "KL/std": 107.19631958007812, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004085645545274019, "fcm_dpo/delta": 0.013821247965097427, "fcm_dpo/margin": 94.6522216796875, "fcm_dpo/q_t": 0.4093363285064697, "grad_norm": 12.116225242614746, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.9086768627166748, "logits/rejected": -1.466553807258606, "logps/chosen": -280.97601318359375, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -382.6529541015625, "loss": 1.0914, "margin_dpo/margin_mean": 94.6522216796875, "margin_dpo/margin_std": 117.91465759277344, "step": 481 }, { "KL/chosen_KL_mean": -182.67518615722656, "KL/mean": -229.00897216796875, "KL/rejected_KL_mean": -275.3427429199219, "KL/std": 105.12167358398438, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004095034673810005, "fcm_dpo/delta": 0.02129482850432396, "fcm_dpo/margin": 92.66755676269531, "fcm_dpo/q_t": 0.41059422492980957, "grad_norm": 11.115386962890625, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.592073917388916, "logits/rejected": -1.2328649759292603, "logps/chosen": -281.1953125, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -370.17218017578125, "loss": 1.0992, "margin_dpo/margin_mean": 92.66755676269531, "margin_dpo/margin_std": 118.57544708251953, "step": 482 }, { "KL/chosen_KL_mean": -188.4476776123047, "KL/mean": -237.20565795898438, "KL/rejected_KL_mean": -285.963623046875, "KL/std": 110.55517578125, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004127908032387495, "fcm_dpo/delta": -0.003221064805984497, "fcm_dpo/margin": 97.51594543457031, "fcm_dpo/q_t": 0.40510088205337524, "grad_norm": 7.828423976898193, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.09444260597229, "logits/rejected": -1.4489614963531494, "logps/chosen": -295.5662841796875, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -387.07861328125, "loss": 1.0745, "margin_dpo/margin_mean": 97.51594543457031, "margin_dpo/margin_std": 109.26264190673828, "step": 483 }, { "KL/chosen_KL_mean": -170.21783447265625, "KL/mean": -226.3297882080078, "KL/rejected_KL_mean": -282.4417419433594, "KL/std": 113.4232177734375, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004078528843820095, "fcm_dpo/delta": -0.060454338788986206, "fcm_dpo/margin": 112.22389221191406, "fcm_dpo/q_t": 0.39285874366760254, "grad_norm": 9.091538429260254, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.372498035430908, "logits/rejected": -1.8844008445739746, "logps/chosen": -257.19775390625, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -373.1654052734375, "loss": 1.0415, "margin_dpo/margin_mean": 112.223876953125, "margin_dpo/margin_std": 124.59432983398438, "step": 484 }, { "KL/chosen_KL_mean": -163.52532958984375, "KL/mean": -223.05308532714844, "KL/rejected_KL_mean": -282.5808410644531, "KL/std": 118.2580795288086, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.004007050301879644, "fcm_dpo/delta": -0.08092304319143295, "fcm_dpo/margin": 119.05551147460938, "fcm_dpo/q_t": 0.3885510265827179, "grad_norm": 8.90786075592041, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.792538046836853, "logits/rejected": -1.373389482498169, "logps/chosen": -257.5840759277344, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -391.1438293457031, "loss": 1.0196, "margin_dpo/margin_mean": 119.05551147460938, "margin_dpo/margin_std": 120.69647216796875, "step": 485 }, { "KL/chosen_KL_mean": -186.2275390625, "KL/mean": -239.26644897460938, "KL/rejected_KL_mean": -292.30535888671875, "KL/std": 115.06755065917969, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.003987641539424658, "fcm_dpo/delta": -0.024115145206451416, "fcm_dpo/margin": 106.07785034179688, "fcm_dpo/q_t": 0.400878369808197, "grad_norm": 9.323591232299805, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.137449264526367, "logits/rejected": -1.4359307289123535, "logps/chosen": -279.9734191894531, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -390.37603759765625, "loss": 1.0646, "margin_dpo/margin_mean": 106.07785034179688, "margin_dpo/margin_std": 121.78941345214844, "step": 486 }, { "KL/chosen_KL_mean": -182.8374481201172, "KL/mean": -229.7008514404297, "KL/rejected_KL_mean": -276.5642395019531, "KL/std": 107.610595703125, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.003985828720033169, "fcm_dpo/delta": 0.02745460718870163, "fcm_dpo/margin": 93.7267837524414, "fcm_dpo/q_t": 0.41237351298332214, "grad_norm": 8.849111557006836, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.1997432708740234, "logits/rejected": -1.887646198272705, "logps/chosen": -270.87103271484375, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -375.0363464355469, "loss": 1.099, "margin_dpo/margin_mean": 93.72679138183594, "margin_dpo/margin_std": 116.96118927001953, "step": 487 }, { "KL/chosen_KL_mean": -163.2896728515625, "KL/mean": -215.43246459960938, "KL/rejected_KL_mean": -267.57525634765625, "KL/std": 113.99366760253906, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.003988135606050491, "fcm_dpo/delta": -0.016605138778686523, "fcm_dpo/margin": 104.28558349609375, "fcm_dpo/q_t": 0.4013731777667999, "grad_norm": 8.859611511230469, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.6134033203125, "logits/rejected": -1.3482341766357422, "logps/chosen": -248.07931518554688, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -357.8487243652344, "loss": 1.0543, "margin_dpo/margin_mean": 104.28558349609375, "margin_dpo/margin_std": 102.58961486816406, "step": 488 }, { "KL/chosen_KL_mean": -184.8152313232422, "KL/mean": -231.94927978515625, "KL/rejected_KL_mean": -279.08331298828125, "KL/std": 103.82145690917969, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.003995851147919893, "fcm_dpo/delta": 0.024219054728746414, "fcm_dpo/margin": 94.26812744140625, "fcm_dpo/q_t": 0.41199398040771484, "grad_norm": 9.812134742736816, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.786972999572754, "logits/rejected": -1.3014819622039795, "logps/chosen": -275.2845458984375, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -365.48095703125, "loss": 1.095, "margin_dpo/margin_mean": 94.26812744140625, "margin_dpo/margin_std": 115.49855041503906, "step": 489 }, { "KL/chosen_KL_mean": -183.83555603027344, "KL/mean": -230.20745849609375, "KL/rejected_KL_mean": -276.5793762207031, "KL/std": 113.89126586914062, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004040284547954798, "fcm_dpo/delta": 0.02526799961924553, "fcm_dpo/margin": 92.74380493164062, "fcm_dpo/q_t": 0.4148640036582947, "grad_norm": 9.05355167388916, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1972804069519043, "logits/rejected": -2.2835488319396973, "logps/chosen": -269.15570068359375, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -392.5732421875, "loss": 1.1361, "margin_dpo/margin_mean": 92.74381256103516, "margin_dpo/margin_std": 149.79110717773438, "step": 490 }, { "KL/chosen_KL_mean": -187.4703369140625, "KL/mean": -228.67257690429688, "KL/rejected_KL_mean": -269.87481689453125, "KL/std": 115.35963439941406, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004058500286191702, "fcm_dpo/delta": 0.06785758584737778, "fcm_dpo/margin": 82.40447998046875, "fcm_dpo/q_t": 0.42165571451187134, "grad_norm": 9.705269813537598, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -2.015139102935791, "logits/rejected": -1.5317437648773193, "logps/chosen": -273.6138610839844, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -350.55426025390625, "loss": 1.1342, "margin_dpo/margin_mean": 82.40447998046875, "margin_dpo/margin_std": 117.63522338867188, "step": 491 }, { "KL/chosen_KL_mean": -175.05572509765625, "KL/mean": -209.53367614746094, "KL/rejected_KL_mean": -244.01162719726562, "KL/std": 114.59422302246094, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.00414105923846364, "fcm_dpo/delta": 0.1179397702217102, "fcm_dpo/margin": 68.95591735839844, "fcm_dpo/q_t": 0.434764564037323, "grad_norm": 14.161654472351074, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.1284608840942383, "logits/rejected": -1.7416476011276245, "logps/chosen": -256.1632995605469, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -324.76361083984375, "loss": 1.2029, "margin_dpo/margin_mean": 68.95591735839844, "margin_dpo/margin_std": 141.6307373046875, "step": 492 }, { "KL/chosen_KL_mean": -174.72048950195312, "KL/mean": -218.2819366455078, "KL/rejected_KL_mean": -261.8433837890625, "KL/std": 102.5228271484375, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004175018519163132, "fcm_dpo/delta": 0.03734022378921509, "fcm_dpo/margin": 87.1229019165039, "fcm_dpo/q_t": 0.4143916368484497, "grad_norm": 15.842375755310059, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.6169674396514893, "logits/rejected": -1.9062365293502808, "logps/chosen": -286.9278259277344, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -355.4505615234375, "loss": 1.1068, "margin_dpo/margin_mean": 87.12289428710938, "margin_dpo/margin_std": 108.22197723388672, "step": 493 }, { "KL/chosen_KL_mean": -168.04824829101562, "KL/mean": -226.3206024169922, "KL/rejected_KL_mean": -284.59295654296875, "KL/std": 112.60301208496094, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004127000458538532, "fcm_dpo/delta": -0.0861416757106781, "fcm_dpo/margin": 116.54468536376953, "fcm_dpo/q_t": 0.3875739574432373, "grad_norm": 10.736001014709473, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.5629189014434814, "logits/rejected": -1.490004062652588, "logps/chosen": -241.16314697265625, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -376.75592041015625, "loss": 1.0233, "margin_dpo/margin_mean": 116.54468536376953, "margin_dpo/margin_std": 121.68224334716797, "step": 494 }, { "KL/chosen_KL_mean": -174.65231323242188, "KL/mean": -223.97994995117188, "KL/rejected_KL_mean": -273.3076171875, "KL/std": 106.97380065917969, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.004125487990677357, "fcm_dpo/delta": -0.00730224559083581, "fcm_dpo/margin": 98.65533447265625, "fcm_dpo/q_t": 0.4056897461414337, "grad_norm": 11.742447853088379, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.2542951107025146, "logits/rejected": -1.7824485301971436, "logps/chosen": -274.172607421875, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -371.238525390625, "loss": 1.0868, "margin_dpo/margin_mean": 98.65533447265625, "margin_dpo/margin_std": 129.64596557617188, "step": 495 }, { "KL/chosen_KL_mean": -192.12567138671875, "KL/mean": -235.3259735107422, "KL/rejected_KL_mean": -278.52630615234375, "KL/std": 107.96463775634766, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004160840529948473, "fcm_dpo/delta": 0.041795387864112854, "fcm_dpo/margin": 86.40060424804688, "fcm_dpo/q_t": 0.41523051261901855, "grad_norm": 9.882986068725586, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.2711257934570312, "logits/rejected": -1.6292805671691895, "logps/chosen": -299.982421875, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -371.2968444824219, "loss": 1.1101, "margin_dpo/margin_mean": 86.40060424804688, "margin_dpo/margin_std": 109.64350891113281, "step": 496 }, { "KL/chosen_KL_mean": -204.38055419921875, "KL/mean": -232.70252990722656, "KL/rejected_KL_mean": -261.02447509765625, "KL/std": 110.23169708251953, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004202047362923622, "fcm_dpo/delta": 0.050566066056489944, "fcm_dpo/margin": 56.6439208984375, "fcm_dpo/q_t": 0.4448184669017792, "grad_norm": 8.33975601196289, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.8047343492507935, "logits/rejected": -1.353994607925415, "logps/chosen": -296.46380615234375, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -342.81951904296875, "loss": 1.2237, "margin_dpo/margin_mean": 56.64391326904297, "margin_dpo/margin_std": 118.32649230957031, "step": 497 }, { "KL/chosen_KL_mean": -174.2274169921875, "KL/mean": -220.36846923828125, "KL/rejected_KL_mean": -266.5094909667969, "KL/std": 107.56463623046875, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0042066145688295364, "fcm_dpo/delta": 0.012279531918466091, "fcm_dpo/margin": 92.28205871582031, "fcm_dpo/q_t": 0.40907078981399536, "grad_norm": 12.279354095458984, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.9402216672897339, "logits/rejected": -1.4453167915344238, "logps/chosen": -272.4217834472656, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -357.19696044921875, "loss": 1.0941, "margin_dpo/margin_mean": 92.28205871582031, "margin_dpo/margin_std": 118.37500762939453, "step": 498 }, { "KL/chosen_KL_mean": -181.41525268554688, "KL/mean": -230.64187622070312, "KL/rejected_KL_mean": -279.8684997558594, "KL/std": 116.31420135498047, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0042075724340975285, "fcm_dpo/delta": -0.014872591011226177, "fcm_dpo/margin": 98.45327758789062, "fcm_dpo/q_t": 0.4039517641067505, "grad_norm": 10.034916877746582, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.1260604858398438, "logits/rejected": -2.0786187648773193, "logps/chosen": -262.4551696777344, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -386.79022216796875, "loss": 1.0803, "margin_dpo/margin_mean": 98.45327758789062, "margin_dpo/margin_std": 126.04025268554688, "step": 499 }, { "KL/chosen_KL_mean": -200.36398315429688, "KL/mean": -244.26638793945312, "KL/rejected_KL_mean": -288.16876220703125, "KL/std": 113.14503479003906, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004215183202177286, "fcm_dpo/delta": 0.03104613721370697, "fcm_dpo/margin": 87.8047866821289, "fcm_dpo/q_t": 0.4154951870441437, "grad_norm": 11.248675346374512, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.714082956314087, "logits/rejected": -1.5405797958374023, "logps/chosen": -289.612060546875, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -388.5789794921875, "loss": 1.119, "margin_dpo/margin_mean": 87.80477905273438, "margin_dpo/margin_std": 128.55874633789062, "step": 500 }, { "KL/chosen_KL_mean": -177.24533081054688, "KL/mean": -223.9384765625, "KL/rejected_KL_mean": -270.631591796875, "KL/std": 121.61801147460938, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004257489927113056, "fcm_dpo/delta": 0.0014469251036643982, "fcm_dpo/margin": 93.38627624511719, "fcm_dpo/q_t": 0.4092925786972046, "grad_norm": 10.240721702575684, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.5110124349594116, "logits/rejected": -1.062692642211914, "logps/chosen": -267.8422546386719, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -355.92889404296875, "loss": 1.1135, "margin_dpo/margin_mean": 93.38626861572266, "margin_dpo/margin_std": 138.09828186035156, "step": 501 }, { "KL/chosen_KL_mean": -176.85089111328125, "KL/mean": -219.25973510742188, "KL/rejected_KL_mean": -261.6685791015625, "KL/std": 101.14533996582031, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.004267643205821514, "fcm_dpo/delta": 0.039035193622112274, "fcm_dpo/margin": 84.81770324707031, "fcm_dpo/q_t": 0.4154490828514099, "grad_norm": 9.773853302001953, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.8911011219024658, "logits/rejected": -1.1454780101776123, "logps/chosen": -280.3216247558594, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -350.7127685546875, "loss": 1.1259, "margin_dpo/margin_mean": 84.81770324707031, "margin_dpo/margin_std": 124.6889877319336, "step": 502 }, { "KL/chosen_KL_mean": -174.27731323242188, "KL/mean": -221.48226928710938, "KL/rejected_KL_mean": -268.687255859375, "KL/std": 105.44994354248047, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004275772254914045, "fcm_dpo/delta": -0.004175648093223572, "fcm_dpo/margin": 94.40992736816406, "fcm_dpo/q_t": 0.4056119918823242, "grad_norm": 9.841965675354004, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7462100982666016, "logits/rejected": -1.2403883934020996, "logps/chosen": -268.0072937011719, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -356.29620361328125, "loss": 1.0835, "margin_dpo/margin_mean": 94.40992736816406, "margin_dpo/margin_std": 116.24095916748047, "step": 503 }, { "KL/chosen_KL_mean": -179.78802490234375, "KL/mean": -239.94482421875, "KL/rejected_KL_mean": -300.1015930175781, "KL/std": 112.4724349975586, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0041933730244636536, "fcm_dpo/delta": -0.10996431112289429, "fcm_dpo/margin": 120.3135986328125, "fcm_dpo/q_t": 0.381486177444458, "grad_norm": 10.2820463180542, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.1172213554382324, "logits/rejected": -1.7816128730773926, "logps/chosen": -272.07989501953125, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -409.1392517089844, "loss": 0.9999, "margin_dpo/margin_mean": 120.31359100341797, "margin_dpo/margin_std": 115.76571655273438, "step": 504 }, { "KL/chosen_KL_mean": -198.60671997070312, "KL/mean": -242.07272338867188, "KL/rejected_KL_mean": -285.53875732421875, "KL/std": 111.92926025390625, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.004184329882264137, "fcm_dpo/delta": 0.037633128464221954, "fcm_dpo/margin": 86.93201446533203, "fcm_dpo/q_t": 0.4146425724029541, "grad_norm": 8.632781028747559, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.472946882247925, "logits/rejected": -1.914054274559021, "logps/chosen": -304.8836669921875, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -379.744873046875, "loss": 1.1049, "margin_dpo/margin_mean": 86.93202209472656, "margin_dpo/margin_std": 108.03833770751953, "step": 505 }, { "KL/chosen_KL_mean": -191.17037963867188, "KL/mean": -239.73452758789062, "KL/rejected_KL_mean": -288.2987060546875, "KL/std": 105.11408233642578, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.004176631569862366, "fcm_dpo/delta": -0.006201267242431641, "fcm_dpo/margin": 97.12833404541016, "fcm_dpo/q_t": 0.4037782847881317, "grad_norm": 8.882339477539062, "learning_rate": 9.527690882192635e-08, "logits/chosen": -2.1211307048797607, "logits/rejected": -1.7663815021514893, "logps/chosen": -272.6112365722656, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -373.9631042480469, "loss": 1.074, "margin_dpo/margin_mean": 97.12834167480469, "margin_dpo/margin_std": 111.20382690429688, "step": 506 }, { "KL/chosen_KL_mean": -199.1107177734375, "KL/mean": -248.58865356445312, "KL/rejected_KL_mean": -298.0665588378906, "KL/std": 111.09418487548828, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004183316603302956, "fcm_dpo/delta": -0.01456424780189991, "fcm_dpo/margin": 98.95584106445312, "fcm_dpo/q_t": 0.40459540486335754, "grad_norm": 9.085943222045898, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.2950427532196045, "logits/rejected": -1.8540151119232178, "logps/chosen": -298.53765869140625, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -400.31951904296875, "loss": 1.0877, "margin_dpo/margin_mean": 98.95584106445312, "margin_dpo/margin_std": 131.24609375, "step": 507 }, { "KL/chosen_KL_mean": -195.20225524902344, "KL/mean": -236.54771423339844, "KL/rejected_KL_mean": -277.8931884765625, "KL/std": 118.61614227294922, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.004210569430142641, "fcm_dpo/delta": 0.053708747029304504, "fcm_dpo/margin": 82.69091796875, "fcm_dpo/q_t": 0.4210348427295685, "grad_norm": 15.756381034851074, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.36354398727417, "logits/rejected": -1.9058914184570312, "logps/chosen": -293.117919921875, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -381.6910095214844, "loss": 1.1519, "margin_dpo/margin_mean": 82.69091796875, "margin_dpo/margin_std": 139.587890625, "step": 508 }, { "KL/chosen_KL_mean": -178.75347900390625, "KL/mean": -218.01986694335938, "KL/rejected_KL_mean": -257.2862243652344, "KL/std": 92.8510971069336, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004271560348570347, "fcm_dpo/delta": 0.06665083765983582, "fcm_dpo/margin": 78.53276824951172, "fcm_dpo/q_t": 0.42086923122406006, "grad_norm": 14.400446891784668, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9184513092041016, "logits/rejected": -1.2318346500396729, "logps/chosen": -279.48333740234375, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -344.2295837402344, "loss": 1.146, "margin_dpo/margin_mean": 78.53277587890625, "margin_dpo/margin_std": 123.22199249267578, "step": 509 }, { "KL/chosen_KL_mean": -213.82516479492188, "KL/mean": -261.0078125, "KL/rejected_KL_mean": -308.1904296875, "KL/std": 111.6173095703125, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004283777438104153, "fcm_dpo/delta": -0.004458754323422909, "fcm_dpo/margin": 94.36524200439453, "fcm_dpo/q_t": 0.4054196774959564, "grad_norm": 11.385659217834473, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.5190986394882202, "logits/rejected": -1.3322863578796387, "logps/chosen": -302.99456787109375, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -416.10980224609375, "loss": 1.0792, "margin_dpo/margin_mean": 94.36524200439453, "margin_dpo/margin_std": 111.97315979003906, "step": 510 }, { "KL/chosen_KL_mean": -195.78176879882812, "KL/mean": -245.87701416015625, "KL/rejected_KL_mean": -295.9722900390625, "KL/std": 110.81451416015625, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004267792217433453, "fcm_dpo/delta": -0.028866847977042198, "fcm_dpo/margin": 100.19053649902344, "fcm_dpo/q_t": 0.4014691412448883, "grad_norm": 11.730097770690918, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.1892738342285156, "logits/rejected": -1.6178803443908691, "logps/chosen": -285.53155517578125, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -393.7035827636719, "loss": 1.0696, "margin_dpo/margin_mean": 100.19053649902344, "margin_dpo/margin_std": 123.99414825439453, "step": 511 }, { "KL/chosen_KL_mean": -185.9376220703125, "KL/mean": -234.96481323242188, "KL/rejected_KL_mean": -283.99200439453125, "KL/std": 107.79226684570312, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004259875044226646, "fcm_dpo/delta": -0.01883702725172043, "fcm_dpo/margin": 98.05441284179688, "fcm_dpo/q_t": 0.4027479588985443, "grad_norm": 12.844273567199707, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.9486550092697144, "logits/rejected": -1.5080482959747314, "logps/chosen": -271.67279052734375, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -371.61688232421875, "loss": 1.0847, "margin_dpo/margin_mean": 98.05441284179688, "margin_dpo/margin_std": 128.55126953125, "step": 512 }, { "KL/chosen_KL_mean": -197.2462158203125, "KL/mean": -238.0255126953125, "KL/rejected_KL_mean": -278.8048400878906, "KL/std": 102.64700317382812, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004252881743013859, "fcm_dpo/delta": 0.05492629483342171, "fcm_dpo/margin": 81.55860137939453, "fcm_dpo/q_t": 0.4182282090187073, "grad_norm": 10.787209510803223, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.4328973293304443, "logits/rejected": -1.7441322803497314, "logps/chosen": -293.43670654296875, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -361.72259521484375, "loss": 1.1301, "margin_dpo/margin_mean": 81.55860137939453, "margin_dpo/margin_std": 116.00071716308594, "step": 513 }, { "KL/chosen_KL_mean": -192.15423583984375, "KL/mean": -236.6586151123047, "KL/rejected_KL_mean": -281.1629638671875, "KL/std": 103.19206237792969, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004295860417187214, "fcm_dpo/delta": 0.018321529030799866, "fcm_dpo/margin": 89.00872802734375, "fcm_dpo/q_t": 0.4119195342063904, "grad_norm": 12.24336051940918, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.8497495651245117, "logits/rejected": -1.2820158004760742, "logps/chosen": -289.9778747558594, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -366.5328063964844, "loss": 1.1057, "margin_dpo/margin_mean": 89.00872802734375, "margin_dpo/margin_std": 122.92401123046875, "step": 514 }, { "KL/chosen_KL_mean": -182.26402282714844, "KL/mean": -225.89266967773438, "KL/rejected_KL_mean": -269.52130126953125, "KL/std": 107.48809051513672, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.004311527591198683, "fcm_dpo/delta": 0.02472526766359806, "fcm_dpo/margin": 87.25727844238281, "fcm_dpo/q_t": 0.4125068485736847, "grad_norm": 10.774267196655273, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.8016979694366455, "logits/rejected": -1.570582628250122, "logps/chosen": -264.19757080078125, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -363.09783935546875, "loss": 1.1092, "margin_dpo/margin_mean": 87.25727844238281, "margin_dpo/margin_std": 118.87619018554688, "step": 515 }, { "KL/chosen_KL_mean": -190.41758728027344, "KL/mean": -233.88275146484375, "KL/rejected_KL_mean": -277.347900390625, "KL/std": 124.785888671875, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0043390244245529175, "fcm_dpo/delta": 0.023658277466893196, "fcm_dpo/margin": 86.93028259277344, "fcm_dpo/q_t": 0.4154844284057617, "grad_norm": 13.44446086883545, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.2703497409820557, "logits/rejected": -2.1000850200653076, "logps/chosen": -287.50927734375, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -395.9045715332031, "loss": 1.14, "margin_dpo/margin_mean": 86.93028259277344, "margin_dpo/margin_std": 147.93496704101562, "step": 516 }, { "KL/chosen_KL_mean": -201.16177368164062, "KL/mean": -253.23001098632812, "KL/rejected_KL_mean": -305.29827880859375, "KL/std": 110.51988220214844, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004280781373381615, "fcm_dpo/delta": -0.04908519983291626, "fcm_dpo/margin": 104.136474609375, "fcm_dpo/q_t": 0.3967878222465515, "grad_norm": 9.522708892822266, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.9000929594039917, "logits/rejected": -1.4687031507492065, "logps/chosen": -294.34832763671875, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -400.89105224609375, "loss": 1.0607, "margin_dpo/margin_mean": 104.13648223876953, "margin_dpo/margin_std": 124.54408264160156, "step": 517 }, { "KL/chosen_KL_mean": -209.32077026367188, "KL/mean": -247.96453857421875, "KL/rejected_KL_mean": -286.6082763671875, "KL/std": 103.82592010498047, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004338730126619339, "fcm_dpo/delta": 0.06694076955318451, "fcm_dpo/margin": 77.28751373291016, "fcm_dpo/q_t": 0.4233773350715637, "grad_norm": 10.521740913391113, "learning_rate": 8.348171708068747e-08, "logits/chosen": -2.0635008811950684, "logits/rejected": -1.7380964756011963, "logps/chosen": -296.3421325683594, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -377.85284423828125, "loss": 1.1543, "margin_dpo/margin_mean": 77.28751373291016, "margin_dpo/margin_std": 130.4095458984375, "step": 518 }, { "KL/chosen_KL_mean": -195.26507568359375, "KL/mean": -237.4440460205078, "KL/rejected_KL_mean": -279.6230163574219, "KL/std": 96.91849517822266, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.004368597641587257, "fcm_dpo/delta": 0.03266787528991699, "fcm_dpo/margin": 84.35796356201172, "fcm_dpo/q_t": 0.4140433371067047, "grad_norm": 10.512846946716309, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.9309871196746826, "logits/rejected": -1.5532617568969727, "logps/chosen": -295.95263671875, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -388.6534423828125, "loss": 1.0985, "margin_dpo/margin_mean": 84.35797119140625, "margin_dpo/margin_std": 102.00047302246094, "step": 519 }, { "KL/chosen_KL_mean": -180.96832275390625, "KL/mean": -231.30001831054688, "KL/rejected_KL_mean": -281.6317138671875, "KL/std": 110.69091796875, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004349192604422569, "fcm_dpo/delta": -0.03970557451248169, "fcm_dpo/margin": 100.66338348388672, "fcm_dpo/q_t": 0.3970341682434082, "grad_norm": 9.259831428527832, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.5995028018951416, "logits/rejected": -1.1544158458709717, "logps/chosen": -279.7459716796875, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -388.577392578125, "loss": 1.0478, "margin_dpo/margin_mean": 100.66338348388672, "margin_dpo/margin_std": 107.28077697753906, "step": 520 }, { "KL/chosen_KL_mean": -179.88372802734375, "KL/mean": -232.01339721679688, "KL/rejected_KL_mean": -284.1430969238281, "KL/std": 102.542236328125, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004333083517849445, "fcm_dpo/delta": -0.05435986444354057, "fcm_dpo/margin": 104.25936126708984, "fcm_dpo/q_t": 0.3939448595046997, "grad_norm": 10.628755569458008, "learning_rate": 8.062991975753378e-08, "logits/chosen": -2.1092841625213623, "logits/rejected": -1.7741756439208984, "logps/chosen": -266.23480224609375, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -374.5362548828125, "loss": 1.0344, "margin_dpo/margin_mean": 104.25936126708984, "margin_dpo/margin_std": 104.25616455078125, "step": 521 }, { "KL/chosen_KL_mean": -191.06764221191406, "KL/mean": -236.00685119628906, "KL/rejected_KL_mean": -280.94610595703125, "KL/std": 112.46234893798828, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.004319024737924337, "fcm_dpo/delta": 0.01217893697321415, "fcm_dpo/margin": 89.87844848632812, "fcm_dpo/q_t": 0.4095935821533203, "grad_norm": 10.399538040161133, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.6738702058792114, "logits/rejected": -1.3025612831115723, "logps/chosen": -267.07568359375, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -355.3873291015625, "loss": 1.0981, "margin_dpo/margin_mean": 89.87844848632812, "margin_dpo/margin_std": 118.70707702636719, "step": 522 }, { "KL/chosen_KL_mean": -211.54798889160156, "KL/mean": -245.0439910888672, "KL/rejected_KL_mean": -278.53997802734375, "KL/std": 111.36172485351562, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.004379163030534983, "fcm_dpo/delta": 0.10995499789714813, "fcm_dpo/margin": 66.99199676513672, "fcm_dpo/q_t": 0.4330518841743469, "grad_norm": 11.5806303024292, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.098353624343872, "logits/rejected": -1.3979312181472778, "logps/chosen": -308.04425048828125, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -369.5552673339844, "loss": 1.1896, "margin_dpo/margin_mean": 66.99198913574219, "margin_dpo/margin_std": 128.467041015625, "step": 523 }, { "KL/chosen_KL_mean": -193.17877197265625, "KL/mean": -240.19509887695312, "KL/rejected_KL_mean": -287.21142578125, "KL/std": 110.6829605102539, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.004394913092255592, "fcm_dpo/delta": -0.013875662349164486, "fcm_dpo/margin": 94.03265380859375, "fcm_dpo/q_t": 0.4047653079032898, "grad_norm": 13.050933837890625, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.2942776679992676, "logits/rejected": -1.602282166481018, "logps/chosen": -296.58428955078125, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -375.12158203125, "loss": 1.0781, "margin_dpo/margin_mean": 94.03265380859375, "margin_dpo/margin_std": 118.09622955322266, "step": 524 }, { "KL/chosen_KL_mean": -180.47857666015625, "KL/mean": -235.26129150390625, "KL/rejected_KL_mean": -290.04400634765625, "KL/std": 111.2188720703125, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0043082088232040405, "fcm_dpo/delta": -0.07783657312393188, "fcm_dpo/margin": 109.56543731689453, "fcm_dpo/q_t": 0.39272141456604004, "grad_norm": 10.241365432739258, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.6826257705688477, "logits/rejected": -1.5947628021240234, "logps/chosen": -264.68505859375, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -396.30560302734375, "loss": 1.0473, "margin_dpo/margin_mean": 109.56543731689453, "margin_dpo/margin_std": 129.88880920410156, "step": 525 }, { "KL/chosen_KL_mean": -185.9484405517578, "KL/mean": -235.00726318359375, "KL/rejected_KL_mean": -284.06610107421875, "KL/std": 105.1544418334961, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.004312381148338318, "fcm_dpo/delta": -0.02414672076702118, "fcm_dpo/margin": 98.11766052246094, "fcm_dpo/q_t": 0.4018915295600891, "grad_norm": 7.981964588165283, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.794324517250061, "logits/rejected": -1.136673927307129, "logps/chosen": -288.80255126953125, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -377.7175598144531, "loss": 1.0757, "margin_dpo/margin_mean": 98.11766052246094, "margin_dpo/margin_std": 124.7968978881836, "step": 526 }, { "KL/chosen_KL_mean": -187.45962524414062, "KL/mean": -237.12197875976562, "KL/rejected_KL_mean": -286.7843322753906, "KL/std": 109.56422424316406, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004288290627300739, "fcm_dpo/delta": -0.027105331420898438, "fcm_dpo/margin": 99.32472229003906, "fcm_dpo/q_t": 0.398837685585022, "grad_norm": 9.680211067199707, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.139167547225952, "logits/rejected": -2.3070554733276367, "logps/chosen": -284.2250671386719, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -372.17950439453125, "loss": 1.0451, "margin_dpo/margin_mean": 99.32472229003906, "margin_dpo/margin_std": 95.76783752441406, "step": 527 }, { "KL/chosen_KL_mean": -175.07550048828125, "KL/mean": -224.99072265625, "KL/rejected_KL_mean": -274.9059753417969, "KL/std": 111.56260681152344, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004272320307791233, "fcm_dpo/delta": -0.02768673002719879, "fcm_dpo/margin": 99.8304672241211, "fcm_dpo/q_t": 0.40124598145484924, "grad_norm": 9.486064910888672, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.259384870529175, "logits/rejected": -1.745857834815979, "logps/chosen": -269.24041748046875, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -375.2782897949219, "loss": 1.0718, "margin_dpo/margin_mean": 99.83047485351562, "margin_dpo/margin_std": 125.18672180175781, "step": 528 }, { "KL/chosen_KL_mean": -204.37759399414062, "KL/mean": -249.1416015625, "KL/rejected_KL_mean": -293.9056091308594, "KL/std": 111.83256530761719, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.004279471002519131, "fcm_dpo/delta": 0.017444506287574768, "fcm_dpo/margin": 89.52803039550781, "fcm_dpo/q_t": 0.41241908073425293, "grad_norm": 11.053693771362305, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.696305751800537, "logits/rejected": -1.5192456245422363, "logps/chosen": -285.1543884277344, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -391.0967712402344, "loss": 1.1167, "margin_dpo/margin_mean": 89.52803039550781, "margin_dpo/margin_std": 133.93035888671875, "step": 529 }, { "KL/chosen_KL_mean": -186.2950439453125, "KL/mean": -231.241455078125, "KL/rejected_KL_mean": -276.1878662109375, "KL/std": 106.78868103027344, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.00427293311804533, "fcm_dpo/delta": 0.016384001821279526, "fcm_dpo/margin": 89.892822265625, "fcm_dpo/q_t": 0.41018176078796387, "grad_norm": 9.80273723602295, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.7856957912445068, "logits/rejected": -1.4705675840377808, "logps/chosen": -266.0707702636719, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -368.24176025390625, "loss": 1.0845, "margin_dpo/margin_mean": 89.892822265625, "margin_dpo/margin_std": 102.54185485839844, "step": 530 }, { "KL/chosen_KL_mean": -179.4569854736328, "KL/mean": -235.7604217529297, "KL/rejected_KL_mean": -292.0638427734375, "KL/std": 113.32392883300781, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004234970547258854, "fcm_dpo/delta": -0.08081808686256409, "fcm_dpo/margin": 112.60685729980469, "fcm_dpo/q_t": 0.3877102732658386, "grad_norm": 12.241541862487793, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.9442431926727295, "logits/rejected": -1.3433287143707275, "logps/chosen": -258.77618408203125, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -370.17218017578125, "loss": 1.0199, "margin_dpo/margin_mean": 112.60685729980469, "margin_dpo/margin_std": 113.838134765625, "step": 531 }, { "KL/chosen_KL_mean": -194.2032470703125, "KL/mean": -254.5747833251953, "KL/rejected_KL_mean": -314.94635009765625, "KL/std": 107.7076644897461, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.004163362085819244, "fcm_dpo/delta": -0.10799334943294525, "fcm_dpo/margin": 120.74311828613281, "fcm_dpo/q_t": 0.3826107978820801, "grad_norm": 11.09316635131836, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.8975014686584473, "logits/rejected": -1.2828367948532104, "logps/chosen": -295.84014892578125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -408.291748046875, "loss": 1.0058, "margin_dpo/margin_mean": 120.74311828613281, "margin_dpo/margin_std": 120.03959655761719, "step": 532 }, { "KL/chosen_KL_mean": -186.34715270996094, "KL/mean": -221.4824981689453, "KL/rejected_KL_mean": -256.61785888671875, "KL/std": 113.3179931640625, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.00420135073363781, "fcm_dpo/delta": 0.10785086452960968, "fcm_dpo/margin": 70.27070617675781, "fcm_dpo/q_t": 0.4329884648323059, "grad_norm": 10.501534461975098, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.5394585132598877, "logits/rejected": -1.944779634475708, "logps/chosen": -277.09381103515625, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -339.50634765625, "loss": 1.1768, "margin_dpo/margin_mean": 70.27070617675781, "margin_dpo/margin_std": 123.86640930175781, "step": 533 }, { "KL/chosen_KL_mean": -165.78724670410156, "KL/mean": -218.48812866210938, "KL/rejected_KL_mean": -271.18896484375, "KL/std": 107.39715576171875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004197611473500729, "fcm_dpo/delta": -0.04437590762972832, "fcm_dpo/margin": 105.40174865722656, "fcm_dpo/q_t": 0.3954181671142578, "grad_norm": 10.022228240966797, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.930671215057373, "logits/rejected": -1.5240843296051025, "logps/chosen": -252.4180908203125, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -356.0609436035156, "loss": 1.0367, "margin_dpo/margin_mean": 105.40174865722656, "margin_dpo/margin_std": 103.25664520263672, "step": 534 }, { "KL/chosen_KL_mean": -167.16305541992188, "KL/mean": -222.51524353027344, "KL/rejected_KL_mean": -277.867431640625, "KL/std": 106.85429382324219, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.004161850083619356, "fcm_dpo/delta": -0.0637688934803009, "fcm_dpo/margin": 110.7043685913086, "fcm_dpo/q_t": 0.392774760723114, "grad_norm": 7.636528968811035, "learning_rate": 6.786329772205246e-08, "logits/chosen": -2.0237112045288086, "logits/rejected": -1.8429303169250488, "logps/chosen": -257.5084533691406, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -378.6453857421875, "loss": 1.0379, "margin_dpo/margin_mean": 110.7043685913086, "margin_dpo/margin_std": 120.06649780273438, "step": 535 }, { "KL/chosen_KL_mean": -166.84432983398438, "KL/mean": -221.38522338867188, "KL/rejected_KL_mean": -275.9261169433594, "KL/std": 107.9281005859375, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0040939003229141235, "fcm_dpo/delta": -0.048880912363529205, "fcm_dpo/margin": 109.08181762695312, "fcm_dpo/q_t": 0.39477846026420593, "grad_norm": 8.790689468383789, "learning_rate": 6.698662514899638e-08, "logits/chosen": -2.0505764484405518, "logits/rejected": -1.806133508682251, "logps/chosen": -245.80389404296875, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -369.013916015625, "loss": 1.0405, "margin_dpo/margin_mean": 109.08181762695312, "margin_dpo/margin_std": 113.99948120117188, "step": 536 }, { "KL/chosen_KL_mean": -145.86102294921875, "KL/mean": -202.533203125, "KL/rejected_KL_mean": -259.20538330078125, "KL/std": 111.95986938476562, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0040747374296188354, "fcm_dpo/delta": -0.06516027450561523, "fcm_dpo/margin": 113.34437561035156, "fcm_dpo/q_t": 0.39119261503219604, "grad_norm": 9.319231033325195, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.848917007446289, "logits/rejected": -1.1820181608200073, "logps/chosen": -232.57040405273438, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -338.113037109375, "loss": 1.0277, "margin_dpo/margin_mean": 113.34437561035156, "margin_dpo/margin_std": 109.42387390136719, "step": 537 }, { "KL/chosen_KL_mean": -199.399658203125, "KL/mean": -245.74227905273438, "KL/rejected_KL_mean": -292.08489990234375, "KL/std": 100.41688537597656, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.004054270684719086, "fcm_dpo/delta": 0.025160444900393486, "fcm_dpo/margin": 92.68526458740234, "fcm_dpo/q_t": 0.4115146994590759, "grad_norm": 8.960223197937012, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.820164680480957, "logits/rejected": -1.2694764137268066, "logps/chosen": -297.841796875, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -383.1795654296875, "loss": 1.0896, "margin_dpo/margin_mean": 92.68527221679688, "margin_dpo/margin_std": 106.3125, "step": 538 }, { "KL/chosen_KL_mean": -175.23257446289062, "KL/mean": -225.11611938476562, "KL/rejected_KL_mean": -274.9996643066406, "KL/std": 110.34243774414062, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.004055923782289028, "fcm_dpo/delta": -0.004843953065574169, "fcm_dpo/margin": 99.76710510253906, "fcm_dpo/q_t": 0.405444860458374, "grad_norm": 8.570791244506836, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.6142425537109375, "logits/rejected": -2.140502452850342, "logps/chosen": -266.1646728515625, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -372.44512939453125, "loss": 1.0829, "margin_dpo/margin_mean": 99.76710510253906, "margin_dpo/margin_std": 125.08678436279297, "step": 539 }, { "KL/chosen_KL_mean": -187.48997497558594, "KL/mean": -246.75460815429688, "KL/rejected_KL_mean": -306.01922607421875, "KL/std": 120.60459899902344, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.004022745415568352, "fcm_dpo/delta": -0.0806887224316597, "fcm_dpo/margin": 118.5292739868164, "fcm_dpo/q_t": 0.39139121770858765, "grad_norm": 10.366264343261719, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.182539939880371, "logits/rejected": -1.9528584480285645, "logps/chosen": -295.35986328125, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -427.4127502441406, "loss": 1.0413, "margin_dpo/margin_mean": 118.5292739868164, "margin_dpo/margin_std": 142.02444458007812, "step": 540 }, { "KL/chosen_KL_mean": -208.48077392578125, "KL/mean": -245.89437866210938, "KL/rejected_KL_mean": -283.3079833984375, "KL/std": 99.52751159667969, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0039839535020291805, "fcm_dpo/delta": -0.0062293098308146, "fcm_dpo/margin": 74.82720947265625, "fcm_dpo/q_t": 0.43003278970718384, "grad_norm": 9.064143180847168, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.5246334075927734, "logits/rejected": -2.0710487365722656, "logps/chosen": -315.04827880859375, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -392.37432861328125, "loss": 1.1585, "margin_dpo/margin_mean": 74.82720947265625, "margin_dpo/margin_std": 106.73542785644531, "step": 541 }, { "KL/chosen_KL_mean": -175.36514282226562, "KL/mean": -232.37423706054688, "KL/rejected_KL_mean": -289.38336181640625, "KL/std": 111.55144500732422, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0039509800262749195, "fcm_dpo/delta": -0.05289233475923538, "fcm_dpo/margin": 114.0181884765625, "fcm_dpo/q_t": 0.3936605453491211, "grad_norm": 8.876738548278809, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.374492645263672, "logits/rejected": -2.254364013671875, "logps/chosen": -262.73443603515625, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -392.20367431640625, "loss": 1.0383, "margin_dpo/margin_mean": 114.0181884765625, "margin_dpo/margin_std": 118.94149780273438, "step": 542 }, { "KL/chosen_KL_mean": -202.9133758544922, "KL/mean": -233.67112731933594, "KL/rejected_KL_mean": -264.42889404296875, "KL/std": 119.41917419433594, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.003974448423832655, "fcm_dpo/delta": 0.040949247777462006, "fcm_dpo/margin": 61.515506744384766, "fcm_dpo/q_t": 0.4439527988433838, "grad_norm": 12.601298332214355, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.786360263824463, "logits/rejected": -1.8364989757537842, "logps/chosen": -290.06597900390625, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -375.96063232421875, "loss": 1.2372, "margin_dpo/margin_mean": 61.51551055908203, "margin_dpo/margin_std": 145.4685821533203, "step": 543 }, { "KL/chosen_KL_mean": -211.18374633789062, "KL/mean": -250.33059692382812, "KL/rejected_KL_mean": -289.4774169921875, "KL/std": 112.02928161621094, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.004023072775453329, "fcm_dpo/delta": 0.08784949779510498, "fcm_dpo/margin": 78.29368591308594, "fcm_dpo/q_t": 0.4272671341896057, "grad_norm": 9.190691947937012, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.7565805912017822, "logits/rejected": -1.6641802787780762, "logps/chosen": -297.3306579589844, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -395.871337890625, "loss": 1.1491, "margin_dpo/margin_mean": 78.29368591308594, "margin_dpo/margin_std": 118.38993835449219, "step": 544 }, { "KL/chosen_KL_mean": -188.42410278320312, "KL/mean": -235.53717041015625, "KL/rejected_KL_mean": -282.6502685546875, "KL/std": 110.67372131347656, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004062117543071508, "fcm_dpo/delta": 0.017848581075668335, "fcm_dpo/margin": 94.22618865966797, "fcm_dpo/q_t": 0.4110974669456482, "grad_norm": 9.196220397949219, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4836559295654297, "logits/rejected": -1.0497890710830688, "logps/chosen": -284.84832763671875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -383.38433837890625, "loss": 1.1112, "margin_dpo/margin_mean": 94.22618865966797, "margin_dpo/margin_std": 134.9276123046875, "step": 545 }, { "KL/chosen_KL_mean": -194.76907348632812, "KL/mean": -236.8666229248047, "KL/rejected_KL_mean": -278.96417236328125, "KL/std": 112.41192626953125, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0040893517434597015, "fcm_dpo/delta": 0.057723209261894226, "fcm_dpo/margin": 84.19506072998047, "fcm_dpo/q_t": 0.4190519154071808, "grad_norm": 9.832690238952637, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.701326847076416, "logits/rejected": -1.3498517274856567, "logps/chosen": -280.79022216796875, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -366.3367919921875, "loss": 1.1316, "margin_dpo/margin_mean": 84.195068359375, "margin_dpo/margin_std": 122.81524658203125, "step": 546 }, { "KL/chosen_KL_mean": -185.94342041015625, "KL/mean": -236.97146606445312, "KL/rejected_KL_mean": -287.99945068359375, "KL/std": 109.74354553222656, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0041055576875805855, "fcm_dpo/delta": -0.01992580108344555, "fcm_dpo/margin": 102.05604553222656, "fcm_dpo/q_t": 0.4004826545715332, "grad_norm": 9.375100135803223, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -2.0269548892974854, "logits/rejected": -1.5708189010620117, "logps/chosen": -286.59442138671875, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -385.91119384765625, "loss": 1.0502, "margin_dpo/margin_mean": 102.05604553222656, "margin_dpo/margin_std": 98.05469512939453, "step": 547 }, { "KL/chosen_KL_mean": -184.34112548828125, "KL/mean": -225.80908203125, "KL/rejected_KL_mean": -267.2770080566406, "KL/std": 112.59555053710938, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.004127576947212219, "fcm_dpo/delta": 0.0597076490521431, "fcm_dpo/margin": 82.93586730957031, "fcm_dpo/q_t": 0.42124462127685547, "grad_norm": 11.327168464660645, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.166285991668701, "logits/rejected": -1.8003568649291992, "logps/chosen": -280.3712463378906, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -365.69024658203125, "loss": 1.156, "margin_dpo/margin_mean": 82.93587493896484, "margin_dpo/margin_std": 142.95108032226562, "step": 548 }, { "KL/chosen_KL_mean": -173.96829223632812, "KL/mean": -216.8719940185547, "KL/rejected_KL_mean": -259.77569580078125, "KL/std": 105.71980285644531, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004183759912848473, "fcm_dpo/delta": 0.0419439896941185, "fcm_dpo/margin": 85.80741882324219, "fcm_dpo/q_t": 0.416200190782547, "grad_norm": 11.18349838256836, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8468971252441406, "logits/rejected": -1.4243619441986084, "logps/chosen": -257.41925048828125, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -341.15594482421875, "loss": 1.1346, "margin_dpo/margin_mean": 85.80742645263672, "margin_dpo/margin_std": 131.349609375, "step": 549 }, { "KL/chosen_KL_mean": -191.08680725097656, "KL/mean": -233.97369384765625, "KL/rejected_KL_mean": -276.860595703125, "KL/std": 101.20584106445312, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004207301419228315, "fcm_dpo/delta": 0.040440842509269714, "fcm_dpo/margin": 85.7738037109375, "fcm_dpo/q_t": 0.41588765382766724, "grad_norm": 9.447436332702637, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.6737799644470215, "logits/rejected": -1.4633197784423828, "logps/chosen": -280.68115234375, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -378.17608642578125, "loss": 1.1108, "margin_dpo/margin_mean": 85.7738037109375, "margin_dpo/margin_std": 110.5300064086914, "step": 550 }, { "KL/chosen_KL_mean": -171.5891571044922, "KL/mean": -223.37344360351562, "KL/rejected_KL_mean": -275.15771484375, "KL/std": 115.31768798828125, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004167678765952587, "fcm_dpo/delta": -0.033663488924503326, "fcm_dpo/margin": 103.56855773925781, "fcm_dpo/q_t": 0.40039873123168945, "grad_norm": 11.026494979858398, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.7405534982681274, "logits/rejected": -1.6029549837112427, "logps/chosen": -266.67132568359375, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -392.8769836425781, "loss": 1.0574, "margin_dpo/margin_mean": 103.56856536865234, "margin_dpo/margin_std": 117.51785278320312, "step": 551 }, { "KL/chosen_KL_mean": -164.08262634277344, "KL/mean": -236.92544555664062, "KL/rejected_KL_mean": -309.7682800292969, "KL/std": 123.63762664794922, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.004026327282190323, "fcm_dpo/delta": -0.20091430842876434, "fcm_dpo/margin": 145.68563842773438, "fcm_dpo/q_t": 0.3627035617828369, "grad_norm": 10.497523307800293, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.240084171295166, "logits/rejected": -1.8852581977844238, "logps/chosen": -265.4918212890625, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -420.6067199707031, "loss": 0.9493, "margin_dpo/margin_mean": 145.68563842773438, "margin_dpo/margin_std": 128.68348693847656, "step": 552 }, { "KL/chosen_KL_mean": -179.5113067626953, "KL/mean": -222.06015014648438, "KL/rejected_KL_mean": -264.6089782714844, "KL/std": 115.56234741210938, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0040542492642998695, "fcm_dpo/delta": 0.056860677897930145, "fcm_dpo/margin": 85.09767150878906, "fcm_dpo/q_t": 0.4189780354499817, "grad_norm": 9.507705688476562, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.9423251152038574, "logits/rejected": -1.6295682191848755, "logps/chosen": -277.4751892089844, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -368.1861572265625, "loss": 1.1313, "margin_dpo/margin_mean": 85.09767150878906, "margin_dpo/margin_std": 123.11065673828125, "step": 553 }, { "KL/chosen_KL_mean": -186.24365234375, "KL/mean": -234.90419006347656, "KL/rejected_KL_mean": -283.564697265625, "KL/std": 116.09307861328125, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.004077576100826263, "fcm_dpo/delta": 0.0030234865844249725, "fcm_dpo/margin": 97.32106018066406, "fcm_dpo/q_t": 0.408843070268631, "grad_norm": 8.6865873336792, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8401384353637695, "logits/rejected": -1.0450546741485596, "logps/chosen": -289.93048095703125, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -376.55682373046875, "loss": 1.1169, "margin_dpo/margin_mean": 97.32106018066406, "margin_dpo/margin_std": 149.6651611328125, "step": 554 }, { "KL/chosen_KL_mean": -196.34255981445312, "KL/mean": -240.06967163085938, "KL/rejected_KL_mean": -283.7967529296875, "KL/std": 119.64387512207031, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.004089915659278631, "fcm_dpo/delta": 0.043893106281757355, "fcm_dpo/margin": 87.4542236328125, "fcm_dpo/q_t": 0.4185032844543457, "grad_norm": 9.308362007141113, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.801566243171692, "logits/rejected": -1.714731216430664, "logps/chosen": -282.54620361328125, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -398.3439636230469, "loss": 1.1369, "margin_dpo/margin_mean": 87.4542236328125, "margin_dpo/margin_std": 139.8123016357422, "step": 555 }, { "KL/chosen_KL_mean": -175.00946044921875, "KL/mean": -224.28863525390625, "KL/rejected_KL_mean": -273.56781005859375, "KL/std": 105.60594177246094, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004085252992808819, "fcm_dpo/delta": -0.002942195162177086, "fcm_dpo/margin": 98.558349609375, "fcm_dpo/q_t": 0.4057893753051758, "grad_norm": 10.461771011352539, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.9199899435043335, "logits/rejected": -1.6285616159439087, "logps/chosen": -263.4129638671875, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -380.2069396972656, "loss": 1.0798, "margin_dpo/margin_mean": 98.558349609375, "margin_dpo/margin_std": 118.70310974121094, "step": 556 }, { "KL/chosen_KL_mean": -187.9474639892578, "KL/mean": -244.15554809570312, "KL/rejected_KL_mean": -300.3636169433594, "KL/std": 123.55421447753906, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.004043270833790302, "fcm_dpo/delta": -0.057823777198791504, "fcm_dpo/margin": 112.4161605834961, "fcm_dpo/q_t": 0.39547961950302124, "grad_norm": 8.269311904907227, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9680204391479492, "logits/rejected": -1.7928869724273682, "logps/chosen": -286.78271484375, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -416.01251220703125, "loss": 1.0532, "margin_dpo/margin_mean": 112.4161605834961, "margin_dpo/margin_std": 134.6422119140625, "step": 557 }, { "KL/chosen_KL_mean": -185.55831909179688, "KL/mean": -245.080810546875, "KL/rejected_KL_mean": -304.603271484375, "KL/std": 106.08320617675781, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.003993769641965628, "fcm_dpo/delta": -0.07946664839982986, "fcm_dpo/margin": 119.04496765136719, "fcm_dpo/q_t": 0.38778582215309143, "grad_norm": 10.891857147216797, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.6487679481506348, "logits/rejected": -2.0621299743652344, "logps/chosen": -291.5701599121094, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -412.9945983886719, "loss": 1.0116, "margin_dpo/margin_mean": 119.04496765136719, "margin_dpo/margin_std": 107.96421813964844, "step": 558 }, { "KL/chosen_KL_mean": -199.28817749023438, "KL/mean": -246.65101623535156, "KL/rejected_KL_mean": -294.01385498046875, "KL/std": 110.32373046875, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.004002976231276989, "fcm_dpo/delta": 0.021592391654849052, "fcm_dpo/margin": 94.72566223144531, "fcm_dpo/q_t": 0.4107655882835388, "grad_norm": 10.484630584716797, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.425611972808838, "logits/rejected": -1.947913408279419, "logps/chosen": -299.7129211425781, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -394.42657470703125, "loss": 1.1019, "margin_dpo/margin_mean": 94.72566223144531, "margin_dpo/margin_std": 123.9442138671875, "step": 559 }, { "KL/chosen_KL_mean": -190.43069458007812, "KL/mean": -237.13800048828125, "KL/rejected_KL_mean": -283.8453063964844, "KL/std": 106.30087280273438, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.004013483412563801, "fcm_dpo/delta": 0.02605726383626461, "fcm_dpo/margin": 93.41459655761719, "fcm_dpo/q_t": 0.41240373253822327, "grad_norm": 11.043991088867188, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.4511281251907349, "logits/rejected": -1.0058010816574097, "logps/chosen": -278.06622314453125, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -371.22515869140625, "loss": 1.1037, "margin_dpo/margin_mean": 93.41459655761719, "margin_dpo/margin_std": 122.50228881835938, "step": 560 }, { "KL/chosen_KL_mean": -198.71273803710938, "KL/mean": -242.55905151367188, "KL/rejected_KL_mean": -286.4053955078125, "KL/std": 104.87700653076172, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004060794599354267, "fcm_dpo/delta": 0.045307595282793045, "fcm_dpo/margin": 87.69265747070312, "fcm_dpo/q_t": 0.41590213775634766, "grad_norm": 8.315340042114258, "learning_rate": 4.669493178106432e-08, "logits/chosen": -2.166055202484131, "logits/rejected": -2.135782241821289, "logps/chosen": -273.5941162109375, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -392.453369140625, "loss": 1.1012, "margin_dpo/margin_mean": 87.69265747070312, "margin_dpo/margin_std": 100.08743286132812, "step": 561 }, { "KL/chosen_KL_mean": -181.15066528320312, "KL/mean": -224.27371215820312, "KL/rejected_KL_mean": -267.3967590332031, "KL/std": 106.69064331054688, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.004078497178852558, "fcm_dpo/delta": 0.04996158555150032, "fcm_dpo/margin": 86.24610900878906, "fcm_dpo/q_t": 0.41895538568496704, "grad_norm": 10.672560691833496, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.4575424194335938, "logits/rejected": -2.0262508392333984, "logps/chosen": -281.8138732910156, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -369.63763427734375, "loss": 1.135, "margin_dpo/margin_mean": 86.24610900878906, "margin_dpo/margin_std": 133.39053344726562, "step": 562 }, { "KL/chosen_KL_mean": -193.34735107421875, "KL/mean": -228.6373291015625, "KL/rejected_KL_mean": -263.92730712890625, "KL/std": 107.42042541503906, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004182306118309498, "fcm_dpo/delta": 0.10738831758499146, "fcm_dpo/margin": 70.57994842529297, "fcm_dpo/q_t": 0.431059867143631, "grad_norm": 9.836493492126465, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.849326491355896, "logits/rejected": -1.516762614250183, "logps/chosen": -289.6065673828125, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -365.13232421875, "loss": 1.1675, "margin_dpo/margin_mean": 70.57994842529297, "margin_dpo/margin_std": 113.46513366699219, "step": 563 }, { "KL/chosen_KL_mean": -189.71180725097656, "KL/mean": -228.94329833984375, "KL/rejected_KL_mean": -268.1748046875, "KL/std": 106.10342407226562, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.004243284929543734, "fcm_dpo/delta": 0.06912320107221603, "fcm_dpo/margin": 78.46298217773438, "fcm_dpo/q_t": 0.42120468616485596, "grad_norm": 11.98410415649414, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.375516414642334, "logits/rejected": -1.7997002601623535, "logps/chosen": -302.6722106933594, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -367.70281982421875, "loss": 1.1222, "margin_dpo/margin_mean": 78.4629898071289, "margin_dpo/margin_std": 96.65330505371094, "step": 564 }, { "KL/chosen_KL_mean": -184.39523315429688, "KL/mean": -233.93846130371094, "KL/rejected_KL_mean": -283.481689453125, "KL/std": 115.96296691894531, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.00422974769026041, "fcm_dpo/delta": -0.02002754807472229, "fcm_dpo/margin": 99.08645629882812, "fcm_dpo/q_t": 0.40298211574554443, "grad_norm": 9.827868461608887, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.6117150783538818, "logits/rejected": -1.0940276384353638, "logps/chosen": -279.76080322265625, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -375.6990661621094, "loss": 1.0774, "margin_dpo/margin_mean": 99.08645629882812, "margin_dpo/margin_std": 125.90777587890625, "step": 565 }, { "KL/chosen_KL_mean": -193.77210998535156, "KL/mean": -241.14736938476562, "KL/rejected_KL_mean": -288.52264404296875, "KL/std": 119.49641418457031, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004241024609655142, "fcm_dpo/delta": -0.002031531184911728, "fcm_dpo/margin": 94.75050354003906, "fcm_dpo/q_t": 0.405820369720459, "grad_norm": 8.476825714111328, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -2.1209261417388916, "logits/rejected": -1.6969926357269287, "logps/chosen": -292.74530029296875, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -399.3435363769531, "loss": 1.0848, "margin_dpo/margin_mean": 94.75050354003906, "margin_dpo/margin_std": 118.17759704589844, "step": 566 }, { "KL/chosen_KL_mean": -188.708740234375, "KL/mean": -230.3837890625, "KL/rejected_KL_mean": -272.058837890625, "KL/std": 107.45489501953125, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.004240104928612709, "fcm_dpo/delta": 0.04804380610585213, "fcm_dpo/margin": 83.35013580322266, "fcm_dpo/q_t": 0.41667065024375916, "grad_norm": 9.879344940185547, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.4661149978637695, "logits/rejected": -1.8080782890319824, "logps/chosen": -281.4123229980469, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -363.2831726074219, "loss": 1.1234, "margin_dpo/margin_mean": 83.35013580322266, "margin_dpo/margin_std": 114.15693664550781, "step": 567 }, { "KL/chosen_KL_mean": -177.64431762695312, "KL/mean": -234.65660095214844, "KL/rejected_KL_mean": -291.66888427734375, "KL/std": 113.32445526123047, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.004204676952213049, "fcm_dpo/delta": -0.0839148536324501, "fcm_dpo/margin": 114.02455139160156, "fcm_dpo/q_t": 0.38786250352859497, "grad_norm": 8.451993942260742, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.495766282081604, "logits/rejected": -1.3671430349349976, "logps/chosen": -250.77658081054688, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -386.62152099609375, "loss": 1.0179, "margin_dpo/margin_mean": 114.02455139160156, "margin_dpo/margin_std": 114.50048065185547, "step": 568 }, { "KL/chosen_KL_mean": -190.01319885253906, "KL/mean": -234.86807250976562, "KL/rejected_KL_mean": -279.7229309082031, "KL/std": 114.22376251220703, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004220103845000267, "fcm_dpo/delta": 0.022182654589414597, "fcm_dpo/margin": 89.70973205566406, "fcm_dpo/q_t": 0.4120173454284668, "grad_norm": 14.541197776794434, "learning_rate": 4.089328585837512e-08, "logits/chosen": -2.0217533111572266, "logits/rejected": -1.643317699432373, "logps/chosen": -279.23028564453125, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -366.13397216796875, "loss": 1.1235, "margin_dpo/margin_mean": 89.70973205566406, "margin_dpo/margin_std": 136.84793090820312, "step": 569 }, { "KL/chosen_KL_mean": -167.72897338867188, "KL/mean": -226.77548217773438, "KL/rejected_KL_mean": -285.822021484375, "KL/std": 108.75656127929688, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0041577573865652084, "fcm_dpo/delta": -0.09573477506637573, "fcm_dpo/margin": 118.0930404663086, "fcm_dpo/q_t": 0.3840533196926117, "grad_norm": 9.478748321533203, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.536208152770996, "logits/rejected": -1.868765115737915, "logps/chosen": -262.4917297363281, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -374.6689453125, "loss": 1.0062, "margin_dpo/margin_mean": 118.09303283691406, "margin_dpo/margin_std": 112.74952697753906, "step": 570 }, { "KL/chosen_KL_mean": -176.25250244140625, "KL/mean": -228.1181640625, "KL/rejected_KL_mean": -279.9838562011719, "KL/std": 107.37240600585938, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.004130367189645767, "fcm_dpo/delta": -0.029768578708171844, "fcm_dpo/margin": 103.73133850097656, "fcm_dpo/q_t": 0.4007578194141388, "grad_norm": 12.821880340576172, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7348816394805908, "logits/rejected": -1.5773625373840332, "logps/chosen": -257.07379150390625, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -376.9991455078125, "loss": 1.0721, "margin_dpo/margin_mean": 103.7313461303711, "margin_dpo/margin_std": 130.45559692382812, "step": 571 }, { "KL/chosen_KL_mean": -179.30978393554688, "KL/mean": -232.7744903564453, "KL/rejected_KL_mean": -286.23919677734375, "KL/std": 114.60835266113281, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0040854779072105885, "fcm_dpo/delta": -0.03862118721008301, "fcm_dpo/margin": 106.92941284179688, "fcm_dpo/q_t": 0.39918047189712524, "grad_norm": 8.538667678833008, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.6574701070785522, "logits/rejected": -1.4827027320861816, "logps/chosen": -256.689697265625, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -376.80517578125, "loss": 1.0558, "margin_dpo/margin_mean": 106.92941284179688, "margin_dpo/margin_std": 123.46943664550781, "step": 572 }, { "KL/chosen_KL_mean": -174.64370727539062, "KL/mean": -223.53176879882812, "KL/rejected_KL_mean": -272.41986083984375, "KL/std": 113.25565338134766, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.004105414263904095, "fcm_dpo/delta": -0.002169378101825714, "fcm_dpo/margin": 97.77615356445312, "fcm_dpo/q_t": 0.40578413009643555, "grad_norm": 8.71941089630127, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.206594228744507, "logits/rejected": -1.9811270236968994, "logps/chosen": -258.34979248046875, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -363.1173095703125, "loss": 1.085, "margin_dpo/margin_mean": 97.77615356445312, "margin_dpo/margin_std": 119.63339233398438, "step": 573 }, { "KL/chosen_KL_mean": -183.52645874023438, "KL/mean": -236.83056640625, "KL/rejected_KL_mean": -290.13470458984375, "KL/std": 112.23715209960938, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.004072434268891811, "fcm_dpo/delta": -0.035808220505714417, "fcm_dpo/margin": 106.60823059082031, "fcm_dpo/q_t": 0.3986174464225769, "grad_norm": 9.343003273010254, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.520824909210205, "logits/rejected": -2.2105178833007812, "logps/chosen": -268.65594482421875, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -389.188720703125, "loss": 1.0621, "margin_dpo/margin_mean": 106.60822296142578, "margin_dpo/margin_std": 126.27757263183594, "step": 574 }, { "KL/chosen_KL_mean": -192.8800048828125, "KL/mean": -241.285888671875, "KL/rejected_KL_mean": -289.6917724609375, "KL/std": 115.3101806640625, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.004054746590554714, "fcm_dpo/delta": 0.007757302839308977, "fcm_dpo/margin": 96.81177520751953, "fcm_dpo/q_t": 0.409275621175766, "grad_norm": 10.893427848815918, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.5410020351409912, "logits/rejected": -1.0779401063919067, "logps/chosen": -298.6508483886719, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -390.6412353515625, "loss": 1.103, "margin_dpo/margin_mean": 96.811767578125, "margin_dpo/margin_std": 135.85992431640625, "step": 575 }, { "KL/chosen_KL_mean": -188.8015899658203, "KL/mean": -235.95462036132812, "KL/rejected_KL_mean": -283.107666015625, "KL/std": 107.0743637084961, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.004074057564139366, "fcm_dpo/delta": 0.016349155455827713, "fcm_dpo/margin": 94.30607604980469, "fcm_dpo/q_t": 0.4102601110935211, "grad_norm": 10.506529808044434, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.043593406677246, "logits/rejected": -1.989152431488037, "logps/chosen": -270.006103515625, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -388.51708984375, "loss": 1.1038, "margin_dpo/margin_mean": 94.30609130859375, "margin_dpo/margin_std": 128.0098114013672, "step": 576 }, { "KL/chosen_KL_mean": -169.4656219482422, "KL/mean": -229.055419921875, "KL/rejected_KL_mean": -288.64520263671875, "KL/std": 115.49400329589844, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.004008019343018532, "fcm_dpo/delta": -0.08209189027547836, "fcm_dpo/margin": 119.17958068847656, "fcm_dpo/q_t": 0.3886150121688843, "grad_norm": 10.71567153930664, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.0056052207946777, "logits/rejected": -1.2994003295898438, "logps/chosen": -270.9298095703125, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -377.9715881347656, "loss": 1.032, "margin_dpo/margin_mean": 119.17958068847656, "margin_dpo/margin_std": 132.78355407714844, "step": 577 }, { "KL/chosen_KL_mean": -203.45555114746094, "KL/mean": -245.52447509765625, "KL/rejected_KL_mean": -287.5933837890625, "KL/std": 109.64051818847656, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.004041189793497324, "fcm_dpo/delta": 0.06210838258266449, "fcm_dpo/margin": 84.13783264160156, "fcm_dpo/q_t": 0.4207271337509155, "grad_norm": 12.33154010772705, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.4667751789093018, "logits/rejected": -1.228208303451538, "logps/chosen": -280.74798583984375, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -375.84039306640625, "loss": 1.1361, "margin_dpo/margin_mean": 84.13783264160156, "margin_dpo/margin_std": 125.31484985351562, "step": 578 }, { "KL/chosen_KL_mean": -181.93585205078125, "KL/mean": -225.53314208984375, "KL/rejected_KL_mean": -269.13043212890625, "KL/std": 110.27000427246094, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0040673259645700455, "fcm_dpo/delta": 0.04688463360071182, "fcm_dpo/margin": 87.194580078125, "fcm_dpo/q_t": 0.41760504245758057, "grad_norm": 8.789334297180176, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.640284538269043, "logits/rejected": -1.341177225112915, "logps/chosen": -269.2789001464844, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -371.2514343261719, "loss": 1.1184, "margin_dpo/margin_mean": 87.194580078125, "margin_dpo/margin_std": 114.00082397460938, "step": 579 }, { "KL/chosen_KL_mean": -174.81494140625, "KL/mean": -229.07266235351562, "KL/rejected_KL_mean": -283.33038330078125, "KL/std": 109.96595764160156, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.004051432013511658, "fcm_dpo/delta": -0.041960835456848145, "fcm_dpo/margin": 108.51542663574219, "fcm_dpo/q_t": 0.39895501732826233, "grad_norm": 10.137391090393066, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.7137794494628906, "logits/rejected": -1.405045747756958, "logps/chosen": -263.7437438964844, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -372.0270080566406, "loss": 1.0624, "margin_dpo/margin_mean": 108.51542663574219, "margin_dpo/margin_std": 132.77352905273438, "step": 580 }, { "KL/chosen_KL_mean": -183.92608642578125, "KL/mean": -232.64208984375, "KL/rejected_KL_mean": -281.35809326171875, "KL/std": 110.395263671875, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004064774140715599, "fcm_dpo/delta": 0.004107022657990456, "fcm_dpo/margin": 97.4320068359375, "fcm_dpo/q_t": 0.4079989194869995, "grad_norm": 10.978434562683105, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.0431127548217773, "logits/rejected": -1.6960697174072266, "logps/chosen": -276.1493225097656, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -380.7475280761719, "loss": 1.1034, "margin_dpo/margin_mean": 97.4320068359375, "margin_dpo/margin_std": 137.46701049804688, "step": 581 }, { "KL/chosen_KL_mean": -162.09918212890625, "KL/mean": -206.79100036621094, "KL/rejected_KL_mean": -251.48281860351562, "KL/std": 115.68901062011719, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.004036447964608669, "fcm_dpo/delta": -0.06399291753768921, "fcm_dpo/margin": 89.38363647460938, "fcm_dpo/q_t": 0.41700369119644165, "grad_norm": 10.023308753967285, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.395697593688965, "logits/rejected": -2.3870773315429688, "logps/chosen": -249.43479919433594, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -366.6372375488281, "loss": 1.1258, "margin_dpo/margin_mean": 89.38363647460938, "margin_dpo/margin_std": 127.701904296875, "step": 582 }, { "KL/chosen_KL_mean": -193.9169921875, "KL/mean": -243.56077575683594, "KL/rejected_KL_mean": -293.20452880859375, "KL/std": 109.24276733398438, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.003996574319899082, "fcm_dpo/delta": 0.00301278755068779, "fcm_dpo/margin": 99.28753662109375, "fcm_dpo/q_t": 0.40823161602020264, "grad_norm": 9.612485885620117, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4049155712127686, "logits/rejected": -1.1623666286468506, "logps/chosen": -277.6340637207031, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -387.7891845703125, "loss": 1.0923, "margin_dpo/margin_mean": 99.28753662109375, "margin_dpo/margin_std": 129.1490020751953, "step": 583 }, { "KL/chosen_KL_mean": -173.615966796875, "KL/mean": -226.0709228515625, "KL/rejected_KL_mean": -278.5259094238281, "KL/std": 113.8797836303711, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.004025900270789862, "fcm_dpo/delta": -0.024113763123750687, "fcm_dpo/margin": 104.90992736816406, "fcm_dpo/q_t": 0.4013599157333374, "grad_norm": 8.879335403442383, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9192804098129272, "logits/rejected": -1.4578561782836914, "logps/chosen": -267.87554931640625, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -383.00860595703125, "loss": 1.0774, "margin_dpo/margin_mean": 104.90992736816406, "margin_dpo/margin_std": 129.94912719726562, "step": 584 }, { "KL/chosen_KL_mean": -185.697509765625, "KL/mean": -244.35079956054688, "KL/rejected_KL_mean": -303.00408935546875, "KL/std": 129.81309509277344, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.003963273484259844, "fcm_dpo/delta": -0.06803968548774719, "fcm_dpo/margin": 117.30657958984375, "fcm_dpo/q_t": 0.3939836621284485, "grad_norm": 9.19937801361084, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.839807391166687, "logits/rejected": -1.5561017990112305, "logps/chosen": -271.33428955078125, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -393.28955078125, "loss": 1.0586, "margin_dpo/margin_mean": 117.30657958984375, "margin_dpo/margin_std": 151.78761291503906, "step": 585 }, { "KL/chosen_KL_mean": -204.75018310546875, "KL/mean": -244.48345947265625, "KL/rejected_KL_mean": -284.2167663574219, "KL/std": 111.50582122802734, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.003989547491073608, "fcm_dpo/delta": 0.08569972217082977, "fcm_dpo/margin": 79.46658325195312, "fcm_dpo/q_t": 0.4252578914165497, "grad_norm": 10.594883918762207, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.9271612167358398, "logits/rejected": -1.6638318300247192, "logps/chosen": -288.43133544921875, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -382.575927734375, "loss": 1.1386, "margin_dpo/margin_mean": 79.46658325195312, "margin_dpo/margin_std": 106.62471771240234, "step": 586 }, { "KL/chosen_KL_mean": -164.82427978515625, "KL/mean": -208.59393310546875, "KL/rejected_KL_mean": -252.36358642578125, "KL/std": 110.41917419433594, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.004041352309286594, "fcm_dpo/delta": 0.0478321798145771, "fcm_dpo/margin": 87.53933715820312, "fcm_dpo/q_t": 0.4172610342502594, "grad_norm": 9.208721160888672, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.458909511566162, "logits/rejected": -2.172611951828003, "logps/chosen": -242.4017791748047, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -341.6839294433594, "loss": 1.1146, "margin_dpo/margin_mean": 87.53933715820312, "margin_dpo/margin_std": 114.77027130126953, "step": 587 }, { "KL/chosen_KL_mean": -180.7640838623047, "KL/mean": -221.15957641601562, "KL/rejected_KL_mean": -261.5550537109375, "KL/std": 111.83136749267578, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004082599654793739, "fcm_dpo/delta": 0.0726204365491867, "fcm_dpo/margin": 80.79093933105469, "fcm_dpo/q_t": 0.4248579442501068, "grad_norm": 13.712924003601074, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.8691706657409668, "logits/rejected": -1.8332127332687378, "logps/chosen": -264.568359375, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -374.6744689941406, "loss": 1.1559, "margin_dpo/margin_mean": 80.79093933105469, "margin_dpo/margin_std": 135.17100524902344, "step": 588 }, { "KL/chosen_KL_mean": -193.25018310546875, "KL/mean": -246.08694458007812, "KL/rejected_KL_mean": -298.9237060546875, "KL/std": 109.84137725830078, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004086260683834553, "fcm_dpo/delta": -0.033238545060157776, "fcm_dpo/margin": 105.67351531982422, "fcm_dpo/q_t": 0.39795875549316406, "grad_norm": 8.546754837036133, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.7471801042556763, "logits/rejected": -1.355210304260254, "logps/chosen": -289.7152099609375, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -414.96087646484375, "loss": 1.0455, "margin_dpo/margin_mean": 105.67350769042969, "margin_dpo/margin_std": 106.55941009521484, "step": 589 }, { "KL/chosen_KL_mean": -177.5114288330078, "KL/mean": -230.699951171875, "KL/rejected_KL_mean": -283.88848876953125, "KL/std": 102.91893005371094, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004072621930390596, "fcm_dpo/delta": -0.03489726409316063, "fcm_dpo/margin": 106.37704467773438, "fcm_dpo/q_t": 0.39842212200164795, "grad_norm": 13.065176963806152, "learning_rate": 2.736501028272095e-08, "logits/chosen": -2.1156227588653564, "logits/rejected": -1.9748711585998535, "logps/chosen": -265.93927001953125, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -394.08721923828125, "loss": 1.0455, "margin_dpo/margin_mean": 106.37704467773438, "margin_dpo/margin_std": 107.93649291992188, "step": 590 }, { "KL/chosen_KL_mean": -170.37203979492188, "KL/mean": -226.2574462890625, "KL/rejected_KL_mean": -282.1428527832031, "KL/std": 110.25895690917969, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.004012429621070623, "fcm_dpo/delta": -0.05089985579252243, "fcm_dpo/margin": 111.77080535888672, "fcm_dpo/q_t": 0.39408475160598755, "grad_norm": 7.287550449371338, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9527333974838257, "logits/rejected": -1.3316266536712646, "logps/chosen": -258.22174072265625, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -371.7220458984375, "loss": 1.0309, "margin_dpo/margin_mean": 111.77081298828125, "margin_dpo/margin_std": 106.5495834350586, "step": 591 }, { "KL/chosen_KL_mean": -202.34869384765625, "KL/mean": -245.450927734375, "KL/rejected_KL_mean": -288.55316162109375, "KL/std": 106.58428955078125, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004042393993586302, "fcm_dpo/delta": 0.05336785316467285, "fcm_dpo/margin": 86.20445251464844, "fcm_dpo/q_t": 0.4189147353172302, "grad_norm": 11.773332595825195, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.9155915975570679, "logits/rejected": -1.5887186527252197, "logps/chosen": -277.3244323730469, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -370.10577392578125, "loss": 1.1396, "margin_dpo/margin_mean": 86.20445251464844, "margin_dpo/margin_std": 135.76019287109375, "step": 592 }, { "KL/chosen_KL_mean": -184.27528381347656, "KL/mean": -220.46759033203125, "KL/rejected_KL_mean": -256.659912109375, "KL/std": 111.53401184082031, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004105973988771439, "fcm_dpo/delta": 0.10607777535915375, "fcm_dpo/margin": 72.38459777832031, "fcm_dpo/q_t": 0.4315840005874634, "grad_norm": 14.052986145019531, "learning_rate": 2.564009866938349e-08, "logits/chosen": -1.0323822498321533, "logits/rejected": -0.7319837212562561, "logps/chosen": -256.5143127441406, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -326.75640869140625, "loss": 1.1743, "margin_dpo/margin_mean": 72.38459777832031, "margin_dpo/margin_std": 124.56719970703125, "step": 593 }, { "KL/chosen_KL_mean": -164.63681030273438, "KL/mean": -221.2125244140625, "KL/rejected_KL_mean": -277.7882385253906, "KL/std": 107.58360290527344, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.004112704657018185, "fcm_dpo/delta": -0.06858627498149872, "fcm_dpo/margin": 113.15145111083984, "fcm_dpo/q_t": 0.3904545307159424, "grad_norm": 9.389117240905762, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9529743194580078, "logits/rejected": -1.419319748878479, "logps/chosen": -255.07452392578125, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -370.0558166503906, "loss": 1.0208, "margin_dpo/margin_mean": 113.15144348144531, "margin_dpo/margin_std": 106.26850891113281, "step": 594 }, { "KL/chosen_KL_mean": -179.94378662109375, "KL/mean": -226.30545043945312, "KL/rejected_KL_mean": -272.6671142578125, "KL/std": 108.42716979980469, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.00410500168800354, "fcm_dpo/delta": 0.019957292824983597, "fcm_dpo/margin": 92.72335815429688, "fcm_dpo/q_t": 0.412435382604599, "grad_norm": 10.090431213378906, "learning_rate": 2.451969280180849e-08, "logits/chosen": -2.0598249435424805, "logits/rejected": -1.6654869318008423, "logps/chosen": -261.4585876464844, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -359.1876220703125, "loss": 1.1064, "margin_dpo/margin_mean": 92.72335815429688, "margin_dpo/margin_std": 127.38136291503906, "step": 595 }, { "KL/chosen_KL_mean": -184.9493865966797, "KL/mean": -226.26556396484375, "KL/rejected_KL_mean": -267.58172607421875, "KL/std": 108.11846923828125, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.00413933489471674, "fcm_dpo/delta": 0.05994009971618652, "fcm_dpo/margin": 82.6323471069336, "fcm_dpo/q_t": 0.4216810464859009, "grad_norm": 12.351500511169434, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.607701301574707, "logits/rejected": -1.0172823667526245, "logps/chosen": -273.1574401855469, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -353.67547607421875, "loss": 1.1499, "margin_dpo/margin_mean": 82.63233947753906, "margin_dpo/margin_std": 137.04678344726562, "step": 596 }, { "KL/chosen_KL_mean": -179.78782653808594, "KL/mean": -234.74386596679688, "KL/rejected_KL_mean": -289.6999206542969, "KL/std": 116.84693908691406, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.004105809144675732, "fcm_dpo/delta": -0.053928203880786896, "fcm_dpo/margin": 109.9120864868164, "fcm_dpo/q_t": 0.39556410908699036, "grad_norm": 8.498239517211914, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.372464656829834, "logits/rejected": -1.2551164627075195, "logps/chosen": -263.3778076171875, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -396.6443176269531, "loss": 1.049, "margin_dpo/margin_mean": 109.9120864868164, "margin_dpo/margin_std": 127.33842468261719, "step": 597 }, { "KL/chosen_KL_mean": -172.37979125976562, "KL/mean": -218.36749267578125, "KL/rejected_KL_mean": -264.355224609375, "KL/std": 108.08756256103516, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.004102311562746763, "fcm_dpo/delta": 0.02336534485220909, "fcm_dpo/margin": 91.97543334960938, "fcm_dpo/q_t": 0.41108351945877075, "grad_norm": 9.12246036529541, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.8903148174285889, "logits/rejected": -1.5724514722824097, "logps/chosen": -262.5624084472656, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -359.15380859375, "loss": 1.1061, "margin_dpo/margin_mean": 91.97543334960938, "margin_dpo/margin_std": 121.35367584228516, "step": 598 }, { "KL/chosen_KL_mean": -172.87823486328125, "KL/mean": -215.1025390625, "KL/rejected_KL_mean": -257.3268737792969, "KL/std": 112.94737243652344, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0041346475481987, "fcm_dpo/delta": 0.05242312699556351, "fcm_dpo/margin": 84.44866943359375, "fcm_dpo/q_t": 0.41838622093200684, "grad_norm": 9.776275634765625, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.6142070293426514, "logits/rejected": -1.489001989364624, "logps/chosen": -250.72048950195312, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -346.5245056152344, "loss": 1.1316, "margin_dpo/margin_mean": 84.44866943359375, "margin_dpo/margin_std": 122.41178894042969, "step": 599 }, { "KL/chosen_KL_mean": -194.56576538085938, "KL/mean": -241.29513549804688, "KL/rejected_KL_mean": -288.0245361328125, "KL/std": 106.9513168334961, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004176343325525522, "fcm_dpo/delta": 0.010053128004074097, "fcm_dpo/margin": 93.45875549316406, "fcm_dpo/q_t": 0.40920281410217285, "grad_norm": 8.599320411682129, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.4492430686950684, "logits/rejected": -2.1027684211730957, "logps/chosen": -286.7535705566406, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -380.97882080078125, "loss": 1.0903, "margin_dpo/margin_mean": 93.45875549316406, "margin_dpo/margin_std": 116.01634216308594, "step": 600 }, { "KL/chosen_KL_mean": -198.09544372558594, "KL/mean": -247.28463745117188, "KL/rejected_KL_mean": -296.47381591796875, "KL/std": 108.16612243652344, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0041562458500266075, "fcm_dpo/delta": -0.009482888504862785, "fcm_dpo/margin": 98.37838745117188, "fcm_dpo/q_t": 0.40386176109313965, "grad_norm": 8.867517471313477, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.911363124847412, "logits/rejected": -1.5914725065231323, "logps/chosen": -287.31158447265625, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -396.6443786621094, "loss": 1.0639, "margin_dpo/margin_mean": 98.37838745117188, "margin_dpo/margin_std": 104.06755828857422, "step": 601 }, { "KL/chosen_KL_mean": -155.97250366210938, "KL/mean": -219.24282836914062, "KL/rejected_KL_mean": -282.51318359375, "KL/std": 117.71324157714844, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.004092435352504253, "fcm_dpo/delta": -0.12424849718809128, "fcm_dpo/margin": 126.54065704345703, "fcm_dpo/q_t": 0.38046348094940186, "grad_norm": 8.37907886505127, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.2150826454162598, "logits/rejected": -1.954132080078125, "logps/chosen": -236.0301055908203, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -375.710693359375, "loss": 1.0041, "margin_dpo/margin_mean": 126.5406494140625, "margin_dpo/margin_std": 135.1009979248047, "step": 602 }, { "KL/chosen_KL_mean": -165.99282836914062, "KL/mean": -219.2417755126953, "KL/rejected_KL_mean": -272.49072265625, "KL/std": 106.90812683105469, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.004049020819365978, "fcm_dpo/delta": -0.03260667994618416, "fcm_dpo/margin": 106.4979248046875, "fcm_dpo/q_t": 0.39958661794662476, "grad_norm": 8.262897491455078, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1421027183532715, "logits/rejected": -1.55470609664917, "logps/chosen": -268.3023986816406, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -380.5595703125, "loss": 1.0586, "margin_dpo/margin_mean": 106.4979248046875, "margin_dpo/margin_std": 123.26119995117188, "step": 603 }, { "KL/chosen_KL_mean": -173.67581176757812, "KL/mean": -228.51123046875, "KL/rejected_KL_mean": -283.3466796875, "KL/std": 112.52799987792969, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0040023913607001305, "fcm_dpo/delta": -0.04101036116480827, "fcm_dpo/margin": 109.67085266113281, "fcm_dpo/q_t": 0.3957962095737457, "grad_norm": 8.259285926818848, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.8294146060943604, "logits/rejected": -1.6799557209014893, "logps/chosen": -251.84988403320312, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -382.8427734375, "loss": 1.0464, "margin_dpo/margin_mean": 109.67085266113281, "margin_dpo/margin_std": 115.56230163574219, "step": 604 }, { "KL/chosen_KL_mean": -188.81961059570312, "KL/mean": -234.1514892578125, "KL/rejected_KL_mean": -279.4833679199219, "KL/std": 99.96769714355469, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004026109352707863, "fcm_dpo/delta": 0.036309871822595596, "fcm_dpo/margin": 90.66377258300781, "fcm_dpo/q_t": 0.4145576059818268, "grad_norm": 9.728354454040527, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.9479005336761475, "logits/rejected": -1.4563846588134766, "logps/chosen": -283.5929260253906, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -386.78826904296875, "loss": 1.1019, "margin_dpo/margin_mean": 90.66377258300781, "margin_dpo/margin_std": 111.20294189453125, "step": 605 }, { "KL/chosen_KL_mean": -182.92303466796875, "KL/mean": -229.58848571777344, "KL/rejected_KL_mean": -276.25390625, "KL/std": 110.25325775146484, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004068125505000353, "fcm_dpo/delta": 0.02044859156012535, "fcm_dpo/margin": 93.33086395263672, "fcm_dpo/q_t": 0.4105270206928253, "grad_norm": 8.441485404968262, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -2.007131338119507, "logits/rejected": -1.6282564401626587, "logps/chosen": -270.67645263671875, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -380.4961242675781, "loss": 1.0994, "margin_dpo/margin_mean": 93.33086395263672, "margin_dpo/margin_std": 117.64743041992188, "step": 606 }, { "KL/chosen_KL_mean": -176.15921020507812, "KL/mean": -229.25767517089844, "KL/rejected_KL_mean": -282.35614013671875, "KL/std": 117.61363220214844, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.004034915007650852, "fcm_dpo/delta": -0.029761768877506256, "fcm_dpo/margin": 106.1968994140625, "fcm_dpo/q_t": 0.40105605125427246, "grad_norm": 10.403225898742676, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7476400136947632, "logits/rejected": -1.209754228591919, "logps/chosen": -264.4882507324219, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -369.12420654296875, "loss": 1.0827, "margin_dpo/margin_mean": 106.1968994140625, "margin_dpo/margin_std": 144.11721801757812, "step": 607 }, { "KL/chosen_KL_mean": -161.58456420898438, "KL/mean": -216.97747802734375, "KL/rejected_KL_mean": -272.37042236328125, "KL/std": 115.47309875488281, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.003977589774876833, "fcm_dpo/delta": -0.043492261320352554, "fcm_dpo/margin": 110.78582763671875, "fcm_dpo/q_t": 0.39591413736343384, "grad_norm": 8.927714347839355, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.863581895828247, "logits/rejected": -1.5609511137008667, "logps/chosen": -247.35394287109375, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -372.60321044921875, "loss": 1.0553, "margin_dpo/margin_mean": 110.78582763671875, "margin_dpo/margin_std": 123.92117309570312, "step": 608 }, { "KL/chosen_KL_mean": -180.017333984375, "KL/mean": -231.4561767578125, "KL/rejected_KL_mean": -282.89501953125, "KL/std": 116.58786010742188, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.003970222547650337, "fcm_dpo/delta": -0.009191669523715973, "fcm_dpo/margin": 102.87765502929688, "fcm_dpo/q_t": 0.4052579700946808, "grad_norm": 10.313264846801758, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.4407684803009033, "logits/rejected": -2.0321435928344727, "logps/chosen": -272.9839172363281, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -378.81317138671875, "loss": 1.08, "margin_dpo/margin_mean": 102.87765502929688, "margin_dpo/margin_std": 122.93418884277344, "step": 609 }, { "KL/chosen_KL_mean": -164.86219787597656, "KL/mean": -220.55718994140625, "KL/rejected_KL_mean": -276.252197265625, "KL/std": 110.83695983886719, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.003951625898480415, "fcm_dpo/delta": -0.042194806039333344, "fcm_dpo/margin": 111.38999938964844, "fcm_dpo/q_t": 0.395657479763031, "grad_norm": 8.365357398986816, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.8999133110046387, "logits/rejected": -1.363194465637207, "logps/chosen": -259.5625, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -365.9395751953125, "loss": 1.0441, "margin_dpo/margin_mean": 111.38999938964844, "margin_dpo/margin_std": 115.97005462646484, "step": 610 }, { "KL/chosen_KL_mean": -174.7987518310547, "KL/mean": -228.76992797851562, "KL/rejected_KL_mean": -282.7410888671875, "KL/std": 115.02599334716797, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.00395344290882349, "fcm_dpo/delta": -0.02826276421546936, "fcm_dpo/margin": 107.94233703613281, "fcm_dpo/q_t": 0.40053707361221313, "grad_norm": 11.092637062072754, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.2929861545562744, "logits/rejected": -1.9596469402313232, "logps/chosen": -261.58209228515625, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -372.58416748046875, "loss": 1.0577, "margin_dpo/margin_mean": 107.94232940673828, "margin_dpo/margin_std": 117.93943786621094, "step": 611 }, { "KL/chosen_KL_mean": -191.0244140625, "KL/mean": -247.26327514648438, "KL/rejected_KL_mean": -303.50213623046875, "KL/std": 109.7452163696289, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0039003510028123856, "fcm_dpo/delta": -0.04056285694241524, "fcm_dpo/margin": 112.47770690917969, "fcm_dpo/q_t": 0.39674028754234314, "grad_norm": 9.567357063293457, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.233349084854126, "logits/rejected": -1.4018044471740723, "logps/chosen": -292.0445556640625, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -397.28515625, "loss": 1.0422, "margin_dpo/margin_mean": 112.47769927978516, "margin_dpo/margin_std": 114.98200988769531, "step": 612 }, { "KL/chosen_KL_mean": -154.35121154785156, "KL/mean": -216.7435302734375, "KL/rejected_KL_mean": -279.1358337402344, "KL/std": 111.03958129882812, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0038442490622401237, "fcm_dpo/delta": -0.08385127037763596, "fcm_dpo/margin": 124.78461456298828, "fcm_dpo/q_t": 0.38628652691841125, "grad_norm": 7.95382833480835, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.9761018753051758, "logits/rejected": -1.7813019752502441, "logps/chosen": -243.33984375, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -389.13134765625, "loss": 1.0036, "margin_dpo/margin_mean": 124.78462219238281, "margin_dpo/margin_std": 108.30632019042969, "step": 613 }, { "KL/chosen_KL_mean": -193.02383422851562, "KL/mean": -243.55471801757812, "KL/rejected_KL_mean": -294.0855712890625, "KL/std": 105.59566497802734, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0038365358486771584, "fcm_dpo/delta": 0.01276468113064766, "fcm_dpo/margin": 101.06173706054688, "fcm_dpo/q_t": 0.40904563665390015, "grad_norm": 9.698623657226562, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.4823282957077026, "logits/rejected": -1.475990891456604, "logps/chosen": -273.223876953125, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -403.9479675292969, "loss": 1.0972, "margin_dpo/margin_mean": 101.06173706054688, "margin_dpo/margin_std": 131.5426483154297, "step": 614 }, { "KL/chosen_KL_mean": -180.7034912109375, "KL/mean": -222.38426208496094, "KL/rejected_KL_mean": -264.0650329589844, "KL/std": 107.35983276367188, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0038346643559634686, "fcm_dpo/delta": -0.019658269360661507, "fcm_dpo/margin": 83.36151885986328, "fcm_dpo/q_t": 0.4235570728778839, "grad_norm": 9.126237869262695, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.8665204048156738, "logits/rejected": -1.473127007484436, "logps/chosen": -281.1387634277344, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -365.2450866699219, "loss": 1.1316, "margin_dpo/margin_mean": 83.36151885986328, "margin_dpo/margin_std": 100.91619110107422, "step": 615 }, { "KL/chosen_KL_mean": -184.83575439453125, "KL/mean": -240.40695190429688, "KL/rejected_KL_mean": -295.9781188964844, "KL/std": 106.6855697631836, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0038266004994511604, "fcm_dpo/delta": -0.026616863906383514, "fcm_dpo/margin": 111.14237213134766, "fcm_dpo/q_t": 0.4004327952861786, "grad_norm": 8.612765312194824, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.891554355621338, "logits/rejected": -1.3936419486999512, "logps/chosen": -277.32867431640625, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -378.03875732421875, "loss": 1.0624, "margin_dpo/margin_mean": 111.14236450195312, "margin_dpo/margin_std": 127.1973876953125, "step": 616 }, { "KL/chosen_KL_mean": -190.70895385742188, "KL/mean": -235.07211303710938, "KL/rejected_KL_mean": -279.435302734375, "KL/std": 103.89315795898438, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0038407244719564915, "fcm_dpo/delta": 0.06133158504962921, "fcm_dpo/margin": 88.72637939453125, "fcm_dpo/q_t": 0.41962820291519165, "grad_norm": 9.418571472167969, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.7256907224655151, "logits/rejected": -1.091673731803894, "logps/chosen": -281.8059387207031, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -349.8453674316406, "loss": 1.1237, "margin_dpo/margin_mean": 88.72637939453125, "margin_dpo/margin_std": 117.33831787109375, "step": 617 }, { "KL/chosen_KL_mean": -192.83261108398438, "KL/mean": -242.9971923828125, "KL/rejected_KL_mean": -293.1617431640625, "KL/std": 115.65933227539062, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.003856419585645199, "fcm_dpo/delta": 0.01359693892300129, "fcm_dpo/margin": 100.32913208007812, "fcm_dpo/q_t": 0.4085526466369629, "grad_norm": 8.299123764038086, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.9794960021972656, "logits/rejected": -1.6083101034164429, "logps/chosen": -294.85321044921875, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -392.96295166015625, "loss": 1.0956, "margin_dpo/margin_mean": 100.3291244506836, "margin_dpo/margin_std": 128.28640747070312, "step": 618 }, { "KL/chosen_KL_mean": -185.37481689453125, "KL/mean": -230.65933227539062, "KL/rejected_KL_mean": -275.94384765625, "KL/std": 113.65373992919922, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.003890527877956629, "fcm_dpo/delta": 0.049400992691516876, "fcm_dpo/margin": 90.56901550292969, "fcm_dpo/q_t": 0.419327974319458, "grad_norm": 8.495819091796875, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.584770917892456, "logits/rejected": -1.2194254398345947, "logps/chosen": -275.1161804199219, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -375.84521484375, "loss": 1.1296, "margin_dpo/margin_mean": 90.56902313232422, "margin_dpo/margin_std": 135.59197998046875, "step": 619 }, { "KL/chosen_KL_mean": -177.029296875, "KL/mean": -227.11651611328125, "KL/rejected_KL_mean": -277.2037353515625, "KL/std": 119.36036682128906, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0038387158419936895, "fcm_dpo/delta": -0.08259420841932297, "fcm_dpo/margin": 100.17442321777344, "fcm_dpo/q_t": 0.4094107151031494, "grad_norm": 8.985579490661621, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.2178003787994385, "logits/rejected": -1.6842093467712402, "logps/chosen": -262.1535949707031, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -362.61627197265625, "loss": 1.1025, "margin_dpo/margin_mean": 100.17443084716797, "margin_dpo/margin_std": 127.7076187133789, "step": 620 }, { "KL/chosen_KL_mean": -182.30662536621094, "KL/mean": -230.0196533203125, "KL/rejected_KL_mean": -277.73272705078125, "KL/std": 114.39295959472656, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.003814116818830371, "fcm_dpo/delta": -0.06449554860591888, "fcm_dpo/margin": 95.42610931396484, "fcm_dpo/q_t": 0.4148963391780853, "grad_norm": 8.249038696289062, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -2.1150190830230713, "logits/rejected": -1.8575360774993896, "logps/chosen": -271.5550537109375, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -373.1973571777344, "loss": 1.1256, "margin_dpo/margin_mean": 95.42610931396484, "margin_dpo/margin_std": 135.45458984375, "step": 621 }, { "KL/chosen_KL_mean": -178.2733154296875, "KL/mean": -230.05718994140625, "KL/rejected_KL_mean": -281.8410949707031, "KL/std": 102.78477478027344, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0037874968256801367, "fcm_dpo/delta": 0.00800613034516573, "fcm_dpo/margin": 103.56777954101562, "fcm_dpo/q_t": 0.4071420431137085, "grad_norm": 9.912426948547363, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.6918493509292603, "logits/rejected": -1.0490416288375854, "logps/chosen": -278.06744384765625, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -372.6693115234375, "loss": 1.0752, "margin_dpo/margin_mean": 103.5677719116211, "margin_dpo/margin_std": 111.64562225341797, "step": 622 }, { "KL/chosen_KL_mean": -185.13986206054688, "KL/mean": -221.69297790527344, "KL/rejected_KL_mean": -258.24609375, "KL/std": 111.67076110839844, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0038553238846361637, "fcm_dpo/delta": 0.12175789475440979, "fcm_dpo/margin": 73.10620880126953, "fcm_dpo/q_t": 0.4353444576263428, "grad_norm": 9.366002082824707, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.3189127445220947, "logits/rejected": -1.8803666830062866, "logps/chosen": -275.5954284667969, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -349.56884765625, "loss": 1.1926, "margin_dpo/margin_mean": 73.10621643066406, "margin_dpo/margin_std": 137.8582763671875, "step": 623 }, { "KL/chosen_KL_mean": -194.68923950195312, "KL/mean": -252.433837890625, "KL/rejected_KL_mean": -310.1784362792969, "KL/std": 110.85626983642578, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0038621448911726475, "fcm_dpo/delta": -0.04817651957273483, "fcm_dpo/margin": 115.48921203613281, "fcm_dpo/q_t": 0.3958989381790161, "grad_norm": 9.54487133026123, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.5515079498291016, "logits/rejected": -1.629817008972168, "logps/chosen": -304.564453125, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -414.95166015625, "loss": 1.0477, "margin_dpo/margin_mean": 115.48921203613281, "margin_dpo/margin_std": 129.01551818847656, "step": 624 }, { "KL/chosen_KL_mean": -199.49636840820312, "KL/mean": -240.47018432617188, "KL/rejected_KL_mean": -281.44403076171875, "KL/std": 108.75747680664062, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0039050844497978687, "fcm_dpo/delta": 0.08247450739145279, "fcm_dpo/margin": 81.94761657714844, "fcm_dpo/q_t": 0.42522335052490234, "grad_norm": 9.0634183883667, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.8231406211853027, "logits/rejected": -1.4502251148223877, "logps/chosen": -286.66455078125, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -373.30548095703125, "loss": 1.1409, "margin_dpo/margin_mean": 81.94761657714844, "margin_dpo/margin_std": 116.41514587402344, "step": 625 }, { "KL/chosen_KL_mean": -188.19851684570312, "KL/mean": -230.0714569091797, "KL/rejected_KL_mean": -271.94439697265625, "KL/std": 121.47804260253906, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0039656637236475945, "fcm_dpo/delta": 0.06997434049844742, "fcm_dpo/margin": 83.7458724975586, "fcm_dpo/q_t": 0.42354124784469604, "grad_norm": 11.721783638000488, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.3157954216003418, "logits/rejected": -0.8656997680664062, "logps/chosen": -292.4227294921875, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -376.1218566894531, "loss": 1.1468, "margin_dpo/margin_mean": 83.7458724975586, "margin_dpo/margin_std": 132.2644500732422, "step": 626 }, { "KL/chosen_KL_mean": -168.0554656982422, "KL/mean": -220.60317993164062, "KL/rejected_KL_mean": -273.15093994140625, "KL/std": 123.74507141113281, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.00396343506872654, "fcm_dpo/delta": -0.017255175858736038, "fcm_dpo/margin": 105.095458984375, "fcm_dpo/q_t": 0.4047045111656189, "grad_norm": 7.592726230621338, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.6768958568572998, "logits/rejected": -1.8258049488067627, "logps/chosen": -242.99008178710938, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -385.72381591796875, "loss": 1.0813, "margin_dpo/margin_mean": 105.095458984375, "margin_dpo/margin_std": 137.81634521484375, "step": 627 }, { "KL/chosen_KL_mean": -178.6898193359375, "KL/mean": -232.95144653320312, "KL/rejected_KL_mean": -287.21307373046875, "KL/std": 112.09091186523438, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.003941023722290993, "fcm_dpo/delta": -0.0289263054728508, "fcm_dpo/margin": 108.52330017089844, "fcm_dpo/q_t": 0.4002716541290283, "grad_norm": 9.484331130981445, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.0296919345855713, "logits/rejected": -1.7697314023971558, "logps/chosen": -272.3808898925781, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -394.5570373535156, "loss": 1.0619, "margin_dpo/margin_mean": 108.52330017089844, "margin_dpo/margin_std": 125.49978637695312, "step": 628 }, { "KL/chosen_KL_mean": -191.25076293945312, "KL/mean": -240.8883819580078, "KL/rejected_KL_mean": -290.5260314941406, "KL/std": 111.755859375, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.003927278332412243, "fcm_dpo/delta": 0.010326949879527092, "fcm_dpo/margin": 99.27525329589844, "fcm_dpo/q_t": 0.4078764319419861, "grad_norm": 9.442082405090332, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.1486945152282715, "logits/rejected": -1.7155827283859253, "logps/chosen": -294.48114013671875, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -387.6944580078125, "loss": 1.0883, "margin_dpo/margin_mean": 99.27525329589844, "margin_dpo/margin_std": 119.87527465820312, "step": 629 }, { "KL/chosen_KL_mean": -173.87506103515625, "KL/mean": -205.5396728515625, "KL/rejected_KL_mean": -237.20431518554688, "KL/std": 103.52656555175781, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.003960174508392811, "fcm_dpo/delta": 0.04403216391801834, "fcm_dpo/margin": 63.329254150390625, "fcm_dpo/q_t": 0.4405936896800995, "grad_norm": 10.985085487365723, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.1335201263427734, "logits/rejected": -1.6137484312057495, "logps/chosen": -267.7725830078125, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -326.5787048339844, "loss": 1.2039, "margin_dpo/margin_mean": 63.329254150390625, "margin_dpo/margin_std": 116.61585235595703, "step": 630 }, { "KL/chosen_KL_mean": -169.75119018554688, "KL/mean": -218.516357421875, "KL/rejected_KL_mean": -267.2815246582031, "KL/std": 104.50186157226562, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.003981543239206076, "fcm_dpo/delta": 0.012146728113293648, "fcm_dpo/margin": 97.53034973144531, "fcm_dpo/q_t": 0.40854722261428833, "grad_norm": 7.87180233001709, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.616711139678955, "logits/rejected": -2.1954004764556885, "logps/chosen": -257.5220031738281, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -355.96392822265625, "loss": 1.0894, "margin_dpo/margin_mean": 97.53034973144531, "margin_dpo/margin_std": 119.45866394042969, "step": 631 }, { "KL/chosen_KL_mean": -168.02880859375, "KL/mean": -219.97882080078125, "KL/rejected_KL_mean": -271.9288330078125, "KL/std": 108.82746124267578, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.003961225505918264, "fcm_dpo/delta": -0.012576870620250702, "fcm_dpo/margin": 103.9000015258789, "fcm_dpo/q_t": 0.4044049382209778, "grad_norm": 11.642831802368164, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.9771146774291992, "logits/rejected": -1.539427399635315, "logps/chosen": -256.65533447265625, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -363.3797302246094, "loss": 1.0857, "margin_dpo/margin_mean": 103.9000015258789, "margin_dpo/margin_std": 134.85719299316406, "step": 632 }, { "KL/chosen_KL_mean": -208.78170776367188, "KL/mean": -238.86834716796875, "KL/rejected_KL_mean": -268.95501708984375, "KL/std": 110.056396484375, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.003990051802247763, "fcm_dpo/delta": 0.03203321620821953, "fcm_dpo/margin": 60.17329406738281, "fcm_dpo/q_t": 0.445614755153656, "grad_norm": 10.350008010864258, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.244903802871704, "logits/rejected": -2.0556321144104004, "logps/chosen": -290.1561279296875, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -367.58074951171875, "loss": 1.2429, "margin_dpo/margin_mean": 60.17329788208008, "margin_dpo/margin_std": 146.69613647460938, "step": 633 }, { "KL/chosen_KL_mean": -192.9940185546875, "KL/mean": -232.87289428710938, "KL/rejected_KL_mean": -272.75177001953125, "KL/std": 100.20779418945312, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0039918990805745125, "fcm_dpo/delta": -0.02739102393388748, "fcm_dpo/margin": 79.75776672363281, "fcm_dpo/q_t": 0.42442965507507324, "grad_norm": 10.371672630310059, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.5726510286331177, "logits/rejected": -1.404700756072998, "logps/chosen": -289.11688232421875, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -385.5995788574219, "loss": 1.1365, "margin_dpo/margin_mean": 79.75776672363281, "margin_dpo/margin_std": 101.3698501586914, "step": 634 }, { "KL/chosen_KL_mean": -202.0726776123047, "KL/mean": -233.09791564941406, "KL/rejected_KL_mean": -264.1231384277344, "KL/std": 110.18873596191406, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.003995340317487717, "fcm_dpo/delta": 0.035981494933366776, "fcm_dpo/margin": 62.05045700073242, "fcm_dpo/q_t": 0.44263267517089844, "grad_norm": 13.941206932067871, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.8405745029449463, "logits/rejected": -1.2990353107452393, "logps/chosen": -300.75677490234375, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -354.0223388671875, "loss": 1.2213, "margin_dpo/margin_mean": 62.05045700073242, "margin_dpo/margin_std": 132.36175537109375, "step": 635 }, { "KL/chosen_KL_mean": -178.16909790039062, "KL/mean": -231.85238647460938, "KL/rejected_KL_mean": -285.53564453125, "KL/std": 113.04296112060547, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.003996372222900391, "fcm_dpo/delta": -0.030373020097613335, "fcm_dpo/margin": 107.36656188964844, "fcm_dpo/q_t": 0.39931249618530273, "grad_norm": 8.551154136657715, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.5116078853607178, "logits/rejected": -1.0246648788452148, "logps/chosen": -268.5870361328125, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -373.2425537109375, "loss": 1.0507, "margin_dpo/margin_mean": 107.36656188964844, "margin_dpo/margin_std": 112.85643005371094, "step": 636 }, { "KL/chosen_KL_mean": -179.79299926757812, "KL/mean": -237.13363647460938, "KL/rejected_KL_mean": -294.47430419921875, "KL/std": 126.12345886230469, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.003958066925406456, "fcm_dpo/delta": -0.05644138902425766, "fcm_dpo/margin": 114.6812973022461, "fcm_dpo/q_t": 0.393381267786026, "grad_norm": 8.233260154724121, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.1410350799560547, "logits/rejected": -2.0404295921325684, "logps/chosen": -262.24273681640625, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -398.5028991699219, "loss": 1.0386, "margin_dpo/margin_mean": 114.6812973022461, "margin_dpo/margin_std": 121.02161407470703, "step": 637 }, { "KL/chosen_KL_mean": -167.48101806640625, "KL/mean": -222.94651794433594, "KL/rejected_KL_mean": -278.4120178222656, "KL/std": 111.3506088256836, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.003937150351703167, "fcm_dpo/delta": -0.038722481578588486, "fcm_dpo/margin": 110.93099975585938, "fcm_dpo/q_t": 0.396528959274292, "grad_norm": 7.841807842254639, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.6616134643554688, "logits/rejected": -0.9813757538795471, "logps/chosen": -259.406005859375, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -364.69903564453125, "loss": 1.0395, "margin_dpo/margin_mean": 110.93099975585938, "margin_dpo/margin_std": 104.21211242675781, "step": 638 }, { "KL/chosen_KL_mean": -224.26531982421875, "KL/mean": -249.7328643798828, "KL/rejected_KL_mean": -275.2004089355469, "KL/std": 102.64078521728516, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0039391228929162025, "fcm_dpo/delta": 0.07318165153265, "fcm_dpo/margin": 50.93510437011719, "fcm_dpo/q_t": 0.452598512172699, "grad_norm": 9.876532554626465, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.5673177242279053, "logits/rejected": -0.816768229007721, "logps/chosen": -328.7928466796875, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -356.6807556152344, "loss": 1.2573, "margin_dpo/margin_mean": 50.935096740722656, "margin_dpo/margin_std": 127.72814178466797, "step": 639 }, { "KL/chosen_KL_mean": -189.7541046142578, "KL/mean": -240.61605834960938, "KL/rejected_KL_mean": -291.4779968261719, "KL/std": 112.2113037109375, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.003997094929218292, "fcm_dpo/delta": -0.008289456367492676, "fcm_dpo/margin": 101.72390747070312, "fcm_dpo/q_t": 0.4041714668273926, "grad_norm": 11.88459300994873, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8559362888336182, "logits/rejected": -1.336440086364746, "logps/chosen": -287.6393737792969, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -404.18304443359375, "loss": 1.0768, "margin_dpo/margin_mean": 101.72390747070312, "margin_dpo/margin_std": 112.64619445800781, "step": 640 }, { "KL/chosen_KL_mean": -173.19712829589844, "KL/mean": -225.78286743164062, "KL/rejected_KL_mean": -278.36859130859375, "KL/std": 130.15615844726562, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.00395325617864728, "fcm_dpo/delta": -0.016450634226202965, "fcm_dpo/margin": 105.17149353027344, "fcm_dpo/q_t": 0.4063078761100769, "grad_norm": 8.498259544372559, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.4093432426452637, "logits/rejected": -1.8730708360671997, "logps/chosen": -269.6427917480469, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -373.5043029785156, "loss": 1.0866, "margin_dpo/margin_mean": 105.17149353027344, "margin_dpo/margin_std": 145.051025390625, "step": 641 }, { "KL/chosen_KL_mean": -193.46871948242188, "KL/mean": -241.56509399414062, "KL/rejected_KL_mean": -289.6614990234375, "KL/std": 111.49632263183594, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.00396847166121006, "fcm_dpo/delta": 0.01883857510983944, "fcm_dpo/margin": 96.19275665283203, "fcm_dpo/q_t": 0.41323286294937134, "grad_norm": 10.56634521484375, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.607203722000122, "logits/rejected": -1.2723290920257568, "logps/chosen": -294.22857666015625, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -404.369140625, "loss": 1.1104, "margin_dpo/margin_mean": 96.1927490234375, "margin_dpo/margin_std": 139.09646606445312, "step": 642 }, { "KL/chosen_KL_mean": -180.5625762939453, "KL/mean": -226.94200134277344, "KL/rejected_KL_mean": -273.3214111328125, "KL/std": 103.8188705444336, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.003958440385758877, "fcm_dpo/delta": 0.033485498279333115, "fcm_dpo/margin": 92.75883483886719, "fcm_dpo/q_t": 0.41362398862838745, "grad_norm": 11.628656387329102, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.185035228729248, "logits/rejected": -1.8050799369812012, "logps/chosen": -265.3062438964844, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -367.63983154296875, "loss": 1.1101, "margin_dpo/margin_mean": 92.75883483886719, "margin_dpo/margin_std": 119.328369140625, "step": 643 }, { "KL/chosen_KL_mean": -190.39022827148438, "KL/mean": -226.90316772460938, "KL/rejected_KL_mean": -263.4161071777344, "KL/std": 105.73284149169922, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004054566845297813, "fcm_dpo/delta": 0.10707315802574158, "fcm_dpo/margin": 73.02589416503906, "fcm_dpo/q_t": 0.4308372139930725, "grad_norm": 10.074755668640137, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.9174877405166626, "logits/rejected": -1.6521148681640625, "logps/chosen": -275.7129821777344, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -358.02471923828125, "loss": 1.1647, "margin_dpo/margin_mean": 73.02589416503906, "margin_dpo/margin_std": 115.69828033447266, "step": 644 }, { "KL/chosen_KL_mean": -198.63180541992188, "KL/mean": -224.41680908203125, "KL/rejected_KL_mean": -250.20181274414062, "KL/std": 113.77066040039062, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.004108809400349855, "fcm_dpo/delta": 0.07921098917722702, "fcm_dpo/margin": 51.569984436035156, "fcm_dpo/q_t": 0.4508335590362549, "grad_norm": 10.686655044555664, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.610945701599121, "logits/rejected": -1.4476780891418457, "logps/chosen": -281.2220458984375, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -353.20556640625, "loss": 1.2535, "margin_dpo/margin_mean": 51.569984436035156, "margin_dpo/margin_std": 128.64085388183594, "step": 645 }, { "KL/chosen_KL_mean": -176.72103881835938, "KL/mean": -225.80923461914062, "KL/rejected_KL_mean": -274.89739990234375, "KL/std": 98.71224975585938, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.004123710095882416, "fcm_dpo/delta": -0.005321785807609558, "fcm_dpo/margin": 98.17638397216797, "fcm_dpo/q_t": 0.40454936027526855, "grad_norm": 8.713913917541504, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.8150607347488403, "logits/rejected": -1.2016782760620117, "logps/chosen": -267.9130859375, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -358.6757507324219, "loss": 1.0695, "margin_dpo/margin_mean": 98.17638397216797, "margin_dpo/margin_std": 107.13117980957031, "step": 646 }, { "KL/chosen_KL_mean": -191.77938842773438, "KL/mean": -234.74676513671875, "KL/rejected_KL_mean": -277.71417236328125, "KL/std": 119.11856842041016, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004162761382758617, "fcm_dpo/delta": 0.043854326009750366, "fcm_dpo/margin": 85.9347915649414, "fcm_dpo/q_t": 0.4180751144886017, "grad_norm": 10.131415367126465, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.5013996362686157, "logits/rejected": -1.365624189376831, "logps/chosen": -280.62384033203125, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -377.2125244140625, "loss": 1.1317, "margin_dpo/margin_mean": 85.9347915649414, "margin_dpo/margin_std": 132.09080505371094, "step": 647 }, { "KL/chosen_KL_mean": -179.02273559570312, "KL/mean": -238.31134033203125, "KL/rejected_KL_mean": -297.59991455078125, "KL/std": 121.99029541015625, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.004124972969293594, "fcm_dpo/delta": -0.09360457211732864, "fcm_dpo/margin": 118.57717895507812, "fcm_dpo/q_t": 0.3862631022930145, "grad_norm": 9.538021087646484, "learning_rate": 3.798061746947995e-09, "logits/chosen": -2.057560920715332, "logits/rejected": -1.9873476028442383, "logps/chosen": -266.870849609375, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -402.26995849609375, "loss": 1.0245, "margin_dpo/margin_mean": 118.57717895507812, "margin_dpo/margin_std": 130.91688537597656, "step": 648 }, { "KL/chosen_KL_mean": -163.51776123046875, "KL/mean": -212.4190673828125, "KL/rejected_KL_mean": -261.3203430175781, "KL/std": 105.4999008178711, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.004078328143805265, "fcm_dpo/delta": 0.0007754676043987274, "fcm_dpo/margin": 97.8026123046875, "fcm_dpo/q_t": 0.4058907628059387, "grad_norm": 11.018086433410645, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.5053539276123047, "logits/rejected": -2.1028976440429688, "logps/chosen": -253.21026611328125, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -350.02691650390625, "loss": 1.0714, "margin_dpo/margin_mean": 97.8026123046875, "margin_dpo/margin_std": 103.08965301513672, "step": 649 }, { "KL/chosen_KL_mean": -193.8750762939453, "KL/mean": -241.3687744140625, "KL/rejected_KL_mean": -288.8624572753906, "KL/std": 115.9395751953125, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.00410448620095849, "fcm_dpo/delta": 0.010530292056500912, "fcm_dpo/margin": 94.98738098144531, "fcm_dpo/q_t": 0.40980586409568787, "grad_norm": 9.769255638122559, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.5492500066757202, "logits/rejected": -1.354164958000183, "logps/chosen": -283.8056640625, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -391.47528076171875, "loss": 1.1048, "margin_dpo/margin_mean": 94.98738098144531, "margin_dpo/margin_std": 134.60646057128906, "step": 650 }, { "KL/chosen_KL_mean": -159.7874755859375, "KL/mean": -218.44410705566406, "KL/rejected_KL_mean": -277.1007385253906, "KL/std": 111.49384307861328, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.004053032025694847, "fcm_dpo/delta": -0.07934260368347168, "fcm_dpo/margin": 117.31327819824219, "fcm_dpo/q_t": 0.3874052166938782, "grad_norm": 9.146835327148438, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.14914608001709, "logits/rejected": -1.9731441736221313, "logps/chosen": -238.97479248046875, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -383.0340576171875, "loss": 1.013, "margin_dpo/margin_mean": 117.31327819824219, "margin_dpo/margin_std": 108.98707580566406, "step": 651 }, { "KL/chosen_KL_mean": -167.6063690185547, "KL/mean": -215.79043579101562, "KL/rejected_KL_mean": -263.9744873046875, "KL/std": 102.97781372070312, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0040542855858802795, "fcm_dpo/delta": 0.009578289464116096, "fcm_dpo/margin": 96.36812591552734, "fcm_dpo/q_t": 0.40761759877204895, "grad_norm": 9.722610473632812, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.7876081466674805, "logits/rejected": -2.5110831260681152, "logps/chosen": -269.3966064453125, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -380.299072265625, "loss": 1.0718, "margin_dpo/margin_mean": 96.36812591552734, "margin_dpo/margin_std": 99.33796691894531, "step": 652 }, { "KL/chosen_KL_mean": -191.7828826904297, "KL/mean": -239.32199096679688, "KL/rejected_KL_mean": -286.8611145019531, "KL/std": 114.07211303710938, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004062423948198557, "fcm_dpo/delta": 0.014248719438910484, "fcm_dpo/margin": 95.07823944091797, "fcm_dpo/q_t": 0.4081156253814697, "grad_norm": 9.394801139831543, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.3320093154907227, "logits/rejected": -2.0807199478149414, "logps/chosen": -285.136474609375, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -395.984375, "loss": 1.0872, "margin_dpo/margin_mean": 95.0782470703125, "margin_dpo/margin_std": 111.96937561035156, "step": 653 }, { "KL/chosen_KL_mean": -192.05471801757812, "KL/mean": -233.37423706054688, "KL/rejected_KL_mean": -274.6937561035156, "KL/std": 108.77951049804688, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.004097599536180496, "fcm_dpo/delta": 0.06354405730962753, "fcm_dpo/margin": 82.63905334472656, "fcm_dpo/q_t": 0.4232095777988434, "grad_norm": 12.241462707519531, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.241218090057373, "logits/rejected": -2.056581974029541, "logps/chosen": -281.17022705078125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -379.61370849609375, "loss": 1.1484, "margin_dpo/margin_mean": 82.63905334472656, "margin_dpo/margin_std": 135.84942626953125, "step": 654 }, { "KL/chosen_KL_mean": -176.69546508789062, "KL/mean": -225.2274169921875, "KL/rejected_KL_mean": -273.7593688964844, "KL/std": 118.79559326171875, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0041176192462444305, "fcm_dpo/delta": 0.0003037545830011368, "fcm_dpo/margin": 97.06387329101562, "fcm_dpo/q_t": 0.4066257178783417, "grad_norm": 8.597825050354004, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.9758461713790894, "logits/rejected": -1.9831876754760742, "logps/chosen": -257.7315673828125, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -376.56170654296875, "loss": 1.0926, "margin_dpo/margin_mean": 97.06387329101562, "margin_dpo/margin_std": 127.79817199707031, "step": 655 }, { "KL/chosen_KL_mean": -178.1404571533203, "KL/mean": -239.09280395507812, "KL/rejected_KL_mean": -300.0451354980469, "KL/std": 118.36692810058594, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.004047637339681387, "fcm_dpo/delta": -0.09841868281364441, "fcm_dpo/margin": 121.90470123291016, "fcm_dpo/q_t": 0.3851315677165985, "grad_norm": 10.15025520324707, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.4454021453857422, "logits/rejected": -1.5177350044250488, "logps/chosen": -263.4658203125, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -418.3837890625, "loss": 1.0077, "margin_dpo/margin_mean": 121.90469360351562, "margin_dpo/margin_std": 120.98646545410156, "step": 656 }, { "KL/chosen_KL_mean": -192.5126190185547, "KL/mean": -239.36492919921875, "KL/rejected_KL_mean": -286.21722412109375, "KL/std": 109.62409973144531, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.004026750102639198, "fcm_dpo/delta": 0.02321067452430725, "fcm_dpo/margin": 93.70463562011719, "fcm_dpo/q_t": 0.41337689757347107, "grad_norm": 10.449422836303711, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.978181004524231, "logits/rejected": -1.8103981018066406, "logps/chosen": -272.7103271484375, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -388.79852294921875, "loss": 1.1152, "margin_dpo/margin_mean": 93.70462799072266, "margin_dpo/margin_std": 134.04403686523438, "step": 657 }, { "KL/chosen_KL_mean": -190.110107421875, "KL/mean": -232.64801025390625, "KL/rejected_KL_mean": -275.1858825683594, "KL/std": 102.90364074707031, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.004094169475138187, "fcm_dpo/delta": 0.05325368419289589, "fcm_dpo/margin": 85.07577514648438, "fcm_dpo/q_t": 0.41826874017715454, "grad_norm": 8.98663330078125, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.1919946670532227, "logits/rejected": -1.7307937145233154, "logps/chosen": -284.74462890625, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -376.8175048828125, "loss": 1.1174, "margin_dpo/margin_mean": 85.07577514648438, "margin_dpo/margin_std": 109.95591735839844, "step": 658 }, { "KL/chosen_KL_mean": -171.71493530273438, "KL/mean": -223.228515625, "KL/rejected_KL_mean": -274.74212646484375, "KL/std": 111.15776824951172, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.004066976252943277, "fcm_dpo/delta": -0.02018512599170208, "fcm_dpo/margin": 103.02717590332031, "fcm_dpo/q_t": 0.4036858081817627, "grad_norm": 8.604510307312012, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.18768310546875, "logits/rejected": -1.2148463726043701, "logps/chosen": -284.27081298828125, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -368.6637268066406, "loss": 1.0937, "margin_dpo/margin_mean": 103.02717590332031, "margin_dpo/margin_std": 143.40936279296875, "step": 659 }, { "KL/chosen_KL_mean": -180.9205322265625, "KL/mean": -234.12216186523438, "KL/rejected_KL_mean": -287.3237609863281, "KL/std": 108.43683624267578, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0040566930547356606, "fcm_dpo/delta": -0.03308086469769478, "fcm_dpo/margin": 106.40322875976562, "fcm_dpo/q_t": 0.39907893538475037, "grad_norm": 11.16364860534668, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.9939277172088623, "logits/rejected": -1.3743653297424316, "logps/chosen": -274.2979736328125, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -368.71856689453125, "loss": 1.0552, "margin_dpo/margin_mean": 106.40322875976562, "margin_dpo/margin_std": 119.30760192871094, "step": 660 }, { "KL/chosen_KL_mean": -178.259765625, "KL/mean": -236.94805908203125, "KL/rejected_KL_mean": -295.6363525390625, "KL/std": 123.20396423339844, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.004016554448753595, "fcm_dpo/delta": -0.07493022084236145, "fcm_dpo/margin": 117.37657165527344, "fcm_dpo/q_t": 0.3915262520313263, "grad_norm": 10.60836124420166, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -2.0488831996917725, "logits/rejected": -1.5506341457366943, "logps/chosen": -266.11492919921875, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -388.0396728515625, "loss": 1.0417, "margin_dpo/margin_mean": 117.3765640258789, "margin_dpo/margin_std": 137.22247314453125, "step": 661 }, { "KL/chosen_KL_mean": -175.47518920898438, "KL/mean": -228.83383178710938, "KL/rejected_KL_mean": -282.1925048828125, "KL/std": 104.6898193359375, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.003982154186815023, "fcm_dpo/delta": -0.026091977953910828, "fcm_dpo/margin": 106.71732330322266, "fcm_dpo/q_t": 0.3999849259853363, "grad_norm": 10.604673385620117, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.9311115741729736, "logits/rejected": -1.3715747594833374, "logps/chosen": -278.2549743652344, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -377.4178161621094, "loss": 1.0534, "margin_dpo/margin_mean": 106.71733093261719, "margin_dpo/margin_std": 111.18516540527344, "step": 662 }, { "KL/chosen_KL_mean": -176.2903594970703, "KL/mean": -226.06210327148438, "KL/rejected_KL_mean": -275.8338317871094, "KL/std": 110.68124389648438, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.00398049782961607, "fcm_dpo/delta": 0.0038205645978450775, "fcm_dpo/margin": 99.54345703125, "fcm_dpo/q_t": 0.40785789489746094, "grad_norm": 10.145652770996094, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.269473075866699, "logits/rejected": -2.0540032386779785, "logps/chosen": -272.636962890625, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -396.36029052734375, "loss": 1.092, "margin_dpo/margin_mean": 99.54346466064453, "margin_dpo/margin_std": 128.92494201660156, "step": 663 }, { "KL/chosen_KL_mean": -194.470458984375, "KL/mean": -240.77163696289062, "KL/rejected_KL_mean": -287.07281494140625, "KL/std": 113.80953216552734, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.003998877480626106, "fcm_dpo/delta": 0.03069480136036873, "fcm_dpo/margin": 92.60240173339844, "fcm_dpo/q_t": 0.4130041003227234, "grad_norm": 8.466816902160645, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.7803696393966675, "logits/rejected": -1.6094014644622803, "logps/chosen": -286.3128662109375, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -398.9095458984375, "loss": 1.1001, "margin_dpo/margin_mean": 92.60240173339844, "margin_dpo/margin_std": 112.78471374511719, "step": 664 }, { "KL/chosen_KL_mean": -166.8582763671875, "KL/mean": -215.3350067138672, "KL/rejected_KL_mean": -263.811767578125, "KL/std": 106.89512634277344, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004011192359030247, "fcm_dpo/delta": 0.011464295908808708, "fcm_dpo/margin": 96.95347595214844, "fcm_dpo/q_t": 0.40784311294555664, "grad_norm": 7.9474921226501465, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.665969133377075, "logits/rejected": -2.252330780029297, "logps/chosen": -255.04446411132812, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -355.7237548828125, "loss": 1.0822, "margin_dpo/margin_mean": 96.95348358154297, "margin_dpo/margin_std": 110.98664855957031, "step": 665 }, { "KL/chosen_KL_mean": -183.67689514160156, "KL/mean": -234.58139038085938, "KL/rejected_KL_mean": -285.48590087890625, "KL/std": 106.78742218017578, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.003991521429270506, "fcm_dpo/delta": -0.0068099964410066605, "fcm_dpo/margin": 101.80900573730469, "fcm_dpo/q_t": 0.40567559003829956, "grad_norm": 10.734161376953125, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.4872748851776123, "logits/rejected": -1.0946418046951294, "logps/chosen": -284.6514892578125, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -386.73583984375, "loss": 1.0825, "margin_dpo/margin_mean": 101.80900573730469, "margin_dpo/margin_std": 126.11776733398438, "step": 666 }, { "KL/chosen_KL_mean": -200.7777099609375, "KL/mean": -251.9720458984375, "KL/rejected_KL_mean": -303.1663818359375, "KL/std": 111.79800415039062, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.003990326076745987, "fcm_dpo/delta": -0.008975849486887455, "fcm_dpo/margin": 102.38865661621094, "fcm_dpo/q_t": 0.4034135639667511, "grad_norm": 8.865206718444824, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.454331874847412, "logits/rejected": -0.7864596843719482, "logps/chosen": -298.348876953125, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -396.75115966796875, "loss": 1.0646, "margin_dpo/margin_mean": 102.38865661621094, "margin_dpo/margin_std": 109.56192016601562, "step": 667 }, { "KL/chosen_KL_mean": -178.02706909179688, "KL/mean": -237.51568603515625, "KL/rejected_KL_mean": -297.00433349609375, "KL/std": 118.31504821777344, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.003954906016588211, "fcm_dpo/delta": -0.07396724820137024, "fcm_dpo/margin": 118.97724914550781, "fcm_dpo/q_t": 0.39137357473373413, "grad_norm": 10.627269744873047, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.3260974884033203, "logits/rejected": -1.8018579483032227, "logps/chosen": -280.60211181640625, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -405.822021484375, "loss": 1.0301, "margin_dpo/margin_mean": 118.97724914550781, "margin_dpo/margin_std": 128.97518920898438, "step": 668 }, { "KL/chosen_KL_mean": -162.53472900390625, "KL/mean": -219.58013916015625, "KL/rejected_KL_mean": -276.6255798339844, "KL/std": 115.88079071044922, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.003908197395503521, "fcm_dpo/delta": -0.04801907390356064, "fcm_dpo/margin": 114.09083557128906, "fcm_dpo/q_t": 0.3955235779285431, "grad_norm": 9.786161422729492, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.7025192975997925, "logits/rejected": -1.3039329051971436, "logps/chosen": -252.02951049804688, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -359.14508056640625, "loss": 1.0442, "margin_dpo/margin_mean": 114.09083557128906, "margin_dpo/margin_std": 123.54478454589844, "step": 669 }, { "KL/chosen_KL_mean": -164.00192260742188, "KL/mean": -225.89703369140625, "KL/rejected_KL_mean": -287.7921447753906, "KL/std": 117.611328125, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0038563506677746773, "fcm_dpo/delta": -0.0811828225851059, "fcm_dpo/margin": 123.79022216796875, "fcm_dpo/q_t": 0.38705527782440186, "grad_norm": 8.082456588745117, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.6331467628479004, "logits/rejected": -1.0851571559906006, "logps/chosen": -259.45654296875, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -389.3250732421875, "loss": 1.0187, "margin_dpo/margin_mean": 123.79022216796875, "margin_dpo/margin_std": 123.02424621582031, "step": 670 }, { "KL/chosen_KL_mean": -199.59059143066406, "KL/mean": -241.79470825195312, "KL/rejected_KL_mean": -283.998779296875, "KL/std": 109.87653350830078, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.00389591371640563, "fcm_dpo/delta": 0.07270292192697525, "fcm_dpo/margin": 84.40821838378906, "fcm_dpo/q_t": 0.42364585399627686, "grad_norm": 8.856520652770996, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.692121982574463, "logits/rejected": -1.4689905643463135, "logps/chosen": -281.7137145996094, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -374.218505859375, "loss": 1.1416, "margin_dpo/margin_mean": 84.40821838378906, "margin_dpo/margin_std": 123.49403381347656, "step": 671 }, { "KL/chosen_KL_mean": -198.68405151367188, "KL/mean": -235.6800537109375, "KL/rejected_KL_mean": -272.676025390625, "KL/std": 104.73110961914062, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.003950595390051603, "fcm_dpo/delta": 0.1110440045595169, "fcm_dpo/margin": 73.99197387695312, "fcm_dpo/q_t": 0.43172261118888855, "grad_norm": 11.905909538269043, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.0739834308624268, "logits/rejected": -1.7459323406219482, "logps/chosen": -288.7459716796875, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -373.1292724609375, "loss": 1.1664, "margin_dpo/margin_mean": 73.99197387695312, "margin_dpo/margin_std": 113.03207397460938, "step": 672 }, { "KL/chosen_KL_mean": -179.32730102539062, "KL/mean": -220.52256774902344, "KL/rejected_KL_mean": -261.71783447265625, "KL/std": 107.99259948730469, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0040287841111421585, "fcm_dpo/delta": 0.07015722990036011, "fcm_dpo/margin": 82.39053344726562, "fcm_dpo/q_t": 0.42233896255493164, "grad_norm": 10.533862113952637, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -2.01727294921875, "logits/rejected": -1.6506682634353638, "logps/chosen": -258.5904541015625, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -353.06707763671875, "loss": 1.1296, "margin_dpo/margin_mean": 82.3905258178711, "margin_dpo/margin_std": 111.0585708618164, "step": 673 }, { "KL/chosen_KL_mean": -164.99069213867188, "KL/mean": -224.96255493164062, "KL/rejected_KL_mean": -284.9344482421875, "KL/std": 108.76111602783203, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.00398373743519187, "fcm_dpo/delta": -0.08177487552165985, "fcm_dpo/margin": 119.94371795654297, "fcm_dpo/q_t": 0.38867950439453125, "grad_norm": 7.751524448394775, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.7294471263885498, "logits/rejected": -1.3281581401824951, "logps/chosen": -240.44900512695312, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -361.1380615234375, "loss": 1.0169, "margin_dpo/margin_mean": 119.9437255859375, "margin_dpo/margin_std": 119.86534118652344, "step": 674 }, { "KL/chosen_KL_mean": -182.4979705810547, "KL/mean": -235.23931884765625, "KL/rejected_KL_mean": -287.9806823730469, "KL/std": 116.98486328125, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.003936853259801865, "fcm_dpo/delta": -0.016681402921676636, "fcm_dpo/margin": 105.48270416259766, "fcm_dpo/q_t": 0.40334174036979675, "grad_norm": 9.690571784973145, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.5953117609024048, "logits/rejected": -1.2543377876281738, "logps/chosen": -262.4075012207031, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -369.19891357421875, "loss": 1.0913, "margin_dpo/margin_mean": 105.48271179199219, "margin_dpo/margin_std": 143.41036987304688, "step": 675 }, { "KL/chosen_KL_mean": -195.85867309570312, "KL/mean": -247.6715087890625, "KL/rejected_KL_mean": -299.48431396484375, "KL/std": 112.41414642333984, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.00394262932240963, "fcm_dpo/delta": -0.009021111764013767, "fcm_dpo/margin": 103.62567138671875, "fcm_dpo/q_t": 0.4040564298629761, "grad_norm": 8.638101577758789, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -2.0950560569763184, "logits/rejected": -1.5139997005462646, "logps/chosen": -294.02978515625, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -394.5090026855469, "loss": 1.0776, "margin_dpo/margin_mean": 103.62567138671875, "margin_dpo/margin_std": 124.52552795410156, "step": 676 }, { "KL/chosen_KL_mean": -174.96514892578125, "KL/mean": -227.3082275390625, "KL/rejected_KL_mean": -279.65130615234375, "KL/std": 110.74288940429688, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.003933532163500786, "fcm_dpo/delta": -0.0123984320089221, "fcm_dpo/margin": 104.68614196777344, "fcm_dpo/q_t": 0.40288046002388, "grad_norm": 8.027260780334473, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.8801970481872559, "logits/rejected": -1.213300108909607, "logps/chosen": -266.34442138671875, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -362.5290832519531, "loss": 1.0612, "margin_dpo/margin_mean": 104.68614196777344, "margin_dpo/margin_std": 108.81454467773438, "step": 677 }, { "KL/chosen_KL_mean": -165.76266479492188, "KL/mean": -226.8055419921875, "KL/rejected_KL_mean": -287.84844970703125, "KL/std": 117.56758117675781, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.003908194601535797, "fcm_dpo/delta": -0.08102075010538101, "fcm_dpo/margin": 122.0857925415039, "fcm_dpo/q_t": 0.3878611922264099, "grad_norm": 9.967924118041992, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.8607419729232788, "logits/rejected": -1.540893316268921, "logps/chosen": -241.4085235595703, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -374.8145446777344, "loss": 1.0185, "margin_dpo/margin_mean": 122.0857925415039, "margin_dpo/margin_std": 119.80363464355469, "step": 678 }, { "KL/chosen_KL_mean": -192.77783203125, "KL/mean": -243.42864990234375, "KL/rejected_KL_mean": -294.0794677734375, "KL/std": 98.64926147460938, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0038760651368647814, "fcm_dpo/delta": 0.007635399233549833, "fcm_dpo/margin": 101.3016357421875, "fcm_dpo/q_t": 0.40775835514068604, "grad_norm": 8.507211685180664, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.366889476776123, "logits/rejected": -1.855118989944458, "logps/chosen": -273.55126953125, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -376.95794677734375, "loss": 1.0722, "margin_dpo/margin_mean": 101.30164337158203, "margin_dpo/margin_std": 107.3071060180664, "step": 679 }, { "KL/chosen_KL_mean": -192.68081665039062, "KL/mean": -248.15017700195312, "KL/rejected_KL_mean": -303.61956787109375, "KL/std": 116.31010437011719, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0038864612579345703, "fcm_dpo/delta": -0.03313957899808884, "fcm_dpo/margin": 110.93873596191406, "fcm_dpo/q_t": 0.3997064530849457, "grad_norm": 7.605684757232666, "learning_rate": 1.31753782067201e-11, "logits/chosen": -2.08420729637146, "logits/rejected": -1.7553476095199585, "logps/chosen": -300.3637390136719, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -419.71441650390625, "loss": 1.0596, "margin_dpo/margin_mean": 110.93873596191406, "margin_dpo/margin_std": 124.41973876953125, "step": 680 }, { "KL/chosen_KL_mean": -176.94906616210938, "KL/mean": -225.59840393066406, "KL/rejected_KL_mean": -274.24774169921875, "KL/std": 110.86672973632812, "epoch": 1.0, "fcm_dpo/beta": 0.003876092843711376, "fcm_dpo/delta": 0.023657280951738358, "fcm_dpo/margin": 97.29869079589844, "fcm_dpo/q_t": 0.41202202439308167, "grad_norm": 9.133564949035645, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.460603952407837, "logits/rejected": -2.153721332550049, "logps/chosen": -269.96014404296875, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -369.0699157714844, "loss": 1.0983, "margin_dpo/margin_mean": 97.29869079589844, "margin_dpo/margin_std": 122.32638549804688, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1445424128734067, "train_runtime": 1965.7552, "train_samples_per_second": 22.179, "train_steps_per_second": 0.346 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }