{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.822547912597656, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.756685256958008, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.0178375244140625, "KL/mean": 0.003913797438144684, "KL/rejected_KL_mean": -0.010009765625, "KL/std": 0.304345965385437, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027849331498146057, "fcm_dpo/q_t": 0.49930402636528015, "grad_norm": 26.00975227355957, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42684465646743774, "logits/rejected": 0.6500589847564697, "logps/chosen": -93.79315185546875, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.23951721191406, "loss": 1.3841, "margin_dpo/margin_mean": 0.027849599719047546, "margin_dpo/margin_std": 0.4626951515674591, "step": 3 }, { "KL/chosen_KL_mean": -0.001087188720703125, "KL/mean": 0.00609811395406723, "KL/rejected_KL_mean": 0.01328277587890625, "KL/std": 0.29494738578796387, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014365509152412415, "fcm_dpo/q_t": 0.5003583431243896, "grad_norm": 32.85332489013672, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6471682786941528, "logits/rejected": 0.7788763046264648, "logps/chosen": -87.2935562133789, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.81097412109375, "loss": 1.3881, "margin_dpo/margin_mean": -0.01436544954776764, "margin_dpo/margin_std": 0.3908473551273346, "step": 4 }, { "KL/chosen_KL_mean": 0.005428314208984375, "KL/mean": -0.0018766820430755615, "KL/rejected_KL_mean": -0.009181976318359375, "KL/std": 0.2923153042793274, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0146104097366333, "fcm_dpo/q_t": 0.4996333718299866, "grad_norm": 35.1636848449707, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4455898702144623, "logits/rejected": 0.6571816205978394, "logps/chosen": -89.33132934570312, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.75701904296875, "loss": 1.3853, "margin_dpo/margin_mean": 0.014610469341278076, "margin_dpo/margin_std": 0.4390101730823517, "step": 5 }, { "KL/chosen_KL_mean": -0.07225799560546875, "KL/mean": -0.03147159516811371, "KL/rejected_KL_mean": 0.00931549072265625, "KL/std": 0.29866325855255127, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08157947659492493, "fcm_dpo/q_t": 0.5020367503166199, "grad_norm": 36.23944854736328, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.7280426025390625, "logits/rejected": 1.065699815750122, "logps/chosen": -97.39373779296875, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.87413024902344, "loss": 1.395, "margin_dpo/margin_mean": -0.0815800130367279, "margin_dpo/margin_std": 0.4576171040534973, "step": 6 }, { "KL/chosen_KL_mean": 0.000690460205078125, "KL/mean": -0.03987530618906021, "KL/rejected_KL_mean": -0.08044052124023438, "KL/std": 0.3455958366394043, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08112584054470062, "fcm_dpo/q_t": 0.49797192215919495, "grad_norm": 37.91194152832031, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.497615247964859, "logits/rejected": 0.5452033281326294, "logps/chosen": -86.64784240722656, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.69662475585938, "loss": 1.3787, "margin_dpo/margin_mean": 0.08112582564353943, "margin_dpo/margin_std": 0.44557279348373413, "step": 7 }, { "KL/chosen_KL_mean": -0.06268310546875, "KL/mean": -0.030106835067272186, "KL/rejected_KL_mean": 0.002471923828125, "KL/std": 0.30801481008529663, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06515611708164215, "fcm_dpo/q_t": 0.5016287565231323, "grad_norm": 32.87696075439453, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4197435677051544, "logits/rejected": 0.7316989302635193, "logps/chosen": -89.98636627197266, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.22557067871094, "loss": 1.3933, "margin_dpo/margin_mean": -0.06515583395957947, "margin_dpo/margin_std": 0.42019298672676086, "step": 8 }, { "KL/chosen_KL_mean": 0.05603790283203125, "KL/mean": 0.042095430195331573, "KL/rejected_KL_mean": 0.02816009521484375, "KL/std": 0.3223811388015747, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027874067425727844, "fcm_dpo/q_t": 0.49930375814437866, "grad_norm": 36.03216552734375, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7898930311203003, "logits/rejected": 0.9765688180923462, "logps/chosen": -103.80109405517578, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.2911605834961, "loss": 1.3841, "margin_dpo/margin_mean": 0.027873992919921875, "margin_dpo/margin_std": 0.4638426899909973, "step": 9 }, { "KL/chosen_KL_mean": -0.007755279541015625, "KL/mean": 0.05543231964111328, "KL/rejected_KL_mean": 0.11861801147460938, "KL/std": 0.2821580767631531, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12637776136398315, "fcm_dpo/q_t": 0.5031581521034241, "grad_norm": 33.62808609008789, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.7064789533615112, "logits/rejected": 0.928640604019165, "logps/chosen": -76.21212005615234, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.03348541259766, "loss": 1.3993, "margin_dpo/margin_mean": -0.12637758255004883, "margin_dpo/margin_std": 0.3746614456176758, "step": 10 }, { "KL/chosen_KL_mean": -0.002895355224609375, "KL/mean": -0.012741833925247192, "KL/rejected_KL_mean": -0.0225830078125, "KL/std": 0.32289862632751465, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019684791564941406, "fcm_dpo/q_t": 0.4995085597038269, "grad_norm": 36.94208526611328, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.30024954676628113, "logits/rejected": 0.5162527561187744, "logps/chosen": -82.36939239501953, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.28720092773438, "loss": 1.3848, "margin_dpo/margin_mean": 0.01968526840209961, "margin_dpo/margin_std": 0.4334602653980255, "step": 11 }, { "KL/chosen_KL_mean": 0.042079925537109375, "KL/mean": 0.02708115428686142, "KL/rejected_KL_mean": 0.012081146240234375, "KL/std": 0.35585978627204895, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02999238669872284, "fcm_dpo/q_t": 0.499248743057251, "grad_norm": 40.166481018066406, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.38247498869895935, "logits/rejected": 0.5102629661560059, "logps/chosen": -99.06341552734375, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.25933074951172, "loss": 1.3842, "margin_dpo/margin_mean": 0.02999258041381836, "margin_dpo/margin_std": 0.5800290107727051, "step": 12 }, { "KL/chosen_KL_mean": 0.003894805908203125, "KL/mean": 0.006069682538509369, "KL/rejected_KL_mean": 0.00824737548828125, "KL/std": 0.30687087774276733, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.00435011088848114, "fcm_dpo/q_t": 0.5001076459884644, "grad_norm": 41.75973129272461, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.6182687282562256, "logits/rejected": 0.8849186897277832, "logps/chosen": -90.55583190917969, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.68284606933594, "loss": 1.3873, "margin_dpo/margin_mean": -0.004350066184997559, "margin_dpo/margin_std": 0.46980738639831543, "step": 13 }, { "KL/chosen_KL_mean": -0.058956146240234375, "KL/mean": -0.017445534467697144, "KL/rejected_KL_mean": 0.024066925048828125, "KL/std": 0.30931776762008667, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08303016424179077, "fcm_dpo/q_t": 0.5020747184753418, "grad_norm": 36.09109115600586, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.8514296412467957, "logits/rejected": 0.938631534576416, "logps/chosen": -99.8861312866211, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.91793823242188, "loss": 1.3951, "margin_dpo/margin_mean": -0.083030104637146, "margin_dpo/margin_std": 0.42584455013275146, "step": 14 }, { "KL/chosen_KL_mean": 0.04967498779296875, "KL/mean": 0.03853350132703781, "KL/rejected_KL_mean": 0.02739715576171875, "KL/std": 0.28670358657836914, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.022279784083366394, "fcm_dpo/q_t": 0.4994439482688904, "grad_norm": 31.63225555419922, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.3800371289253235, "logits/rejected": 0.5467950701713562, "logps/chosen": -78.86029052734375, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.03494262695312, "loss": 1.3844, "margin_dpo/margin_mean": 0.022279635071754456, "margin_dpo/margin_std": 0.37670475244522095, "step": 15 }, { "KL/chosen_KL_mean": 0.01479339599609375, "KL/mean": -0.0008888393640518188, "KL/rejected_KL_mean": -0.016574859619140625, "KL/std": 0.3203931450843811, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03136792778968811, "fcm_dpo/q_t": 0.49921584129333496, "grad_norm": 34.533775329589844, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.5583735108375549, "logits/rejected": 0.8152099847793579, "logps/chosen": -97.40848541259766, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.61603546142578, "loss": 1.3837, "margin_dpo/margin_mean": 0.03136765956878662, "margin_dpo/margin_std": 0.4386933445930481, "step": 16 }, { "KL/chosen_KL_mean": 0.017818450927734375, "KL/mean": -0.0016682744026184082, "KL/rejected_KL_mean": -0.02115631103515625, "KL/std": 0.332327276468277, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03897756338119507, "fcm_dpo/q_t": 0.4990251064300537, "grad_norm": 35.74559783935547, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.4846668839454651, "logits/rejected": 0.7148842215538025, "logps/chosen": -104.34648895263672, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.48887634277344, "loss": 1.3831, "margin_dpo/margin_mean": 0.03897660970687866, "margin_dpo/margin_std": 0.5136370658874512, "step": 17 }, { "KL/chosen_KL_mean": 0.00605010986328125, "KL/mean": -0.02283298224210739, "KL/rejected_KL_mean": -0.051708221435546875, "KL/std": 0.3226258158683777, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05774863064289093, "fcm_dpo/q_t": 0.4985583424568176, "grad_norm": 41.769351959228516, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.36288249492645264, "logits/rejected": 0.7034786939620972, "logps/chosen": -87.08590698242188, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.90242767333984, "loss": 1.3811, "margin_dpo/margin_mean": 0.057748809456825256, "margin_dpo/margin_std": 0.4633875787258148, "step": 18 }, { "KL/chosen_KL_mean": 0.0035552978515625, "KL/mean": -0.028663814067840576, "KL/rejected_KL_mean": -0.060882568359375, "KL/std": 0.284589946269989, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06442779302597046, "fcm_dpo/q_t": 0.49838972091674805, "grad_norm": 31.232036590576172, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6800117492675781, "logits/rejected": 0.9637447595596313, "logps/chosen": -105.86998748779297, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.99111938476562, "loss": 1.3804, "margin_dpo/margin_mean": 0.06442800164222717, "margin_dpo/margin_std": 0.4454364776611328, "step": 19 }, { "KL/chosen_KL_mean": -0.021755218505859375, "KL/mean": -0.017552152276039124, "KL/rejected_KL_mean": -0.013355255126953125, "KL/std": 0.30829310417175293, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0083942711353302, "fcm_dpo/q_t": 0.5002096891403198, "grad_norm": 32.888671875, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.49024587869644165, "logits/rejected": 0.7916733026504517, "logps/chosen": -90.77987670898438, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.92567443847656, "loss": 1.3876, "margin_dpo/margin_mean": -0.0083942711353302, "margin_dpo/margin_std": 0.4098876416683197, "step": 20 }, { "KL/chosen_KL_mean": 0.06244659423828125, "KL/mean": 0.021179810166358948, "KL/rejected_KL_mean": -0.02008819580078125, "KL/std": 0.26202642917633057, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08253756165504456, "fcm_dpo/q_t": 0.4979369044303894, "grad_norm": 31.52644920349121, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.3749101758003235, "logits/rejected": 0.539322555065155, "logps/chosen": -80.27101135253906, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.95382690429688, "loss": 1.3784, "margin_dpo/margin_mean": 0.08253741264343262, "margin_dpo/margin_std": 0.3521482050418854, "step": 21 }, { "KL/chosen_KL_mean": 0.043365478515625, "KL/mean": -0.007114529609680176, "KL/rejected_KL_mean": -0.057590484619140625, "KL/std": 0.3476327657699585, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1009489893913269, "fcm_dpo/q_t": 0.4974777400493622, "grad_norm": 42.53053665161133, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.8603641986846924, "logits/rejected": 0.9916726350784302, "logps/chosen": -95.35193634033203, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.53111267089844, "loss": 1.3767, "margin_dpo/margin_mean": 0.10094878077507019, "margin_dpo/margin_std": 0.41960203647613525, "step": 22 }, { "KL/chosen_KL_mean": 0.01694488525390625, "KL/mean": 0.006412632763385773, "KL/rejected_KL_mean": -0.00411224365234375, "KL/std": 0.30847597122192383, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.021054580807685852, "fcm_dpo/q_t": 0.4994734525680542, "grad_norm": 31.42880630493164, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7944221496582031, "logits/rejected": 0.9766426086425781, "logps/chosen": -90.62056732177734, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.59837341308594, "loss": 1.3847, "margin_dpo/margin_mean": 0.02105468511581421, "margin_dpo/margin_std": 0.4509127736091614, "step": 23 }, { "KL/chosen_KL_mean": 0.08630180358886719, "KL/mean": 0.05158620327711105, "KL/rejected_KL_mean": 0.016876220703125, "KL/std": 0.30128204822540283, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06942640244960785, "fcm_dpo/q_t": 0.49826565384864807, "grad_norm": 43.98313903808594, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.5761356353759766, "logits/rejected": 0.608249306678772, "logps/chosen": -69.83099365234375, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.59302520751953, "loss": 1.3798, "margin_dpo/margin_mean": 0.06942710280418396, "margin_dpo/margin_std": 0.4226471781730652, "step": 24 }, { "KL/chosen_KL_mean": -0.005115509033203125, "KL/mean": -0.018337689340114594, "KL/rejected_KL_mean": -0.03156280517578125, "KL/std": 0.29717016220092773, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.026436403393745422, "fcm_dpo/q_t": 0.4993394613265991, "grad_norm": 37.152408599853516, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.32355797290802, "logits/rejected": 0.48436790704727173, "logps/chosen": -80.83059692382812, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -95.9886703491211, "loss": 1.3841, "margin_dpo/margin_mean": 0.026436209678649902, "margin_dpo/margin_std": 0.4401368796825409, "step": 25 }, { "KL/chosen_KL_mean": -0.044551849365234375, "KL/mean": -0.05042964220046997, "KL/rejected_KL_mean": -0.056304931640625, "KL/std": 0.3430444598197937, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01175624132156372, "fcm_dpo/q_t": 0.4997081458568573, "grad_norm": 40.781532287597656, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.3756452798843384, "logits/rejected": 0.5856554508209229, "logps/chosen": -88.94572448730469, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.8844985961914, "loss": 1.3858, "margin_dpo/margin_mean": 0.011756688356399536, "margin_dpo/margin_std": 0.4871821999549866, "step": 26 }, { "KL/chosen_KL_mean": 0.035480499267578125, "KL/mean": -0.03736785054206848, "KL/rejected_KL_mean": -0.11021804809570312, "KL/std": 0.3032664358615875, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14569908380508423, "fcm_dpo/q_t": 0.49635937809944153, "grad_norm": 43.724117279052734, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.2613159418106079, "logits/rejected": 0.45179516077041626, "logps/chosen": -77.56053161621094, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.04872131347656, "loss": 1.3722, "margin_dpo/margin_mean": 0.14569932222366333, "margin_dpo/margin_std": 0.4155786633491516, "step": 27 }, { "KL/chosen_KL_mean": 0.017139434814453125, "KL/mean": 0.0019927918910980225, "KL/rejected_KL_mean": -0.013149261474609375, "KL/std": 0.28540778160095215, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.030287563800811768, "fcm_dpo/q_t": 0.49924230575561523, "grad_norm": 36.94441604614258, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.6262137293815613, "logits/rejected": 0.8496136665344238, "logps/chosen": -102.21142578125, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -96.97264099121094, "loss": 1.3837, "margin_dpo/margin_mean": 0.030288219451904297, "margin_dpo/margin_std": 0.40886539220809937, "step": 28 }, { "KL/chosen_KL_mean": -0.042491912841796875, "KL/mean": -0.0938672348856926, "KL/rejected_KL_mean": -0.14524078369140625, "KL/std": 0.3711159825325012, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10275976359844208, "fcm_dpo/q_t": 0.4974325895309448, "grad_norm": 42.3587760925293, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.1861664056777954, "logits/rejected": 0.4359433352947235, "logps/chosen": -88.68954467773438, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.10535430908203, "loss": 1.3766, "margin_dpo/margin_mean": 0.10276031494140625, "margin_dpo/margin_std": 0.4728996753692627, "step": 29 }, { "KL/chosen_KL_mean": -0.012622833251953125, "KL/mean": -0.05746837705373764, "KL/rejected_KL_mean": -0.10232162475585938, "KL/std": 0.3367045819759369, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08969850838184357, "fcm_dpo/q_t": 0.4977586269378662, "grad_norm": 39.04236602783203, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.7552507519721985, "logits/rejected": 0.8772361278533936, "logps/chosen": -88.40100860595703, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.42121887207031, "loss": 1.3778, "margin_dpo/margin_mean": 0.08969917893409729, "margin_dpo/margin_std": 0.4408782124519348, "step": 30 }, { "KL/chosen_KL_mean": 0.00852203369140625, "KL/mean": -0.05681120604276657, "KL/rejected_KL_mean": -0.1221466064453125, "KL/std": 0.3417091965675354, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13066129386425018, "fcm_dpo/q_t": 0.49673551321029663, "grad_norm": 31.153759002685547, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.579674482345581, "logits/rejected": 0.8094455003738403, "logps/chosen": -101.11714172363281, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.9805679321289, "loss": 1.3737, "margin_dpo/margin_mean": 0.13066184520721436, "margin_dpo/margin_std": 0.4235449433326721, "step": 31 }, { "KL/chosen_KL_mean": -0.05690765380859375, "KL/mean": -0.07819276303052902, "KL/rejected_KL_mean": -0.0994720458984375, "KL/std": 0.287200927734375, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04256699979305267, "fcm_dpo/q_t": 0.49893563985824585, "grad_norm": 35.390411376953125, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.477569043636322, "logits/rejected": 0.6216788291931152, "logps/chosen": -96.67393493652344, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.32383728027344, "loss": 1.3825, "margin_dpo/margin_mean": 0.042567700147628784, "margin_dpo/margin_std": 0.421936571598053, "step": 32 }, { "KL/chosen_KL_mean": 0.020603179931640625, "KL/mean": -0.04929838329553604, "KL/rejected_KL_mean": -0.11920166015625, "KL/std": 0.31584644317626953, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.139804407954216, "fcm_dpo/q_t": 0.4965066909790039, "grad_norm": 33.54821014404297, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.4737398624420166, "logits/rejected": 0.6510694622993469, "logps/chosen": -81.50041198730469, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.92515563964844, "loss": 1.3728, "margin_dpo/margin_mean": 0.13980549573898315, "margin_dpo/margin_std": 0.4062957167625427, "step": 33 }, { "KL/chosen_KL_mean": 0.05941009521484375, "KL/mean": -0.07057561725378036, "KL/rejected_KL_mean": -0.2005615234375, "KL/std": 0.3939353823661804, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2599775195121765, "fcm_dpo/q_t": 0.4935128688812256, "grad_norm": 40.73752212524414, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.7153167128562927, "logits/rejected": 0.8874403834342957, "logps/chosen": -77.14262390136719, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.91932678222656, "loss": 1.3612, "margin_dpo/margin_mean": 0.25997766852378845, "margin_dpo/margin_std": 0.54261314868927, "step": 34 }, { "KL/chosen_KL_mean": -0.02227020263671875, "KL/mean": -0.1421138048171997, "KL/rejected_KL_mean": -0.2619590759277344, "KL/std": 0.37461405992507935, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2396947145462036, "fcm_dpo/q_t": 0.49401408433914185, "grad_norm": 42.73909378051758, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.31547415256500244, "logits/rejected": 0.47016456723213196, "logps/chosen": -77.59262084960938, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.45051574707031, "loss": 1.3632, "margin_dpo/margin_mean": 0.23969528079032898, "margin_dpo/margin_std": 0.5240889191627502, "step": 35 }, { "KL/chosen_KL_mean": -0.0523834228515625, "KL/mean": -0.0986747294664383, "KL/rejected_KL_mean": -0.14496994018554688, "KL/std": 0.4046453833580017, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09258332848548889, "fcm_dpo/q_t": 0.49768954515457153, "grad_norm": 32.60264587402344, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.41901838779449463, "logits/rejected": 0.6083250045776367, "logps/chosen": -83.34062194824219, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.90238952636719, "loss": 1.3779, "margin_dpo/margin_mean": 0.09258322417736053, "margin_dpo/margin_std": 0.5844389200210571, "step": 36 }, { "KL/chosen_KL_mean": -0.02469635009765625, "KL/mean": -0.15922805666923523, "KL/rejected_KL_mean": -0.2937660217285156, "KL/std": 0.43276894092559814, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26906347274780273, "fcm_dpo/q_t": 0.4932851195335388, "grad_norm": 35.876285552978516, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.8121761679649353, "logits/rejected": 0.9844027757644653, "logps/chosen": -94.79579162597656, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01103210449219, "loss": 1.3604, "margin_dpo/margin_mean": 0.26906365156173706, "margin_dpo/margin_std": 0.5784456729888916, "step": 37 }, { "KL/chosen_KL_mean": 0.07448196411132812, "KL/mean": -0.05365963280200958, "KL/rejected_KL_mean": -0.181793212890625, "KL/std": 0.45101338624954224, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.256277471780777, "fcm_dpo/q_t": 0.49360552430152893, "grad_norm": 36.92803192138672, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.39100658893585205, "logits/rejected": 0.5291630029678345, "logps/chosen": -75.90402221679688, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.28581237792969, "loss": 1.3617, "margin_dpo/margin_mean": 0.25627779960632324, "margin_dpo/margin_std": 0.5744462609291077, "step": 38 }, { "KL/chosen_KL_mean": 0.04575347900390625, "KL/mean": -0.1388418823480606, "KL/rejected_KL_mean": -0.32343292236328125, "KL/std": 0.42838531732559204, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3691839277744293, "fcm_dpo/q_t": 0.4907814562320709, "grad_norm": 36.005001068115234, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.4559093117713928, "logits/rejected": 0.6429603099822998, "logps/chosen": -81.14002990722656, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.51934814453125, "loss": 1.3506, "margin_dpo/margin_mean": 0.3691841959953308, "margin_dpo/margin_std": 0.5734850764274597, "step": 39 }, { "KL/chosen_KL_mean": 0.03179931640625, "KL/mean": -0.0900685042142868, "KL/rejected_KL_mean": -0.21193313598632812, "KL/std": 0.3715287446975708, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2437363564968109, "fcm_dpo/q_t": 0.4939113259315491, "grad_norm": 29.054807662963867, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.7182142734527588, "logits/rejected": 0.9224390387535095, "logps/chosen": -83.30077362060547, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.46784973144531, "loss": 1.3627, "margin_dpo/margin_mean": 0.24373579025268555, "margin_dpo/margin_std": 0.5138312578201294, "step": 40 }, { "KL/chosen_KL_mean": -0.05670166015625, "KL/mean": -0.18418022990226746, "KL/rejected_KL_mean": -0.3116607666015625, "KL/std": 0.45756882429122925, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25496429204940796, "fcm_dpo/q_t": 0.49363958835601807, "grad_norm": 33.33799743652344, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6881804466247559, "logits/rejected": 0.8769144415855408, "logps/chosen": -93.20536804199219, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.390869140625, "loss": 1.362, "margin_dpo/margin_mean": 0.2549642026424408, "margin_dpo/margin_std": 0.6404173374176025, "step": 41 }, { "KL/chosen_KL_mean": 0.03771209716796875, "KL/mean": -0.16136547923088074, "KL/rejected_KL_mean": -0.360443115234375, "KL/std": 0.47513124346733093, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3981584906578064, "fcm_dpo/q_t": 0.4900566637516022, "grad_norm": 40.84376525878906, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.19536203145980835, "logits/rejected": 0.3331182599067688, "logps/chosen": -90.66390991210938, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.22509002685547, "loss": 1.3478, "margin_dpo/margin_mean": 0.39815831184387207, "margin_dpo/margin_std": 0.6086747646331787, "step": 42 }, { "KL/chosen_KL_mean": -0.016628265380859375, "KL/mean": -0.22303588688373566, "KL/rejected_KL_mean": -0.429443359375, "KL/std": 0.5302529335021973, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.41281160712242126, "fcm_dpo/q_t": 0.48970967531204224, "grad_norm": 37.84836959838867, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.6356506943702698, "logits/rejected": 0.8291884660720825, "logps/chosen": -89.66065979003906, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.38130187988281, "loss": 1.3469, "margin_dpo/margin_mean": 0.4128119945526123, "margin_dpo/margin_std": 0.7402452230453491, "step": 43 }, { "KL/chosen_KL_mean": -0.0759124755859375, "KL/mean": -0.27672359347343445, "KL/rejected_KL_mean": -0.4775390625, "KL/std": 0.4525594711303711, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40163010358810425, "fcm_dpo/q_t": 0.48997339606285095, "grad_norm": 34.487823486328125, "learning_rate": 3.115942028985507e-07, "logits/chosen": 1.0510084629058838, "logits/rejected": 1.1668331623077393, "logps/chosen": -81.95423126220703, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.899169921875, "loss": 1.3474, "margin_dpo/margin_mean": 0.4016307592391968, "margin_dpo/margin_std": 0.5492493510246277, "step": 44 }, { "KL/chosen_KL_mean": -0.087005615234375, "KL/mean": -0.192674919962883, "KL/rejected_KL_mean": -0.29833984375, "KL/std": 0.4497334957122803, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21133527159690857, "fcm_dpo/q_t": 0.49472421407699585, "grad_norm": 24.32515525817871, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.44442206621170044, "logits/rejected": 0.5521092414855957, "logps/chosen": -77.43160247802734, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.54608154296875, "loss": 1.3662, "margin_dpo/margin_mean": 0.21133577823638916, "margin_dpo/margin_std": 0.5942250490188599, "step": 45 }, { "KL/chosen_KL_mean": 0.00968170166015625, "KL/mean": -0.2235029935836792, "KL/rejected_KL_mean": -0.4566841125488281, "KL/std": 0.5296105742454529, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4663686156272888, "fcm_dpo/q_t": 0.4883562922477722, "grad_norm": 33.14401626586914, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.5547580718994141, "logits/rejected": 0.7996883392333984, "logps/chosen": -90.33114624023438, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.1332015991211, "loss": 1.341, "margin_dpo/margin_mean": 0.4663684368133545, "margin_dpo/margin_std": 0.57627272605896, "step": 46 }, { "KL/chosen_KL_mean": -0.023273468017578125, "KL/mean": -0.2193836122751236, "KL/rejected_KL_mean": -0.41548919677734375, "KL/std": 0.5713561773300171, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.39222225546836853, "fcm_dpo/q_t": 0.4902188181877136, "grad_norm": 33.990516662597656, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.27963316440582275, "logits/rejected": 0.45156657695770264, "logps/chosen": -104.4345703125, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.6703872680664, "loss": 1.3491, "margin_dpo/margin_mean": 0.39222198724746704, "margin_dpo/margin_std": 0.7943464517593384, "step": 47 }, { "KL/chosen_KL_mean": -0.01111602783203125, "KL/mean": -0.29631292819976807, "KL/rejected_KL_mean": -0.581512451171875, "KL/std": 0.7119265794754028, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5703924298286438, "fcm_dpo/q_t": 0.4857747256755829, "grad_norm": 38.662052154541016, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.08642253279685974, "logits/rejected": 0.27748626470565796, "logps/chosen": -91.11138916015625, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.66209411621094, "loss": 1.3324, "margin_dpo/margin_mean": 0.5703927874565125, "margin_dpo/margin_std": 0.9583979845046997, "step": 48 }, { "KL/chosen_KL_mean": 0.0429534912109375, "KL/mean": -0.3334777355194092, "KL/rejected_KL_mean": -0.7099113464355469, "KL/std": 0.7896941900253296, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7528567314147949, "fcm_dpo/q_t": 0.48124662041664124, "grad_norm": 39.64509582519531, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.007220529019832611, "logits/rejected": 0.46800845861434937, "logps/chosen": -92.96072387695312, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.45890808105469, "loss": 1.3147, "margin_dpo/margin_mean": 0.7528566122055054, "margin_dpo/margin_std": 0.9518918991088867, "step": 49 }, { "KL/chosen_KL_mean": -0.10455703735351562, "KL/mean": -0.4379521608352661, "KL/rejected_KL_mean": -0.7713432312011719, "KL/std": 0.7604032754898071, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6667819619178772, "fcm_dpo/q_t": 0.4833870530128479, "grad_norm": 33.32814025878906, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.5869323015213013, "logits/rejected": 0.75873202085495, "logps/chosen": -94.73136901855469, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.345703125, "loss": 1.3229, "margin_dpo/margin_mean": 0.6667814254760742, "margin_dpo/margin_std": 0.9237608313560486, "step": 50 }, { "KL/chosen_KL_mean": -0.01895904541015625, "KL/mean": -0.36135223507881165, "KL/rejected_KL_mean": -0.7037467956542969, "KL/std": 0.8597716093063354, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6847754120826721, "fcm_dpo/q_t": 0.4829750061035156, "grad_norm": 29.811017990112305, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.6957095861434937, "logits/rejected": 0.840387761592865, "logps/chosen": -87.52623748779297, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.17610168457031, "loss": 1.322, "margin_dpo/margin_mean": 0.684775173664093, "margin_dpo/margin_std": 1.086775779724121, "step": 51 }, { "KL/chosen_KL_mean": 0.006282806396484375, "KL/mean": -0.4337540864944458, "KL/rejected_KL_mean": -0.8737983703613281, "KL/std": 0.7699867486953735, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8800760507583618, "fcm_dpo/q_t": 0.4780733585357666, "grad_norm": 32.131961822509766, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.1260317862033844, "logits/rejected": 0.3173823654651642, "logps/chosen": -90.62397766113281, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.91285705566406, "loss": 1.3022, "margin_dpo/margin_mean": 0.8800765872001648, "margin_dpo/margin_std": 0.8856418132781982, "step": 52 }, { "KL/chosen_KL_mean": -0.015636444091796875, "KL/mean": -0.5477332472801208, "KL/rejected_KL_mean": -1.0798301696777344, "KL/std": 1.0665379762649536, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0641931295394897, "fcm_dpo/q_t": 0.4736406207084656, "grad_norm": 39.42461013793945, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.2841842472553253, "logits/rejected": 0.40194982290267944, "logps/chosen": -81.59870910644531, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.74136352539062, "loss": 1.2875, "margin_dpo/margin_mean": 1.064192771911621, "margin_dpo/margin_std": 1.3264973163604736, "step": 53 }, { "KL/chosen_KL_mean": 0.031391143798828125, "KL/mean": -0.577863335609436, "KL/rejected_KL_mean": -1.1871185302734375, "KL/std": 1.351470708847046, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2185075283050537, "fcm_dpo/q_t": 0.46977293491363525, "grad_norm": 39.59609603881836, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.45085379481315613, "logits/rejected": 0.6335223913192749, "logps/chosen": -88.87877655029297, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.30467224121094, "loss": 1.2726, "margin_dpo/margin_mean": 1.2185072898864746, "margin_dpo/margin_std": 1.3391330242156982, "step": 54 }, { "KL/chosen_KL_mean": 0.13379669189453125, "KL/mean": -0.4512465000152588, "KL/rejected_KL_mean": -1.0362892150878906, "KL/std": 1.1628375053405762, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1700855493545532, "fcm_dpo/q_t": 0.47104203701019287, "grad_norm": 35.451961517333984, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.10927199572324753, "logits/rejected": 0.41715580224990845, "logps/chosen": -92.32212829589844, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.0072250366211, "loss": 1.2787, "margin_dpo/margin_mean": 1.1700853109359741, "margin_dpo/margin_std": 1.5021584033966064, "step": 55 }, { "KL/chosen_KL_mean": -0.12094879150390625, "KL/mean": -0.655026912689209, "KL/rejected_KL_mean": -1.1891136169433594, "KL/std": 1.1773183345794678, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0681629180908203, "fcm_dpo/q_t": 0.47360652685165405, "grad_norm": 33.158226013183594, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.19475769996643066, "logits/rejected": 0.5313428640365601, "logps/chosen": -87.35760498046875, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.83465576171875, "loss": 1.2891, "margin_dpo/margin_mean": 1.0681627988815308, "margin_dpo/margin_std": 1.6377336978912354, "step": 56 }, { "KL/chosen_KL_mean": -0.038440704345703125, "KL/mean": -0.6105505228042603, "KL/rejected_KL_mean": -1.1826629638671875, "KL/std": 1.2375710010528564, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1442389488220215, "fcm_dpo/q_t": 0.4716712534427643, "grad_norm": 33.56289291381836, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.2519586384296417, "logits/rejected": 0.4669032394886017, "logps/chosen": -98.18917846679688, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.23550415039062, "loss": 1.2812, "margin_dpo/margin_mean": 1.1442394256591797, "margin_dpo/margin_std": 1.5591473579406738, "step": 57 }, { "KL/chosen_KL_mean": 0.24883651733398438, "KL/mean": -0.515122652053833, "KL/rejected_KL_mean": -1.2790756225585938, "KL/std": 1.3940134048461914, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5279054641723633, "fcm_dpo/q_t": 0.4623526930809021, "grad_norm": 38.1209602355957, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.48203712701797485, "logits/rejected": 0.8335832357406616, "logps/chosen": -99.33213806152344, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.50135040283203, "loss": 1.2479, "margin_dpo/margin_mean": 1.5279054641723633, "margin_dpo/margin_std": 1.802613377571106, "step": 58 }, { "KL/chosen_KL_mean": 0.030872344970703125, "KL/mean": -0.6342012882232666, "KL/rejected_KL_mean": -1.2992744445800781, "KL/std": 1.2341996431350708, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.330143928527832, "fcm_dpo/q_t": 0.4670875072479248, "grad_norm": 33.938438415527344, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.22905437648296356, "logits/rejected": 0.4315071702003479, "logps/chosen": -89.77145385742188, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.5759506225586, "loss": 1.2635, "margin_dpo/margin_mean": 1.3301435708999634, "margin_dpo/margin_std": 1.5024757385253906, "step": 59 }, { "KL/chosen_KL_mean": 0.07645416259765625, "KL/mean": -0.561975359916687, "KL/rejected_KL_mean": -1.200408935546875, "KL/std": 1.2536375522613525, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2768651247024536, "fcm_dpo/q_t": 0.46831074357032776, "grad_norm": 28.53378677368164, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.29867804050445557, "logits/rejected": 0.561711847782135, "logps/chosen": -95.07925415039062, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.57284545898438, "loss": 1.2673, "margin_dpo/margin_mean": 1.2768653631210327, "margin_dpo/margin_std": 1.3698031902313232, "step": 60 }, { "KL/chosen_KL_mean": -0.21953582763671875, "KL/mean": -0.7926335334777832, "KL/rejected_KL_mean": -1.3657302856445312, "KL/std": 1.4385727643966675, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.146198034286499, "fcm_dpo/q_t": 0.47191426157951355, "grad_norm": 28.410425186157227, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.19770316779613495, "logits/rejected": 0.33750465512275696, "logps/chosen": -85.95184326171875, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.33148956298828, "loss": 1.2839, "margin_dpo/margin_mean": 1.1461985111236572, "margin_dpo/margin_std": 1.8459854125976562, "step": 61 }, { "KL/chosen_KL_mean": -0.13912582397460938, "KL/mean": -0.7023835182189941, "KL/rejected_KL_mean": -1.2656402587890625, "KL/std": 1.3832619190216064, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1265177726745605, "fcm_dpo/q_t": 0.4721405506134033, "grad_norm": 28.711816787719727, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.18332479894161224, "logits/rejected": 0.3774925172328949, "logps/chosen": -81.77450561523438, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.303955078125, "loss": 1.2864, "margin_dpo/margin_mean": 1.1265171766281128, "margin_dpo/margin_std": 1.8616397380828857, "step": 62 }, { "KL/chosen_KL_mean": 0.037113189697265625, "KL/mean": -0.7623429298400879, "KL/rejected_KL_mean": -1.5618019104003906, "KL/std": 1.3683931827545166, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5989201068878174, "fcm_dpo/q_t": 0.4604193866252899, "grad_norm": 34.911197662353516, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.38393160700798035, "logits/rejected": 0.5965884327888489, "logps/chosen": -103.58694458007812, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.96482849121094, "loss": 1.239, "margin_dpo/margin_mean": 1.5989201068878174, "margin_dpo/margin_std": 1.55733060836792, "step": 63 }, { "KL/chosen_KL_mean": -0.11870574951171875, "KL/mean": -1.0189087390899658, "KL/rejected_KL_mean": -1.9191169738769531, "KL/std": 1.8777999877929688, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10966499894857407, "fcm_dpo/delta": 0.09226006269454956, "fcm_dpo/margin": 1.800411581993103, "fcm_dpo/q_t": 0.4542243182659149, "grad_norm": 37.27589797973633, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.6055777668952942, "logits/rejected": 0.8308544158935547, "logps/chosen": -87.12029266357422, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.50459289550781, "loss": 1.2126, "margin_dpo/margin_mean": 1.8004118204116821, "margin_dpo/margin_std": 2.1201162338256836, "step": 64 }, { "KL/chosen_KL_mean": -0.091766357421875, "KL/mean": -0.9485232830047607, "KL/rejected_KL_mean": -1.8052902221679688, "KL/std": 1.6350948810577393, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10966499894857407, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.713515043258667, "fcm_dpo/q_t": 0.45396238565444946, "grad_norm": 41.39570617675781, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.05268655717372894, "logits/rejected": 0.0911976546049118, "logps/chosen": -91.31368255615234, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.14082336425781, "loss": 1.2199, "margin_dpo/margin_mean": 1.713515043258667, "margin_dpo/margin_std": 2.067049026489258, "step": 65 }, { "KL/chosen_KL_mean": -0.3515052795410156, "KL/mean": -0.9875293374061584, "KL/rejected_KL_mean": -1.6235580444335938, "KL/std": 1.5623242855072021, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10966499894857407, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.272049069404602, "fcm_dpo/q_t": 0.465734601020813, "grad_norm": 27.6107120513916, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.23577246069908142, "logits/rejected": 0.44414442777633667, "logps/chosen": -84.13572692871094, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.37565612792969, "loss": 1.2627, "margin_dpo/margin_mean": 1.2720489501953125, "margin_dpo/margin_std": 1.8467779159545898, "step": 66 }, { "KL/chosen_KL_mean": -0.8448104858398438, "KL/mean": -1.4403412342071533, "KL/rejected_KL_mean": -2.0358657836914062, "KL/std": 2.037557601928711, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10966499894857407, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1910598278045654, "fcm_dpo/q_t": 0.46860572695732117, "grad_norm": 26.36980628967285, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.3534145653247833, "logits/rejected": 0.6318387985229492, "logps/chosen": -88.51777648925781, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.12261962890625, "loss": 1.2823, "margin_dpo/margin_mean": 1.1910603046417236, "margin_dpo/margin_std": 2.7553486824035645, "step": 67 }, { "KL/chosen_KL_mean": -0.22165679931640625, "KL/mean": -1.3029649257659912, "KL/rejected_KL_mean": -2.3842697143554688, "KL/std": 2.208989143371582, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.12342697381973267, "fcm_dpo/delta": 0.1524980068206787, "fcm_dpo/margin": 2.1626110076904297, "fcm_dpo/q_t": 0.4408666491508484, "grad_norm": 32.79283142089844, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.040355291217565536, "logits/rejected": 0.25334814190864563, "logps/chosen": -97.30050659179688, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.50299835205078, "loss": 1.1643, "margin_dpo/margin_mean": 2.162611246109009, "margin_dpo/margin_std": 2.7016665935516357, "step": 68 }, { "KL/chosen_KL_mean": -0.3457489013671875, "KL/mean": -1.4175262451171875, "KL/rejected_KL_mean": -2.489299774169922, "KL/std": 2.379279136657715, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.14027002453804016, "fcm_dpo/delta": 0.11547283828258514, "fcm_dpo/margin": 2.143554210662842, "fcm_dpo/q_t": 0.43300873041152954, "grad_norm": 38.51386642456055, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.2411491572856903, "logits/rejected": 0.35059475898742676, "logps/chosen": -86.06546020507812, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.9695053100586, "loss": 1.1474, "margin_dpo/margin_mean": 2.143554210662842, "margin_dpo/margin_std": 2.9336843490600586, "step": 69 }, { "KL/chosen_KL_mean": -0.568634033203125, "KL/mean": -1.8904913663864136, "KL/rejected_KL_mean": -3.2123451232910156, "KL/std": 2.805058240890503, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.14585396647453308, "fcm_dpo/delta": 0.017679179087281227, "fcm_dpo/margin": 2.6437108516693115, "fcm_dpo/q_t": 0.4137248992919922, "grad_norm": 36.225582122802734, "learning_rate": 5e-07, "logits/chosen": 0.3409077823162079, "logits/rejected": 0.5218537449836731, "logps/chosen": -95.57858276367188, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.42506408691406, "loss": 1.0977, "margin_dpo/margin_mean": 2.6437110900878906, "margin_dpo/margin_std": 3.493382453918457, "step": 70 }, { "KL/chosen_KL_mean": -0.0187530517578125, "KL/mean": -1.8102540969848633, "KL/rejected_KL_mean": -3.601757049560547, "KL/std": 3.4162793159484863, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.1318536251783371, "fcm_dpo/delta": -0.10150254517793655, "fcm_dpo/margin": 3.5830001831054688, "fcm_dpo/q_t": 0.3878694772720337, "grad_norm": 40.60038757324219, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.00948343239724636, "logits/rejected": 0.36147817969322205, "logps/chosen": -90.99610900878906, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.19278717041016, "loss": 1.0257, "margin_dpo/margin_mean": 3.5829999446868896, "margin_dpo/margin_std": 3.879199504852295, "step": 71 }, { "KL/chosen_KL_mean": -0.5177268981933594, "KL/mean": -2.00016450881958, "KL/rejected_KL_mean": -3.48260498046875, "KL/std": 2.94858717918396, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.1315806806087494, "fcm_dpo/delta": 0.011890824884176254, "fcm_dpo/margin": 2.9648780822753906, "fcm_dpo/q_t": 0.4094286561012268, "grad_norm": 34.87541580200195, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.18947246670722961, "logits/rejected": 0.13340775668621063, "logps/chosen": -98.40925598144531, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.67431640625, "loss": 1.0794, "margin_dpo/margin_mean": 2.9648780822753906, "margin_dpo/margin_std": 3.350485324859619, "step": 72 }, { "KL/chosen_KL_mean": -0.7375717163085938, "KL/mean": -2.2939672470092773, "KL/rejected_KL_mean": -3.8503646850585938, "KL/std": 3.7848873138427734, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.13220612704753876, "fcm_dpo/delta": -0.014505185186862946, "fcm_dpo/margin": 3.1127872467041016, "fcm_dpo/q_t": 0.41045886278152466, "grad_norm": 33.018165588378906, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.3778142035007477, "logits/rejected": -0.07695995271205902, "logps/chosen": -96.506591796875, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.78280639648438, "loss": 1.1035, "margin_dpo/margin_mean": 3.1127867698669434, "margin_dpo/margin_std": 4.785640716552734, "step": 73 }, { "KL/chosen_KL_mean": -0.949737548828125, "KL/mean": -2.7258875370025635, "KL/rejected_KL_mean": -4.502040863037109, "KL/std": 3.7541213035583496, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.12464331090450287, "fcm_dpo/delta": -0.05642575025558472, "fcm_dpo/margin": 3.552306652069092, "fcm_dpo/q_t": 0.3977097272872925, "grad_norm": 34.37622833251953, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.274570107460022, "logits/rejected": -0.1320061832666397, "logps/chosen": -79.75813293457031, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.14881134033203, "loss": 1.0594, "margin_dpo/margin_mean": 3.552306652069092, "margin_dpo/margin_std": 4.393442153930664, "step": 74 }, { "KL/chosen_KL_mean": -1.0490913391113281, "KL/mean": -3.3547377586364746, "KL/rejected_KL_mean": -5.6603851318359375, "KL/std": 4.583926200866699, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.11271272599697113, "fcm_dpo/delta": -0.16040067374706268, "fcm_dpo/margin": 4.611293315887451, "fcm_dpo/q_t": 0.3792864978313446, "grad_norm": 33.74631881713867, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.1460169553756714, "logits/rejected": 0.053659845143556595, "logps/chosen": -79.33094787597656, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.0634994506836, "loss": 1.0155, "margin_dpo/margin_mean": 4.611293792724609, "margin_dpo/margin_std": 5.408316135406494, "step": 75 }, { "KL/chosen_KL_mean": -1.4506568908691406, "KL/mean": -2.9831204414367676, "KL/rejected_KL_mean": -4.515586853027344, "KL/std": 3.810370922088623, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.11174099147319794, "fcm_dpo/delta": 0.06908534467220306, "fcm_dpo/margin": 3.0649337768554688, "fcm_dpo/q_t": 0.42428457736968994, "grad_norm": 24.963457107543945, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.5667819976806641, "logits/rejected": -0.18554571270942688, "logps/chosen": -89.32779693603516, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.23526763916016, "loss": 1.1331, "margin_dpo/margin_mean": 3.0649337768554688, "margin_dpo/margin_std": 4.608339309692383, "step": 76 }, { "KL/chosen_KL_mean": -1.5554580688476562, "KL/mean": -3.2375295162200928, "KL/rejected_KL_mean": -4.9196014404296875, "KL/std": 4.366453647613525, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.11409208178520203, "fcm_dpo/delta": 0.019854050129652023, "fcm_dpo/margin": 3.364147424697876, "fcm_dpo/q_t": 0.41346073150634766, "grad_norm": 29.9552001953125, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.04986449331045151, "logits/rejected": 0.33108559250831604, "logps/chosen": -101.25580596923828, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.1253662109375, "loss": 1.113, "margin_dpo/margin_mean": 3.364147663116455, "margin_dpo/margin_std": 4.9455976486206055, "step": 77 }, { "KL/chosen_KL_mean": -0.7004432678222656, "KL/mean": -2.4978766441345215, "KL/rejected_KL_mean": -4.295307159423828, "KL/std": 3.687453269958496, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.113992840051651, "fcm_dpo/delta": -0.012542225420475006, "fcm_dpo/margin": 3.5948657989501953, "fcm_dpo/q_t": 0.40555503964424133, "grad_norm": 27.088991165161133, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.000656425952911377, "logits/rejected": 0.23149517178535461, "logps/chosen": -90.99715423583984, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.43302917480469, "loss": 1.0742, "margin_dpo/margin_mean": 3.5948657989501953, "margin_dpo/margin_std": 4.38848876953125, "step": 78 }, { "KL/chosen_KL_mean": -0.8678131103515625, "KL/mean": -3.2033865451812744, "KL/rejected_KL_mean": -5.538963317871094, "KL/std": 5.261655330657959, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.10509280860424042, "fcm_dpo/delta": -0.12027428299188614, "fcm_dpo/margin": 4.671147346496582, "fcm_dpo/q_t": 0.38557493686676025, "grad_norm": 27.996864318847656, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.46761998534202576, "logits/rejected": -0.07393725216388702, "logps/chosen": -87.24613952636719, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.646728515625, "loss": 1.0351, "margin_dpo/margin_mean": 4.671147346496582, "margin_dpo/margin_std": 5.772523403167725, "step": 79 }, { "KL/chosen_KL_mean": -1.573944091796875, "KL/mean": -3.4320662021636963, "KL/rejected_KL_mean": -5.290187835693359, "KL/std": 4.665433883666992, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.10297004878520966, "fcm_dpo/delta": 0.02142416313290596, "fcm_dpo/margin": 3.7162439823150635, "fcm_dpo/q_t": 0.4144880771636963, "grad_norm": 26.10356903076172, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.6565202474594116, "logits/rejected": -0.29686030745506287, "logps/chosen": -95.54426574707031, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.86460876464844, "loss": 1.1033, "margin_dpo/margin_mean": 3.7162439823150635, "margin_dpo/margin_std": 5.142528057098389, "step": 80 }, { "KL/chosen_KL_mean": -0.9917259216308594, "KL/mean": -3.668639659881592, "KL/rejected_KL_mean": -6.345546722412109, "KL/std": 5.801922798156738, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.09109668433666229, "fcm_dpo/delta": -0.1269640177488327, "fcm_dpo/margin": 5.353824615478516, "fcm_dpo/q_t": 0.38837116956710815, "grad_norm": 26.538944244384766, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6056756377220154, "logits/rejected": -0.18365435302257538, "logps/chosen": -100.82185363769531, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.01800537109375, "loss": 1.0417, "margin_dpo/margin_mean": 5.353825092315674, "margin_dpo/margin_std": 6.821974754333496, "step": 81 }, { "KL/chosen_KL_mean": -1.9211769104003906, "KL/mean": -3.794064521789551, "KL/rejected_KL_mean": -5.666950225830078, "KL/std": 4.52863883972168, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.096860371530056, "fcm_dpo/delta": 0.03975168988108635, "fcm_dpo/margin": 3.745772361755371, "fcm_dpo/q_t": 0.417072594165802, "grad_norm": 25.51350975036621, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.34076768159866333, "logits/rejected": -0.1277839094400406, "logps/chosen": -84.9671630859375, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.19290161132812, "loss": 1.1129, "margin_dpo/margin_mean": 3.745772361755371, "margin_dpo/margin_std": 4.930633544921875, "step": 82 }, { "KL/chosen_KL_mean": -2.281627655029297, "KL/mean": -4.37533712387085, "KL/rejected_KL_mean": -6.469043731689453, "KL/std": 5.102325916290283, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.09487678855657578, "fcm_dpo/delta": 0.0033668370451778173, "fcm_dpo/margin": 4.187417030334473, "fcm_dpo/q_t": 0.41191110014915466, "grad_norm": 25.401294708251953, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.4662584066390991, "logits/rejected": -0.04236942529678345, "logps/chosen": -94.45783996582031, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.21368408203125, "loss": 1.1122, "margin_dpo/margin_mean": 4.187417030334473, "margin_dpo/margin_std": 6.441137790679932, "step": 83 }, { "KL/chosen_KL_mean": -2.973358154296875, "KL/mean": -4.239016056060791, "KL/rejected_KL_mean": -5.504680633544922, "KL/std": 4.624900817871094, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.09545224159955978, "fcm_dpo/delta": 0.002787574427202344, "fcm_dpo/margin": 2.5313215255737305, "fcm_dpo/q_t": 0.4454571008682251, "grad_norm": 27.507658004760742, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.36730635166168213, "logits/rejected": 0.027123063802719116, "logps/chosen": -104.49984741210938, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.93077087402344, "loss": 1.2191, "margin_dpo/margin_mean": 2.5313222408294678, "margin_dpo/margin_std": 5.0041117668151855, "step": 84 }, { "KL/chosen_KL_mean": -2.5952186584472656, "KL/mean": -4.304271697998047, "KL/rejected_KL_mean": -6.0133209228515625, "KL/std": 5.060808181762695, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.10199123620986938, "fcm_dpo/delta": 0.058307766914367676, "fcm_dpo/margin": 3.41810941696167, "fcm_dpo/q_t": 0.4234800338745117, "grad_norm": 27.35652732849121, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.3776005506515503, "logits/rejected": -0.049516502767801285, "logps/chosen": -98.72259521484375, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.06851196289062, "loss": 1.1582, "margin_dpo/margin_mean": 3.4181106090545654, "margin_dpo/margin_std": 6.087031364440918, "step": 85 }, { "KL/chosen_KL_mean": -2.1110916137695312, "KL/mean": -4.2994232177734375, "KL/rejected_KL_mean": -6.487758636474609, "KL/std": 5.892319679260254, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.09767895936965942, "fcm_dpo/delta": -0.03573746979236603, "fcm_dpo/margin": 4.376669883728027, "fcm_dpo/q_t": 0.40929052233695984, "grad_norm": 26.546979904174805, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.5939052700996399, "logits/rejected": -0.3517759144306183, "logps/chosen": -83.81535339355469, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.14325714111328, "loss": 1.1108, "margin_dpo/margin_mean": 4.376669883728027, "margin_dpo/margin_std": 7.325562953948975, "step": 86 }, { "KL/chosen_KL_mean": -2.4160385131835938, "KL/mean": -4.7936811447143555, "KL/rejected_KL_mean": -7.171321868896484, "KL/std": 6.496661186218262, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.09226903319358826, "fcm_dpo/delta": -0.05227233096957207, "fcm_dpo/margin": 4.75528621673584, "fcm_dpo/q_t": 0.3996947407722473, "grad_norm": 24.918502807617188, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.944890022277832, "logits/rejected": -0.5775765180587769, "logps/chosen": -95.10200500488281, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.0895004272461, "loss": 1.1088, "margin_dpo/margin_mean": 4.75528621673584, "margin_dpo/margin_std": 7.7260589599609375, "step": 87 }, { "KL/chosen_KL_mean": -3.011322021484375, "KL/mean": -4.890939712524414, "KL/rejected_KL_mean": -6.770557403564453, "KL/std": 6.302268028259277, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.08926030993461609, "fcm_dpo/delta": -0.03736710920929909, "fcm_dpo/margin": 3.759223222732544, "fcm_dpo/q_t": 0.4232925772666931, "grad_norm": 22.35008430480957, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.2898971140384674, "logits/rejected": -0.10037471354007721, "logps/chosen": -103.77430725097656, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.92092895507812, "loss": 1.1609, "margin_dpo/margin_mean": 3.7592227458953857, "margin_dpo/margin_std": 6.578047752380371, "step": 88 }, { "KL/chosen_KL_mean": -2.3683700561523438, "KL/mean": -4.403493881225586, "KL/rejected_KL_mean": -6.438617706298828, "KL/std": 5.248843193054199, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.09078041464090347, "fcm_dpo/delta": 0.03732104226946831, "fcm_dpo/margin": 4.070240497589111, "fcm_dpo/q_t": 0.42152321338653564, "grad_norm": 22.52037811279297, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.975236713886261, "logits/rejected": -0.6606465578079224, "logps/chosen": -92.0594482421875, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.00695037841797, "loss": 1.1363, "margin_dpo/margin_mean": 4.070240497589111, "margin_dpo/margin_std": 6.989348888397217, "step": 89 }, { "KL/chosen_KL_mean": -2.169452667236328, "KL/mean": -4.8530097007751465, "KL/rejected_KL_mean": -7.536571502685547, "KL/std": 6.20382022857666, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.08811815083026886, "fcm_dpo/delta": -0.09523273259401321, "fcm_dpo/margin": 5.367116928100586, "fcm_dpo/q_t": 0.393224835395813, "grad_norm": 24.926958084106445, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.5365524291992188, "logits/rejected": -0.41147178411483765, "logps/chosen": -83.55200958251953, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.31371307373047, "loss": 1.0643, "margin_dpo/margin_mean": 5.367117881774902, "margin_dpo/margin_std": 7.4539079666137695, "step": 90 }, { "KL/chosen_KL_mean": -2.4058609008789062, "KL/mean": -4.714499473571777, "KL/rejected_KL_mean": -7.023139953613281, "KL/std": 6.117995262145996, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.08516527712345123, "fcm_dpo/delta": 0.008271850645542145, "fcm_dpo/margin": 4.617286682128906, "fcm_dpo/q_t": 0.414218544960022, "grad_norm": 22.544620513916016, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.41911572217941284, "logits/rejected": -0.007490828633308411, "logps/chosen": -99.62774658203125, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.99988555908203, "loss": 1.1135, "margin_dpo/margin_mean": 4.61728572845459, "margin_dpo/margin_std": 7.188141345977783, "step": 91 }, { "KL/chosen_KL_mean": -2.6820640563964844, "KL/mean": -4.708426475524902, "KL/rejected_KL_mean": -6.734794616699219, "KL/std": 6.029052734375, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.08879489451646805, "fcm_dpo/delta": 0.04808255285024643, "fcm_dpo/margin": 4.052727699279785, "fcm_dpo/q_t": 0.42085373401641846, "grad_norm": 21.61829948425293, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8539000749588013, "logits/rejected": -0.4905719757080078, "logps/chosen": -87.21234130859375, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.23611450195312, "loss": 1.1315, "margin_dpo/margin_mean": 4.052726745605469, "margin_dpo/margin_std": 6.412793159484863, "step": 92 }, { "KL/chosen_KL_mean": -2.9983787536621094, "KL/mean": -4.893575191497803, "KL/rejected_KL_mean": -6.788776397705078, "KL/std": 5.775480270385742, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.09381753951311111, "fcm_dpo/delta": 0.05251936987042427, "fcm_dpo/margin": 3.7904012203216553, "fcm_dpo/q_t": 0.42270439863204956, "grad_norm": 25.563844680786133, "learning_rate": 4.982595640958425e-07, "logits/chosen": -1.1335906982421875, "logits/rejected": -0.5887936353683472, "logps/chosen": -93.24880981445312, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.88299560546875, "loss": 1.1637, "margin_dpo/margin_mean": 3.790400505065918, "margin_dpo/margin_std": 7.06359338760376, "step": 93 }, { "KL/chosen_KL_mean": -3.0379791259765625, "KL/mean": -5.989664077758789, "KL/rejected_KL_mean": -8.941352844238281, "KL/std": 6.772228240966797, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.08164404332637787, "fcm_dpo/delta": -0.12585344910621643, "fcm_dpo/margin": 5.903378009796143, "fcm_dpo/q_t": 0.3849952220916748, "grad_norm": 23.04861831665039, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.7914024591445923, "logits/rejected": -0.49446022510528564, "logps/chosen": -95.49220275878906, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.18864440917969, "loss": 1.0558, "margin_dpo/margin_mean": 5.903378009796143, "margin_dpo/margin_std": 7.753124713897705, "step": 94 }, { "KL/chosen_KL_mean": -2.4876670837402344, "KL/mean": -4.802790641784668, "KL/rejected_KL_mean": -7.117912292480469, "KL/std": 5.6994500160217285, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.08500619232654572, "fcm_dpo/delta": 0.005726486444473267, "fcm_dpo/margin": 4.630244255065918, "fcm_dpo/q_t": 0.4137350916862488, "grad_norm": 20.226961135864258, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8820583820343018, "logits/rejected": -0.5447894334793091, "logps/chosen": -81.36137390136719, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.70912170410156, "loss": 1.1149, "margin_dpo/margin_mean": 4.630244255065918, "margin_dpo/margin_std": 7.096403121948242, "step": 95 }, { "KL/chosen_KL_mean": -2.1548309326171875, "KL/mean": -4.986147880554199, "KL/rejected_KL_mean": -7.817466735839844, "KL/std": 6.826396942138672, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.07861308753490448, "fcm_dpo/delta": -0.059963975101709366, "fcm_dpo/margin": 5.6626482009887695, "fcm_dpo/q_t": 0.3983404040336609, "grad_norm": 20.37750816345215, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.9891648292541504, "logits/rejected": -0.5680550932884216, "logps/chosen": -108.74699401855469, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.58549499511719, "loss": 1.0708, "margin_dpo/margin_mean": 5.6626482009887695, "margin_dpo/margin_std": 7.546749114990234, "step": 96 }, { "KL/chosen_KL_mean": -3.268970489501953, "KL/mean": -5.825924873352051, "KL/rejected_KL_mean": -8.38287353515625, "KL/std": 5.900346755981445, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.08026112616062164, "fcm_dpo/delta": -0.01806424930691719, "fcm_dpo/margin": 5.113907814025879, "fcm_dpo/q_t": 0.4046017527580261, "grad_norm": 22.69223403930664, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.5443437695503235, "logits/rejected": -0.142826646566391, "logps/chosen": -91.9488525390625, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.1951675415039, "loss": 1.0819, "margin_dpo/margin_mean": 5.113907814025879, "margin_dpo/margin_std": 6.255102157592773, "step": 97 }, { "KL/chosen_KL_mean": -2.5263900756835938, "KL/mean": -5.461396217346191, "KL/rejected_KL_mean": -8.396400451660156, "KL/std": 5.741586685180664, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.07302533090114594, "fcm_dpo/delta": -0.040231652557849884, "fcm_dpo/margin": 5.87000846862793, "fcm_dpo/q_t": 0.39952319860458374, "grad_norm": 20.505746841430664, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.7217051982879639, "logits/rejected": -0.4568660259246826, "logps/chosen": -94.76889038085938, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.91588592529297, "loss": 1.0565, "margin_dpo/margin_mean": 5.87000846862793, "margin_dpo/margin_std": 6.459377288818359, "step": 98 }, { "KL/chosen_KL_mean": -1.8695907592773438, "KL/mean": -5.092023849487305, "KL/rejected_KL_mean": -8.314460754394531, "KL/std": 6.849026679992676, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.06998633593320847, "fcm_dpo/delta": -0.06673847138881683, "fcm_dpo/margin": 6.444874286651611, "fcm_dpo/q_t": 0.3988569378852844, "grad_norm": 20.471097946166992, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.5464094877243042, "logits/rejected": -0.3336743414402008, "logps/chosen": -74.05424499511719, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.19578552246094, "loss": 1.0639, "margin_dpo/margin_mean": 6.444873809814453, "margin_dpo/margin_std": 8.70749282836914, "step": 99 }, { "KL/chosen_KL_mean": -2.177379608154297, "KL/mean": -4.515026569366455, "KL/rejected_KL_mean": -6.852668762207031, "KL/std": 5.2488203048706055, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.07227616012096405, "fcm_dpo/delta": 0.07497376948595047, "fcm_dpo/margin": 4.675288677215576, "fcm_dpo/q_t": 0.4232574999332428, "grad_norm": 17.493696212768555, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.698930561542511, "logits/rejected": -0.2161497175693512, "logps/chosen": -91.69120788574219, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -88.06979370117188, "loss": 1.1168, "margin_dpo/margin_mean": 4.675288677215576, "margin_dpo/margin_std": 5.602867126464844, "step": 100 }, { "KL/chosen_KL_mean": -3.5789451599121094, "KL/mean": -5.679132461547852, "KL/rejected_KL_mean": -7.779315948486328, "KL/std": 6.789219379425049, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.07820059359073639, "fcm_dpo/delta": 0.0858079195022583, "fcm_dpo/margin": 4.200361251831055, "fcm_dpo/q_t": 0.4271366596221924, "grad_norm": 22.138336181640625, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.824146032333374, "logits/rejected": -0.5540787577629089, "logps/chosen": -97.98213958740234, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.6893310546875, "loss": 1.1637, "margin_dpo/margin_mean": 4.200361251831055, "margin_dpo/margin_std": 7.24616813659668, "step": 101 }, { "KL/chosen_KL_mean": -4.077091217041016, "KL/mean": -5.880828857421875, "KL/rejected_KL_mean": -7.684562683105469, "KL/std": 6.507264137268066, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.08007986098527908, "fcm_dpo/delta": -0.030535273253917694, "fcm_dpo/margin": 3.6074752807617188, "fcm_dpo/q_t": 0.43360280990600586, "grad_norm": 27.30049705505371, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.9352939128875732, "logits/rejected": -0.572573184967041, "logps/chosen": -95.68651580810547, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.45348358154297, "loss": 1.2038, "margin_dpo/margin_mean": 3.6074750423431396, "margin_dpo/margin_std": 7.349678993225098, "step": 102 }, { "KL/chosen_KL_mean": -3.2285842895507812, "KL/mean": -6.28793478012085, "KL/rejected_KL_mean": -9.347286224365234, "KL/std": 6.969038009643555, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.07408405095338821, "fcm_dpo/delta": -0.0704045444726944, "fcm_dpo/margin": 6.118701934814453, "fcm_dpo/q_t": 0.3941284716129303, "grad_norm": 20.653127670288086, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.7487130165100098, "logits/rejected": -0.3508497476577759, "logps/chosen": -100.63096618652344, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.32897186279297, "loss": 1.0619, "margin_dpo/margin_mean": 6.1187028884887695, "margin_dpo/margin_std": 7.7995758056640625, "step": 103 }, { "KL/chosen_KL_mean": -2.4273719787597656, "KL/mean": -5.771093368530273, "KL/rejected_KL_mean": -9.114810943603516, "KL/std": 7.591581344604492, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.06870924681425095, "fcm_dpo/delta": -0.078645721077919, "fcm_dpo/margin": 6.68743896484375, "fcm_dpo/q_t": 0.39612478017807007, "grad_norm": 16.89374542236328, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.0259485244750977, "logits/rejected": -0.6767420768737793, "logps/chosen": -97.28373718261719, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -106.14151000976562, "loss": 1.0588, "margin_dpo/margin_mean": 6.68743896484375, "margin_dpo/margin_std": 9.01892375946045, "step": 104 }, { "KL/chosen_KL_mean": -3.301971435546875, "KL/mean": -6.584873199462891, "KL/rejected_KL_mean": -9.867774963378906, "KL/std": 6.882030487060547, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.06694912910461426, "fcm_dpo/delta": -0.05177009850740433, "fcm_dpo/margin": 6.565798759460449, "fcm_dpo/q_t": 0.39785873889923096, "grad_norm": 20.101085662841797, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.0759086608886719, "logits/rejected": -0.604017972946167, "logps/chosen": -100.05987548828125, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.44526672363281, "loss": 1.0732, "margin_dpo/margin_mean": 6.565799236297607, "margin_dpo/margin_std": 8.619524002075195, "step": 105 }, { "KL/chosen_KL_mean": -3.297027587890625, "KL/mean": -6.419658184051514, "KL/rejected_KL_mean": -9.542278289794922, "KL/std": 7.081768035888672, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.06513407826423645, "fcm_dpo/delta": -0.009557196870446205, "fcm_dpo/margin": 6.245247840881348, "fcm_dpo/q_t": 0.4084540903568268, "grad_norm": 16.995929718017578, "learning_rate": 4.957432749209755e-07, "logits/chosen": -1.0123717784881592, "logits/rejected": -0.5952422022819519, "logps/chosen": -95.73493957519531, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.5519790649414, "loss": 1.0919, "margin_dpo/margin_mean": 6.245246887207031, "margin_dpo/margin_std": 8.603517532348633, "step": 106 }, { "KL/chosen_KL_mean": -2.481060028076172, "KL/mean": -6.129586696624756, "KL/rejected_KL_mean": -9.778114318847656, "KL/std": 9.013992309570312, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.061054885387420654, "fcm_dpo/delta": -0.05881115049123764, "fcm_dpo/margin": 7.297057151794434, "fcm_dpo/q_t": 0.4040898382663727, "grad_norm": 17.12337875366211, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.7476747632026672, "logits/rejected": -0.2195618748664856, "logps/chosen": -107.60606384277344, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -110.96061706542969, "loss": 1.0841, "margin_dpo/margin_mean": 7.297056674957275, "margin_dpo/margin_std": 11.249258041381836, "step": 107 }, { "KL/chosen_KL_mean": -3.637096405029297, "KL/mean": -7.081631660461426, "KL/rejected_KL_mean": -10.526165008544922, "KL/std": 8.420123100280762, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.058732353150844574, "fcm_dpo/delta": -0.00774291530251503, "fcm_dpo/margin": 6.889064311981201, "fcm_dpo/q_t": 0.40853023529052734, "grad_norm": 17.601348876953125, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.9315465688705444, "logits/rejected": -0.4219909608364105, "logps/chosen": -92.8464584350586, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -112.364013671875, "loss": 1.0913, "margin_dpo/margin_mean": 6.889063835144043, "margin_dpo/margin_std": 9.232198715209961, "step": 108 }, { "KL/chosen_KL_mean": -2.9428787231445312, "KL/mean": -7.2346954345703125, "KL/rejected_KL_mean": -11.52651596069336, "KL/std": 8.477420806884766, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.05480396747589111, "fcm_dpo/delta": -0.09542834013700485, "fcm_dpo/margin": 8.583632469177246, "fcm_dpo/q_t": 0.3895283341407776, "grad_norm": 16.257457733154297, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.616068959236145, "logits/rejected": -0.37900426983833313, "logps/chosen": -103.12996673583984, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -115.06391143798828, "loss": 1.0432, "margin_dpo/margin_mean": 8.583633422851562, "margin_dpo/margin_std": 10.46832275390625, "step": 109 }, { "KL/chosen_KL_mean": -3.8190231323242188, "KL/mean": -6.351284503936768, "KL/rejected_KL_mean": -8.883552551269531, "KL/std": 7.056370735168457, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.05537309870123863, "fcm_dpo/delta": 0.01881362497806549, "fcm_dpo/margin": 5.064527988433838, "fcm_dpo/q_t": 0.4351041913032532, "grad_norm": 16.412124633789062, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.45676684379577637, "logits/rejected": -0.05104871094226837, "logps/chosen": -110.71473693847656, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.69475555419922, "loss": 1.1731, "margin_dpo/margin_mean": 5.06452751159668, "margin_dpo/margin_std": 7.96699333190918, "step": 110 }, { "KL/chosen_KL_mean": -3.4242630004882812, "KL/mean": -8.444239616394043, "KL/rejected_KL_mean": -13.464214324951172, "KL/std": 9.899216651916504, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.048961907625198364, "fcm_dpo/delta": -0.129607692360878, "fcm_dpo/margin": 10.039949417114258, "fcm_dpo/q_t": 0.38147810101509094, "grad_norm": 15.058576583862305, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.9807999134063721, "logits/rejected": -0.6876587867736816, "logps/chosen": -96.75363159179688, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -117.6417007446289, "loss": 1.0218, "margin_dpo/margin_mean": 10.039949417114258, "margin_dpo/margin_std": 11.011577606201172, "step": 111 }, { "KL/chosen_KL_mean": -3.909557342529297, "KL/mean": -6.489706039428711, "KL/rejected_KL_mean": -9.069854736328125, "KL/std": 5.839533805847168, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.05392807722091675, "fcm_dpo/delta": 0.1415294110774994, "fcm_dpo/margin": 5.160292625427246, "fcm_dpo/q_t": 0.4386399984359741, "grad_norm": 14.429816246032715, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.2216668128967285, "logits/rejected": -0.7013336420059204, "logps/chosen": -86.65660095214844, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -80.62830352783203, "loss": 1.1729, "margin_dpo/margin_mean": 5.1602935791015625, "margin_dpo/margin_std": 7.994180679321289, "step": 112 }, { "KL/chosen_KL_mean": -5.028282165527344, "KL/mean": -7.763637542724609, "KL/rejected_KL_mean": -10.49898910522461, "KL/std": 7.7972822189331055, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.060039445757865906, "fcm_dpo/delta": 0.08476116508245468, "fcm_dpo/margin": 5.470708847045898, "fcm_dpo/q_t": 0.42645731568336487, "grad_norm": 19.74599838256836, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.9751421213150024, "logits/rejected": -0.42994487285614014, "logps/chosen": -102.92778015136719, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -94.91858673095703, "loss": 1.1364, "margin_dpo/margin_mean": 5.470708847045898, "margin_dpo/margin_std": 7.588527679443359, "step": 113 }, { "KL/chosen_KL_mean": -4.748241424560547, "KL/mean": -8.660430908203125, "KL/rejected_KL_mean": -12.572624206542969, "KL/std": 9.109458923339844, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.05719109997153282, "fcm_dpo/delta": -0.06390418112277985, "fcm_dpo/margin": 7.82438325881958, "fcm_dpo/q_t": 0.40046727657318115, "grad_norm": 18.30594825744629, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.9462922811508179, "logits/rejected": -0.338051438331604, "logps/chosen": -101.24336242675781, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -104.58798217773438, "loss": 1.0923, "margin_dpo/margin_mean": 7.824383735656738, "margin_dpo/margin_std": 11.942830085754395, "step": 114 }, { "KL/chosen_KL_mean": -4.874778747558594, "KL/mean": -8.872907638549805, "KL/rejected_KL_mean": -12.871036529541016, "KL/std": 9.595939636230469, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.05524079501628876, "fcm_dpo/delta": -0.053423553705215454, "fcm_dpo/margin": 7.996259689331055, "fcm_dpo/q_t": 0.39892691373825073, "grad_norm": 15.786572456359863, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.9631392955780029, "logits/rejected": -0.5420498251914978, "logps/chosen": -108.05490112304688, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -118.58699035644531, "loss": 1.0703, "margin_dpo/margin_mean": 7.996259689331055, "margin_dpo/margin_std": 10.467954635620117, "step": 115 }, { "KL/chosen_KL_mean": -4.356288909912109, "KL/mean": -7.689609527587891, "KL/rejected_KL_mean": -11.022926330566406, "KL/std": 7.621214866638184, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.05641636997461319, "fcm_dpo/delta": 0.027186207473278046, "fcm_dpo/margin": 6.6666388511657715, "fcm_dpo/q_t": 0.4142521619796753, "grad_norm": 15.703874588012695, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.0944675207138062, "logits/rejected": -0.6601686477661133, "logps/chosen": -85.51466369628906, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -97.32878112792969, "loss": 1.1021, "margin_dpo/margin_mean": 6.66663932800293, "margin_dpo/margin_std": 8.647909164428711, "step": 116 }, { "KL/chosen_KL_mean": -5.282848358154297, "KL/mean": -7.92410945892334, "KL/rejected_KL_mean": -10.565372467041016, "KL/std": 7.7878875732421875, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.05565632879734039, "fcm_dpo/delta": -0.0014393724268302321, "fcm_dpo/margin": 5.282529354095459, "fcm_dpo/q_t": 0.43198567628860474, "grad_norm": 22.120391845703125, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.8035591840744019, "logits/rejected": -0.37195509672164917, "logps/chosen": -102.36380004882812, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -85.70529174804688, "loss": 1.1836, "margin_dpo/margin_mean": 5.282528877258301, "margin_dpo/margin_std": 9.644733428955078, "step": 117 }, { "KL/chosen_KL_mean": -3.9702720642089844, "KL/mean": -7.812582015991211, "KL/rejected_KL_mean": -11.654891967773438, "KL/std": 8.529531478881836, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.054393358528614044, "fcm_dpo/delta": -0.023072468116879463, "fcm_dpo/margin": 7.684615612030029, "fcm_dpo/q_t": 0.4022981822490692, "grad_norm": 14.745206832885742, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.2866241931915283, "logits/rejected": -0.9531787037849426, "logps/chosen": -90.50650787353516, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -94.59053039550781, "loss": 1.0663, "margin_dpo/margin_mean": 7.684615612030029, "margin_dpo/margin_std": 8.983680725097656, "step": 118 }, { "KL/chosen_KL_mean": -5.899219512939453, "KL/mean": -9.935869216918945, "KL/rejected_KL_mean": -13.972515106201172, "KL/std": 10.175472259521484, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.05246628075838089, "fcm_dpo/delta": -0.03115876391530037, "fcm_dpo/margin": 8.073290824890137, "fcm_dpo/q_t": 0.4025569558143616, "grad_norm": 16.316574096679688, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.2231206893920898, "logits/rejected": -0.8906387090682983, "logps/chosen": -110.50830078125, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -123.13032531738281, "loss": 1.1093, "margin_dpo/margin_mean": 8.073290824890137, "margin_dpo/margin_std": 12.850178718566895, "step": 119 }, { "KL/chosen_KL_mean": -4.808917999267578, "KL/mean": -10.124603271484375, "KL/rejected_KL_mean": -15.44028091430664, "KL/std": 12.088314056396484, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.04721060395240784, "fcm_dpo/delta": -0.13835002481937408, "fcm_dpo/margin": 10.631363868713379, "fcm_dpo/q_t": 0.3848896026611328, "grad_norm": 17.04759407043457, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.4383158683776855, "logits/rejected": -0.8894621133804321, "logps/chosen": -119.49634552001953, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -126.62614440917969, "loss": 1.0611, "margin_dpo/margin_mean": 10.631362915039062, "margin_dpo/margin_std": 15.411432266235352, "step": 120 }, { "KL/chosen_KL_mean": -4.3831939697265625, "KL/mean": -9.282760620117188, "KL/rejected_KL_mean": -14.182331085205078, "KL/std": 9.501340866088867, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.043836843222379684, "fcm_dpo/delta": -0.039743851870298386, "fcm_dpo/margin": 9.799141883850098, "fcm_dpo/q_t": 0.398059606552124, "grad_norm": 15.672893524169922, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.0442187786102295, "logits/rejected": -0.707910418510437, "logps/chosen": -96.4998550415039, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -129.1552276611328, "loss": 1.052, "margin_dpo/margin_mean": 9.799141883850098, "margin_dpo/margin_std": 10.525093078613281, "step": 121 }, { "KL/chosen_KL_mean": -4.765674591064453, "KL/mean": -10.408536911010742, "KL/rejected_KL_mean": -16.0513916015625, "KL/std": 10.358043670654297, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.041880324482917786, "fcm_dpo/delta": -0.09473232924938202, "fcm_dpo/margin": 11.28571891784668, "fcm_dpo/q_t": 0.38653290271759033, "grad_norm": 16.07025718688965, "learning_rate": 4.911461260693638e-07, "logits/chosen": -1.0575600862503052, "logits/rejected": -0.9688279628753662, "logps/chosen": -89.64324951171875, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -127.1343765258789, "loss": 1.0317, "margin_dpo/margin_mean": 11.28571891784668, "margin_dpo/margin_std": 12.233473777770996, "step": 122 }, { "KL/chosen_KL_mean": -5.257568359375, "KL/mean": -9.335735321044922, "KL/rejected_KL_mean": -13.413890838623047, "KL/std": 9.494452476501465, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.04144714027643204, "fcm_dpo/delta": 0.07397902011871338, "fcm_dpo/margin": 8.156316757202148, "fcm_dpo/q_t": 0.4234795570373535, "grad_norm": 13.160378456115723, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.274225115776062, "logits/rejected": -0.9531005620956421, "logps/chosen": -103.73631286621094, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -107.26806640625, "loss": 1.1308, "margin_dpo/margin_mean": 8.156315803527832, "margin_dpo/margin_std": 10.986837387084961, "step": 123 }, { "KL/chosen_KL_mean": -5.278892517089844, "KL/mean": -10.263033866882324, "KL/rejected_KL_mean": -15.247169494628906, "KL/std": 11.924956321716309, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.04236108809709549, "fcm_dpo/delta": -0.028215467929840088, "fcm_dpo/margin": 9.968271255493164, "fcm_dpo/q_t": 0.40783774852752686, "grad_norm": 13.909711837768555, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.2021403312683105, "logits/rejected": -0.9369191527366638, "logps/chosen": -108.04179382324219, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -138.25277709960938, "loss": 1.1023, "margin_dpo/margin_mean": 9.968271255493164, "margin_dpo/margin_std": 15.532505989074707, "step": 124 }, { "KL/chosen_KL_mean": -6.652351379394531, "KL/mean": -10.209712982177734, "KL/rejected_KL_mean": -13.767074584960938, "KL/std": 9.741104125976562, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.04505613073706627, "fcm_dpo/delta": 0.09451193362474442, "fcm_dpo/margin": 7.114715576171875, "fcm_dpo/q_t": 0.42968934774398804, "grad_norm": 14.71399974822998, "learning_rate": 4.90102044194588e-07, "logits/chosen": -1.122762680053711, "logits/rejected": -0.9022940397262573, "logps/chosen": -97.07862854003906, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -103.0160903930664, "loss": 1.1623, "margin_dpo/margin_mean": 7.114715576171875, "margin_dpo/margin_std": 12.071730613708496, "step": 125 }, { "KL/chosen_KL_mean": -5.638824462890625, "KL/mean": -10.920883178710938, "KL/rejected_KL_mean": -16.20294189453125, "KL/std": 10.732868194580078, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.04353801906108856, "fcm_dpo/delta": -0.07810616493225098, "fcm_dpo/margin": 10.564114570617676, "fcm_dpo/q_t": 0.389649897813797, "grad_norm": 14.064850807189941, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.9380927085876465, "logits/rejected": -0.6466289758682251, "logps/chosen": -100.06170654296875, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -120.08450317382812, "loss": 1.0345, "margin_dpo/margin_mean": 10.564115524291992, "margin_dpo/margin_std": 11.310665130615234, "step": 126 }, { "KL/chosen_KL_mean": -5.706539154052734, "KL/mean": -9.53803825378418, "KL/rejected_KL_mean": -13.36953353881836, "KL/std": 9.644844055175781, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.04533717781305313, "fcm_dpo/delta": 0.06171207129955292, "fcm_dpo/margin": 7.6629958152771, "fcm_dpo/q_t": 0.4230027198791504, "grad_norm": 14.91133975982666, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.862733006477356, "logits/rejected": -0.4981622099876404, "logps/chosen": -90.45759582519531, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -96.91476440429688, "loss": 1.138, "margin_dpo/margin_mean": 7.662996292114258, "margin_dpo/margin_std": 12.089923858642578, "step": 127 }, { "KL/chosen_KL_mean": -5.589611053466797, "KL/mean": -9.69711685180664, "KL/rejected_KL_mean": -13.804616928100586, "KL/std": 12.145851135253906, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.047133758664131165, "fcm_dpo/delta": 0.012025699019432068, "fcm_dpo/margin": 8.215008735656738, "fcm_dpo/q_t": 0.4136927127838135, "grad_norm": 17.388444900512695, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7748000025749207, "logits/rejected": -0.27430593967437744, "logps/chosen": -96.20925903320312, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -96.90809631347656, "loss": 1.1461, "margin_dpo/margin_mean": 8.215008735656738, "margin_dpo/margin_std": 15.223422050476074, "step": 128 }, { "KL/chosen_KL_mean": -6.291412353515625, "KL/mean": -10.550219535827637, "KL/rejected_KL_mean": -14.809028625488281, "KL/std": 9.936651229858398, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.046685267239809036, "fcm_dpo/delta": 0.002050258219242096, "fcm_dpo/margin": 8.517616271972656, "fcm_dpo/q_t": 0.4135642647743225, "grad_norm": 14.013954162597656, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.99830162525177, "logits/rejected": -0.6885870099067688, "logps/chosen": -105.21012115478516, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -118.09322357177734, "loss": 1.1224, "margin_dpo/margin_mean": 8.517616271972656, "margin_dpo/margin_std": 13.672584533691406, "step": 129 }, { "KL/chosen_KL_mean": -4.889881134033203, "KL/mean": -10.375089645385742, "KL/rejected_KL_mean": -15.860298156738281, "KL/std": 11.192024230957031, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.044131435453891754, "fcm_dpo/delta": -0.11279173195362091, "fcm_dpo/margin": 10.970418930053711, "fcm_dpo/q_t": 0.3878905177116394, "grad_norm": 12.749807357788086, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.2828559875488281, "logits/rejected": -1.0426405668258667, "logps/chosen": -78.07357025146484, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -103.8818359375, "loss": 1.0473, "margin_dpo/margin_mean": 10.970418930053711, "margin_dpo/margin_std": 13.445935249328613, "step": 130 }, { "KL/chosen_KL_mean": -4.26873779296875, "KL/mean": -9.423567771911621, "KL/rejected_KL_mean": -14.578399658203125, "KL/std": 11.42249870300293, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.041312411427497864, "fcm_dpo/delta": -0.03565748780965805, "fcm_dpo/margin": 10.30965805053711, "fcm_dpo/q_t": 0.4012097716331482, "grad_norm": 15.178191184997559, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.2398402690887451, "logits/rejected": -1.0306782722473145, "logps/chosen": -78.22518157958984, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -109.03422546386719, "loss": 1.0789, "margin_dpo/margin_mean": 10.309657096862793, "margin_dpo/margin_std": 13.13311767578125, "step": 131 }, { "KL/chosen_KL_mean": -6.158870697021484, "KL/mean": -11.158656120300293, "KL/rejected_KL_mean": -16.158435821533203, "KL/std": 11.929384231567383, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.03926509991288185, "fcm_dpo/delta": 0.007965883240103722, "fcm_dpo/margin": 9.999558448791504, "fcm_dpo/q_t": 0.4098545014858246, "grad_norm": 13.97481632232666, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.391385555267334, "logits/rejected": -0.9650901556015015, "logps/chosen": -98.59835815429688, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -110.29095458984375, "loss": 1.103, "margin_dpo/margin_mean": 9.99955940246582, "margin_dpo/margin_std": 13.984575271606445, "step": 132 }, { "KL/chosen_KL_mean": -5.313484191894531, "KL/mean": -10.530542373657227, "KL/rejected_KL_mean": -15.747600555419922, "KL/std": 11.825967788696289, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.03966645896434784, "fcm_dpo/delta": -0.01750118099153042, "fcm_dpo/margin": 10.434115409851074, "fcm_dpo/q_t": 0.40446826815605164, "grad_norm": 13.228242874145508, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.6455813646316528, "logits/rejected": -1.24617338180542, "logps/chosen": -98.43014526367188, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -117.84251403808594, "loss": 1.0714, "margin_dpo/margin_mean": 10.434114456176758, "margin_dpo/margin_std": 12.543338775634766, "step": 133 }, { "KL/chosen_KL_mean": -5.489070892333984, "KL/mean": -11.091697692871094, "KL/rejected_KL_mean": -16.69432830810547, "KL/std": 11.129806518554688, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.039201922714710236, "fcm_dpo/delta": -0.053764164447784424, "fcm_dpo/margin": 11.205256462097168, "fcm_dpo/q_t": 0.3955914378166199, "grad_norm": 13.055850982666016, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.73142671585083, "logits/rejected": -1.3928914070129395, "logps/chosen": -89.03919982910156, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -110.57589721679688, "loss": 1.0555, "margin_dpo/margin_mean": 11.205256462097168, "margin_dpo/margin_std": 12.164222717285156, "step": 134 }, { "KL/chosen_KL_mean": -5.147111892700195, "KL/mean": -10.892704010009766, "KL/rejected_KL_mean": -16.6383056640625, "KL/std": 12.523448944091797, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.03685580939054489, "fcm_dpo/delta": -0.029993582516908646, "fcm_dpo/margin": 11.49118423461914, "fcm_dpo/q_t": 0.40367603302001953, "grad_norm": 16.24430274963379, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.3129987716674805, "logits/rejected": -1.2074275016784668, "logps/chosen": -82.729736328125, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -137.69479370117188, "loss": 1.0723, "margin_dpo/margin_mean": 11.491186141967773, "margin_dpo/margin_std": 14.243597984313965, "step": 135 }, { "KL/chosen_KL_mean": -6.055454254150391, "KL/mean": -11.51393985748291, "KL/rejected_KL_mean": -16.972431182861328, "KL/std": 12.704488754272461, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.03624006360769272, "fcm_dpo/delta": 0.005436797626316547, "fcm_dpo/margin": 10.916970252990723, "fcm_dpo/q_t": 0.4089515805244446, "grad_norm": 12.204792976379395, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.5238163471221924, "logits/rejected": -1.0878732204437256, "logps/chosen": -100.22818756103516, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -120.23672485351562, "loss": 1.0805, "margin_dpo/margin_mean": 10.916969299316406, "margin_dpo/margin_std": 13.137273788452148, "step": 136 }, { "KL/chosen_KL_mean": -4.742633819580078, "KL/mean": -13.568758964538574, "KL/rejected_KL_mean": -22.394882202148438, "KL/std": 16.540176391601562, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.030293885618448257, "fcm_dpo/delta": -0.19980694353580475, "fcm_dpo/margin": 17.652252197265625, "fcm_dpo/q_t": 0.3680573105812073, "grad_norm": 14.358153343200684, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.6943318843841553, "logits/rejected": -1.4849953651428223, "logps/chosen": -82.86137390136719, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -140.3172607421875, "loss": 0.9877, "margin_dpo/margin_mean": 17.652254104614258, "margin_dpo/margin_std": 18.226600646972656, "step": 137 }, { "KL/chosen_KL_mean": -7.726005554199219, "KL/mean": -13.050067901611328, "KL/rejected_KL_mean": -18.374126434326172, "KL/std": 11.96668815612793, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.03145552799105644, "fcm_dpo/delta": 0.07831856608390808, "fcm_dpo/margin": 10.648128509521484, "fcm_dpo/q_t": 0.42406925559043884, "grad_norm": 10.967446327209473, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.9240460395812988, "logits/rejected": -0.6425586938858032, "logps/chosen": -100.55595397949219, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -117.02252960205078, "loss": 1.1223, "margin_dpo/margin_mean": 10.648128509521484, "margin_dpo/margin_std": 13.526496887207031, "step": 138 }, { "KL/chosen_KL_mean": -5.306785583496094, "KL/mean": -11.523063659667969, "KL/rejected_KL_mean": -17.73934555053711, "KL/std": 12.881488800048828, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.03197995573282242, "fcm_dpo/delta": 0.002651616930961609, "fcm_dpo/margin": 12.432565689086914, "fcm_dpo/q_t": 0.40631160140037537, "grad_norm": 11.436823844909668, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.255253553390503, "logits/rejected": -0.915815532207489, "logps/chosen": -84.30979919433594, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -103.56668090820312, "loss": 1.0844, "margin_dpo/margin_mean": 12.432564735412598, "margin_dpo/margin_std": 15.422233581542969, "step": 139 }, { "KL/chosen_KL_mean": -5.975318908691406, "KL/mean": -11.600736618041992, "KL/rejected_KL_mean": -17.226154327392578, "KL/std": 12.166454315185547, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.03320346772670746, "fcm_dpo/delta": 0.032281871885061264, "fcm_dpo/margin": 11.250846862792969, "fcm_dpo/q_t": 0.41346126794815063, "grad_norm": 11.045943260192871, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.7294979095458984, "logits/rejected": -1.315974235534668, "logps/chosen": -94.32776641845703, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -107.8009262084961, "loss": 1.0921, "margin_dpo/margin_mean": 11.250845909118652, "margin_dpo/margin_std": 13.126893997192383, "step": 140 }, { "KL/chosen_KL_mean": -8.328506469726562, "KL/mean": -14.580768585205078, "KL/rejected_KL_mean": -20.833030700683594, "KL/std": 16.498218536376953, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.03305599465966225, "fcm_dpo/delta": -0.016817986965179443, "fcm_dpo/margin": 12.504520416259766, "fcm_dpo/q_t": 0.4033375084400177, "grad_norm": 14.025592803955078, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.3751758337020874, "logits/rejected": -0.9677672982215881, "logps/chosen": -115.54866790771484, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -125.54525756835938, "loss": 1.0868, "margin_dpo/margin_mean": 12.504520416259766, "margin_dpo/margin_std": 16.89531135559082, "step": 141 }, { "KL/chosen_KL_mean": -8.10693359375, "KL/mean": -14.37746810913086, "KL/rejected_KL_mean": -20.64800262451172, "KL/std": 15.011102676391602, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.032375965267419815, "fcm_dpo/delta": -0.007865412160754204, "fcm_dpo/margin": 12.541069030761719, "fcm_dpo/q_t": 0.4074018597602844, "grad_norm": 11.0122709274292, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.152029275894165, "logits/rejected": -0.8840595483779907, "logps/chosen": -96.83587646484375, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -120.37838745117188, "loss": 1.0791, "margin_dpo/margin_mean": 12.541069030761719, "margin_dpo/margin_std": 15.735469818115234, "step": 142 }, { "KL/chosen_KL_mean": -6.350429534912109, "KL/mean": -11.379204750061035, "KL/rejected_KL_mean": -16.407974243164062, "KL/std": 12.301786422729492, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.03378184139728546, "fcm_dpo/delta": 0.07258955389261246, "fcm_dpo/margin": 10.05754280090332, "fcm_dpo/q_t": 0.4230427145957947, "grad_norm": 11.739222526550293, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.8390339612960815, "logits/rejected": -1.3921382427215576, "logps/chosen": -91.85501098632812, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -102.21363830566406, "loss": 1.1259, "margin_dpo/margin_mean": 10.057541847229004, "margin_dpo/margin_std": 13.367086410522461, "step": 143 }, { "KL/chosen_KL_mean": -7.836193084716797, "KL/mean": -12.773773193359375, "KL/rejected_KL_mean": -17.711353302001953, "KL/std": 13.22342300415039, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.03710213676095009, "fcm_dpo/delta": 0.035222604870796204, "fcm_dpo/margin": 9.875158309936523, "fcm_dpo/q_t": 0.41745567321777344, "grad_norm": 17.035795211791992, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.6299495697021484, "logits/rejected": -1.0775026082992554, "logps/chosen": -112.52328491210938, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -105.43043518066406, "loss": 1.143, "margin_dpo/margin_mean": 9.875158309936523, "margin_dpo/margin_std": 16.47283935546875, "step": 144 }, { "KL/chosen_KL_mean": -6.375873565673828, "KL/mean": -12.409714698791504, "KL/rejected_KL_mean": -18.443557739257812, "KL/std": 12.504430770874023, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.03535092994570732, "fcm_dpo/delta": -0.03384307399392128, "fcm_dpo/margin": 12.0676851272583, "fcm_dpo/q_t": 0.39995336532592773, "grad_norm": 12.51016616821289, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.1608127355575562, "logits/rejected": -0.5706623196601868, "logps/chosen": -94.89576721191406, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -102.20869445800781, "loss": 1.0622, "margin_dpo/margin_mean": 12.067686080932617, "margin_dpo/margin_std": 14.101991653442383, "step": 145 }, { "KL/chosen_KL_mean": -7.114696502685547, "KL/mean": -13.66038990020752, "KL/rejected_KL_mean": -20.206085205078125, "KL/std": 13.602307319641113, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.0330333337187767, "fcm_dpo/delta": -0.04571498930454254, "fcm_dpo/margin": 13.091388702392578, "fcm_dpo/q_t": 0.39727506041526794, "grad_norm": 12.048237800598145, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.6899479627609253, "logits/rejected": -1.0224535465240479, "logps/chosen": -110.02056121826172, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -109.15544891357422, "loss": 1.0602, "margin_dpo/margin_mean": 13.091388702392578, "margin_dpo/margin_std": 15.205850601196289, "step": 146 }, { "KL/chosen_KL_mean": -6.706836700439453, "KL/mean": -13.515523910522461, "KL/rejected_KL_mean": -20.32421112060547, "KL/std": 12.95321273803711, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.03229077160358429, "fcm_dpo/delta": -0.05088489130139351, "fcm_dpo/margin": 13.617376327514648, "fcm_dpo/q_t": 0.39549872279167175, "grad_norm": 11.222355842590332, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.748637318611145, "logits/rejected": -1.1770691871643066, "logps/chosen": -95.96902465820312, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -96.82432556152344, "loss": 1.0421, "margin_dpo/margin_mean": 13.617377281188965, "margin_dpo/margin_std": 14.129131317138672, "step": 147 }, { "KL/chosen_KL_mean": -5.013446807861328, "KL/mean": -11.57408332824707, "KL/rejected_KL_mean": -18.134719848632812, "KL/std": 13.838018417358398, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.032298021018505096, "fcm_dpo/delta": -0.03590024262666702, "fcm_dpo/margin": 13.121277809143066, "fcm_dpo/q_t": 0.403020977973938, "grad_norm": 11.59742259979248, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.744011640548706, "logits/rejected": -1.3514835834503174, "logps/chosen": -96.77632904052734, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -128.52024841308594, "loss": 1.0761, "margin_dpo/margin_mean": 13.121277809143066, "margin_dpo/margin_std": 15.740398406982422, "step": 148 }, { "KL/chosen_KL_mean": -8.353858947753906, "KL/mean": -14.527746200561523, "KL/rejected_KL_mean": -20.701629638671875, "KL/std": 14.797009468078613, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.030692007392644882, "fcm_dpo/delta": 0.02488698437809944, "fcm_dpo/margin": 12.347770690917969, "fcm_dpo/q_t": 0.41692036390304565, "grad_norm": 11.461216926574707, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.4702894687652588, "logits/rejected": -1.1338486671447754, "logps/chosen": -101.83787536621094, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -126.34532928466797, "loss": 1.1191, "margin_dpo/margin_mean": 12.347771644592285, "margin_dpo/margin_std": 18.69577407836914, "step": 149 }, { "KL/chosen_KL_mean": -6.8068695068359375, "KL/mean": -13.210695266723633, "KL/rejected_KL_mean": -19.614521026611328, "KL/std": 15.506492614746094, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.031135190278291702, "fcm_dpo/delta": 0.0009706467390060425, "fcm_dpo/margin": 12.807655334472656, "fcm_dpo/q_t": 0.4092704951763153, "grad_norm": 10.339449882507324, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.5506547689437866, "logits/rejected": -1.2951431274414062, "logps/chosen": -82.58555603027344, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -109.64128112792969, "loss": 1.0921, "margin_dpo/margin_mean": 12.807655334472656, "margin_dpo/margin_std": 17.250259399414062, "step": 150 }, { "KL/chosen_KL_mean": -8.748283386230469, "KL/mean": -14.980594635009766, "KL/rejected_KL_mean": -21.21289825439453, "KL/std": 14.852688789367676, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.03160766139626503, "fcm_dpo/delta": 0.007497953251004219, "fcm_dpo/margin": 12.464618682861328, "fcm_dpo/q_t": 0.41037604212760925, "grad_norm": 12.870458602905273, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.3628439903259277, "logits/rejected": -1.7876429557800293, "logps/chosen": -111.48250579833984, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -127.2507095336914, "loss": 1.096, "margin_dpo/margin_mean": 12.464617729187012, "margin_dpo/margin_std": 17.052486419677734, "step": 151 }, { "KL/chosen_KL_mean": -5.981044769287109, "KL/mean": -13.556360244750977, "KL/rejected_KL_mean": -21.131675720214844, "KL/std": 14.90648365020752, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.029856868088245392, "fcm_dpo/delta": -0.06903348863124847, "fcm_dpo/margin": 15.150629043579102, "fcm_dpo/q_t": 0.39234402775764465, "grad_norm": 11.063338279724121, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.6887303590774536, "logits/rejected": -1.2993872165679932, "logps/chosen": -103.57179260253906, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -131.63079833984375, "loss": 1.0487, "margin_dpo/margin_mean": 15.150629043579102, "margin_dpo/margin_std": 17.75276756286621, "step": 152 }, { "KL/chosen_KL_mean": -8.239131927490234, "KL/mean": -13.717041015625, "KL/rejected_KL_mean": -19.194957733154297, "KL/std": 14.55648422241211, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.03128330409526825, "fcm_dpo/delta": 0.06868214160203934, "fcm_dpo/margin": 10.955827713012695, "fcm_dpo/q_t": 0.42307090759277344, "grad_norm": 11.456689834594727, "learning_rate": 4.776497044244016e-07, "logits/chosen": -2.094259023666382, "logits/rejected": -1.786583423614502, "logps/chosen": -95.26473236083984, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -110.26349639892578, "loss": 1.1397, "margin_dpo/margin_mean": 10.955827713012695, "margin_dpo/margin_std": 16.96314239501953, "step": 153 }, { "KL/chosen_KL_mean": -6.660789489746094, "KL/mean": -15.43112564086914, "KL/rejected_KL_mean": -24.20146942138672, "KL/std": 15.92460823059082, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.029300162568688393, "fcm_dpo/delta": -0.1525585949420929, "fcm_dpo/margin": 17.54067611694336, "fcm_dpo/q_t": 0.37518295645713806, "grad_norm": 10.90088176727295, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.501826286315918, "logits/rejected": -1.179551362991333, "logps/chosen": -97.7664566040039, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -135.5738525390625, "loss": 1.0045, "margin_dpo/margin_mean": 17.54067611694336, "margin_dpo/margin_std": 17.358489990234375, "step": 154 }, { "KL/chosen_KL_mean": -8.802101135253906, "KL/mean": -15.240921020507812, "KL/rejected_KL_mean": -21.679733276367188, "KL/std": 16.325271606445312, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.027737107127904892, "fcm_dpo/delta": 0.051422543823719025, "fcm_dpo/margin": 12.877633094787598, "fcm_dpo/q_t": 0.4169430434703827, "grad_norm": 12.51191234588623, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.6491388082504272, "logits/rejected": -1.2653968334197998, "logps/chosen": -103.36307525634766, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -106.45332336425781, "loss": 1.1374, "margin_dpo/margin_mean": 12.877634048461914, "margin_dpo/margin_std": 20.486942291259766, "step": 155 }, { "KL/chosen_KL_mean": -10.472888946533203, "KL/mean": -16.265338897705078, "KL/rejected_KL_mean": -22.057781219482422, "KL/std": 14.660511016845703, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.02952522225677967, "fcm_dpo/delta": 0.06930956244468689, "fcm_dpo/margin": 11.584893226623535, "fcm_dpo/q_t": 0.42500463128089905, "grad_norm": 12.215765953063965, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.695134162902832, "logits/rejected": -1.1871190071105957, "logps/chosen": -105.12828063964844, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -106.47712707519531, "loss": 1.1445, "margin_dpo/margin_mean": 11.584893226623535, "margin_dpo/margin_std": 18.20134925842285, "step": 156 }, { "KL/chosen_KL_mean": -9.882583618164062, "KL/mean": -18.22896957397461, "KL/rejected_KL_mean": -26.57535171508789, "KL/std": 15.878637313842773, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.02858750708401203, "fcm_dpo/delta": -0.10139432549476624, "fcm_dpo/margin": 16.692768096923828, "fcm_dpo/q_t": 0.3844242990016937, "grad_norm": 11.946284294128418, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.242360830307007, "logits/rejected": -1.795778751373291, "logps/chosen": -106.89497375488281, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -138.27670288085938, "loss": 1.02, "margin_dpo/margin_mean": 16.692768096923828, "margin_dpo/margin_std": 16.960887908935547, "step": 157 }, { "KL/chosen_KL_mean": -9.01821517944336, "KL/mean": -17.68844985961914, "KL/rejected_KL_mean": -26.358680725097656, "KL/std": 17.18183135986328, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.02642417699098587, "fcm_dpo/delta": -0.0752686932682991, "fcm_dpo/margin": 17.340465545654297, "fcm_dpo/q_t": 0.3915228843688965, "grad_norm": 10.48434066772461, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.5603117942810059, "logits/rejected": -1.303189754486084, "logps/chosen": -103.7311782836914, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -147.2960205078125, "loss": 1.0361, "margin_dpo/margin_mean": 17.340465545654297, "margin_dpo/margin_std": 18.844329833984375, "step": 158 }, { "KL/chosen_KL_mean": -8.63034439086914, "KL/mean": -16.157297134399414, "KL/rejected_KL_mean": -23.684246063232422, "KL/std": 16.34818458557129, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.025986962020397186, "fcm_dpo/delta": 0.010914142243564129, "fcm_dpo/margin": 15.053915023803711, "fcm_dpo/q_t": 0.40958553552627563, "grad_norm": 9.778227806091309, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.2328193187713623, "logits/rejected": -1.7503135204315186, "logps/chosen": -116.84858703613281, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -118.52326965332031, "loss": 1.0886, "margin_dpo/margin_mean": 15.053915023803711, "margin_dpo/margin_std": 18.865768432617188, "step": 159 }, { "KL/chosen_KL_mean": -7.338710784912109, "KL/mean": -14.93021297454834, "KL/rejected_KL_mean": -22.52172088623047, "KL/std": 15.042978286743164, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.025954831391572952, "fcm_dpo/delta": 0.007132204249501228, "fcm_dpo/margin": 15.182998657226562, "fcm_dpo/q_t": 0.40725454688072205, "grad_norm": 9.95501708984375, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.7437938451766968, "logits/rejected": -1.1986379623413086, "logps/chosen": -105.50212097167969, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -124.80270385742188, "loss": 1.0734, "margin_dpo/margin_mean": 15.182998657226562, "margin_dpo/margin_std": 16.355247497558594, "step": 160 }, { "KL/chosen_KL_mean": -9.328296661376953, "KL/mean": -16.03545570373535, "KL/rejected_KL_mean": -22.74261474609375, "KL/std": 13.65327262878418, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.026775486767292023, "fcm_dpo/delta": 0.04946756735444069, "fcm_dpo/margin": 13.414326667785645, "fcm_dpo/q_t": 0.417966365814209, "grad_norm": 10.30246353149414, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -2.061227321624756, "logits/rejected": -1.5780619382858276, "logps/chosen": -95.920166015625, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -111.75904846191406, "loss": 1.1163, "margin_dpo/margin_mean": 13.414325714111328, "margin_dpo/margin_std": 18.00604248046875, "step": 161 }, { "KL/chosen_KL_mean": -8.443050384521484, "KL/mean": -15.984201431274414, "KL/rejected_KL_mean": -23.525352478027344, "KL/std": 16.024250030517578, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.02694505825638771, "fcm_dpo/delta": -0.009272031486034393, "fcm_dpo/margin": 15.082305908203125, "fcm_dpo/q_t": 0.4043654501438141, "grad_norm": 9.978202819824219, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.4984841346740723, "logits/rejected": -1.1039178371429443, "logps/chosen": -105.84593200683594, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -119.39458465576172, "loss": 1.0726, "margin_dpo/margin_mean": 15.082305908203125, "margin_dpo/margin_std": 17.367443084716797, "step": 162 }, { "KL/chosen_KL_mean": -8.183639526367188, "KL/mean": -17.03965187072754, "KL/rejected_KL_mean": -25.895668029785156, "KL/std": 18.026674270629883, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.025536291301250458, "fcm_dpo/delta": -0.06996553391218185, "fcm_dpo/margin": 17.712032318115234, "fcm_dpo/q_t": 0.39307117462158203, "grad_norm": 8.8844575881958, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.7190968990325928, "logits/rejected": -1.318759799003601, "logps/chosen": -90.3590316772461, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -108.76040649414062, "loss": 1.0406, "margin_dpo/margin_mean": 17.712032318115234, "margin_dpo/margin_std": 19.59677505493164, "step": 163 }, { "KL/chosen_KL_mean": -12.025035858154297, "KL/mean": -18.09355926513672, "KL/rejected_KL_mean": -24.16208267211914, "KL/std": 14.223909378051758, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.025103319436311722, "fcm_dpo/delta": -0.013811816461384296, "fcm_dpo/margin": 12.137043952941895, "fcm_dpo/q_t": 0.4269532263278961, "grad_norm": 10.278435707092285, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.8588850498199463, "logits/rejected": -1.4150574207305908, "logps/chosen": -102.49930572509766, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -111.19017028808594, "loss": 1.147, "margin_dpo/margin_mean": 12.137044906616211, "margin_dpo/margin_std": 16.108354568481445, "step": 164 }, { "KL/chosen_KL_mean": -8.357200622558594, "KL/mean": -17.087276458740234, "KL/rejected_KL_mean": -25.817352294921875, "KL/std": 16.669830322265625, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.024621862918138504, "fcm_dpo/delta": -0.03806956112384796, "fcm_dpo/margin": 17.460153579711914, "fcm_dpo/q_t": 0.3996947407722473, "grad_norm": 8.409309387207031, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.9583961963653564, "logits/rejected": -1.5361154079437256, "logps/chosen": -95.67042541503906, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -107.71994018554688, "loss": 1.0522, "margin_dpo/margin_mean": 17.46015167236328, "margin_dpo/margin_std": 19.091873168945312, "step": 165 }, { "KL/chosen_KL_mean": -13.024986267089844, "KL/mean": -21.7943115234375, "KL/rejected_KL_mean": -30.563636779785156, "KL/std": 18.736536026000977, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.023765943944454193, "fcm_dpo/delta": -0.021287992596626282, "fcm_dpo/margin": 17.53864097595215, "fcm_dpo/q_t": 0.4059121012687683, "grad_norm": 9.579103469848633, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.854757308959961, "logits/rejected": -1.6598014831542969, "logps/chosen": -94.61105346679688, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -133.72882080078125, "loss": 1.0764, "margin_dpo/margin_mean": 17.53864097595215, "margin_dpo/margin_std": 22.872329711914062, "step": 166 }, { "KL/chosen_KL_mean": -11.706672668457031, "KL/mean": -21.002195358276367, "KL/rejected_KL_mean": -30.297718048095703, "KL/std": 17.076831817626953, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.02260380983352661, "fcm_dpo/delta": -0.029087122529745102, "fcm_dpo/margin": 18.591049194335938, "fcm_dpo/q_t": 0.3994421362876892, "grad_norm": 8.685297966003418, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.1954982280731201, "logits/rejected": -1.0285547971725464, "logps/chosen": -95.26834106445312, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -129.35076904296875, "loss": 1.0492, "margin_dpo/margin_mean": 18.591047286987305, "margin_dpo/margin_std": 17.77145767211914, "step": 167 }, { "KL/chosen_KL_mean": -11.17022705078125, "KL/mean": -19.209747314453125, "KL/rejected_KL_mean": -27.249271392822266, "KL/std": 16.860321044921875, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.02360011637210846, "fcm_dpo/delta": 0.024780407547950745, "fcm_dpo/margin": 16.079044342041016, "fcm_dpo/q_t": 0.41144290566444397, "grad_norm": 9.503809928894043, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.8133610486984253, "logits/rejected": -1.4783084392547607, "logps/chosen": -91.1014633178711, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -107.3514404296875, "loss": 1.0819, "margin_dpo/margin_mean": 16.07904624938965, "margin_dpo/margin_std": 17.243244171142578, "step": 168 }, { "KL/chosen_KL_mean": -9.694145202636719, "KL/mean": -18.956661224365234, "KL/rejected_KL_mean": -28.219181060791016, "KL/std": 18.29271125793457, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.022906193509697914, "fcm_dpo/delta": -0.031197769567370415, "fcm_dpo/margin": 18.52503204345703, "fcm_dpo/q_t": 0.3999554514884949, "grad_norm": 9.250802040100098, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.725956678390503, "logits/rejected": -1.5073273181915283, "logps/chosen": -92.59785461425781, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -118.34109497070312, "loss": 1.0528, "margin_dpo/margin_mean": 18.52503204345703, "margin_dpo/margin_std": 19.53860855102539, "step": 169 }, { "KL/chosen_KL_mean": -11.246002197265625, "KL/mean": -19.23270034790039, "KL/rejected_KL_mean": -27.21939468383789, "KL/std": 15.14326286315918, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.023020930588245392, "fcm_dpo/delta": 0.03851843625307083, "fcm_dpo/margin": 15.973390579223633, "fcm_dpo/q_t": 0.41338926553726196, "grad_norm": 9.70813274383545, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.5175697803497314, "logits/rejected": -1.3046817779541016, "logps/chosen": -86.22797393798828, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -109.38499450683594, "loss": 1.0847, "margin_dpo/margin_mean": 15.973390579223633, "margin_dpo/margin_std": 15.195291519165039, "step": 170 }, { "KL/chosen_KL_mean": -14.640754699707031, "KL/mean": -23.50275993347168, "KL/rejected_KL_mean": -32.36477279663086, "KL/std": 22.283531188964844, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.023487437516450882, "fcm_dpo/delta": -0.020574606955051422, "fcm_dpo/margin": 17.724008560180664, "fcm_dpo/q_t": 0.4085419774055481, "grad_norm": 11.532255172729492, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.7726773023605347, "logits/rejected": -1.2215464115142822, "logps/chosen": -133.61927795410156, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -148.1340789794922, "loss": 1.0996, "margin_dpo/margin_mean": 17.724010467529297, "margin_dpo/margin_std": 27.127548217773438, "step": 171 }, { "KL/chosen_KL_mean": -15.617515563964844, "KL/mean": -23.349994659423828, "KL/rejected_KL_mean": -31.082469940185547, "KL/std": 15.75918197631836, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.023721888661384583, "fcm_dpo/delta": 0.04057793691754341, "fcm_dpo/margin": 15.464958190917969, "fcm_dpo/q_t": 0.41323837637901306, "grad_norm": 9.596761703491211, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.2844831943511963, "logits/rejected": -1.4943002462387085, "logps/chosen": -116.77130126953125, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -122.08815002441406, "loss": 1.077, "margin_dpo/margin_mean": 15.464958190917969, "margin_dpo/margin_std": 13.010668754577637, "step": 172 }, { "KL/chosen_KL_mean": -13.783706665039062, "KL/mean": -20.679168701171875, "KL/rejected_KL_mean": -27.574630737304688, "KL/std": 14.951168060302734, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.025508373975753784, "fcm_dpo/delta": 0.057387758046388626, "fcm_dpo/margin": 13.790926933288574, "fcm_dpo/q_t": 0.4185292720794678, "grad_norm": 9.617586135864258, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.8402680158615112, "logits/rejected": -1.3737025260925293, "logps/chosen": -112.84039306640625, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -111.42536163330078, "loss": 1.1037, "margin_dpo/margin_mean": 13.790926933288574, "margin_dpo/margin_std": 15.762795448303223, "step": 173 }, { "KL/chosen_KL_mean": -13.510988235473633, "KL/mean": -23.187362670898438, "KL/rejected_KL_mean": -32.863739013671875, "KL/std": 20.68968963623047, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.02414734847843647, "fcm_dpo/delta": -0.08764201402664185, "fcm_dpo/margin": 19.352750778198242, "fcm_dpo/q_t": 0.3884669542312622, "grad_norm": 10.640708923339844, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.9299228191375732, "logits/rejected": -1.8951292037963867, "logps/chosen": -93.91989135742188, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -141.06546020507812, "loss": 1.0315, "margin_dpo/margin_mean": 19.352752685546875, "margin_dpo/margin_std": 21.118885040283203, "step": 174 }, { "KL/chosen_KL_mean": -17.026233673095703, "KL/mean": -26.24807357788086, "KL/rejected_KL_mean": -35.46991729736328, "KL/std": 18.233856201171875, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.023151367902755737, "fcm_dpo/delta": -0.03444796800613403, "fcm_dpo/margin": 18.443683624267578, "fcm_dpo/q_t": 0.39746299386024475, "grad_norm": 10.261703491210938, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.2694547176361084, "logits/rejected": -1.0000406503677368, "logps/chosen": -99.74272155761719, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -132.13705444335938, "loss": 1.0448, "margin_dpo/margin_mean": 18.443683624267578, "margin_dpo/margin_std": 17.517757415771484, "step": 175 }, { "KL/chosen_KL_mean": -14.07122802734375, "KL/mean": -23.643081665039062, "KL/rejected_KL_mean": -33.21493148803711, "KL/std": 19.77219581604004, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.022144177928566933, "fcm_dpo/delta": -0.030458027496933937, "fcm_dpo/margin": 19.14370346069336, "fcm_dpo/q_t": 0.40107303857803345, "grad_norm": 10.001806259155273, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.574167013168335, "logits/rejected": -1.4204909801483154, "logps/chosen": -93.70053100585938, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -135.90353393554688, "loss": 1.0669, "margin_dpo/margin_mean": 19.14370346069336, "margin_dpo/margin_std": 23.025402069091797, "step": 176 }, { "KL/chosen_KL_mean": -16.02703857421875, "KL/mean": -27.14234733581543, "KL/rejected_KL_mean": -38.25765609741211, "KL/std": 22.841716766357422, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.02102130651473999, "fcm_dpo/delta": -0.08760670572519302, "fcm_dpo/margin": 22.230615615844727, "fcm_dpo/q_t": 0.3901672661304474, "grad_norm": 8.968939781188965, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.708207368850708, "logits/rejected": -1.3717527389526367, "logps/chosen": -108.588134765625, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -141.59603881835938, "loss": 1.0301, "margin_dpo/margin_mean": 22.230613708496094, "margin_dpo/margin_std": 23.80647850036621, "step": 177 }, { "KL/chosen_KL_mean": -14.664066314697266, "KL/mean": -27.89319610595703, "KL/rejected_KL_mean": -41.12232208251953, "KL/std": 22.152904510498047, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.01839258149266243, "fcm_dpo/delta": -0.11595679819583893, "fcm_dpo/margin": 26.458251953125, "fcm_dpo/q_t": 0.38095957040786743, "grad_norm": 9.25490665435791, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.6748722791671753, "logits/rejected": -1.465683937072754, "logps/chosen": -98.92986297607422, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -151.9196319580078, "loss": 1.0014, "margin_dpo/margin_mean": 26.458251953125, "margin_dpo/margin_std": 23.887147903442383, "step": 178 }, { "KL/chosen_KL_mean": -18.774662017822266, "KL/mean": -27.11644172668457, "KL/rejected_KL_mean": -35.45821762084961, "KL/std": 21.37868881225586, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.01909565180540085, "fcm_dpo/delta": 0.09737280011177063, "fcm_dpo/margin": 16.683565139770508, "fcm_dpo/q_t": 0.4290935695171356, "grad_norm": 8.929889678955078, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.3933687210083008, "logits/rejected": -0.9525288343429565, "logps/chosen": -112.22264099121094, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -129.713623046875, "loss": 1.1496, "margin_dpo/margin_mean": 16.683563232421875, "margin_dpo/margin_std": 25.12432861328125, "step": 179 }, { "KL/chosen_KL_mean": -15.641124725341797, "KL/mean": -26.620159149169922, "KL/rejected_KL_mean": -37.59919738769531, "KL/std": 20.529296875, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.01915919780731201, "fcm_dpo/delta": -0.027312349528074265, "fcm_dpo/margin": 21.958078384399414, "fcm_dpo/q_t": 0.3999558091163635, "grad_norm": 10.984769821166992, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.3352171182632446, "logits/rejected": -1.0998756885528564, "logps/chosen": -104.47666931152344, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -141.50616455078125, "loss": 1.0468, "margin_dpo/margin_mean": 21.95807647705078, "margin_dpo/margin_std": 21.14169692993164, "step": 180 }, { "KL/chosen_KL_mean": -15.403095245361328, "KL/mean": -23.651805877685547, "KL/rejected_KL_mean": -31.900508880615234, "KL/std": 19.677555084228516, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.020471904426813126, "fcm_dpo/delta": 0.07417376339435577, "fcm_dpo/margin": 16.497411727905273, "fcm_dpo/q_t": 0.42375648021698, "grad_norm": 8.679154396057129, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.7934125661849976, "logits/rejected": -1.3403092622756958, "logps/chosen": -94.63705444335938, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -101.07493591308594, "loss": 1.1201, "margin_dpo/margin_mean": 16.497413635253906, "margin_dpo/margin_std": 21.13150405883789, "step": 181 }, { "KL/chosen_KL_mean": -19.270896911621094, "KL/mean": -30.23662757873535, "KL/rejected_KL_mean": -41.20235824584961, "KL/std": 22.852035522460938, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.019751984626054764, "fcm_dpo/delta": -0.04442457854747772, "fcm_dpo/margin": 21.93145751953125, "fcm_dpo/q_t": 0.39787203073501587, "grad_norm": 9.238212585449219, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.3798832893371582, "logits/rejected": -1.1711434125900269, "logps/chosen": -95.35308074951172, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -136.17095947265625, "loss": 1.0569, "margin_dpo/margin_mean": 21.93145751953125, "margin_dpo/margin_std": 25.202974319458008, "step": 182 }, { "KL/chosen_KL_mean": -20.518768310546875, "KL/mean": -30.923627853393555, "KL/rejected_KL_mean": -41.3284912109375, "KL/std": 20.78168487548828, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.01937912032008171, "fcm_dpo/delta": -0.005916904658079147, "fcm_dpo/margin": 20.809715270996094, "fcm_dpo/q_t": 0.4043998420238495, "grad_norm": 9.696259498596191, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.4464740753173828, "logits/rejected": -0.8211211562156677, "logps/chosen": -114.42667388916016, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -124.88910675048828, "loss": 1.0655, "margin_dpo/margin_mean": 20.809715270996094, "margin_dpo/margin_std": 21.25434112548828, "step": 183 }, { "KL/chosen_KL_mean": -18.11540985107422, "KL/mean": -25.275827407836914, "KL/rejected_KL_mean": -32.436241149902344, "KL/std": 21.461109161376953, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.021522991359233856, "fcm_dpo/delta": 0.10718071460723877, "fcm_dpo/margin": 14.32082748413086, "fcm_dpo/q_t": 0.43053269386291504, "grad_norm": 9.42421817779541, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.723520278930664, "logits/rejected": -1.4406230449676514, "logps/chosen": -104.98332214355469, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -116.72079467773438, "loss": 1.1612, "margin_dpo/margin_mean": 14.32082748413086, "margin_dpo/margin_std": 23.366987228393555, "step": 184 }, { "KL/chosen_KL_mean": -19.26616668701172, "KL/mean": -27.779523849487305, "KL/rejected_KL_mean": -36.29288864135742, "KL/std": 21.036174774169922, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.02259145677089691, "fcm_dpo/delta": 0.017603283748030663, "fcm_dpo/margin": 17.026723861694336, "fcm_dpo/q_t": 0.4127582311630249, "grad_norm": 11.019499778747559, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.0956642627716064, "logits/rejected": -1.4612774848937988, "logps/chosen": -113.28759002685547, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -117.74957275390625, "loss": 1.1077, "margin_dpo/margin_mean": 17.026723861694336, "margin_dpo/margin_std": 24.20270347595215, "step": 185 }, { "KL/chosen_KL_mean": -24.184925079345703, "KL/mean": -33.42481231689453, "KL/rejected_KL_mean": -42.66469192504883, "KL/std": 20.490568161010742, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.02188299223780632, "fcm_dpo/delta": -0.006783852353692055, "fcm_dpo/margin": 18.479766845703125, "fcm_dpo/q_t": 0.40534502267837524, "grad_norm": 11.6025972366333, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.3166654109954834, "logits/rejected": -1.3908374309539795, "logps/chosen": -138.37661743164062, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -136.00381469726562, "loss": 1.0796, "margin_dpo/margin_mean": 18.479766845703125, "margin_dpo/margin_std": 22.405405044555664, "step": 186 }, { "KL/chosen_KL_mean": -20.316513061523438, "KL/mean": -29.33014678955078, "KL/rejected_KL_mean": -38.34377670288086, "KL/std": 22.862106323242188, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.022458814084529877, "fcm_dpo/delta": -0.007275024428963661, "fcm_dpo/margin": 18.027267456054688, "fcm_dpo/q_t": 0.40739840269088745, "grad_norm": 10.186311721801758, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.9312413930892944, "logits/rejected": -1.6999621391296387, "logps/chosen": -97.18606567382812, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -126.50872039794922, "loss": 1.0961, "margin_dpo/margin_mean": 18.027267456054688, "margin_dpo/margin_std": 25.019088745117188, "step": 187 }, { "KL/chosen_KL_mean": -15.72100830078125, "KL/mean": -26.020931243896484, "KL/rejected_KL_mean": -36.32086181640625, "KL/std": 22.402416229248047, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.020931556820869446, "fcm_dpo/delta": -0.042580496519804, "fcm_dpo/margin": 20.599851608276367, "fcm_dpo/q_t": 0.3959561288356781, "grad_norm": 10.116601943969727, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.277616500854492, "logits/rejected": -2.0453851222991943, "logps/chosen": -94.91580963134766, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -133.14974975585938, "loss": 1.051, "margin_dpo/margin_mean": 20.599853515625, "margin_dpo/margin_std": 21.22943115234375, "step": 188 }, { "KL/chosen_KL_mean": -20.79247283935547, "KL/mean": -31.227237701416016, "KL/rejected_KL_mean": -41.66200256347656, "KL/std": 22.354028701782227, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.02093755081295967, "fcm_dpo/delta": -0.04910081997513771, "fcm_dpo/margin": 20.869535446166992, "fcm_dpo/q_t": 0.3986428380012512, "grad_norm": 9.260082244873047, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.7928179502487183, "logits/rejected": -1.1443138122558594, "logps/chosen": -104.68527221679688, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -121.12789916992188, "loss": 1.0618, "margin_dpo/margin_mean": 20.86953353881836, "margin_dpo/margin_std": 24.95367431640625, "step": 189 }, { "KL/chosen_KL_mean": -16.802154541015625, "KL/mean": -29.507232666015625, "KL/rejected_KL_mean": -42.212310791015625, "KL/std": 24.9478702545166, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.0186519343405962, "fcm_dpo/delta": -0.09755348414182663, "fcm_dpo/margin": 25.410160064697266, "fcm_dpo/q_t": 0.38471317291259766, "grad_norm": 9.161209106445312, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.9582674503326416, "logits/rejected": -1.3673369884490967, "logps/chosen": -120.24362182617188, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -151.42733764648438, "loss": 1.0137, "margin_dpo/margin_mean": 25.4101619720459, "margin_dpo/margin_std": 23.836109161376953, "step": 190 }, { "KL/chosen_KL_mean": -16.06429672241211, "KL/mean": -30.53830909729004, "KL/rejected_KL_mean": -45.0123291015625, "KL/std": 27.228267669677734, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.016832303255796432, "fcm_dpo/delta": -0.11516901105642319, "fcm_dpo/margin": 28.948030471801758, "fcm_dpo/q_t": 0.38184839487075806, "grad_norm": 9.607026100158691, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.6919198036193848, "logits/rejected": -1.3913637399673462, "logps/chosen": -115.84980773925781, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -161.01260375976562, "loss": 1.008, "margin_dpo/margin_mean": 28.94803237915039, "margin_dpo/margin_std": 27.8498592376709, "step": 191 }, { "KL/chosen_KL_mean": -25.423294067382812, "KL/mean": -35.52320861816406, "KL/rejected_KL_mean": -45.623111724853516, "KL/std": 21.664939880371094, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.01704908162355423, "fcm_dpo/delta": 0.06724658608436584, "fcm_dpo/margin": 20.199813842773438, "fcm_dpo/q_t": 0.4216330051422119, "grad_norm": 9.75805377960205, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.8191382884979248, "logits/rejected": -1.5773956775665283, "logps/chosen": -126.28361511230469, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -155.068115234375, "loss": 1.1159, "margin_dpo/margin_mean": 20.199813842773438, "margin_dpo/margin_std": 25.36754608154297, "step": 192 }, { "KL/chosen_KL_mean": -21.269168853759766, "KL/mean": -29.382198333740234, "KL/rejected_KL_mean": -37.495235443115234, "KL/std": 23.00933074951172, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.01907082460820675, "fcm_dpo/delta": 0.10415051877498627, "fcm_dpo/margin": 16.22606658935547, "fcm_dpo/q_t": 0.43063807487487793, "grad_norm": 10.1223783493042, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.2537434101104736, "logits/rejected": -1.6822829246520996, "logps/chosen": -114.20748901367188, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -120.60441589355469, "loss": 1.1683, "margin_dpo/margin_mean": 16.22606658935547, "margin_dpo/margin_std": 27.835372924804688, "step": 193 }, { "KL/chosen_KL_mean": -21.495811462402344, "KL/mean": -31.002561569213867, "KL/rejected_KL_mean": -40.509315490722656, "KL/std": 23.438098907470703, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.01995539292693138, "fcm_dpo/delta": 0.023750171065330505, "fcm_dpo/margin": 19.013505935668945, "fcm_dpo/q_t": 0.4129156470298767, "grad_norm": 8.866279602050781, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.506181240081787, "logits/rejected": -2.1333847045898438, "logps/chosen": -85.28738403320312, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -109.23596954345703, "loss": 1.1003, "margin_dpo/margin_mean": 19.013505935668945, "margin_dpo/margin_std": 24.768600463867188, "step": 194 }, { "KL/chosen_KL_mean": -20.74060821533203, "KL/mean": -33.536415100097656, "KL/rejected_KL_mean": -46.33222579956055, "KL/std": 23.98350715637207, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.018584923818707466, "fcm_dpo/delta": -0.09868627041578293, "fcm_dpo/margin": 25.59161949157715, "fcm_dpo/q_t": 0.3852018117904663, "grad_norm": 8.841790199279785, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.479853868484497, "logits/rejected": -1.3680099248886108, "logps/chosen": -104.62606811523438, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -150.33348083496094, "loss": 1.028, "margin_dpo/margin_mean": 25.59161949157715, "margin_dpo/margin_std": 27.398292541503906, "step": 195 }, { "KL/chosen_KL_mean": -24.056381225585938, "KL/mean": -35.98994827270508, "KL/rejected_KL_mean": -47.923519134521484, "KL/std": 25.61292839050293, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.01756369322538376, "fcm_dpo/delta": -0.024323102086782455, "fcm_dpo/margin": 23.867141723632812, "fcm_dpo/q_t": 0.39935576915740967, "grad_norm": 8.40381145477295, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.5628294944763184, "logits/rejected": -1.124363899230957, "logps/chosen": -113.45774841308594, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -131.2708740234375, "loss": 1.0503, "margin_dpo/margin_mean": 23.867141723632812, "margin_dpo/margin_std": 23.267929077148438, "step": 196 }, { "KL/chosen_KL_mean": -24.48284149169922, "KL/mean": -31.698562622070312, "KL/rejected_KL_mean": -38.91427993774414, "KL/std": 20.539302825927734, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.019432900473475456, "fcm_dpo/delta": 0.1382276862859726, "fcm_dpo/margin": 14.431436538696289, "fcm_dpo/q_t": 0.43718987703323364, "grad_norm": 9.28950023651123, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.5858526229858398, "logits/rejected": -1.2593833208084106, "logps/chosen": -111.32354736328125, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -121.7071762084961, "loss": 1.168, "margin_dpo/margin_mean": 14.431437492370605, "margin_dpo/margin_std": 21.790342330932617, "step": 197 }, { "KL/chosen_KL_mean": -28.20901107788086, "KL/mean": -40.219207763671875, "KL/rejected_KL_mean": -52.229408264160156, "KL/std": 24.555408477783203, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.01901150681078434, "fcm_dpo/delta": -0.07349881529808044, "fcm_dpo/margin": 24.020397186279297, "fcm_dpo/q_t": 0.39112406969070435, "grad_norm": 10.598127365112305, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.9616641998291016, "logits/rejected": -1.7987546920776367, "logps/chosen": -113.25725555419922, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -156.33306884765625, "loss": 1.0334, "margin_dpo/margin_mean": 24.020397186279297, "margin_dpo/margin_std": 25.253231048583984, "step": 198 }, { "KL/chosen_KL_mean": -27.378582000732422, "KL/mean": -37.238189697265625, "KL/rejected_KL_mean": -47.09780502319336, "KL/std": 23.82937240600586, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.019146274775266647, "fcm_dpo/delta": 0.027230389416217804, "fcm_dpo/margin": 19.719219207763672, "fcm_dpo/q_t": 0.4128786325454712, "grad_norm": 9.3964204788208, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.7952451705932617, "logits/rejected": -1.539876103401184, "logps/chosen": -109.165283203125, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -137.88858032226562, "loss": 1.0984, "margin_dpo/margin_mean": 19.719219207763672, "margin_dpo/margin_std": 25.033199310302734, "step": 199 }, { "KL/chosen_KL_mean": -26.850914001464844, "KL/mean": -38.4180908203125, "KL/rejected_KL_mean": -49.98526382446289, "KL/std": 25.483978271484375, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.01827327162027359, "fcm_dpo/delta": -0.032152414321899414, "fcm_dpo/margin": 23.13435173034668, "fcm_dpo/q_t": 0.40113285183906555, "grad_norm": 9.323463439941406, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.8229202032089233, "logits/rejected": -1.2880196571350098, "logps/chosen": -111.11900329589844, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -136.953125, "loss": 1.0775, "margin_dpo/margin_mean": 23.134349822998047, "margin_dpo/margin_std": 29.485624313354492, "step": 200 }, { "KL/chosen_KL_mean": -29.169261932373047, "KL/mean": -40.44519805908203, "KL/rejected_KL_mean": -51.72113800048828, "KL/std": 27.836441040039062, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.018521422520279884, "fcm_dpo/delta": -0.023059772327542305, "fcm_dpo/margin": 22.551876068115234, "fcm_dpo/q_t": 0.4033951759338379, "grad_norm": 9.633829116821289, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.2738564014434814, "logits/rejected": -0.8835486173629761, "logps/chosen": -136.94175720214844, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -171.51361083984375, "loss": 1.0852, "margin_dpo/margin_mean": 22.551877975463867, "margin_dpo/margin_std": 30.370878219604492, "step": 201 }, { "KL/chosen_KL_mean": -26.848426818847656, "KL/mean": -37.39369583129883, "KL/rejected_KL_mean": -47.93896484375, "KL/std": 22.92142105102539, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.018419362604618073, "fcm_dpo/delta": 0.01404392346739769, "fcm_dpo/margin": 21.090543746948242, "fcm_dpo/q_t": 0.4105417728424072, "grad_norm": 9.657011985778809, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.8463599681854248, "logits/rejected": -1.724034309387207, "logps/chosen": -102.82087707519531, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -142.39895629882812, "loss": 1.091, "margin_dpo/margin_mean": 21.09054183959961, "margin_dpo/margin_std": 26.002574920654297, "step": 202 }, { "KL/chosen_KL_mean": -24.270061492919922, "KL/mean": -35.97858428955078, "KL/rejected_KL_mean": -47.68711853027344, "KL/std": 23.847951889038086, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.01810358837246895, "fcm_dpo/delta": -0.030372433364391327, "fcm_dpo/margin": 23.41704750061035, "fcm_dpo/q_t": 0.39936673641204834, "grad_norm": 8.818370819091797, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.9847279787063599, "logits/rejected": -1.5345661640167236, "logps/chosen": -119.23721313476562, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -140.57479858398438, "loss": 1.0552, "margin_dpo/margin_mean": 23.417049407958984, "margin_dpo/margin_std": 25.02086639404297, "step": 203 }, { "KL/chosen_KL_mean": -30.43415069580078, "KL/mean": -40.488441467285156, "KL/rejected_KL_mean": -50.542724609375, "KL/std": 23.903648376464844, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.018523653969168663, "fcm_dpo/delta": 0.03234053775668144, "fcm_dpo/margin": 20.108572006225586, "fcm_dpo/q_t": 0.4148145318031311, "grad_norm": 9.832338333129883, "learning_rate": 4.431508065452897e-07, "logits/chosen": -2.064927577972412, "logits/rejected": -1.4193929433822632, "logps/chosen": -146.79135131835938, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -143.9403076171875, "loss": 1.1129, "margin_dpo/margin_mean": 20.108572006225586, "margin_dpo/margin_std": 27.739965438842773, "step": 204 }, { "KL/chosen_KL_mean": -28.256710052490234, "KL/mean": -41.42533874511719, "KL/rejected_KL_mean": -54.59397888183594, "KL/std": 26.590599060058594, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.017227891832590103, "fcm_dpo/delta": -0.07162527740001678, "fcm_dpo/margin": 26.337263107299805, "fcm_dpo/q_t": 0.38857996463775635, "grad_norm": 8.479545593261719, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.3530163764953613, "logits/rejected": -1.9280385971069336, "logps/chosen": -117.11605834960938, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -146.44847106933594, "loss": 1.0188, "margin_dpo/margin_mean": 26.337261199951172, "margin_dpo/margin_std": 22.48405647277832, "step": 205 }, { "KL/chosen_KL_mean": -28.71802520751953, "KL/mean": -40.87794494628906, "KL/rejected_KL_mean": -53.03786849975586, "KL/std": 25.582393646240234, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.016735248267650604, "fcm_dpo/delta": -0.01020396500825882, "fcm_dpo/margin": 24.319839477539062, "fcm_dpo/q_t": 0.4020794630050659, "grad_norm": 9.486464500427246, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.6704683303833008, "logits/rejected": -1.5286383628845215, "logps/chosen": -105.26437377929688, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -155.99099731445312, "loss": 1.0529, "margin_dpo/margin_mean": 24.319839477539062, "margin_dpo/margin_std": 21.6246337890625, "step": 206 }, { "KL/chosen_KL_mean": -30.22890853881836, "KL/mean": -43.83028030395508, "KL/rejected_KL_mean": -57.43165969848633, "KL/std": 31.60708999633789, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.016296718269586563, "fcm_dpo/delta": -0.05567023903131485, "fcm_dpo/margin": 27.20275115966797, "fcm_dpo/q_t": 0.3981319069862366, "grad_norm": 9.372552871704102, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.0517992973327637, "logits/rejected": -1.6258461475372314, "logps/chosen": -116.46055603027344, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -148.08677673339844, "loss": 1.0703, "margin_dpo/margin_mean": 27.20275115966797, "margin_dpo/margin_std": 36.28471755981445, "step": 207 }, { "KL/chosen_KL_mean": -31.306777954101562, "KL/mean": -45.2238655090332, "KL/rejected_KL_mean": -59.14094543457031, "KL/std": 28.4910888671875, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.015761833637952805, "fcm_dpo/delta": -0.05012091249227524, "fcm_dpo/margin": 27.83415985107422, "fcm_dpo/q_t": 0.39358043670654297, "grad_norm": 9.314495086669922, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.8671371936798096, "logits/rejected": -1.5173243284225464, "logps/chosen": -125.46632385253906, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -160.103271484375, "loss": 1.037, "margin_dpo/margin_mean": 27.83416175842285, "margin_dpo/margin_std": 25.886476516723633, "step": 208 }, { "KL/chosen_KL_mean": -26.681255340576172, "KL/mean": -38.987945556640625, "KL/rejected_KL_mean": -51.29462814331055, "KL/std": 25.54229736328125, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.015339231118559837, "fcm_dpo/delta": 0.027087349444627762, "fcm_dpo/margin": 24.61337661743164, "fcm_dpo/q_t": 0.41243764758110046, "grad_norm": 8.196447372436523, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.2242431640625, "logits/rejected": -1.7563769817352295, "logps/chosen": -110.85182189941406, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -138.9141845703125, "loss": 1.093, "margin_dpo/margin_mean": 24.61337661743164, "margin_dpo/margin_std": 29.186100006103516, "step": 209 }, { "KL/chosen_KL_mean": -30.322654724121094, "KL/mean": -41.985595703125, "KL/rejected_KL_mean": -53.64853286743164, "KL/std": 26.853891372680664, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.015991918742656708, "fcm_dpo/delta": 0.03321721404790878, "fcm_dpo/margin": 23.32588005065918, "fcm_dpo/q_t": 0.4137533903121948, "grad_norm": 7.905354022979736, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.9902300834655762, "logits/rejected": -1.6045615673065186, "logps/chosen": -111.49382781982422, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -137.8233184814453, "loss": 1.0927, "margin_dpo/margin_mean": 23.325881958007812, "margin_dpo/margin_std": 27.211631774902344, "step": 210 }, { "KL/chosen_KL_mean": -34.065128326416016, "KL/mean": -45.36610412597656, "KL/rejected_KL_mean": -56.66708755493164, "KL/std": 28.563034057617188, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.0167279951274395, "fcm_dpo/delta": 0.025996726006269455, "fcm_dpo/margin": 22.601966857910156, "fcm_dpo/q_t": 0.41329389810562134, "grad_norm": 9.987271308898926, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.1086535453796387, "logits/rejected": -1.4828133583068848, "logps/chosen": -142.77783203125, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -150.22274780273438, "loss": 1.0987, "margin_dpo/margin_mean": 22.601966857910156, "margin_dpo/margin_std": 29.056612014770508, "step": 211 }, { "KL/chosen_KL_mean": -24.332000732421875, "KL/mean": -41.114952087402344, "KL/rejected_KL_mean": -57.89790344238281, "KL/std": 32.84299087524414, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.014816425740718842, "fcm_dpo/delta": -0.13415198028087616, "fcm_dpo/margin": 33.56590270996094, "fcm_dpo/q_t": 0.3784555196762085, "grad_norm": 8.152562141418457, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.6908276081085205, "logits/rejected": -1.2613362073898315, "logps/chosen": -122.69395446777344, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -167.78790283203125, "loss": 1.0042, "margin_dpo/margin_mean": 33.56590270996094, "margin_dpo/margin_std": 33.09062194824219, "step": 212 }, { "KL/chosen_KL_mean": -28.221744537353516, "KL/mean": -40.88191223144531, "KL/rejected_KL_mean": -53.542083740234375, "KL/std": 29.766315460205078, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.014773603528738022, "fcm_dpo/delta": 0.03186316788196564, "fcm_dpo/margin": 25.320335388183594, "fcm_dpo/q_t": 0.41338711977005005, "grad_norm": 10.03795051574707, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.9192556142807007, "logits/rejected": -1.1192853450775146, "logps/chosen": -136.2770538330078, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -153.6862335205078, "loss": 1.1026, "margin_dpo/margin_mean": 25.320335388183594, "margin_dpo/margin_std": 32.70939636230469, "step": 213 }, { "KL/chosen_KL_mean": -29.98089599609375, "KL/mean": -41.815269470214844, "KL/rejected_KL_mean": -53.64964294433594, "KL/std": 31.887283325195312, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.01555601879954338, "fcm_dpo/delta": 0.038486022502183914, "fcm_dpo/margin": 23.668752670288086, "fcm_dpo/q_t": 0.4171031713485718, "grad_norm": 8.305341720581055, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -2.0943803787231445, "logits/rejected": -1.9772772789001465, "logps/chosen": -107.78562927246094, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -137.92543029785156, "loss": 1.1214, "margin_dpo/margin_mean": 23.668752670288086, "margin_dpo/margin_std": 35.17040252685547, "step": 214 }, { "KL/chosen_KL_mean": -26.23699951171875, "KL/mean": -42.418006896972656, "KL/rejected_KL_mean": -58.59901809692383, "KL/std": 31.333694458007812, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.014558718539774418, "fcm_dpo/delta": -0.09340120851993561, "fcm_dpo/margin": 32.36200714111328, "fcm_dpo/q_t": 0.3860856890678406, "grad_norm": 8.016737937927246, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.6894843578338623, "logits/rejected": -1.2108550071716309, "logps/chosen": -111.35089111328125, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -155.46054077148438, "loss": 1.017, "margin_dpo/margin_mean": 32.36200714111328, "margin_dpo/margin_std": 31.150489807128906, "step": 215 }, { "KL/chosen_KL_mean": -28.50570297241211, "KL/mean": -42.535614013671875, "KL/rejected_KL_mean": -56.565528869628906, "KL/std": 28.157943725585938, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.014428189024329185, "fcm_dpo/delta": -0.007730651646852493, "fcm_dpo/margin": 28.059829711914062, "fcm_dpo/q_t": 0.40440452098846436, "grad_norm": 7.582639217376709, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.825791597366333, "logits/rejected": -1.4182853698730469, "logps/chosen": -109.09800720214844, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -137.98536682128906, "loss": 1.066, "margin_dpo/margin_mean": 28.05982780456543, "margin_dpo/margin_std": 29.215532302856445, "step": 216 }, { "KL/chosen_KL_mean": -34.54163360595703, "KL/mean": -45.978572845458984, "KL/rejected_KL_mean": -57.4155158996582, "KL/std": 29.22164535522461, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.014982433058321476, "fcm_dpo/delta": 0.06790442764759064, "fcm_dpo/margin": 22.87388038635254, "fcm_dpo/q_t": 0.4231342375278473, "grad_norm": 9.506776809692383, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.2902331352233887, "logits/rejected": -1.9937427043914795, "logps/chosen": -128.01422119140625, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -160.90403747558594, "loss": 1.1314, "margin_dpo/margin_mean": 22.873878479003906, "margin_dpo/margin_std": 33.55830383300781, "step": 217 }, { "KL/chosen_KL_mean": -33.67940902709961, "KL/mean": -47.55498504638672, "KL/rejected_KL_mean": -61.43055725097656, "KL/std": 29.582746505737305, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.014761775732040405, "fcm_dpo/delta": -0.013108644634485245, "fcm_dpo/margin": 27.751144409179688, "fcm_dpo/q_t": 0.40482863783836365, "grad_norm": 8.229127883911133, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.6078345775604248, "logits/rejected": -1.2357655763626099, "logps/chosen": -122.73823547363281, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -155.73736572265625, "loss": 1.084, "margin_dpo/margin_mean": 27.751142501831055, "margin_dpo/margin_std": 35.74803161621094, "step": 218 }, { "KL/chosen_KL_mean": -30.408729553222656, "KL/mean": -42.84320831298828, "KL/rejected_KL_mean": -55.27769470214844, "KL/std": 34.472251892089844, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.015513407066464424, "fcm_dpo/delta": 0.013083156198263168, "fcm_dpo/margin": 24.868967056274414, "fcm_dpo/q_t": 0.41158172488212585, "grad_norm": 9.688722610473633, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.747894287109375, "logits/rejected": -1.2835466861724854, "logps/chosen": -131.41607666015625, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -147.74563598632812, "loss": 1.1168, "margin_dpo/margin_mean": 24.86896514892578, "margin_dpo/margin_std": 36.8939208984375, "step": 219 }, { "KL/chosen_KL_mean": -31.275924682617188, "KL/mean": -42.937034606933594, "KL/rejected_KL_mean": -54.59815216064453, "KL/std": 28.443553924560547, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.015653913840651512, "fcm_dpo/delta": 0.04236077517271042, "fcm_dpo/margin": 23.322235107421875, "fcm_dpo/q_t": 0.41550636291503906, "grad_norm": 9.878349304199219, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.6682710647583008, "logits/rejected": -1.3280028104782104, "logps/chosen": -126.81314086914062, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -148.90518188476562, "loss": 1.1006, "margin_dpo/margin_mean": 23.322235107421875, "margin_dpo/margin_std": 27.966110229492188, "step": 220 }, { "KL/chosen_KL_mean": -29.12652587890625, "KL/mean": -47.259735107421875, "KL/rejected_KL_mean": -65.3929443359375, "KL/std": 35.63246154785156, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.014401989988982677, "fcm_dpo/delta": -0.16427062451839447, "fcm_dpo/margin": 36.26641845703125, "fcm_dpo/q_t": 0.37131625413894653, "grad_norm": 9.387941360473633, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.071643590927124, "logits/rejected": -0.9364381432533264, "logps/chosen": -111.97138977050781, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -176.20474243164062, "loss": 0.9849, "margin_dpo/margin_mean": 36.26641845703125, "margin_dpo/margin_std": 31.379037857055664, "step": 221 }, { "KL/chosen_KL_mean": -37.42237854003906, "KL/mean": -52.01066589355469, "KL/rejected_KL_mean": -66.59895324707031, "KL/std": 34.023983001708984, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.013657873496413231, "fcm_dpo/delta": -0.00017971545457839966, "fcm_dpo/margin": 29.17656898498535, "fcm_dpo/q_t": 0.40772745013237, "grad_norm": 8.896951675415039, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.9998897314071655, "logits/rejected": -1.5702409744262695, "logps/chosen": -132.56436157226562, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -173.4033660888672, "loss": 1.0987, "margin_dpo/margin_mean": 29.17656707763672, "margin_dpo/margin_std": 39.702056884765625, "step": 222 }, { "KL/chosen_KL_mean": -36.42448043823242, "KL/mean": -46.7750244140625, "KL/rejected_KL_mean": -57.12557601928711, "KL/std": 30.077072143554688, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.01452045887708664, "fcm_dpo/delta": 0.11732238531112671, "fcm_dpo/margin": 20.701099395751953, "fcm_dpo/q_t": 0.43321359157562256, "grad_norm": 9.171944618225098, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.2255377769470215, "logits/rejected": -1.8388512134552002, "logps/chosen": -121.99958801269531, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -143.5779571533203, "loss": 1.1632, "margin_dpo/margin_mean": 20.70109748840332, "margin_dpo/margin_std": 32.923397064208984, "step": 223 }, { "KL/chosen_KL_mean": -33.27123260498047, "KL/mean": -48.062278747558594, "KL/rejected_KL_mean": -62.85333251953125, "KL/std": 33.345367431640625, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.014528336003422737, "fcm_dpo/delta": -0.03834237903356552, "fcm_dpo/margin": 29.582101821899414, "fcm_dpo/q_t": 0.39912548661231995, "grad_norm": 8.862432479858398, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.7713361978530884, "logits/rejected": -1.3091644048690796, "logps/chosen": -115.99504089355469, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -145.44871520996094, "loss": 1.0618, "margin_dpo/margin_mean": 29.58209991455078, "margin_dpo/margin_std": 34.84231948852539, "step": 224 }, { "KL/chosen_KL_mean": -37.421260833740234, "KL/mean": -52.7893180847168, "KL/rejected_KL_mean": -68.1573715209961, "KL/std": 35.16419982910156, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.014226853847503662, "fcm_dpo/delta": -0.04829014092683792, "fcm_dpo/margin": 30.736112594604492, "fcm_dpo/q_t": 0.39854174852371216, "grad_norm": 9.451468467712402, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.018327236175537, "logits/rejected": -1.5413098335266113, "logps/chosen": -133.09893798828125, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -173.25424194335938, "loss": 1.0704, "margin_dpo/margin_mean": 30.736114501953125, "margin_dpo/margin_std": 39.747215270996094, "step": 225 }, { "KL/chosen_KL_mean": -30.908401489257812, "KL/mean": -45.666969299316406, "KL/rejected_KL_mean": -60.425537109375, "KL/std": 34.32215118408203, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.013445444405078888, "fcm_dpo/delta": 0.0009674355387687683, "fcm_dpo/margin": 29.51712417602539, "fcm_dpo/q_t": 0.4062495827674866, "grad_norm": 7.777832984924316, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.8186979293823242, "logits/rejected": -1.2765536308288574, "logps/chosen": -124.36932373046875, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -147.12728881835938, "loss": 1.0713, "margin_dpo/margin_mean": 29.517126083374023, "margin_dpo/margin_std": 30.549583435058594, "step": 226 }, { "KL/chosen_KL_mean": -41.61910629272461, "KL/mean": -56.93528747558594, "KL/rejected_KL_mean": -72.25147247314453, "KL/std": 33.41253662109375, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.013608145527541637, "fcm_dpo/delta": -0.021262839436531067, "fcm_dpo/margin": 30.63235855102539, "fcm_dpo/q_t": 0.40327054262161255, "grad_norm": 8.672306060791016, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.433508276939392, "logits/rejected": -1.0804963111877441, "logps/chosen": -134.43338012695312, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -176.9884033203125, "loss": 1.0808, "margin_dpo/margin_mean": 30.63235855102539, "margin_dpo/margin_std": 39.62535095214844, "step": 227 }, { "KL/chosen_KL_mean": -39.32029724121094, "KL/mean": -50.80522155761719, "KL/rejected_KL_mean": -62.29014205932617, "KL/std": 31.226049423217773, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.014359613880515099, "fcm_dpo/delta": 0.08401058614253998, "fcm_dpo/margin": 22.969839096069336, "fcm_dpo/q_t": 0.42543885111808777, "grad_norm": 9.254663467407227, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.994230031967163, "logits/rejected": -1.5689135789871216, "logps/chosen": -133.357421875, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -158.31166076660156, "loss": 1.1446, "margin_dpo/margin_mean": 22.969839096069336, "margin_dpo/margin_std": 35.186256408691406, "step": 228 }, { "KL/chosen_KL_mean": -34.937503814697266, "KL/mean": -53.8047981262207, "KL/rejected_KL_mean": -72.67210388183594, "KL/std": 37.754783630371094, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.013269356451928616, "fcm_dpo/delta": -0.1352168619632721, "fcm_dpo/margin": 37.734596252441406, "fcm_dpo/q_t": 0.38035333156585693, "grad_norm": 8.725262641906738, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.9720571041107178, "logits/rejected": -1.350530743598938, "logps/chosen": -130.51516723632812, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -161.84588623046875, "loss": 1.0159, "margin_dpo/margin_mean": 37.734596252441406, "margin_dpo/margin_std": 41.62085723876953, "step": 229 }, { "KL/chosen_KL_mean": -36.101531982421875, "KL/mean": -55.69793701171875, "KL/rejected_KL_mean": -75.29434204101562, "KL/std": 37.454833984375, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.011966872960329056, "fcm_dpo/delta": -0.09069934487342834, "fcm_dpo/margin": 39.19281005859375, "fcm_dpo/q_t": 0.38571298122406006, "grad_norm": 9.206086158752441, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8256072998046875, "logits/rejected": -1.768280267715454, "logps/chosen": -99.81515502929688, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -167.20306396484375, "loss": 1.0179, "margin_dpo/margin_mean": 39.19281005859375, "margin_dpo/margin_std": 37.31896209716797, "step": 230 }, { "KL/chosen_KL_mean": -39.511749267578125, "KL/mean": -56.65168762207031, "KL/rejected_KL_mean": -73.7916259765625, "KL/std": 36.115196228027344, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.011529543437063694, "fcm_dpo/delta": 0.00431978702545166, "fcm_dpo/margin": 34.279869079589844, "fcm_dpo/q_t": 0.4057861864566803, "grad_norm": 7.16748046875, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.8528066873550415, "logits/rejected": -1.2918373346328735, "logps/chosen": -134.96742248535156, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -154.747314453125, "loss": 1.0626, "margin_dpo/margin_mean": 34.279869079589844, "margin_dpo/margin_std": 31.2620906829834, "step": 231 }, { "KL/chosen_KL_mean": -42.28727722167969, "KL/mean": -56.36078643798828, "KL/rejected_KL_mean": -70.43429565429688, "KL/std": 33.354820251464844, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.012327105738222599, "fcm_dpo/delta": 0.06418103724718094, "fcm_dpo/margin": 28.147018432617188, "fcm_dpo/q_t": 0.42000409960746765, "grad_norm": 8.479443550109863, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.228865385055542, "logits/rejected": -0.7198700904846191, "logps/chosen": -139.18453979492188, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -160.19891357421875, "loss": 1.1095, "margin_dpo/margin_mean": 28.147018432617188, "margin_dpo/margin_std": 33.141517639160156, "step": 232 }, { "KL/chosen_KL_mean": -43.833152770996094, "KL/mean": -63.683387756347656, "KL/rejected_KL_mean": -83.53361511230469, "KL/std": 42.543121337890625, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.011685685254633427, "fcm_dpo/delta": -0.08464187383651733, "fcm_dpo/margin": 39.700469970703125, "fcm_dpo/q_t": 0.3879812955856323, "grad_norm": 8.722740173339844, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.8170032501220703, "logits/rejected": -1.622929573059082, "logps/chosen": -132.88348388671875, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -196.29278564453125, "loss": 1.0294, "margin_dpo/margin_mean": 39.700469970703125, "margin_dpo/margin_std": 41.24516677856445, "step": 233 }, { "KL/chosen_KL_mean": -43.56699752807617, "KL/mean": -64.6952133178711, "KL/rejected_KL_mean": -85.82342529296875, "KL/std": 40.5099983215332, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.01058598980307579, "fcm_dpo/delta": -0.06926058232784271, "fcm_dpo/margin": 42.256431579589844, "fcm_dpo/q_t": 0.392082154750824, "grad_norm": 7.538671493530273, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.378786325454712, "logits/rejected": -1.3142715692520142, "logps/chosen": -117.88536071777344, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -192.2110137939453, "loss": 1.0433, "margin_dpo/margin_mean": 42.256431579589844, "margin_dpo/margin_std": 44.720237731933594, "step": 234 }, { "KL/chosen_KL_mean": -52.03403091430664, "KL/mean": -67.93826293945312, "KL/rejected_KL_mean": -83.84249877929688, "KL/std": 36.435001373291016, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.011249369010329247, "fcm_dpo/delta": 0.05069228261709213, "fcm_dpo/margin": 31.80847930908203, "fcm_dpo/q_t": 0.41667789220809937, "grad_norm": 7.962204456329346, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.5340381860733032, "logits/rejected": -0.9661629796028137, "logps/chosen": -150.25106811523438, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -181.08926391601562, "loss": 1.0959, "margin_dpo/margin_mean": 31.80847930908203, "margin_dpo/margin_std": 34.40613555908203, "step": 235 }, { "KL/chosen_KL_mean": -43.275516510009766, "KL/mean": -64.23094177246094, "KL/rejected_KL_mean": -85.18637084960938, "KL/std": 46.075714111328125, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.010838273912668228, "fcm_dpo/delta": -0.069915771484375, "fcm_dpo/margin": 41.910850524902344, "fcm_dpo/q_t": 0.39556747674942017, "grad_norm": 8.581408500671387, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.4453034400939941, "logits/rejected": -1.4540538787841797, "logps/chosen": -122.11325073242188, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -194.24981689453125, "loss": 1.0474, "margin_dpo/margin_mean": 41.910850524902344, "margin_dpo/margin_std": 50.12990951538086, "step": 236 }, { "KL/chosen_KL_mean": -52.34904861450195, "KL/mean": -67.39190673828125, "KL/rejected_KL_mean": -82.43476104736328, "KL/std": 40.46875762939453, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.01119772344827652, "fcm_dpo/delta": 0.07549043744802475, "fcm_dpo/margin": 30.08570098876953, "fcm_dpo/q_t": 0.4264224171638489, "grad_norm": 7.9057536125183105, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.6765233278274536, "logits/rejected": -1.334644079208374, "logps/chosen": -137.56033325195312, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -182.34475708007812, "loss": 1.1563, "margin_dpo/margin_mean": 30.08570098876953, "margin_dpo/margin_std": 52.309200286865234, "step": 237 }, { "KL/chosen_KL_mean": -57.32621383666992, "KL/mean": -74.94511413574219, "KL/rejected_KL_mean": -92.56401824951172, "KL/std": 42.023948669433594, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.011323714628815651, "fcm_dpo/delta": 0.0010739434510469437, "fcm_dpo/margin": 35.23780059814453, "fcm_dpo/q_t": 0.40792223811149597, "grad_norm": 7.639669895172119, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2956106662750244, "logits/rejected": -1.1492812633514404, "logps/chosen": -136.0574493408203, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -187.982421875, "loss": 1.0837, "margin_dpo/margin_mean": 35.23780059814453, "margin_dpo/margin_std": 43.827293395996094, "step": 238 }, { "KL/chosen_KL_mean": -51.97614669799805, "KL/mean": -68.33161926269531, "KL/rejected_KL_mean": -84.68709564208984, "KL/std": 37.53894805908203, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.011355066671967506, "fcm_dpo/delta": 0.03257526457309723, "fcm_dpo/margin": 32.710960388183594, "fcm_dpo/q_t": 0.4142038822174072, "grad_norm": 8.52668571472168, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.8286402225494385, "logits/rejected": -1.3520989418029785, "logps/chosen": -144.57708740234375, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -185.2047882080078, "loss": 1.1086, "margin_dpo/margin_mean": 32.710960388183594, "margin_dpo/margin_std": 41.71491622924805, "step": 239 }, { "KL/chosen_KL_mean": -53.8466911315918, "KL/mean": -68.30577087402344, "KL/rejected_KL_mean": -82.76486206054688, "KL/std": 38.06828308105469, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.012177902273833752, "fcm_dpo/delta": 0.05823323875665665, "fcm_dpo/margin": 28.918167114257812, "fcm_dpo/q_t": 0.4200113117694855, "grad_norm": 10.253983497619629, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.2072818279266357, "logits/rejected": -1.8491199016571045, "logps/chosen": -140.0064697265625, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -163.22052001953125, "loss": 1.1252, "margin_dpo/margin_mean": 28.918167114257812, "margin_dpo/margin_std": 41.01519012451172, "step": 240 }, { "KL/chosen_KL_mean": -50.0117073059082, "KL/mean": -75.92140197753906, "KL/rejected_KL_mean": -101.83108520507812, "KL/std": 43.155296325683594, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.010831963270902634, "fcm_dpo/delta": -0.22201445698738098, "fcm_dpo/margin": 51.819374084472656, "fcm_dpo/q_t": 0.36074212193489075, "grad_norm": 8.25784683227539, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.6948285102844238, "logits/rejected": -1.0031019449234009, "logps/chosen": -151.05764770507812, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -195.88043212890625, "loss": 0.9688, "margin_dpo/margin_mean": 51.819374084472656, "margin_dpo/margin_std": 48.466888427734375, "step": 241 }, { "KL/chosen_KL_mean": -58.887454986572266, "KL/mean": -77.54991912841797, "KL/rejected_KL_mean": -96.21239471435547, "KL/std": 38.969879150390625, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.010279927402734756, "fcm_dpo/delta": 0.01893443800508976, "fcm_dpo/margin": 37.324951171875, "fcm_dpo/q_t": 0.40978217124938965, "grad_norm": 8.201447486877441, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.8516682386398315, "logits/rejected": -1.1807255744934082, "logps/chosen": -154.56011962890625, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -186.866943359375, "loss": 1.0872, "margin_dpo/margin_mean": 37.32494354248047, "margin_dpo/margin_std": 43.27454376220703, "step": 242 }, { "KL/chosen_KL_mean": -58.56468200683594, "KL/mean": -77.09098815917969, "KL/rejected_KL_mean": -95.61730194091797, "KL/std": 41.739112854003906, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.010607500560581684, "fcm_dpo/delta": 0.0018581226468086243, "fcm_dpo/margin": 37.05261993408203, "fcm_dpo/q_t": 0.4085524380207062, "grad_norm": 8.111590385437012, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.8422305583953857, "logits/rejected": -1.0021212100982666, "logps/chosen": -156.59608459472656, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -178.80535888671875, "loss": 1.1047, "margin_dpo/margin_mean": 37.05261993408203, "margin_dpo/margin_std": 49.85070037841797, "step": 243 }, { "KL/chosen_KL_mean": -61.15730285644531, "KL/mean": -80.81985473632812, "KL/rejected_KL_mean": -100.48240661621094, "KL/std": 45.986351013183594, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.010135053656995296, "fcm_dpo/delta": 0.0015381239354610443, "fcm_dpo/margin": 39.325103759765625, "fcm_dpo/q_t": 0.4069027304649353, "grad_norm": 7.744978904724121, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.223412036895752, "logits/rejected": -1.8975272178649902, "logps/chosen": -150.0512237548828, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -203.05517578125, "loss": 1.0772, "margin_dpo/margin_mean": 39.325103759765625, "margin_dpo/margin_std": 45.613990783691406, "step": 244 }, { "KL/chosen_KL_mean": -52.40642166137695, "KL/mean": -73.05278015136719, "KL/rejected_KL_mean": -93.69912719726562, "KL/std": 42.23783874511719, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.010123731568455696, "fcm_dpo/delta": -0.022869443520903587, "fcm_dpo/margin": 41.292701721191406, "fcm_dpo/q_t": 0.4010651111602783, "grad_norm": 7.282863616943359, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.468716859817505, "logits/rejected": -1.6550054550170898, "logps/chosen": -156.6004180908203, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -186.3555908203125, "loss": 1.0582, "margin_dpo/margin_mean": 41.292701721191406, "margin_dpo/margin_std": 43.73301696777344, "step": 245 }, { "KL/chosen_KL_mean": -68.0627670288086, "KL/mean": -84.51531219482422, "KL/rejected_KL_mean": -100.96786499023438, "KL/std": 41.89714813232422, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.010377575643360615, "fcm_dpo/delta": 0.07071786373853683, "fcm_dpo/margin": 32.905113220214844, "fcm_dpo/q_t": 0.4228881299495697, "grad_norm": 7.639719486236572, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.6596500873565674, "logits/rejected": -1.0910500288009644, "logps/chosen": -171.2818145751953, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -191.9600830078125, "loss": 1.1285, "margin_dpo/margin_mean": 32.905113220214844, "margin_dpo/margin_std": 45.34526824951172, "step": 246 }, { "KL/chosen_KL_mean": -61.51020812988281, "KL/mean": -88.72766876220703, "KL/rejected_KL_mean": -115.94512939453125, "KL/std": 46.88505172729492, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.00932776927947998, "fcm_dpo/delta": -0.15308821201324463, "fcm_dpo/margin": 54.434913635253906, "fcm_dpo/q_t": 0.3754442036151886, "grad_norm": 7.546737194061279, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.2121925354003906, "logits/rejected": -2.042255163192749, "logps/chosen": -148.50457763671875, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -227.28314208984375, "loss": 0.9989, "margin_dpo/margin_mean": 54.434913635253906, "margin_dpo/margin_std": 52.950172424316406, "step": 247 }, { "KL/chosen_KL_mean": -71.36326599121094, "KL/mean": -89.35107421875, "KL/rejected_KL_mean": -107.3388671875, "KL/std": 41.13702392578125, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.009744415059685707, "fcm_dpo/delta": 0.05793558061122894, "fcm_dpo/margin": 35.97561264038086, "fcm_dpo/q_t": 0.4188319444656372, "grad_norm": 8.204211235046387, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.8887605667114258, "logits/rejected": -1.8674348592758179, "logps/chosen": -146.14877319335938, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -205.61575317382812, "loss": 1.107, "margin_dpo/margin_mean": 35.97561264038086, "margin_dpo/margin_std": 42.23295593261719, "step": 248 }, { "KL/chosen_KL_mean": -79.62918090820312, "KL/mean": -95.87835693359375, "KL/rejected_KL_mean": -112.12753295898438, "KL/std": 44.25087356567383, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.0101999556645751, "fcm_dpo/delta": 0.08236635476350784, "fcm_dpo/margin": 32.49835968017578, "fcm_dpo/q_t": 0.42579174041748047, "grad_norm": 9.469715118408203, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.1768670082092285, "logits/rejected": -1.6281009912490845, "logps/chosen": -175.33297729492188, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -199.27401733398438, "loss": 1.1493, "margin_dpo/margin_mean": 32.49835968017578, "margin_dpo/margin_std": 50.757835388183594, "step": 249 }, { "KL/chosen_KL_mean": -70.50202941894531, "KL/mean": -89.09284973144531, "KL/rejected_KL_mean": -107.68367004394531, "KL/std": 45.04582214355469, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.01043069176375866, "fcm_dpo/delta": 0.012804217636585236, "fcm_dpo/margin": 37.1816520690918, "fcm_dpo/q_t": 0.40953969955444336, "grad_norm": 8.82200813293457, "learning_rate": 4.006586590948141e-07, "logits/chosen": -2.039700984954834, "logits/rejected": -1.1060597896575928, "logps/chosen": -184.55422973632812, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -188.7713623046875, "loss": 1.1007, "margin_dpo/margin_mean": 37.1816520690918, "margin_dpo/margin_std": 48.32110595703125, "step": 250 }, { "KL/chosen_KL_mean": -70.81432342529297, "KL/mean": -87.14140319824219, "KL/rejected_KL_mean": -103.46849060058594, "KL/std": 41.88313293457031, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.010854813270270824, "fcm_dpo/delta": 0.053538694977760315, "fcm_dpo/margin": 32.65416717529297, "fcm_dpo/q_t": 0.42047691345214844, "grad_norm": 9.806711196899414, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.9602625370025635, "logits/rejected": -1.2665648460388184, "logps/chosen": -168.52560424804688, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -186.99591064453125, "loss": 1.1399, "margin_dpo/margin_mean": 32.65416717529297, "margin_dpo/margin_std": 50.13427734375, "step": 251 }, { "KL/chosen_KL_mean": -71.17076873779297, "KL/mean": -87.47100067138672, "KL/rejected_KL_mean": -103.771240234375, "KL/std": 44.95682907104492, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.01163908839225769, "fcm_dpo/delta": 0.024557186290621758, "fcm_dpo/margin": 32.60048294067383, "fcm_dpo/q_t": 0.41286611557006836, "grad_norm": 10.25337028503418, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.0894744396209717, "logits/rejected": -1.5449550151824951, "logps/chosen": -180.15687561035156, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -195.33547973632812, "loss": 1.1165, "margin_dpo/margin_mean": 32.60048294067383, "margin_dpo/margin_std": 47.365684509277344, "step": 252 }, { "KL/chosen_KL_mean": -78.54226684570312, "KL/mean": -93.84425354003906, "KL/rejected_KL_mean": -109.146240234375, "KL/std": 48.38672637939453, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.012233496643602848, "fcm_dpo/delta": 0.024961821734905243, "fcm_dpo/margin": 30.603965759277344, "fcm_dpo/q_t": 0.41287171840667725, "grad_norm": 10.56840705871582, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.351504325866699, "logits/rejected": -2.156203269958496, "logps/chosen": -178.75857543945312, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -214.82293701171875, "loss": 1.1144, "margin_dpo/margin_mean": 30.603965759277344, "margin_dpo/margin_std": 41.948150634765625, "step": 253 }, { "KL/chosen_KL_mean": -75.96542358398438, "KL/mean": -91.75752258300781, "KL/rejected_KL_mean": -107.54962158203125, "KL/std": 45.78758239746094, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.011882639490067959, "fcm_dpo/delta": 0.028975360095500946, "fcm_dpo/margin": 31.58419418334961, "fcm_dpo/q_t": 0.41372185945510864, "grad_norm": 10.43439769744873, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.1060571670532227, "logits/rejected": -1.585012435913086, "logps/chosen": -174.7006072998047, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -201.28787231445312, "loss": 1.1297, "margin_dpo/margin_mean": 31.58419418334961, "margin_dpo/margin_std": 48.98828125, "step": 254 }, { "KL/chosen_KL_mean": -76.91981506347656, "KL/mean": -101.33998107910156, "KL/rejected_KL_mean": -125.7601318359375, "KL/std": 51.35126495361328, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.010773645713925362, "fcm_dpo/delta": -0.1722373068332672, "fcm_dpo/margin": 48.84031677246094, "fcm_dpo/q_t": 0.3714355230331421, "grad_norm": 9.272868156433105, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.895479440689087, "logits/rejected": -1.5531659126281738, "logps/chosen": -167.30239868164062, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -222.83639526367188, "loss": 1.0023, "margin_dpo/margin_mean": 48.84031677246094, "margin_dpo/margin_std": 52.50782012939453, "step": 255 }, { "KL/chosen_KL_mean": -75.59397888183594, "KL/mean": -96.99990844726562, "KL/rejected_KL_mean": -118.40582275390625, "KL/std": 53.611915588378906, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.010104618035256863, "fcm_dpo/delta": -0.04168049246072769, "fcm_dpo/margin": 42.81184387207031, "fcm_dpo/q_t": 0.40072283148765564, "grad_norm": 9.222391128540039, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.8925092220306396, "logits/rejected": -1.5562992095947266, "logps/chosen": -164.3468017578125, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -216.899658203125, "loss": 1.0949, "margin_dpo/margin_mean": 42.81184387207031, "margin_dpo/margin_std": 63.16502380371094, "step": 256 }, { "KL/chosen_KL_mean": -77.52838897705078, "KL/mean": -99.31514739990234, "KL/rejected_KL_mean": -121.10191345214844, "KL/std": 54.78676223754883, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.009342794306576252, "fcm_dpo/delta": -0.018091648817062378, "fcm_dpo/margin": 43.573524475097656, "fcm_dpo/q_t": 0.4038873612880707, "grad_norm": 8.018754005432129, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.7399544715881348, "logits/rejected": -1.3242884874343872, "logps/chosen": -160.3319091796875, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -206.96969604492188, "loss": 1.0867, "margin_dpo/margin_mean": 43.573524475097656, "margin_dpo/margin_std": 53.172088623046875, "step": 257 }, { "KL/chosen_KL_mean": -81.0065689086914, "KL/mean": -100.8837890625, "KL/rejected_KL_mean": -120.76099395751953, "KL/std": 52.19588851928711, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.010006777942180634, "fcm_dpo/delta": -0.0018219947814941406, "fcm_dpo/margin": 39.754425048828125, "fcm_dpo/q_t": 0.4065181612968445, "grad_norm": 9.079876899719238, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.308764934539795, "logits/rejected": -1.962160348892212, "logps/chosen": -171.19430541992188, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -224.231689453125, "loss": 1.0926, "margin_dpo/margin_mean": 39.754425048828125, "margin_dpo/margin_std": 50.09750747680664, "step": 258 }, { "KL/chosen_KL_mean": -87.27112579345703, "KL/mean": -104.31996154785156, "KL/rejected_KL_mean": -121.36878967285156, "KL/std": 52.47517395019531, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.0099498201161623, "fcm_dpo/delta": 0.0720149502158165, "fcm_dpo/margin": 34.09767150878906, "fcm_dpo/q_t": 0.4235168695449829, "grad_norm": 8.976011276245117, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.4399800300598145, "logits/rejected": -1.1899868249893188, "logps/chosen": -178.04367065429688, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -215.95693969726562, "loss": 1.1643, "margin_dpo/margin_mean": 34.09767150878906, "margin_dpo/margin_std": 59.560760498046875, "step": 259 }, { "KL/chosen_KL_mean": -89.86571502685547, "KL/mean": -104.22332000732422, "KL/rejected_KL_mean": -118.58092498779297, "KL/std": 52.65705871582031, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.011164011433720589, "fcm_dpo/delta": 0.0944385826587677, "fcm_dpo/margin": 28.7152099609375, "fcm_dpo/q_t": 0.42937105894088745, "grad_norm": 11.56718921661377, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.69889497756958, "logits/rejected": -1.733196496963501, "logps/chosen": -165.45840454101562, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -219.4264678955078, "loss": 1.1882, "margin_dpo/margin_mean": 28.715211868286133, "margin_dpo/margin_std": 57.53290557861328, "step": 260 }, { "KL/chosen_KL_mean": -70.02986145019531, "KL/mean": -95.24043273925781, "KL/rejected_KL_mean": -120.45100402832031, "KL/std": 53.50575637817383, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.009888763539493084, "fcm_dpo/delta": -0.14528384804725647, "fcm_dpo/margin": 50.421142578125, "fcm_dpo/q_t": 0.37728244066238403, "grad_norm": 8.428342819213867, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.9075289964675903, "logits/rejected": -1.5339674949645996, "logps/chosen": -164.28477478027344, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -214.10800170898438, "loss": 1.0116, "margin_dpo/margin_mean": 50.421142578125, "margin_dpo/margin_std": 52.29637908935547, "step": 261 }, { "KL/chosen_KL_mean": -75.42882537841797, "KL/mean": -99.1553955078125, "KL/rejected_KL_mean": -122.88198852539062, "KL/std": 56.442344665527344, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.009301488287746906, "fcm_dpo/delta": -0.056236740201711655, "fcm_dpo/margin": 47.45315170288086, "fcm_dpo/q_t": 0.3952695429325104, "grad_norm": 8.028851509094238, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.5547170639038086, "logits/rejected": -1.0034492015838623, "logps/chosen": -160.69613647460938, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -213.70806884765625, "loss": 1.0581, "margin_dpo/margin_mean": 47.453155517578125, "margin_dpo/margin_std": 56.58887481689453, "step": 262 }, { "KL/chosen_KL_mean": -61.85809326171875, "KL/mean": -93.75872802734375, "KL/rejected_KL_mean": -125.65936279296875, "KL/std": 61.49424743652344, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.008263997733592987, "fcm_dpo/delta": -0.17298999428749084, "fcm_dpo/margin": 63.80126953125, "fcm_dpo/q_t": 0.37004828453063965, "grad_norm": 7.488924503326416, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.195141077041626, "logits/rejected": -1.650329828262329, "logps/chosen": -157.61599731445312, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -227.49313354492188, "loss": 0.9971, "margin_dpo/margin_mean": 63.80126953125, "margin_dpo/margin_std": 66.15154266357422, "step": 263 }, { "KL/chosen_KL_mean": -71.39694213867188, "KL/mean": -98.56327819824219, "KL/rejected_KL_mean": -125.72962951660156, "KL/std": 57.7807502746582, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.007684464566409588, "fcm_dpo/delta": -0.022495675832033157, "fcm_dpo/margin": 54.33269500732422, "fcm_dpo/q_t": 0.40082457661628723, "grad_norm": 7.231961727142334, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.0911266803741455, "logits/rejected": -1.8061087131500244, "logps/chosen": -162.01959228515625, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -226.05517578125, "loss": 1.0603, "margin_dpo/margin_mean": 54.33269119262695, "margin_dpo/margin_std": 58.75520706176758, "step": 264 }, { "KL/chosen_KL_mean": -74.0831527709961, "KL/mean": -97.83917236328125, "KL/rejected_KL_mean": -121.59519958496094, "KL/std": 59.00367736816406, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0077673038467764854, "fcm_dpo/delta": 0.03729450702667236, "fcm_dpo/margin": 47.51203918457031, "fcm_dpo/q_t": 0.41560980677604675, "grad_norm": 7.46173620223999, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.217491865158081, "logits/rejected": -1.820622205734253, "logps/chosen": -166.45547485351562, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -216.2227783203125, "loss": 1.1112, "margin_dpo/margin_mean": 47.51203918457031, "margin_dpo/margin_std": 63.81163024902344, "step": 265 }, { "KL/chosen_KL_mean": -86.67901611328125, "KL/mean": -108.4008560180664, "KL/rejected_KL_mean": -130.12269592285156, "KL/std": 57.39195251464844, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.008229855448007584, "fcm_dpo/delta": 0.05185263603925705, "fcm_dpo/margin": 43.44367980957031, "fcm_dpo/q_t": 0.41887032985687256, "grad_norm": 9.001091003417969, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.2421295642852783, "logits/rejected": -1.4043391942977905, "logps/chosen": -188.87904357910156, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -204.48912048339844, "loss": 1.1209, "margin_dpo/margin_mean": 43.44368362426758, "margin_dpo/margin_std": 60.60224533081055, "step": 266 }, { "KL/chosen_KL_mean": -76.52633666992188, "KL/mean": -101.81835174560547, "KL/rejected_KL_mean": -127.11035919189453, "KL/std": 57.520538330078125, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.008142136037349701, "fcm_dpo/delta": -0.01700379140675068, "fcm_dpo/margin": 50.584022521972656, "fcm_dpo/q_t": 0.4028496742248535, "grad_norm": 8.096675872802734, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.7978026866912842, "logits/rejected": -1.1687382459640503, "logps/chosen": -180.81234741210938, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -223.09754943847656, "loss": 1.0656, "margin_dpo/margin_mean": 50.584022521972656, "margin_dpo/margin_std": 55.2481689453125, "step": 267 }, { "KL/chosen_KL_mean": -76.04966735839844, "KL/mean": -105.42285919189453, "KL/rejected_KL_mean": -134.79605102539062, "KL/std": 57.640869140625, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.007654883898794651, "fcm_dpo/delta": -0.06922466307878494, "fcm_dpo/margin": 58.74638366699219, "fcm_dpo/q_t": 0.3903321623802185, "grad_norm": 8.371220588684082, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.583635687828064, "logits/rejected": -1.4334795475006104, "logps/chosen": -162.74588012695312, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -243.98789978027344, "loss": 1.0331, "margin_dpo/margin_mean": 58.74638366699219, "margin_dpo/margin_std": 57.09599304199219, "step": 268 }, { "KL/chosen_KL_mean": -80.81178283691406, "KL/mean": -109.66729736328125, "KL/rejected_KL_mean": -138.5228271484375, "KL/std": 62.1236572265625, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.007539992220699787, "fcm_dpo/delta": -0.044903937727212906, "fcm_dpo/margin": 57.71104431152344, "fcm_dpo/q_t": 0.3960384726524353, "grad_norm": 7.322426795959473, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.870739459991455, "logits/rejected": -1.4186980724334717, "logps/chosen": -185.78359985351562, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -250.99928283691406, "loss": 1.0454, "margin_dpo/margin_mean": 57.71104049682617, "margin_dpo/margin_std": 59.161407470703125, "step": 269 }, { "KL/chosen_KL_mean": -89.42219543457031, "KL/mean": -113.59270477294922, "KL/rejected_KL_mean": -137.7632293701172, "KL/std": 61.128662109375, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.007685117423534393, "fcm_dpo/delta": 0.03301813825964928, "fcm_dpo/margin": 48.34103012084961, "fcm_dpo/q_t": 0.4142339825630188, "grad_norm": 8.356219291687012, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.270066022872925, "logits/rejected": -1.8487281799316406, "logps/chosen": -190.49603271484375, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -255.51611328125, "loss": 1.1183, "margin_dpo/margin_mean": 48.341026306152344, "margin_dpo/margin_std": 69.56396484375, "step": 270 }, { "KL/chosen_KL_mean": -73.82388305664062, "KL/mean": -103.92337036132812, "KL/rejected_KL_mean": -134.0228729248047, "KL/std": 57.25267028808594, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.00728934071958065, "fcm_dpo/delta": -0.05041738972067833, "fcm_dpo/margin": 60.19898986816406, "fcm_dpo/q_t": 0.39405661821365356, "grad_norm": 8.101794242858887, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.579874038696289, "logits/rejected": -1.1312823295593262, "logps/chosen": -170.54847717285156, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -232.5472869873047, "loss": 1.0497, "margin_dpo/margin_mean": 60.19899368286133, "margin_dpo/margin_std": 65.44872283935547, "step": 271 }, { "KL/chosen_KL_mean": -63.70800018310547, "KL/mean": -92.50074768066406, "KL/rejected_KL_mean": -121.29350280761719, "KL/std": 63.553619384765625, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.0071580978110432625, "fcm_dpo/delta": -0.015388628467917442, "fcm_dpo/margin": 57.58549118041992, "fcm_dpo/q_t": 0.40347564220428467, "grad_norm": 7.327671527862549, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.5391390323638916, "logits/rejected": -1.4337210655212402, "logps/chosen": -144.68521118164062, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -229.84886169433594, "loss": 1.0738, "margin_dpo/margin_mean": 57.58549499511719, "margin_dpo/margin_std": 68.96357727050781, "step": 272 }, { "KL/chosen_KL_mean": -86.3058090209961, "KL/mean": -115.90664672851562, "KL/rejected_KL_mean": -145.50747680664062, "KL/std": 65.24349975585938, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.0068403687328100204, "fcm_dpo/delta": -0.011820312589406967, "fcm_dpo/margin": 59.201683044433594, "fcm_dpo/q_t": 0.4041402339935303, "grad_norm": 7.268492221832275, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.4451773166656494, "logits/rejected": -1.023038387298584, "logps/chosen": -178.53041076660156, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -242.87054443359375, "loss": 1.0883, "margin_dpo/margin_mean": 59.201683044433594, "margin_dpo/margin_std": 74.26606750488281, "step": 273 }, { "KL/chosen_KL_mean": -88.30934143066406, "KL/mean": -111.10014343261719, "KL/rejected_KL_mean": -133.89096069335938, "KL/std": 63.88292694091797, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.0073954155668616295, "fcm_dpo/delta": 0.0759783536195755, "fcm_dpo/margin": 45.58162307739258, "fcm_dpo/q_t": 0.42334693670272827, "grad_norm": 11.595264434814453, "learning_rate": 3.75e-07, "logits/chosen": -1.9914908409118652, "logits/rejected": -1.6005935668945312, "logps/chosen": -170.57022094726562, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -217.76795959472656, "loss": 1.1239, "margin_dpo/margin_mean": 45.58161926269531, "margin_dpo/margin_std": 58.808143615722656, "step": 274 }, { "KL/chosen_KL_mean": -74.64450073242188, "KL/mean": -100.84567260742188, "KL/rejected_KL_mean": -127.0468521118164, "KL/std": 62.89811325073242, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.007766038179397583, "fcm_dpo/delta": -0.012169171124696732, "fcm_dpo/margin": 52.40233612060547, "fcm_dpo/q_t": 0.4051956534385681, "grad_norm": 8.797940254211426, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.3578405380249023, "logits/rejected": -2.0480215549468994, "logps/chosen": -154.3314666748047, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -225.79782104492188, "loss": 1.1049, "margin_dpo/margin_mean": 52.40233612060547, "margin_dpo/margin_std": 74.95335388183594, "step": 275 }, { "KL/chosen_KL_mean": -88.10804748535156, "KL/mean": -115.79530334472656, "KL/rejected_KL_mean": -143.48255920410156, "KL/std": 63.27790069580078, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.007242959924042225, "fcm_dpo/delta": -0.006368689239025116, "fcm_dpo/margin": 55.37451171875, "fcm_dpo/q_t": 0.4064900875091553, "grad_norm": 9.09972858428955, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.995816707611084, "logits/rejected": -1.4061861038208008, "logps/chosen": -174.64776611328125, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -221.3365020751953, "loss": 1.0943, "margin_dpo/margin_mean": 55.37451171875, "margin_dpo/margin_std": 72.15296936035156, "step": 276 }, { "KL/chosen_KL_mean": -96.068115234375, "KL/mean": -116.16034698486328, "KL/rejected_KL_mean": -136.2525634765625, "KL/std": 61.69487762451172, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.007917188107967377, "fcm_dpo/delta": 0.09784971177577972, "fcm_dpo/margin": 40.184452056884766, "fcm_dpo/q_t": 0.42912858724594116, "grad_norm": 8.882963180541992, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.206472396850586, "logits/rejected": -1.7993063926696777, "logps/chosen": -180.31222534179688, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -235.0867919921875, "loss": 1.1563, "margin_dpo/margin_mean": 40.184452056884766, "margin_dpo/margin_std": 63.47855758666992, "step": 277 }, { "KL/chosen_KL_mean": -91.43848419189453, "KL/mean": -117.1397476196289, "KL/rejected_KL_mean": -142.84100341796875, "KL/std": 65.67283630371094, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.008041847497224808, "fcm_dpo/delta": -0.017597125843167305, "fcm_dpo/margin": 51.40253448486328, "fcm_dpo/q_t": 0.40272557735443115, "grad_norm": 11.617165565490723, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.0351319313049316, "logits/rejected": -1.8173327445983887, "logps/chosen": -173.870361328125, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -240.6979217529297, "loss": 1.0837, "margin_dpo/margin_mean": 51.40253448486328, "margin_dpo/margin_std": 66.2891616821289, "step": 278 }, { "KL/chosen_KL_mean": -93.76026153564453, "KL/mean": -126.07313537597656, "KL/rejected_KL_mean": -158.38601684570312, "KL/std": 70.23944091796875, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.00733610475435853, "fcm_dpo/delta": -0.10241144150495529, "fcm_dpo/margin": 64.625732421875, "fcm_dpo/q_t": 0.385475754737854, "grad_norm": 10.04234504699707, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.7818171977996826, "logits/rejected": -1.5694806575775146, "logps/chosen": -178.92825317382812, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -252.51266479492188, "loss": 1.0291, "margin_dpo/margin_mean": 64.62574005126953, "margin_dpo/margin_std": 70.21957397460938, "step": 279 }, { "KL/chosen_KL_mean": -97.41738891601562, "KL/mean": -129.39630126953125, "KL/rejected_KL_mean": -161.375244140625, "KL/std": 64.18294525146484, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.007031106390058994, "fcm_dpo/delta": -0.06388475000858307, "fcm_dpo/margin": 63.95784378051758, "fcm_dpo/q_t": 0.3920469880104065, "grad_norm": 8.114020347595215, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.8082685470581055, "logits/rejected": -1.611290454864502, "logps/chosen": -183.2738037109375, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -265.49383544921875, "loss": 1.0405, "margin_dpo/margin_mean": 63.95784378051758, "margin_dpo/margin_std": 67.040283203125, "step": 280 }, { "KL/chosen_KL_mean": -103.72454833984375, "KL/mean": -135.58340454101562, "KL/rejected_KL_mean": -167.44223022460938, "KL/std": 74.13990783691406, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.006629610434174538, "fcm_dpo/delta": -0.029396183788776398, "fcm_dpo/margin": 63.717681884765625, "fcm_dpo/q_t": 0.4005139470100403, "grad_norm": 6.3389387130737305, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.0743894577026367, "logits/rejected": -1.3306207656860352, "logps/chosen": -190.27537536621094, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -256.0709228515625, "loss": 1.0659, "margin_dpo/margin_mean": 63.717681884765625, "margin_dpo/margin_std": 74.71734619140625, "step": 281 }, { "KL/chosen_KL_mean": -101.14562225341797, "KL/mean": -141.15447998046875, "KL/rejected_KL_mean": -181.163330078125, "KL/std": 72.099609375, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0060707880184054375, "fcm_dpo/delta": -0.11482174694538116, "fcm_dpo/margin": 80.01770782470703, "fcm_dpo/q_t": 0.3802452087402344, "grad_norm": 8.140931129455566, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.8092694282531738, "logits/rejected": -1.4268803596496582, "logps/chosen": -206.250732421875, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -284.0167236328125, "loss": 1.0032, "margin_dpo/margin_mean": 80.01770782470703, "margin_dpo/margin_std": 72.45964050292969, "step": 282 }, { "KL/chosen_KL_mean": -117.28823852539062, "KL/mean": -145.09426879882812, "KL/rejected_KL_mean": -172.9002685546875, "KL/std": 71.4011459350586, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.00623263418674469, "fcm_dpo/delta": 0.06437629461288452, "fcm_dpo/margin": 55.61204147338867, "fcm_dpo/q_t": 0.42101365327835083, "grad_norm": 8.742179870605469, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.2017266750335693, "logits/rejected": -1.8638412952423096, "logps/chosen": -212.34083557128906, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -276.44482421875, "loss": 1.1282, "margin_dpo/margin_mean": 55.61204147338867, "margin_dpo/margin_std": 79.55592346191406, "step": 283 }, { "KL/chosen_KL_mean": -106.59164428710938, "KL/mean": -142.23692321777344, "KL/rejected_KL_mean": -177.8822021484375, "KL/std": 70.5145263671875, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.006030972581356764, "fcm_dpo/delta": -0.040474165230989456, "fcm_dpo/margin": 71.29055786132812, "fcm_dpo/q_t": 0.395631343126297, "grad_norm": 8.612447738647461, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.6431031227111816, "logits/rejected": -1.294635534286499, "logps/chosen": -194.25811767578125, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -276.63323974609375, "loss": 1.0374, "margin_dpo/margin_mean": 71.29056549072266, "margin_dpo/margin_std": 63.607444763183594, "step": 284 }, { "KL/chosen_KL_mean": -119.69532012939453, "KL/mean": -144.31387329101562, "KL/rejected_KL_mean": -168.9324493408203, "KL/std": 64.98335266113281, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.006569210439920425, "fcm_dpo/delta": 0.08938819169998169, "fcm_dpo/margin": 49.23713302612305, "fcm_dpo/q_t": 0.42542168498039246, "grad_norm": 9.462973594665527, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.4061875343322754, "logits/rejected": -2.128971576690674, "logps/chosen": -214.12835693359375, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -273.00439453125, "loss": 1.1287, "margin_dpo/margin_mean": 49.23713302612305, "margin_dpo/margin_std": 61.95585632324219, "step": 285 }, { "KL/chosen_KL_mean": -124.0464859008789, "KL/mean": -157.87083435058594, "KL/rejected_KL_mean": -191.69515991210938, "KL/std": 69.17616271972656, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.006394201889634132, "fcm_dpo/delta": -0.0421503521502018, "fcm_dpo/margin": 67.64868927001953, "fcm_dpo/q_t": 0.39655354619026184, "grad_norm": 7.823284149169922, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.8647267818450928, "logits/rejected": -1.3920669555664062, "logps/chosen": -227.76687622070312, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -287.95294189453125, "loss": 1.0511, "margin_dpo/margin_mean": 67.64869689941406, "margin_dpo/margin_std": 72.04185485839844, "step": 286 }, { "KL/chosen_KL_mean": -142.4735870361328, "KL/mean": -161.63421630859375, "KL/rejected_KL_mean": -180.79486083984375, "KL/std": 62.46095275878906, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.0064842067658901215, "fcm_dpo/delta": 0.03566659614443779, "fcm_dpo/margin": 38.32127380371094, "fcm_dpo/q_t": 0.44313937425613403, "grad_norm": 7.68184232711792, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.3127171993255615, "logits/rejected": -1.7233219146728516, "logps/chosen": -236.3634796142578, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -264.1285400390625, "loss": 1.2092, "margin_dpo/margin_mean": 38.32127380371094, "margin_dpo/margin_std": 72.92868041992188, "step": 287 }, { "KL/chosen_KL_mean": -134.19581604003906, "KL/mean": -169.46681213378906, "KL/rejected_KL_mean": -204.73780822753906, "KL/std": 71.02359008789062, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.006270209327340126, "fcm_dpo/delta": -0.055538807064294815, "fcm_dpo/margin": 70.54200744628906, "fcm_dpo/q_t": 0.39538609981536865, "grad_norm": 7.082465171813965, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.9104018211364746, "logits/rejected": -1.500800609588623, "logps/chosen": -222.35183715820312, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -298.019775390625, "loss": 1.0492, "margin_dpo/margin_mean": 70.54200744628906, "margin_dpo/margin_std": 79.22444915771484, "step": 288 }, { "KL/chosen_KL_mean": -130.76731872558594, "KL/mean": -165.99050903320312, "KL/rejected_KL_mean": -201.21368408203125, "KL/std": 73.75202941894531, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.006121828220784664, "fcm_dpo/delta": -0.03985625505447388, "fcm_dpo/margin": 70.44636535644531, "fcm_dpo/q_t": 0.39993929862976074, "grad_norm": 6.554727077484131, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.9856581687927246, "logits/rejected": -1.7832098007202148, "logps/chosen": -206.16024780273438, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -294.36798095703125, "loss": 1.0649, "margin_dpo/margin_mean": 70.44636535644531, "margin_dpo/margin_std": 85.47918701171875, "step": 289 }, { "KL/chosen_KL_mean": -139.88604736328125, "KL/mean": -176.86465454101562, "KL/rejected_KL_mean": -213.84326171875, "KL/std": 79.12054443359375, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.005945807322859764, "fcm_dpo/delta": -0.053066737949848175, "fcm_dpo/margin": 73.95724487304688, "fcm_dpo/q_t": 0.39660459756851196, "grad_norm": 7.283459186553955, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.3313839435577393, "logits/rejected": -1.3812696933746338, "logps/chosen": -227.92800903320312, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -337.055419921875, "loss": 1.0554, "margin_dpo/margin_mean": 73.95724487304688, "margin_dpo/margin_std": 83.66102600097656, "step": 290 }, { "KL/chosen_KL_mean": -132.61248779296875, "KL/mean": -164.44447326660156, "KL/rejected_KL_mean": -196.27645874023438, "KL/std": 74.7508544921875, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.00587073527276516, "fcm_dpo/delta": 0.031691212207078934, "fcm_dpo/margin": 63.66398239135742, "fcm_dpo/q_t": 0.4126394987106323, "grad_norm": 8.510637283325195, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.2308287620544434, "logits/rejected": -1.7567081451416016, "logps/chosen": -226.70773315429688, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -292.3265380859375, "loss": 1.0935, "margin_dpo/margin_mean": 63.66398239135742, "margin_dpo/margin_std": 75.01309204101562, "step": 291 }, { "KL/chosen_KL_mean": -137.5259246826172, "KL/mean": -169.74710083007812, "KL/rejected_KL_mean": -201.96826171875, "KL/std": 72.09989929199219, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.005957702174782753, "fcm_dpo/delta": 0.019889622926712036, "fcm_dpo/margin": 64.44235229492188, "fcm_dpo/q_t": 0.4099695384502411, "grad_norm": 7.742751121520996, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0549917221069336, "logits/rejected": -1.6246708631515503, "logps/chosen": -225.77633666992188, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -298.38592529296875, "loss": 1.0773, "margin_dpo/margin_mean": 64.44235229492188, "margin_dpo/margin_std": 67.8692626953125, "step": 292 }, { "KL/chosen_KL_mean": -137.00433349609375, "KL/mean": -163.25811767578125, "KL/rejected_KL_mean": -189.51190185546875, "KL/std": 76.33308410644531, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.006443589925765991, "fcm_dpo/delta": 0.07063733041286469, "fcm_dpo/margin": 52.507568359375, "fcm_dpo/q_t": 0.42419859766960144, "grad_norm": 7.6096086502075195, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.3978490829467773, "logits/rejected": -1.8345913887023926, "logps/chosen": -224.38088989257812, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -275.2677001953125, "loss": 1.1405, "margin_dpo/margin_mean": 52.50757598876953, "margin_dpo/margin_std": 80.62934875488281, "step": 293 }, { "KL/chosen_KL_mean": -139.59945678710938, "KL/mean": -161.21206665039062, "KL/rejected_KL_mean": -182.82469177246094, "KL/std": 71.81602478027344, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.00708289910107851, "fcm_dpo/delta": 0.10729443281888962, "fcm_dpo/margin": 43.22523498535156, "fcm_dpo/q_t": 0.43253186345100403, "grad_norm": 8.532618522644043, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.678802251815796, "logits/rejected": -1.5151461362838745, "logps/chosen": -213.10740661621094, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -270.9134521484375, "loss": 1.1891, "margin_dpo/margin_mean": 43.22523498535156, "margin_dpo/margin_std": 84.1302490234375, "step": 294 }, { "KL/chosen_KL_mean": -133.02474975585938, "KL/mean": -165.92449951171875, "KL/rejected_KL_mean": -198.82427978515625, "KL/std": 73.84162902832031, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.006701253820210695, "fcm_dpo/delta": -0.05678282305598259, "fcm_dpo/margin": 65.79952239990234, "fcm_dpo/q_t": 0.3959708511829376, "grad_norm": 8.537036895751953, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.6673038005828857, "logits/rejected": -1.5104665756225586, "logps/chosen": -210.79022216796875, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -293.071533203125, "loss": 1.0604, "margin_dpo/margin_mean": 65.79952239990234, "margin_dpo/margin_std": 79.82605743408203, "step": 295 }, { "KL/chosen_KL_mean": -143.48297119140625, "KL/mean": -178.45071411132812, "KL/rejected_KL_mean": -213.41847229003906, "KL/std": 73.72811126708984, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.006348126567900181, "fcm_dpo/delta": -0.059592511504888535, "fcm_dpo/margin": 69.93549346923828, "fcm_dpo/q_t": 0.3931013345718384, "grad_norm": 11.730262756347656, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.4922077655792236, "logits/rejected": -2.003821849822998, "logps/chosen": -241.33938598632812, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -314.23480224609375, "loss": 1.045, "margin_dpo/margin_mean": 69.93550109863281, "margin_dpo/margin_std": 74.93051147460938, "step": 296 }, { "KL/chosen_KL_mean": -143.50880432128906, "KL/mean": -173.3553466796875, "KL/rejected_KL_mean": -203.20188903808594, "KL/std": 74.59136199951172, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.00650104321539402, "fcm_dpo/delta": 0.014220818877220154, "fcm_dpo/margin": 59.693084716796875, "fcm_dpo/q_t": 0.4114871621131897, "grad_norm": 10.840648651123047, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.0371147394180298, "logits/rejected": -0.7218179702758789, "logps/chosen": -232.07464599609375, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -304.7584533691406, "loss": 1.0986, "margin_dpo/margin_mean": 59.693084716796875, "margin_dpo/margin_std": 80.18905639648438, "step": 297 }, { "KL/chosen_KL_mean": -122.14201354980469, "KL/mean": -160.1995849609375, "KL/rejected_KL_mean": -198.2571563720703, "KL/std": 80.67182159423828, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.006130347028374672, "fcm_dpo/delta": -0.08647090196609497, "fcm_dpo/margin": 76.11514282226562, "fcm_dpo/q_t": 0.38935068249702454, "grad_norm": 11.45797061920166, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.0638670921325684, "logits/rejected": -2.0472748279571533, "logps/chosen": -217.0224609375, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -320.56817626953125, "loss": 1.0321, "margin_dpo/margin_mean": 76.11514282226562, "margin_dpo/margin_std": 82.27972412109375, "step": 298 }, { "KL/chosen_KL_mean": -126.02423095703125, "KL/mean": -164.16232299804688, "KL/rejected_KL_mean": -202.3004150390625, "KL/std": 77.61238098144531, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.005709235556423664, "fcm_dpo/delta": -0.04565701261162758, "fcm_dpo/margin": 76.27616882324219, "fcm_dpo/q_t": 0.3962002396583557, "grad_norm": 9.023762702941895, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.051196575164795, "logits/rejected": -0.8294165134429932, "logps/chosen": -206.43258666992188, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -291.83758544921875, "loss": 1.0478, "margin_dpo/margin_mean": 76.27616882324219, "margin_dpo/margin_std": 81.02107238769531, "step": 299 }, { "KL/chosen_KL_mean": -133.54019165039062, "KL/mean": -165.068603515625, "KL/rejected_KL_mean": -196.59701538085938, "KL/std": 81.68695068359375, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.005749483592808247, "fcm_dpo/delta": 0.04513074830174446, "fcm_dpo/margin": 63.05683898925781, "fcm_dpo/q_t": 0.4169178903102875, "grad_norm": 7.609052658081055, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.504530429840088, "logits/rejected": -1.2921417951583862, "logps/chosen": -221.69908142089844, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -297.53619384765625, "loss": 1.1204, "margin_dpo/margin_mean": 63.05683898925781, "margin_dpo/margin_std": 88.0323715209961, "step": 300 }, { "KL/chosen_KL_mean": -122.63788604736328, "KL/mean": -162.03237915039062, "KL/rejected_KL_mean": -201.42684936523438, "KL/std": 80.0143051147461, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.005550096742808819, "fcm_dpo/delta": -0.0525687038898468, "fcm_dpo/margin": 78.78895568847656, "fcm_dpo/q_t": 0.3957147002220154, "grad_norm": 8.628950119018555, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7369632720947266, "logits/rejected": -1.3054158687591553, "logps/chosen": -208.14279174804688, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -300.34368896484375, "loss": 1.0567, "margin_dpo/margin_mean": 78.78895568847656, "margin_dpo/margin_std": 90.32650756835938, "step": 301 }, { "KL/chosen_KL_mean": -118.31373596191406, "KL/mean": -151.85110473632812, "KL/rejected_KL_mean": -185.38848876953125, "KL/std": 82.31877136230469, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.005675541236996651, "fcm_dpo/delta": 0.02378086745738983, "fcm_dpo/margin": 67.07474517822266, "fcm_dpo/q_t": 0.4111320376396179, "grad_norm": 7.630757808685303, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.311251163482666, "logits/rejected": -1.1378058195114136, "logps/chosen": -197.76414489746094, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -288.44757080078125, "loss": 1.0915, "margin_dpo/margin_mean": 67.07474517822266, "margin_dpo/margin_std": 80.42845153808594, "step": 302 }, { "KL/chosen_KL_mean": -110.89071655273438, "KL/mean": -147.6366424560547, "KL/rejected_KL_mean": -184.38258361816406, "KL/std": 76.1639404296875, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.005720579996705055, "fcm_dpo/delta": -0.02623974159359932, "fcm_dpo/margin": 73.49186706542969, "fcm_dpo/q_t": 0.39995670318603516, "grad_norm": 7.880323886871338, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -2.050366163253784, "logits/rejected": -1.603386402130127, "logps/chosen": -206.489013671875, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -276.4471435546875, "loss": 1.0627, "margin_dpo/margin_mean": 73.49186706542969, "margin_dpo/margin_std": 81.19976806640625, "step": 303 }, { "KL/chosen_KL_mean": -110.4646224975586, "KL/mean": -148.3409881591797, "KL/rejected_KL_mean": -186.21734619140625, "KL/std": 78.92745971679688, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.005518034100532532, "fcm_dpo/delta": -0.02275390550494194, "fcm_dpo/margin": 75.75274658203125, "fcm_dpo/q_t": 0.4007124900817871, "grad_norm": 8.26273250579834, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.35117769241333, "logits/rejected": -1.948482871055603, "logps/chosen": -209.9177703857422, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -284.29962158203125, "loss": 1.0602, "margin_dpo/margin_mean": 75.75274658203125, "margin_dpo/margin_std": 82.322509765625, "step": 304 }, { "KL/chosen_KL_mean": -120.16925811767578, "KL/mean": -156.27574157714844, "KL/rejected_KL_mean": -192.3822021484375, "KL/std": 82.62189483642578, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.005390538834035397, "fcm_dpo/delta": 0.010848980396986008, "fcm_dpo/margin": 72.21295928955078, "fcm_dpo/q_t": 0.40858912467956543, "grad_norm": 7.042160987854004, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.77750563621521, "logits/rejected": -1.266533613204956, "logps/chosen": -223.43264770507812, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -290.211181640625, "loss": 1.0944, "margin_dpo/margin_mean": 72.21295928955078, "margin_dpo/margin_std": 90.07688903808594, "step": 305 }, { "KL/chosen_KL_mean": -108.23286437988281, "KL/mean": -150.40814208984375, "KL/rejected_KL_mean": -192.58343505859375, "KL/std": 78.016845703125, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.005239395424723625, "fcm_dpo/delta": -0.055834487080574036, "fcm_dpo/margin": 84.35059356689453, "fcm_dpo/q_t": 0.39177146553993225, "grad_norm": 8.511974334716797, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.187769889831543, "logits/rejected": -1.6256468296051025, "logps/chosen": -208.5366973876953, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -284.8143310546875, "loss": 1.0276, "margin_dpo/margin_mean": 84.35059356689453, "margin_dpo/margin_std": 72.43450927734375, "step": 306 }, { "KL/chosen_KL_mean": -113.7959213256836, "KL/mean": -145.59735107421875, "KL/rejected_KL_mean": -177.3988037109375, "KL/std": 79.10685729980469, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.005422373302280903, "fcm_dpo/delta": 0.06663471460342407, "fcm_dpo/margin": 63.60285568237305, "fcm_dpo/q_t": 0.4220127463340759, "grad_norm": 8.425183296203613, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.0650641918182373, "logits/rejected": -1.7226160764694214, "logps/chosen": -210.4456787109375, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -270.7474060058594, "loss": 1.1356, "margin_dpo/margin_mean": 63.60285949707031, "margin_dpo/margin_std": 93.81402587890625, "step": 307 }, { "KL/chosen_KL_mean": -118.91282653808594, "KL/mean": -150.51145935058594, "KL/rejected_KL_mean": -182.11007690429688, "KL/std": 80.61210632324219, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.0056509049609303474, "fcm_dpo/delta": 0.0500740110874176, "fcm_dpo/margin": 63.197235107421875, "fcm_dpo/q_t": 0.41921675205230713, "grad_norm": 10.153379440307617, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.638810157775879, "logits/rejected": -1.3499081134796143, "logps/chosen": -204.63751220703125, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -268.53326416015625, "loss": 1.134, "margin_dpo/margin_mean": 63.197235107421875, "margin_dpo/margin_std": 94.31275177001953, "step": 308 }, { "KL/chosen_KL_mean": -132.63063049316406, "KL/mean": -161.58206176757812, "KL/rejected_KL_mean": -190.53350830078125, "KL/std": 86.3198013305664, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.006151918787509203, "fcm_dpo/delta": 0.05284998193383217, "fcm_dpo/margin": 57.902870178222656, "fcm_dpo/q_t": 0.419302761554718, "grad_norm": 11.839205741882324, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.68941330909729, "logits/rejected": -1.254927396774292, "logps/chosen": -215.78208923339844, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -277.1195068359375, "loss": 1.1318, "margin_dpo/margin_mean": 57.902870178222656, "margin_dpo/margin_std": 86.4367446899414, "step": 309 }, { "KL/chosen_KL_mean": -120.88706970214844, "KL/mean": -152.30587768554688, "KL/rejected_KL_mean": -183.72467041015625, "KL/std": 77.79075622558594, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006229657679796219, "fcm_dpo/delta": 0.010590426623821259, "fcm_dpo/margin": 62.83760452270508, "fcm_dpo/q_t": 0.4094424247741699, "grad_norm": 9.386505126953125, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.3467204570770264, "logits/rejected": -1.939206838607788, "logps/chosen": -210.62506103515625, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -270.7955017089844, "loss": 1.0948, "margin_dpo/margin_mean": 62.83760070800781, "margin_dpo/margin_std": 82.27946472167969, "step": 310 }, { "KL/chosen_KL_mean": -138.75167846679688, "KL/mean": -179.50022888183594, "KL/rejected_KL_mean": -220.24879455566406, "KL/std": 79.37660217285156, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.005835794843733311, "fcm_dpo/delta": -0.09934177994728088, "fcm_dpo/margin": 81.49711608886719, "fcm_dpo/q_t": 0.3852664530277252, "grad_norm": 9.203911781311035, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.742368221282959, "logits/rejected": -1.02065110206604, "logps/chosen": -227.27810668945312, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -302.514892578125, "loss": 1.0176, "margin_dpo/margin_mean": 81.49711608886719, "margin_dpo/margin_std": 80.83274841308594, "step": 311 }, { "KL/chosen_KL_mean": -133.8328399658203, "KL/mean": -168.29788208007812, "KL/rejected_KL_mean": -202.762939453125, "KL/std": 86.90728759765625, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.005779668223112822, "fcm_dpo/delta": 0.0007412508130073547, "fcm_dpo/margin": 68.93009948730469, "fcm_dpo/q_t": 0.4066934287548065, "grad_norm": 9.516599655151367, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.6689212322235107, "logits/rejected": -1.1498750448226929, "logps/chosen": -214.2335205078125, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -285.2100830078125, "loss": 1.0843, "margin_dpo/margin_mean": 68.93009948730469, "margin_dpo/margin_std": 84.05435180664062, "step": 312 }, { "KL/chosen_KL_mean": -129.23672485351562, "KL/mean": -162.3968963623047, "KL/rejected_KL_mean": -195.5570831298828, "KL/std": 85.51274871826172, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.005852111615240574, "fcm_dpo/delta": 0.012751676142215729, "fcm_dpo/margin": 66.32034301757812, "fcm_dpo/q_t": 0.41037997603416443, "grad_norm": 9.87671947479248, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.3273744583129883, "logits/rejected": -1.9183857440948486, "logps/chosen": -213.37863159179688, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -288.9839172363281, "loss": 1.106, "margin_dpo/margin_mean": 66.32034301757812, "margin_dpo/margin_std": 92.53824615478516, "step": 313 }, { "KL/chosen_KL_mean": -131.2313995361328, "KL/mean": -163.8544921875, "KL/rejected_KL_mean": -196.47763061523438, "KL/std": 85.84571838378906, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0057888152077794075, "fcm_dpo/delta": 0.026964198797941208, "fcm_dpo/margin": 65.24623107910156, "fcm_dpo/q_t": 0.4139532148838043, "grad_norm": 9.682923316955566, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8829231262207031, "logits/rejected": -1.6648046970367432, "logps/chosen": -220.04112243652344, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -300.63653564453125, "loss": 1.1066, "margin_dpo/margin_mean": 65.24623107910156, "margin_dpo/margin_std": 88.47806549072266, "step": 314 }, { "KL/chosen_KL_mean": -126.47058868408203, "KL/mean": -160.08599853515625, "KL/rejected_KL_mean": -193.701416015625, "KL/std": 89.92106628417969, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0059029459953308105, "fcm_dpo/delta": 0.0038333097472786903, "fcm_dpo/margin": 67.23080444335938, "fcm_dpo/q_t": 0.4099789261817932, "grad_norm": 9.941434860229492, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.5406198501586914, "logits/rejected": -1.3692675828933716, "logps/chosen": -207.2642822265625, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -285.8385009765625, "loss": 1.117, "margin_dpo/margin_mean": 67.23080444335938, "margin_dpo/margin_std": 104.3451919555664, "step": 315 }, { "KL/chosen_KL_mean": -129.8626708984375, "KL/mean": -166.2532958984375, "KL/rejected_KL_mean": -202.64395141601562, "KL/std": 88.99896240234375, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.005835533607751131, "fcm_dpo/delta": -0.031340524554252625, "fcm_dpo/margin": 72.78126525878906, "fcm_dpo/q_t": 0.40170544385910034, "grad_norm": 9.0072021484375, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.783077597618103, "logits/rejected": -1.7834820747375488, "logps/chosen": -214.04693603515625, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -308.33477783203125, "loss": 1.0757, "margin_dpo/margin_mean": 72.78126525878906, "margin_dpo/margin_std": 94.04297637939453, "step": 316 }, { "KL/chosen_KL_mean": -134.7215576171875, "KL/mean": -179.25302124023438, "KL/rejected_KL_mean": -223.7844696044922, "KL/std": 97.63147735595703, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.00540002528578043, "fcm_dpo/delta": -0.10594746470451355, "fcm_dpo/margin": 89.06292724609375, "fcm_dpo/q_t": 0.38581180572509766, "grad_norm": 10.242342948913574, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -2.0115957260131836, "logits/rejected": -1.542344331741333, "logps/chosen": -222.59506225585938, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -332.0496826171875, "loss": 1.0389, "margin_dpo/margin_mean": 89.06292724609375, "margin_dpo/margin_std": 105.31065368652344, "step": 317 }, { "KL/chosen_KL_mean": -129.66384887695312, "KL/mean": -169.48995971679688, "KL/rejected_KL_mean": -209.3160858154297, "KL/std": 81.92440795898438, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.005114780738949776, "fcm_dpo/delta": -0.009459201246500015, "fcm_dpo/margin": 79.65222930908203, "fcm_dpo/q_t": 0.40404626727104187, "grad_norm": 9.042258262634277, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.1690621376037598, "logits/rejected": -1.4436888694763184, "logps/chosen": -227.86936950683594, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -297.452392578125, "loss": 1.0713, "margin_dpo/margin_mean": 79.6522216796875, "margin_dpo/margin_std": 91.24519348144531, "step": 318 }, { "KL/chosen_KL_mean": -112.32539367675781, "KL/mean": -155.96083068847656, "KL/rejected_KL_mean": -199.5962677001953, "KL/std": 85.32463073730469, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.004933707881718874, "fcm_dpo/delta": -0.0397811084985733, "fcm_dpo/margin": 87.27088928222656, "fcm_dpo/q_t": 0.3975956439971924, "grad_norm": 9.082079887390137, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.7656724452972412, "logits/rejected": -1.7009434700012207, "logps/chosen": -194.82876586914062, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -318.2469482421875, "loss": 1.0554, "margin_dpo/margin_mean": 87.27088928222656, "margin_dpo/margin_std": 97.12173461914062, "step": 319 }, { "KL/chosen_KL_mean": -114.91819763183594, "KL/mean": -150.66384887695312, "KL/rejected_KL_mean": -186.40951538085938, "KL/std": 86.4566650390625, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005067062564194202, "fcm_dpo/delta": 0.04621688276529312, "fcm_dpo/margin": 71.4913330078125, "fcm_dpo/q_t": 0.4164169728755951, "grad_norm": 8.14461612701416, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.238194465637207, "logits/rejected": -2.056528091430664, "logps/chosen": -202.055419921875, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -289.6336364746094, "loss": 1.1004, "margin_dpo/margin_mean": 71.4913330078125, "margin_dpo/margin_std": 84.29984283447266, "step": 320 }, { "KL/chosen_KL_mean": -129.09765625, "KL/mean": -170.3258514404297, "KL/rejected_KL_mean": -211.55404663085938, "KL/std": 88.82239532470703, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.005005875136703253, "fcm_dpo/delta": -0.017796887084841728, "fcm_dpo/margin": 82.45636749267578, "fcm_dpo/q_t": 0.4012599587440491, "grad_norm": 8.134737968444824, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -2.1169023513793945, "logits/rejected": -1.7404584884643555, "logps/chosen": -222.20233154296875, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -309.02215576171875, "loss": 1.0643, "margin_dpo/margin_mean": 82.45636749267578, "margin_dpo/margin_std": 89.35659790039062, "step": 321 }, { "KL/chosen_KL_mean": -120.05974578857422, "KL/mean": -165.01895141601562, "KL/rejected_KL_mean": -209.9781494140625, "KL/std": 88.07916259765625, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0047632623463869095, "fcm_dpo/delta": -0.042056504637002945, "fcm_dpo/margin": 89.91841125488281, "fcm_dpo/q_t": 0.39677509665489197, "grad_norm": 12.511489868164062, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.7582095861434937, "logits/rejected": -1.4030238389968872, "logps/chosen": -212.25265502929688, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -310.8713684082031, "loss": 1.0552, "margin_dpo/margin_mean": 89.91841125488281, "margin_dpo/margin_std": 94.85826110839844, "step": 322 }, { "KL/chosen_KL_mean": -138.58029174804688, "KL/mean": -175.84051513671875, "KL/rejected_KL_mean": -213.10076904296875, "KL/std": 88.87029266357422, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.004958899691700935, "fcm_dpo/delta": 0.03740254044532776, "fcm_dpo/margin": 74.52045440673828, "fcm_dpo/q_t": 0.41524553298950195, "grad_norm": 10.028064727783203, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.1554369926452637, "logits/rejected": -1.7872875928878784, "logps/chosen": -218.85240173339844, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -293.40411376953125, "loss": 1.1015, "margin_dpo/margin_mean": 74.52044677734375, "margin_dpo/margin_std": 93.015625, "step": 323 }, { "KL/chosen_KL_mean": -138.46217346191406, "KL/mean": -176.98995971679688, "KL/rejected_KL_mean": -215.51773071289062, "KL/std": 88.59934997558594, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.005116323009133339, "fcm_dpo/delta": 0.006746111437678337, "fcm_dpo/margin": 77.05555725097656, "fcm_dpo/q_t": 0.406861811876297, "grad_norm": 8.38990306854248, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7440459728240967, "logits/rejected": -1.1018035411834717, "logps/chosen": -234.1915283203125, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -309.6869812011719, "loss": 1.0754, "margin_dpo/margin_mean": 77.05555725097656, "margin_dpo/margin_std": 84.5044174194336, "step": 324 }, { "KL/chosen_KL_mean": -125.80093383789062, "KL/mean": -167.75048828125, "KL/rejected_KL_mean": -209.70001220703125, "KL/std": 84.58573913574219, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.005006608087569475, "fcm_dpo/delta": -0.02534850500524044, "fcm_dpo/margin": 83.89909362792969, "fcm_dpo/q_t": 0.3998144268989563, "grad_norm": 7.732594013214111, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.8797755241394043, "logits/rejected": -1.1536290645599365, "logps/chosen": -224.5078125, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -286.1081237792969, "loss": 1.0565, "margin_dpo/margin_mean": 83.89909362792969, "margin_dpo/margin_std": 87.84051513671875, "step": 325 }, { "KL/chosen_KL_mean": -141.34561157226562, "KL/mean": -179.0806884765625, "KL/rejected_KL_mean": -216.8157958984375, "KL/std": 84.48936462402344, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.005034366622567177, "fcm_dpo/delta": 0.0247870571911335, "fcm_dpo/margin": 75.47016906738281, "fcm_dpo/q_t": 0.41121459007263184, "grad_norm": 7.17478609085083, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.304718255996704, "logits/rejected": -1.6131075620651245, "logps/chosen": -246.00692749023438, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -303.64117431640625, "loss": 1.0932, "margin_dpo/margin_mean": 75.47016906738281, "margin_dpo/margin_std": 91.72134399414062, "step": 326 }, { "KL/chosen_KL_mean": -136.4932861328125, "KL/mean": -172.93618774414062, "KL/rejected_KL_mean": -209.37908935546875, "KL/std": 77.48454284667969, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005131968762725592, "fcm_dpo/delta": 0.031632665544748306, "fcm_dpo/margin": 72.88579559326172, "fcm_dpo/q_t": 0.4138820767402649, "grad_norm": 8.313996315002441, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.6986932754516602, "logits/rejected": -1.1071064472198486, "logps/chosen": -243.9335174560547, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -326.43231201171875, "loss": 1.0958, "margin_dpo/margin_mean": 72.88578796386719, "margin_dpo/margin_std": 88.14754486083984, "step": 327 }, { "KL/chosen_KL_mean": -132.86825561523438, "KL/mean": -180.69216918945312, "KL/rejected_KL_mean": -228.51608276367188, "KL/std": 92.06556701660156, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.004905284382402897, "fcm_dpo/delta": -0.09051632881164551, "fcm_dpo/margin": 95.64781188964844, "fcm_dpo/q_t": 0.3855741322040558, "grad_norm": 9.52945327758789, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.718299388885498, "logits/rejected": -1.4207310676574707, "logps/chosen": -219.47894287109375, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -325.91796875, "loss": 1.017, "margin_dpo/margin_mean": 95.64781188964844, "margin_dpo/margin_std": 89.4625015258789, "step": 328 }, { "KL/chosen_KL_mean": -139.15628051757812, "KL/mean": -176.5908203125, "KL/rejected_KL_mean": -214.02536010742188, "KL/std": 88.76412963867188, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.00485738692805171, "fcm_dpo/delta": 0.043807253241539, "fcm_dpo/margin": 74.86908721923828, "fcm_dpo/q_t": 0.41661280393600464, "grad_norm": 9.588016510009766, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.6088900566101074, "logits/rejected": -1.1167800426483154, "logps/chosen": -239.76934814453125, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -318.97906494140625, "loss": 1.1096, "margin_dpo/margin_mean": 74.86909484863281, "margin_dpo/margin_std": 96.23165893554688, "step": 329 }, { "KL/chosen_KL_mean": -117.57325744628906, "KL/mean": -157.78497314453125, "KL/rejected_KL_mean": -197.99667358398438, "KL/std": 86.26248168945312, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.004972635768353939, "fcm_dpo/delta": -3.9667822420597076e-05, "fcm_dpo/margin": 80.42341613769531, "fcm_dpo/q_t": 0.40421074628829956, "grad_norm": 8.000280380249023, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.9711344242095947, "logits/rejected": -1.8954920768737793, "logps/chosen": -194.5316925048828, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -290.8116149902344, "loss": 1.0623, "margin_dpo/margin_mean": 80.42341613769531, "margin_dpo/margin_std": 76.70714569091797, "step": 330 }, { "KL/chosen_KL_mean": -126.12753295898438, "KL/mean": -171.0685577392578, "KL/rejected_KL_mean": -216.0095672607422, "KL/std": 84.27494049072266, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0048934826627373695, "fcm_dpo/delta": -0.051301538944244385, "fcm_dpo/margin": 89.88203430175781, "fcm_dpo/q_t": 0.3939548134803772, "grad_norm": 9.292428970336914, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.9677704572677612, "logits/rejected": -1.6692938804626465, "logps/chosen": -209.18695068359375, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -310.14068603515625, "loss": 1.0372, "margin_dpo/margin_mean": 89.88204193115234, "margin_dpo/margin_std": 85.2579345703125, "step": 331 }, { "KL/chosen_KL_mean": -135.77740478515625, "KL/mean": -166.34022521972656, "KL/rejected_KL_mean": -196.90304565429688, "KL/std": 93.22622680664062, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005157839972525835, "fcm_dpo/delta": 0.09917619824409485, "fcm_dpo/margin": 61.12565231323242, "fcm_dpo/q_t": 0.4287262260913849, "grad_norm": 8.004188537597656, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.2960219383239746, "logits/rejected": -1.8778109550476074, "logps/chosen": -217.7082977294922, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -279.74896240234375, "loss": 1.1487, "margin_dpo/margin_mean": 61.12565231323242, "margin_dpo/margin_std": 91.44859313964844, "step": 332 }, { "KL/chosen_KL_mean": -155.58975219726562, "KL/mean": -199.17581176757812, "KL/rejected_KL_mean": -242.7618408203125, "KL/std": 97.96177673339844, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.004927680362015963, "fcm_dpo/delta": -0.043494485318660736, "fcm_dpo/margin": 87.17208099365234, "fcm_dpo/q_t": 0.3983389139175415, "grad_norm": 8.339241027832031, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5639601945877075, "logits/rejected": -1.2228820323944092, "logps/chosen": -244.12234497070312, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -345.22784423828125, "loss": 1.0681, "margin_dpo/margin_mean": 87.17207336425781, "margin_dpo/margin_std": 104.8178482055664, "step": 333 }, { "KL/chosen_KL_mean": -141.95364379882812, "KL/mean": -180.70077514648438, "KL/rejected_KL_mean": -219.44789123535156, "KL/std": 92.38648223876953, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.004943979904055595, "fcm_dpo/delta": 0.017589550465345383, "fcm_dpo/margin": 77.49424743652344, "fcm_dpo/q_t": 0.41003161668777466, "grad_norm": 8.692344665527344, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.589308500289917, "logits/rejected": -1.302095890045166, "logps/chosen": -239.71142578125, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -319.4537353515625, "loss": 1.1074, "margin_dpo/margin_mean": 77.49424743652344, "margin_dpo/margin_std": 103.30242919921875, "step": 334 }, { "KL/chosen_KL_mean": -141.2784423828125, "KL/mean": -174.47320556640625, "KL/rejected_KL_mean": -207.66796875, "KL/std": 91.93051147460938, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005339081399142742, "fcm_dpo/delta": 0.055107392370700836, "fcm_dpo/margin": 66.38954162597656, "fcm_dpo/q_t": 0.4195169508457184, "grad_norm": 13.434436798095703, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.243007183074951, "logits/rejected": -1.8118876218795776, "logps/chosen": -237.65447998046875, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -302.59197998046875, "loss": 1.1319, "margin_dpo/margin_mean": 66.38954162597656, "margin_dpo/margin_std": 100.98405456542969, "step": 335 }, { "KL/chosen_KL_mean": -144.08526611328125, "KL/mean": -186.92564392089844, "KL/rejected_KL_mean": -229.76600646972656, "KL/std": 94.19276428222656, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005224978551268578, "fcm_dpo/delta": -0.06128572300076485, "fcm_dpo/margin": 85.68074035644531, "fcm_dpo/q_t": 0.39383822679519653, "grad_norm": 10.656806945800781, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.4706072807312012, "logits/rejected": -1.3197224140167236, "logps/chosen": -232.6147918701172, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -339.32177734375, "loss": 1.0512, "margin_dpo/margin_mean": 85.68074035644531, "margin_dpo/margin_std": 99.94566345214844, "step": 336 }, { "KL/chosen_KL_mean": -142.08499145507812, "KL/mean": -183.0537109375, "KL/rejected_KL_mean": -224.0224609375, "KL/std": 95.30184173583984, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.0050520384684205055, "fcm_dpo/delta": -0.017694959416985512, "fcm_dpo/margin": 81.93748474121094, "fcm_dpo/q_t": 0.40435662865638733, "grad_norm": 8.656922340393066, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.534406900405884, "logits/rejected": -2.2596402168273926, "logps/chosen": -239.39736938476562, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -334.15380859375, "loss": 1.0901, "margin_dpo/margin_mean": 81.93748474121094, "margin_dpo/margin_std": 112.17634582519531, "step": 337 }, { "KL/chosen_KL_mean": -147.04734802246094, "KL/mean": -184.66534423828125, "KL/rejected_KL_mean": -222.28334045410156, "KL/std": 96.31929016113281, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.005059299524873495, "fcm_dpo/delta": 0.02393491007387638, "fcm_dpo/margin": 75.23599243164062, "fcm_dpo/q_t": 0.41124022006988525, "grad_norm": 9.204803466796875, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.8881471157073975, "logits/rejected": -1.7379364967346191, "logps/chosen": -230.67745971679688, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -315.9757080078125, "loss": 1.0973, "margin_dpo/margin_mean": 75.23599243164062, "margin_dpo/margin_std": 95.37352752685547, "step": 338 }, { "KL/chosen_KL_mean": -156.9281463623047, "KL/mean": -196.90182495117188, "KL/rejected_KL_mean": -236.87551879882812, "KL/std": 95.18284606933594, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005209520924836397, "fcm_dpo/delta": -0.025930091738700867, "fcm_dpo/margin": 79.94737243652344, "fcm_dpo/q_t": 0.402057021856308, "grad_norm": 8.157084465026855, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.975715160369873, "logits/rejected": -2.5533595085144043, "logps/chosen": -256.77996826171875, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -345.94989013671875, "loss": 1.0924, "margin_dpo/margin_mean": 79.94737243652344, "margin_dpo/margin_std": 106.02886962890625, "step": 339 }, { "KL/chosen_KL_mean": -168.22434997558594, "KL/mean": -202.9669952392578, "KL/rejected_KL_mean": -237.7096405029297, "KL/std": 87.8060531616211, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.005177933722734451, "fcm_dpo/delta": 0.04882291704416275, "fcm_dpo/margin": 69.48529052734375, "fcm_dpo/q_t": 0.417373925447464, "grad_norm": 9.672468185424805, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.5586137771606445, "logits/rejected": -1.0829503536224365, "logps/chosen": -253.1983184814453, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -316.5099792480469, "loss": 1.1168, "margin_dpo/margin_mean": 69.48529052734375, "margin_dpo/margin_std": 96.5479736328125, "step": 340 }, { "KL/chosen_KL_mean": -175.437744140625, "KL/mean": -209.09356689453125, "KL/rejected_KL_mean": -242.7493896484375, "KL/std": 92.09315490722656, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.005360917653888464, "fcm_dpo/delta": 0.04778551310300827, "fcm_dpo/margin": 67.3116226196289, "fcm_dpo/q_t": 0.4188730716705322, "grad_norm": 9.57992935180664, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.258419156074524, "logits/rejected": -1.0040322542190552, "logps/chosen": -260.70172119140625, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -348.8353271484375, "loss": 1.1354, "margin_dpo/margin_mean": 67.3116226196289, "margin_dpo/margin_std": 105.923583984375, "step": 341 }, { "KL/chosen_KL_mean": -140.89547729492188, "KL/mean": -189.98974609375, "KL/rejected_KL_mean": -239.0840606689453, "KL/std": 92.14039611816406, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.004872831515967846, "fcm_dpo/delta": -0.11370465904474258, "fcm_dpo/margin": 98.18858337402344, "fcm_dpo/q_t": 0.38033878803253174, "grad_norm": 7.536941051483154, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.5700011253356934, "logits/rejected": -0.8244874477386475, "logps/chosen": -241.93406677246094, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -337.9825744628906, "loss": 1.0237, "margin_dpo/margin_mean": 98.18858337402344, "margin_dpo/margin_std": 98.69281005859375, "step": 342 }, { "KL/chosen_KL_mean": -153.5966796875, "KL/mean": -194.4304656982422, "KL/rejected_KL_mean": -235.2642364501953, "KL/std": 95.98880004882812, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.005007544532418251, "fcm_dpo/delta": -0.01563461497426033, "fcm_dpo/margin": 81.66756439208984, "fcm_dpo/q_t": 0.40366697311401367, "grad_norm": 9.826371192932129, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1574318408966064, "logits/rejected": -2.0507731437683105, "logps/chosen": -237.34405517578125, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -351.84332275390625, "loss": 1.085, "margin_dpo/margin_mean": 81.66756439208984, "margin_dpo/margin_std": 100.60665893554688, "step": 343 }, { "KL/chosen_KL_mean": -151.75469970703125, "KL/mean": -195.81814575195312, "KL/rejected_KL_mean": -239.881591796875, "KL/std": 102.260986328125, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.004830214194953442, "fcm_dpo/delta": -0.034847378730773926, "fcm_dpo/margin": 88.12689208984375, "fcm_dpo/q_t": 0.4000549018383026, "grad_norm": 9.220417022705078, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.4777371883392334, "logits/rejected": -1.2250356674194336, "logps/chosen": -230.0086669921875, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -335.310302734375, "loss": 1.0738, "margin_dpo/margin_mean": 88.12689208984375, "margin_dpo/margin_std": 108.75332641601562, "step": 344 }, { "KL/chosen_KL_mean": -158.25970458984375, "KL/mean": -203.97262573242188, "KL/rejected_KL_mean": -249.68551635742188, "KL/std": 102.00994873046875, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.004548564553260803, "fcm_dpo/delta": -0.02051960490643978, "fcm_dpo/margin": 91.4258041381836, "fcm_dpo/q_t": 0.40218982100486755, "grad_norm": 9.40277099609375, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.6107168197631836, "logits/rejected": -1.3184459209442139, "logps/chosen": -256.3392028808594, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -353.72027587890625, "loss": 1.0678, "margin_dpo/margin_mean": 91.42581176757812, "margin_dpo/margin_std": 106.05194091796875, "step": 345 }, { "KL/chosen_KL_mean": -175.39273071289062, "KL/mean": -211.18490600585938, "KL/rejected_KL_mean": -246.97708129882812, "KL/std": 94.53190612792969, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.0047989413142204285, "fcm_dpo/delta": 0.06809362769126892, "fcm_dpo/margin": 71.5843505859375, "fcm_dpo/q_t": 0.4226304292678833, "grad_norm": 7.563831329345703, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.6834132671356201, "logits/rejected": -1.3645910024642944, "logps/chosen": -256.07623291015625, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -332.86968994140625, "loss": 1.1361, "margin_dpo/margin_mean": 71.5843505859375, "margin_dpo/margin_std": 108.64997100830078, "step": 346 }, { "KL/chosen_KL_mean": -165.51107788085938, "KL/mean": -199.71478271484375, "KL/rejected_KL_mean": -233.91847229003906, "KL/std": 94.2136459350586, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.005179152823984623, "fcm_dpo/delta": 0.05252869799733162, "fcm_dpo/margin": 68.40740966796875, "fcm_dpo/q_t": 0.41965988278388977, "grad_norm": 8.774153709411621, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.814160943031311, "logits/rejected": -1.4901604652404785, "logps/chosen": -259.1904602050781, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -331.46099853515625, "loss": 1.1305, "margin_dpo/margin_mean": 68.40740203857422, "margin_dpo/margin_std": 102.20301818847656, "step": 347 }, { "KL/chosen_KL_mean": -143.16952514648438, "KL/mean": -181.97305297851562, "KL/rejected_KL_mean": -220.77659606933594, "KL/std": 88.54947662353516, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.004967576824128628, "fcm_dpo/delta": 0.00989580899477005, "fcm_dpo/margin": 77.60708618164062, "fcm_dpo/q_t": 0.40846896171569824, "grad_norm": 8.027522087097168, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.0985281467437744, "logits/rejected": -1.8332419395446777, "logps/chosen": -233.4217529296875, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -316.546875, "loss": 1.0972, "margin_dpo/margin_mean": 77.60708618164062, "margin_dpo/margin_std": 90.59869384765625, "step": 348 }, { "KL/chosen_KL_mean": -155.95326232910156, "KL/mean": -191.97836303710938, "KL/rejected_KL_mean": -228.00341796875, "KL/std": 97.46289825439453, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.005370058119297028, "fcm_dpo/delta": 0.0136028490960598, "fcm_dpo/margin": 72.05015563964844, "fcm_dpo/q_t": 0.4114587604999542, "grad_norm": 10.33802604675293, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.9580178260803223, "logits/rejected": -1.7308924198150635, "logps/chosen": -248.0113525390625, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -331.96868896484375, "loss": 1.1099, "margin_dpo/margin_mean": 72.05015563964844, "margin_dpo/margin_std": 101.85948944091797, "step": 349 }, { "KL/chosen_KL_mean": -164.3104248046875, "KL/mean": -195.13604736328125, "KL/rejected_KL_mean": -225.96170043945312, "KL/std": 89.29495239257812, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005527984816581011, "fcm_dpo/delta": 0.07157102227210999, "fcm_dpo/margin": 61.651275634765625, "fcm_dpo/q_t": 0.4235541820526123, "grad_norm": 11.38034725189209, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8904534578323364, "logits/rejected": -1.666367530822754, "logps/chosen": -251.3646240234375, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -321.68365478515625, "loss": 1.1448, "margin_dpo/margin_mean": 61.651275634765625, "margin_dpo/margin_std": 97.51602172851562, "step": 350 }, { "KL/chosen_KL_mean": -164.53732299804688, "KL/mean": -208.12725830078125, "KL/rejected_KL_mean": -251.71717834472656, "KL/std": 99.83389282226562, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.005344501696527004, "fcm_dpo/delta": -0.08569169789552689, "fcm_dpo/margin": 87.17984008789062, "fcm_dpo/q_t": 0.389156311750412, "grad_norm": 8.81854248046875, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.3561170101165771, "logits/rejected": -1.0437299013137817, "logps/chosen": -253.70635986328125, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -360.01519775390625, "loss": 1.0507, "margin_dpo/margin_mean": 87.1798324584961, "margin_dpo/margin_std": 106.8216781616211, "step": 351 }, { "KL/chosen_KL_mean": -152.006103515625, "KL/mean": -192.84625244140625, "KL/rejected_KL_mean": -233.68637084960938, "KL/std": 98.2367935180664, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.005102580413222313, "fcm_dpo/delta": -0.021279005333781242, "fcm_dpo/margin": 81.68026733398438, "fcm_dpo/q_t": 0.40443921089172363, "grad_norm": 8.326467514038086, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.9833917617797852, "logits/rejected": -1.441209316253662, "logps/chosen": -248.81759643554688, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -318.0911865234375, "loss": 1.0827, "margin_dpo/margin_mean": 81.68026733398438, "margin_dpo/margin_std": 106.49186706542969, "step": 352 }, { "KL/chosen_KL_mean": -173.87847900390625, "KL/mean": -199.63392639160156, "KL/rejected_KL_mean": -225.38937377929688, "KL/std": 86.9433364868164, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.005247673951089382, "fcm_dpo/delta": 0.031480398029088974, "fcm_dpo/margin": 51.51087188720703, "fcm_dpo/q_t": 0.4373307228088379, "grad_norm": 9.487860679626465, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.0468904972076416, "logits/rejected": -1.6533117294311523, "logps/chosen": -259.240966796875, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -312.7566833496094, "loss": 1.2009, "margin_dpo/margin_mean": 51.51087188720703, "margin_dpo/margin_std": 98.67332458496094, "step": 353 }, { "KL/chosen_KL_mean": -151.95472717285156, "KL/mean": -193.58026123046875, "KL/rejected_KL_mean": -235.20579528808594, "KL/std": 88.13865661621094, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.005161711014807224, "fcm_dpo/delta": -0.03795633465051651, "fcm_dpo/margin": 83.25106811523438, "fcm_dpo/q_t": 0.39747726917266846, "grad_norm": 8.11750602722168, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8631720542907715, "logits/rejected": -1.3921477794647217, "logps/chosen": -247.75588989257812, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -330.0719299316406, "loss": 1.0456, "margin_dpo/margin_mean": 83.25106048583984, "margin_dpo/margin_std": 82.72601318359375, "step": 354 }, { "KL/chosen_KL_mean": -150.09658813476562, "KL/mean": -188.64352416992188, "KL/rejected_KL_mean": -227.1904296875, "KL/std": 89.86520385742188, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0050234501250088215, "fcm_dpo/delta": 0.01478547789156437, "fcm_dpo/margin": 77.0938491821289, "fcm_dpo/q_t": 0.40957561135292053, "grad_norm": 10.140955924987793, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4522151947021484, "logits/rejected": -1.9679625034332275, "logps/chosen": -236.28213500976562, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -315.6520080566406, "loss": 1.0932, "margin_dpo/margin_mean": 77.0938491821289, "margin_dpo/margin_std": 94.97024536132812, "step": 355 }, { "KL/chosen_KL_mean": -176.92230224609375, "KL/mean": -206.52810668945312, "KL/rejected_KL_mean": -236.1338653564453, "KL/std": 87.58965301513672, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005543043836951256, "fcm_dpo/delta": 0.08308710157871246, "fcm_dpo/margin": 59.2115478515625, "fcm_dpo/q_t": 0.4264000654220581, "grad_norm": 9.118013381958008, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8076454401016235, "logits/rejected": -1.3370598554611206, "logps/chosen": -267.74041748046875, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -318.26116943359375, "loss": 1.158, "margin_dpo/margin_mean": 59.2115478515625, "margin_dpo/margin_std": 99.93441772460938, "step": 356 }, { "KL/chosen_KL_mean": -148.65408325195312, "KL/mean": -195.47366333007812, "KL/rejected_KL_mean": -242.29324340820312, "KL/std": 96.29035949707031, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0049818335101008415, "fcm_dpo/delta": -0.09859152138233185, "fcm_dpo/margin": 93.63917541503906, "fcm_dpo/q_t": 0.3847854733467102, "grad_norm": 8.365049362182617, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -2.120944023132324, "logits/rejected": -1.7619775533676147, "logps/chosen": -236.15882873535156, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -334.5426025390625, "loss": 1.0248, "margin_dpo/margin_mean": 93.63916778564453, "margin_dpo/margin_std": 90.29573822021484, "step": 357 }, { "KL/chosen_KL_mean": -159.3295135498047, "KL/mean": -194.50479125976562, "KL/rejected_KL_mean": -229.68006896972656, "KL/std": 89.96337127685547, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.005200651474297047, "fcm_dpo/delta": 0.04179971292614937, "fcm_dpo/margin": 70.35054016113281, "fcm_dpo/q_t": 0.41738834977149963, "grad_norm": 8.755834579467773, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.5443503856658936, "logits/rejected": -1.0692498683929443, "logps/chosen": -254.38497924804688, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -326.15618896484375, "loss": 1.1108, "margin_dpo/margin_mean": 70.35054016113281, "margin_dpo/margin_std": 95.5135726928711, "step": 358 }, { "KL/chosen_KL_mean": -178.2020263671875, "KL/mean": -210.9842529296875, "KL/rejected_KL_mean": -243.7664794921875, "KL/std": 84.37049102783203, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0053596049547195435, "fcm_dpo/delta": 0.057673439383506775, "fcm_dpo/margin": 65.56446838378906, "fcm_dpo/q_t": 0.42082029581069946, "grad_norm": 10.176785469055176, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.3376543521881104, "logits/rejected": -0.7450504899024963, "logps/chosen": -265.5753479003906, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -332.5383605957031, "loss": 1.1357, "margin_dpo/margin_mean": 65.56446838378906, "margin_dpo/margin_std": 98.25286865234375, "step": 359 }, { "KL/chosen_KL_mean": -163.83877563476562, "KL/mean": -205.0393829345703, "KL/rejected_KL_mean": -246.24002075195312, "KL/std": 84.79701232910156, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.005325379781424999, "fcm_dpo/delta": -0.050645507872104645, "fcm_dpo/margin": 82.4012451171875, "fcm_dpo/q_t": 0.394744873046875, "grad_norm": 13.460746765136719, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.6378209590911865, "logits/rejected": -1.2964940071105957, "logps/chosen": -239.06222534179688, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -322.3340148925781, "loss": 1.0495, "margin_dpo/margin_mean": 82.4012451171875, "margin_dpo/margin_std": 90.27430725097656, "step": 360 }, { "KL/chosen_KL_mean": -165.6855926513672, "KL/mean": -202.4282989501953, "KL/rejected_KL_mean": -239.17100524902344, "KL/std": 96.77937316894531, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005352191627025604, "fcm_dpo/delta": 0.008241891860961914, "fcm_dpo/margin": 73.48540496826172, "fcm_dpo/q_t": 0.41045957803726196, "grad_norm": 10.977911949157715, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.6522836685180664, "logits/rejected": -1.408792495727539, "logps/chosen": -247.638916015625, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -335.8502197265625, "loss": 1.1015, "margin_dpo/margin_mean": 73.48540496826172, "margin_dpo/margin_std": 102.55374908447266, "step": 361 }, { "KL/chosen_KL_mean": -171.48989868164062, "KL/mean": -209.89739990234375, "KL/rejected_KL_mean": -248.30494689941406, "KL/std": 91.881103515625, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0052519673481583595, "fcm_dpo/delta": -0.005236415192484856, "fcm_dpo/margin": 76.8150634765625, "fcm_dpo/q_t": 0.40570780634880066, "grad_norm": 9.265073776245117, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.9130197763442993, "logits/rejected": -1.714143991470337, "logps/chosen": -260.82958984375, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -358.8069152832031, "loss": 1.088, "margin_dpo/margin_mean": 76.8150634765625, "margin_dpo/margin_std": 98.45401763916016, "step": 362 }, { "KL/chosen_KL_mean": -158.60055541992188, "KL/mean": -208.75311279296875, "KL/rejected_KL_mean": -258.9056396484375, "KL/std": 96.63841247558594, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.004941675346344709, "fcm_dpo/delta": -0.12633417546749115, "fcm_dpo/margin": 100.30508422851562, "fcm_dpo/q_t": 0.3802333474159241, "grad_norm": 9.689270973205566, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.7999616861343384, "logits/rejected": -1.3671503067016602, "logps/chosen": -239.37554931640625, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -338.33831787109375, "loss": 1.0156, "margin_dpo/margin_mean": 100.30508422851562, "margin_dpo/margin_std": 104.21660614013672, "step": 363 }, { "KL/chosen_KL_mean": -156.61520385742188, "KL/mean": -197.1243896484375, "KL/rejected_KL_mean": -237.63357543945312, "KL/std": 89.92976379394531, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.00485632661730051, "fcm_dpo/delta": 0.0032032281160354614, "fcm_dpo/margin": 81.01837921142578, "fcm_dpo/q_t": 0.4063214659690857, "grad_norm": 11.667214393615723, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0851869583129883, "logits/rejected": -1.9447963237762451, "logps/chosen": -233.14480590820312, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -335.1953125, "loss": 1.079, "margin_dpo/margin_mean": 81.01837158203125, "margin_dpo/margin_std": 86.6229248046875, "step": 364 }, { "KL/chosen_KL_mean": -152.438232421875, "KL/mean": -196.55360412597656, "KL/rejected_KL_mean": -240.66897583007812, "KL/std": 96.5469741821289, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.004718084819614887, "fcm_dpo/delta": -0.021773429587483406, "fcm_dpo/margin": 88.23074340820312, "fcm_dpo/q_t": 0.40334394574165344, "grad_norm": 8.581512451171875, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.634777545928955, "logits/rejected": -1.5104639530181885, "logps/chosen": -232.98312377929688, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -339.99212646484375, "loss": 1.0775, "margin_dpo/margin_mean": 88.23074340820312, "margin_dpo/margin_std": 110.24192810058594, "step": 365 }, { "KL/chosen_KL_mean": -154.55471801757812, "KL/mean": -190.99432373046875, "KL/rejected_KL_mean": -227.43389892578125, "KL/std": 100.02314758300781, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.004813226871192455, "fcm_dpo/delta": 0.05954656004905701, "fcm_dpo/margin": 72.87918090820312, "fcm_dpo/q_t": 0.4213341772556305, "grad_norm": 8.193413734436035, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.7829084396362305, "logits/rejected": -1.8190194368362427, "logps/chosen": -226.7928009033203, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -319.105712890625, "loss": 1.1164, "margin_dpo/margin_mean": 72.8791732788086, "margin_dpo/margin_std": 97.37818908691406, "step": 366 }, { "KL/chosen_KL_mean": -140.16644287109375, "KL/mean": -183.94015502929688, "KL/rejected_KL_mean": -227.71388244628906, "KL/std": 88.03213500976562, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.004802820272743702, "fcm_dpo/delta": -0.025889642536640167, "fcm_dpo/margin": 87.54743957519531, "fcm_dpo/q_t": 0.40007224678993225, "grad_norm": 8.037558555603027, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.851576566696167, "logits/rejected": -1.5068061351776123, "logps/chosen": -225.58843994140625, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -311.231689453125, "loss": 1.0536, "margin_dpo/margin_mean": 87.54743957519531, "margin_dpo/margin_std": 90.57449340820312, "step": 367 }, { "KL/chosen_KL_mean": -130.01263427734375, "KL/mean": -178.4687957763672, "KL/rejected_KL_mean": -226.92495727539062, "KL/std": 96.91070556640625, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.004631819203495979, "fcm_dpo/delta": -0.06337358057498932, "fcm_dpo/margin": 96.91233825683594, "fcm_dpo/q_t": 0.39147093892097473, "grad_norm": 9.086630821228027, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.528329610824585, "logits/rejected": -1.4980268478393555, "logps/chosen": -203.853759765625, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -331.6202392578125, "loss": 1.041, "margin_dpo/margin_mean": 96.91233825683594, "margin_dpo/margin_std": 98.6136474609375, "step": 368 }, { "KL/chosen_KL_mean": -145.07699584960938, "KL/mean": -190.44970703125, "KL/rejected_KL_mean": -235.8224334716797, "KL/std": 89.7952880859375, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.004477568436414003, "fcm_dpo/delta": -0.008193687535822392, "fcm_dpo/margin": 90.74542236328125, "fcm_dpo/q_t": 0.40364909172058105, "grad_norm": 8.085713386535645, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.622049331665039, "logits/rejected": -1.2524824142456055, "logps/chosen": -243.53895568847656, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -334.9791564941406, "loss": 1.0601, "margin_dpo/margin_mean": 90.74542236328125, "margin_dpo/margin_std": 90.787109375, "step": 369 }, { "KL/chosen_KL_mean": -161.63729858398438, "KL/mean": -191.87106323242188, "KL/rejected_KL_mean": -222.1048126220703, "KL/std": 85.90802001953125, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.004878254607319832, "fcm_dpo/delta": 0.12208369374275208, "fcm_dpo/margin": 60.4675178527832, "fcm_dpo/q_t": 0.4336884617805481, "grad_norm": 8.757126808166504, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.9930548667907715, "logits/rejected": -1.7977956533432007, "logps/chosen": -258.41302490234375, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -324.84893798828125, "loss": 1.1699, "margin_dpo/margin_mean": 60.46751403808594, "margin_dpo/margin_std": 100.03948211669922, "step": 370 }, { "KL/chosen_KL_mean": -153.53329467773438, "KL/mean": -187.35546875, "KL/rejected_KL_mean": -221.1776123046875, "KL/std": 86.83732604980469, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005278144031763077, "fcm_dpo/delta": 0.0501711443066597, "fcm_dpo/margin": 67.64431762695312, "fcm_dpo/q_t": 0.41783666610717773, "grad_norm": 11.897764205932617, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.782322645187378, "logits/rejected": -1.4096927642822266, "logps/chosen": -256.52349853515625, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -326.52825927734375, "loss": 1.1225, "margin_dpo/margin_mean": 67.6443099975586, "margin_dpo/margin_std": 95.3824462890625, "step": 371 }, { "KL/chosen_KL_mean": -131.0615234375, "KL/mean": -181.37469482421875, "KL/rejected_KL_mean": -231.68788146972656, "KL/std": 100.80130767822266, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.004844076000154018, "fcm_dpo/delta": -0.11565956473350525, "fcm_dpo/margin": 100.62635803222656, "fcm_dpo/q_t": 0.3820211887359619, "grad_norm": 7.471256732940674, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.93532133102417, "logits/rejected": -1.3045752048492432, "logps/chosen": -222.08775329589844, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -317.298095703125, "loss": 1.016, "margin_dpo/margin_mean": 100.62635803222656, "margin_dpo/margin_std": 103.68220520019531, "step": 372 }, { "KL/chosen_KL_mean": -151.6935272216797, "KL/mean": -188.44186401367188, "KL/rejected_KL_mean": -225.190185546875, "KL/std": 96.10372161865234, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.00485179154202342, "fcm_dpo/delta": 0.052968189120292664, "fcm_dpo/margin": 73.49665832519531, "fcm_dpo/q_t": 0.4184926450252533, "grad_norm": 10.622846603393555, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.1170458793640137, "logits/rejected": -1.5946853160858154, "logps/chosen": -251.7666015625, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -316.3065185546875, "loss": 1.1212, "margin_dpo/margin_mean": 73.49665832519531, "margin_dpo/margin_std": 102.89338684082031, "step": 373 }, { "KL/chosen_KL_mean": -131.01254272460938, "KL/mean": -172.196533203125, "KL/rejected_KL_mean": -213.38050842285156, "KL/std": 91.25364685058594, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.004874797072261572, "fcm_dpo/delta": -0.0025698505342006683, "fcm_dpo/margin": 82.36795806884766, "fcm_dpo/q_t": 0.40562111139297485, "grad_norm": 7.7610883712768555, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.3186330795288086, "logits/rejected": -1.936171293258667, "logps/chosen": -229.02488708496094, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -314.13275146484375, "loss": 1.0678, "margin_dpo/margin_mean": 82.36795806884766, "margin_dpo/margin_std": 87.37416076660156, "step": 374 }, { "KL/chosen_KL_mean": -132.899658203125, "KL/mean": -179.68771362304688, "KL/rejected_KL_mean": -226.47576904296875, "KL/std": 87.86271667480469, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.004609329160302877, "fcm_dpo/delta": -0.046647947281599045, "fcm_dpo/margin": 93.57609558105469, "fcm_dpo/q_t": 0.3950332999229431, "grad_norm": 8.930638313293457, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.5195703506469727, "logits/rejected": -1.2823892831802368, "logps/chosen": -217.2959442138672, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -322.1946105957031, "loss": 1.0485, "margin_dpo/margin_mean": 93.57609558105469, "margin_dpo/margin_std": 93.11183166503906, "step": 375 }, { "KL/chosen_KL_mean": -128.7314453125, "KL/mean": -179.9996337890625, "KL/rejected_KL_mean": -231.267822265625, "KL/std": 101.2147216796875, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004431936889886856, "fcm_dpo/delta": -0.07136962562799454, "fcm_dpo/margin": 102.53638458251953, "fcm_dpo/q_t": 0.3906506299972534, "grad_norm": 8.714337348937988, "learning_rate": 2.5e-07, "logits/chosen": -1.7896634340286255, "logits/rejected": -1.4776968955993652, "logps/chosen": -224.46914672851562, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -343.2411193847656, "loss": 1.0369, "margin_dpo/margin_mean": 102.53638458251953, "margin_dpo/margin_std": 108.4615249633789, "step": 376 }, { "KL/chosen_KL_mean": -136.13803100585938, "KL/mean": -189.80923461914062, "KL/rejected_KL_mean": -243.48046875, "KL/std": 101.97345733642578, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004118419252336025, "fcm_dpo/delta": -0.057123616337776184, "fcm_dpo/margin": 107.34243774414062, "fcm_dpo/q_t": 0.3924797475337982, "grad_norm": 7.169247150421143, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5573700666427612, "logits/rejected": -1.4430251121520996, "logps/chosen": -215.06497192382812, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -349.37939453125, "loss": 1.0343, "margin_dpo/margin_mean": 107.34243774414062, "margin_dpo/margin_std": 101.16941833496094, "step": 377 }, { "KL/chosen_KL_mean": -133.05221557617188, "KL/mean": -175.75714111328125, "KL/rejected_KL_mean": -218.4620361328125, "KL/std": 96.45173645019531, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004335816018283367, "fcm_dpo/delta": 0.03337696194648743, "fcm_dpo/margin": 85.4098129272461, "fcm_dpo/q_t": 0.4125681519508362, "grad_norm": 8.744478225708008, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.1275792121887207, "logits/rejected": -1.8529975414276123, "logps/chosen": -208.42462158203125, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -312.1661376953125, "loss": 1.0963, "margin_dpo/margin_mean": 85.40980529785156, "margin_dpo/margin_std": 99.07200622558594, "step": 378 }, { "KL/chosen_KL_mean": -156.6085968017578, "KL/mean": -193.8806610107422, "KL/rejected_KL_mean": -231.1527099609375, "KL/std": 99.28062438964844, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004522995557636023, "fcm_dpo/delta": 0.07552020251750946, "fcm_dpo/margin": 74.54411315917969, "fcm_dpo/q_t": 0.4244621992111206, "grad_norm": 8.776957511901855, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.845184087753296, "logits/rejected": -1.7740118503570557, "logps/chosen": -245.4642791748047, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -352.53387451171875, "loss": 1.1404, "margin_dpo/margin_mean": 74.54411315917969, "margin_dpo/margin_std": 114.02534484863281, "step": 379 }, { "KL/chosen_KL_mean": -171.58251953125, "KL/mean": -217.8046417236328, "KL/rejected_KL_mean": -264.0267639160156, "KL/std": 102.7412109375, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004527292214334011, "fcm_dpo/delta": -0.023482169955968857, "fcm_dpo/margin": 92.4442367553711, "fcm_dpo/q_t": 0.40313324332237244, "grad_norm": 11.606622695922852, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.4048099517822266, "logits/rejected": -1.3792755603790283, "logps/chosen": -251.41574096679688, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -374.7715148925781, "loss": 1.0751, "margin_dpo/margin_mean": 92.44422912597656, "margin_dpo/margin_std": 116.99021911621094, "step": 380 }, { "KL/chosen_KL_mean": -150.54446411132812, "KL/mean": -190.39254760742188, "KL/rejected_KL_mean": -230.24063110351562, "KL/std": 98.22796630859375, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004641607403755188, "fcm_dpo/delta": 0.03675704076886177, "fcm_dpo/margin": 79.6961669921875, "fcm_dpo/q_t": 0.41416648030281067, "grad_norm": 9.701480865478516, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.7015326023101807, "logits/rejected": -1.145586371421814, "logps/chosen": -237.1112823486328, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -314.6573791503906, "loss": 1.1041, "margin_dpo/margin_mean": 79.6961669921875, "margin_dpo/margin_std": 102.24404907226562, "step": 381 }, { "KL/chosen_KL_mean": -147.53704833984375, "KL/mean": -193.15069580078125, "KL/rejected_KL_mean": -238.76434326171875, "KL/std": 96.33355712890625, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004485836252570152, "fcm_dpo/delta": -0.016123328357934952, "fcm_dpo/margin": 91.22730255126953, "fcm_dpo/q_t": 0.40263742208480835, "grad_norm": 8.351905822753906, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -2.0830140113830566, "logits/rejected": -1.8274853229522705, "logps/chosen": -251.9988555908203, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -353.3665771484375, "loss": 1.069, "margin_dpo/margin_mean": 91.22730255126953, "margin_dpo/margin_std": 99.14009857177734, "step": 382 }, { "KL/chosen_KL_mean": -161.74459838867188, "KL/mean": -200.3641815185547, "KL/rejected_KL_mean": -238.9837646484375, "KL/std": 99.38853454589844, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.004711962305009365, "fcm_dpo/delta": 0.04413574934005737, "fcm_dpo/margin": 77.23917388916016, "fcm_dpo/q_t": 0.41616272926330566, "grad_norm": 9.150345802307129, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.7887623310089111, "logits/rejected": -1.669067144393921, "logps/chosen": -243.86065673828125, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -337.76092529296875, "loss": 1.1076, "margin_dpo/margin_mean": 77.23917388916016, "margin_dpo/margin_std": 98.13470458984375, "step": 383 }, { "KL/chosen_KL_mean": -173.0072021484375, "KL/mean": -211.08004760742188, "KL/rejected_KL_mean": -249.1529083251953, "KL/std": 102.82688903808594, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.005036992020905018, "fcm_dpo/delta": 0.014068394899368286, "fcm_dpo/margin": 76.14570617675781, "fcm_dpo/q_t": 0.41061288118362427, "grad_norm": 11.247259140014648, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7146891355514526, "logits/rejected": -1.489769697189331, "logps/chosen": -253.59243774414062, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -340.6007995605469, "loss": 1.128, "margin_dpo/margin_mean": 76.14570617675781, "margin_dpo/margin_std": 114.12051391601562, "step": 384 }, { "KL/chosen_KL_mean": -143.7839813232422, "KL/mean": -192.79864501953125, "KL/rejected_KL_mean": -241.81332397460938, "KL/std": 92.5248794555664, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004523749928921461, "fcm_dpo/delta": -0.06194532662630081, "fcm_dpo/margin": 98.02932739257812, "fcm_dpo/q_t": 0.39351150393486023, "grad_norm": 9.787887573242188, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.1515557765960693, "logits/rejected": -1.5349156856536865, "logps/chosen": -236.54415893554688, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -326.2733154296875, "loss": 1.0486, "margin_dpo/margin_mean": 98.02932739257812, "margin_dpo/margin_std": 106.84219360351562, "step": 385 }, { "KL/chosen_KL_mean": -152.50816345214844, "KL/mean": -199.96615600585938, "KL/rejected_KL_mean": -247.4241485595703, "KL/std": 94.78890991210938, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0044725071638822556, "fcm_dpo/delta": -0.031192418187856674, "fcm_dpo/margin": 94.91598510742188, "fcm_dpo/q_t": 0.3978579044342041, "grad_norm": 7.261466979980469, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.5630226135253906, "logits/rejected": -1.276221752166748, "logps/chosen": -237.86712646484375, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -344.414794921875, "loss": 1.0573, "margin_dpo/margin_mean": 94.91598510742188, "margin_dpo/margin_std": 103.15345764160156, "step": 386 }, { "KL/chosen_KL_mean": -178.65682983398438, "KL/mean": -220.7406005859375, "KL/rejected_KL_mean": -262.8243408203125, "KL/std": 91.29615783691406, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004552027210593224, "fcm_dpo/delta": 0.020232323557138443, "fcm_dpo/margin": 84.16754150390625, "fcm_dpo/q_t": 0.41018521785736084, "grad_norm": 10.022231101989746, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.4455857276916504, "logits/rejected": -1.7641057968139648, "logps/chosen": -281.344970703125, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -362.2529296875, "loss": 1.0833, "margin_dpo/margin_mean": 84.16753387451172, "margin_dpo/margin_std": 93.94087982177734, "step": 387 }, { "KL/chosen_KL_mean": -178.65330505371094, "KL/mean": -225.36996459960938, "KL/rejected_KL_mean": -272.08660888671875, "KL/std": 102.25494384765625, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0044446163810789585, "fcm_dpo/delta": -0.019525917246937752, "fcm_dpo/margin": 93.43329620361328, "fcm_dpo/q_t": 0.40144819021224976, "grad_norm": 8.355619430541992, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.5332412719726562, "logits/rejected": -1.4158930778503418, "logps/chosen": -262.238525390625, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -382.1869812011719, "loss": 1.0649, "margin_dpo/margin_mean": 93.43328857421875, "margin_dpo/margin_std": 104.42460632324219, "step": 388 }, { "KL/chosen_KL_mean": -184.78273010253906, "KL/mean": -224.53594970703125, "KL/rejected_KL_mean": -264.2891540527344, "KL/std": 103.86572265625, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004579238127917051, "fcm_dpo/delta": 0.043991800397634506, "fcm_dpo/margin": 79.50642395019531, "fcm_dpo/q_t": 0.4167877733707428, "grad_norm": 10.977243423461914, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.3127408027648926, "logits/rejected": -1.8568342924118042, "logps/chosen": -274.60498046875, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -349.1429138183594, "loss": 1.1171, "margin_dpo/margin_mean": 79.50642395019531, "margin_dpo/margin_std": 111.87716674804688, "step": 389 }, { "KL/chosen_KL_mean": -183.372314453125, "KL/mean": -222.46646118164062, "KL/rejected_KL_mean": -261.5605773925781, "KL/std": 101.53630065917969, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004674031399190426, "fcm_dpo/delta": 0.040697962045669556, "fcm_dpo/margin": 78.18827819824219, "fcm_dpo/q_t": 0.4182990789413452, "grad_norm": 8.739748001098633, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.452839970588684, "logits/rejected": -1.2424935102462769, "logps/chosen": -260.9047546386719, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -354.1445007324219, "loss": 1.1264, "margin_dpo/margin_mean": 78.18827056884766, "margin_dpo/margin_std": 116.89789581298828, "step": 390 }, { "KL/chosen_KL_mean": -166.5, "KL/mean": -208.43670654296875, "KL/rejected_KL_mean": -250.3734130859375, "KL/std": 97.52375030517578, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004760085605084896, "fcm_dpo/delta": -0.00023828819394111633, "fcm_dpo/margin": 83.87342071533203, "fcm_dpo/q_t": 0.4066413938999176, "grad_norm": 8.64584732055664, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.8937368392944336, "logits/rejected": -1.4419682025909424, "logps/chosen": -254.73292541503906, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -343.3809814453125, "loss": 1.0854, "margin_dpo/margin_mean": 83.87342071533203, "margin_dpo/margin_std": 102.65730285644531, "step": 391 }, { "KL/chosen_KL_mean": -169.80593872070312, "KL/mean": -221.04783630371094, "KL/rejected_KL_mean": -272.28973388671875, "KL/std": 109.83566284179688, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004594170954078436, "fcm_dpo/delta": -0.09222989529371262, "fcm_dpo/margin": 102.4837646484375, "fcm_dpo/q_t": 0.38698822259902954, "grad_norm": 9.16171932220459, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.678893804550171, "logits/rejected": -1.5637582540512085, "logps/chosen": -257.55035400390625, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -380.8604736328125, "loss": 1.042, "margin_dpo/margin_mean": 102.4837646484375, "margin_dpo/margin_std": 118.87272644042969, "step": 392 }, { "KL/chosen_KL_mean": -150.87298583984375, "KL/mean": -196.84463500976562, "KL/rejected_KL_mean": -242.8162841796875, "KL/std": 102.6601791381836, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004385889507830143, "fcm_dpo/delta": -0.004117501899600029, "fcm_dpo/margin": 91.94329071044922, "fcm_dpo/q_t": 0.4049713611602783, "grad_norm": 7.936796188354492, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.6780462265014648, "logits/rejected": -1.3550350666046143, "logps/chosen": -227.43679809570312, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -322.54248046875, "loss": 1.0704, "margin_dpo/margin_mean": 91.94329071044922, "margin_dpo/margin_std": 101.67320251464844, "step": 393 }, { "KL/chosen_KL_mean": -159.94993591308594, "KL/mean": -210.0718994140625, "KL/rejected_KL_mean": -260.1938781738281, "KL/std": 98.77021789550781, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004272952675819397, "fcm_dpo/delta": -0.036130838096141815, "fcm_dpo/margin": 100.24391174316406, "fcm_dpo/q_t": 0.39690887928009033, "grad_norm": 7.693758964538574, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.6622068881988525, "logits/rejected": -1.434901475906372, "logps/chosen": -244.6486053466797, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -357.6516418457031, "loss": 1.045, "margin_dpo/margin_mean": 100.24391174316406, "margin_dpo/margin_std": 97.96649169921875, "step": 394 }, { "KL/chosen_KL_mean": -165.673095703125, "KL/mean": -204.18621826171875, "KL/rejected_KL_mean": -242.69935607910156, "KL/std": 102.9873046875, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004391923546791077, "fcm_dpo/delta": 0.07411152869462967, "fcm_dpo/margin": 77.02626037597656, "fcm_dpo/q_t": 0.42560938000679016, "grad_norm": 8.423810005187988, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.5770835876464844, "logits/rejected": -2.178558349609375, "logps/chosen": -254.56072998046875, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -333.744873046875, "loss": 1.1681, "margin_dpo/margin_mean": 77.0262680053711, "margin_dpo/margin_std": 139.08111572265625, "step": 395 }, { "KL/chosen_KL_mean": -174.8834228515625, "KL/mean": -209.47169494628906, "KL/rejected_KL_mean": -244.05996704101562, "KL/std": 91.03007507324219, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004838614724576473, "fcm_dpo/delta": 0.07826073467731476, "fcm_dpo/margin": 69.17654418945312, "fcm_dpo/q_t": 0.4247515797615051, "grad_norm": 11.478642463684082, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8755539655685425, "logits/rejected": -1.3808133602142334, "logps/chosen": -277.354736328125, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -337.29473876953125, "loss": 1.1452, "margin_dpo/margin_mean": 69.17654418945312, "margin_dpo/margin_std": 108.41854858398438, "step": 396 }, { "KL/chosen_KL_mean": -157.305419921875, "KL/mean": -218.35189819335938, "KL/rejected_KL_mean": -279.39837646484375, "KL/std": 105.74957275390625, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.004280107095837593, "fcm_dpo/delta": -0.17136073112487793, "fcm_dpo/margin": 122.09295654296875, "fcm_dpo/q_t": 0.3703593611717224, "grad_norm": 7.922102451324463, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.3249011039733887, "logits/rejected": -1.7323898077011108, "logps/chosen": -258.349853515625, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -388.24957275390625, "loss": 0.9981, "margin_dpo/margin_mean": 122.09296417236328, "margin_dpo/margin_std": 126.13517761230469, "step": 397 }, { "KL/chosen_KL_mean": -166.59315490722656, "KL/mean": -210.6360626220703, "KL/rejected_KL_mean": -254.67897033691406, "KL/std": 104.04443359375, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004238451831042767, "fcm_dpo/delta": 0.032827071845531464, "fcm_dpo/margin": 88.08582305908203, "fcm_dpo/q_t": 0.41455817222595215, "grad_norm": 7.659793853759766, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.170881509780884, "logits/rejected": -1.89548659324646, "logps/chosen": -261.64239501953125, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -368.6846008300781, "loss": 1.1127, "margin_dpo/margin_mean": 88.08582305908203, "margin_dpo/margin_std": 123.82085418701172, "step": 398 }, { "KL/chosen_KL_mean": -144.51451110839844, "KL/mean": -200.81150817871094, "KL/rejected_KL_mean": -257.1085205078125, "KL/std": 93.90592956542969, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.004086493514478207, "fcm_dpo/delta": -0.07778343558311462, "fcm_dpo/margin": 112.59400939941406, "fcm_dpo/q_t": 0.3877986669540405, "grad_norm": 7.776646137237549, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -1.9856152534484863, "logits/rejected": -1.1661771535873413, "logps/chosen": -241.15304565429688, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -340.64495849609375, "loss": 1.0197, "margin_dpo/margin_mean": 112.59400939941406, "margin_dpo/margin_std": 101.50218963623047, "step": 399 }, { "KL/chosen_KL_mean": -150.74896240234375, "KL/mean": -207.96499633789062, "KL/rejected_KL_mean": -265.1810607910156, "KL/std": 103.43621826171875, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0038285842165350914, "fcm_dpo/delta": -0.048933133482933044, "fcm_dpo/margin": 114.43209838867188, "fcm_dpo/q_t": 0.3950181305408478, "grad_norm": 10.648974418640137, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.178525924682617, "logits/rejected": -1.6967408657073975, "logps/chosen": -256.0520935058594, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -373.83660888671875, "loss": 1.0467, "margin_dpo/margin_mean": 114.43209838867188, "margin_dpo/margin_std": 119.53279113769531, "step": 400 }, { "KL/chosen_KL_mean": -167.08270263671875, "KL/mean": -202.37286376953125, "KL/rejected_KL_mean": -237.6630401611328, "KL/std": 105.17704772949219, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0041228411719202995, "fcm_dpo/delta": 0.1283659040927887, "fcm_dpo/margin": 70.58032989501953, "fcm_dpo/q_t": 0.4375752806663513, "grad_norm": 10.654839515686035, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.4732662439346313, "logits/rejected": -1.22859787940979, "logps/chosen": -251.3718719482422, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -328.60614013671875, "loss": 1.1835, "margin_dpo/margin_mean": 70.58032989501953, "margin_dpo/margin_std": 128.58880615234375, "step": 401 }, { "KL/chosen_KL_mean": -147.5701141357422, "KL/mean": -214.4874267578125, "KL/rejected_KL_mean": -281.40472412109375, "KL/std": 112.67420959472656, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0038527343422174454, "fcm_dpo/delta": -0.1558227837085724, "fcm_dpo/margin": 133.83462524414062, "fcm_dpo/q_t": 0.37038469314575195, "grad_norm": 7.049633026123047, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.8045716285705566, "logits/rejected": -1.3285980224609375, "logps/chosen": -231.16323852539062, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -369.2149963378906, "loss": 0.9806, "margin_dpo/margin_mean": 133.83462524414062, "margin_dpo/margin_std": 110.3616714477539, "step": 402 }, { "KL/chosen_KL_mean": -151.72555541992188, "KL/mean": -201.3883514404297, "KL/rejected_KL_mean": -251.0511474609375, "KL/std": 107.85037994384766, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0037456275895237923, "fcm_dpo/delta": 0.03434676676988602, "fcm_dpo/margin": 99.32559967041016, "fcm_dpo/q_t": 0.413291871547699, "grad_norm": 7.738055229187012, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.2411837577819824, "logits/rejected": -1.842058777809143, "logps/chosen": -225.4786376953125, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -334.9712829589844, "loss": 1.0944, "margin_dpo/margin_mean": 99.32559967041016, "margin_dpo/margin_std": 115.67559814453125, "step": 403 }, { "KL/chosen_KL_mean": -178.71902465820312, "KL/mean": -221.63088989257812, "KL/rejected_KL_mean": -264.54278564453125, "KL/std": 104.16844940185547, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.003981833811849356, "fcm_dpo/delta": 0.07055004686117172, "fcm_dpo/margin": 85.82374572753906, "fcm_dpo/q_t": 0.4226842522621155, "grad_norm": 11.087850570678711, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.847894549369812, "logits/rejected": -1.5011181831359863, "logps/chosen": -258.39520263671875, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -348.82330322265625, "loss": 1.1358, "margin_dpo/margin_mean": 85.82374572753906, "margin_dpo/margin_std": 127.67596435546875, "step": 404 }, { "KL/chosen_KL_mean": -183.33999633789062, "KL/mean": -226.23289489746094, "KL/rejected_KL_mean": -269.1257629394531, "KL/std": 98.221435546875, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004301545210182667, "fcm_dpo/delta": 0.03416460007429123, "fcm_dpo/margin": 85.7857666015625, "fcm_dpo/q_t": 0.41449791193008423, "grad_norm": 8.135733604431152, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.3810350894927979, "logits/rejected": -0.8926867246627808, "logps/chosen": -279.80194091796875, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -361.9964599609375, "loss": 1.1016, "margin_dpo/margin_mean": 85.7857666015625, "margin_dpo/margin_std": 105.03971862792969, "step": 405 }, { "KL/chosen_KL_mean": -174.10842895507812, "KL/mean": -213.27244567871094, "KL/rejected_KL_mean": -252.4364471435547, "KL/std": 104.73593139648438, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004439661279320717, "fcm_dpo/delta": 0.06309033930301666, "fcm_dpo/margin": 78.32801818847656, "fcm_dpo/q_t": 0.42141997814178467, "grad_norm": 13.681050300598145, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.9638142585754395, "logits/rejected": -1.7979979515075684, "logps/chosen": -252.24240112304688, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -350.72003173828125, "loss": 1.1392, "margin_dpo/margin_mean": 78.32801818847656, "margin_dpo/margin_std": 122.58317565917969, "step": 406 }, { "KL/chosen_KL_mean": -180.16207885742188, "KL/mean": -214.67601013183594, "KL/rejected_KL_mean": -249.18992614746094, "KL/std": 103.01298522949219, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.004803077317774296, "fcm_dpo/delta": 0.08164440840482712, "fcm_dpo/margin": 69.02783203125, "fcm_dpo/q_t": 0.42550182342529297, "grad_norm": 12.030664443969727, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -1.9655282497406006, "logits/rejected": -1.4259536266326904, "logps/chosen": -264.1749267578125, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -331.970947265625, "loss": 1.1469, "margin_dpo/margin_mean": 69.02783203125, "margin_dpo/margin_std": 109.73652648925781, "step": 407 }, { "KL/chosen_KL_mean": -173.51712036132812, "KL/mean": -207.7809600830078, "KL/rejected_KL_mean": -242.04483032226562, "KL/std": 98.60501098632812, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0050923191010952, "fcm_dpo/delta": 0.06201542168855667, "fcm_dpo/margin": 68.52772521972656, "fcm_dpo/q_t": 0.4230646789073944, "grad_norm": 10.49770450592041, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.249680995941162, "logits/rejected": -1.6265431642532349, "logps/chosen": -277.9788818359375, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -338.4170227050781, "loss": 1.159, "margin_dpo/margin_mean": 68.52772521972656, "margin_dpo/margin_std": 122.66981506347656, "step": 408 }, { "KL/chosen_KL_mean": -184.15994262695312, "KL/mean": -220.21133422851562, "KL/rejected_KL_mean": -256.2627258300781, "KL/std": 100.79786682128906, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0052927350625395775, "fcm_dpo/delta": 0.022696368396282196, "fcm_dpo/margin": 72.10279083251953, "fcm_dpo/q_t": 0.4113461375236511, "grad_norm": 9.827393531799316, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5320820808410645, "logits/rejected": -1.0937652587890625, "logps/chosen": -281.8282470703125, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -346.3085632324219, "loss": 1.125, "margin_dpo/margin_mean": 72.10279083251953, "margin_dpo/margin_std": 110.96375274658203, "step": 409 }, { "KL/chosen_KL_mean": -177.80099487304688, "KL/mean": -222.03640747070312, "KL/rejected_KL_mean": -266.27178955078125, "KL/std": 106.89192199707031, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.005057280883193016, "fcm_dpo/delta": -0.061720386147499084, "fcm_dpo/margin": 88.47079467773438, "fcm_dpo/q_t": 0.39273595809936523, "grad_norm": 8.995354652404785, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.9181692600250244, "logits/rejected": -1.7926759719848633, "logps/chosen": -254.27023315429688, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -354.9124450683594, "loss": 1.0567, "margin_dpo/margin_mean": 88.47079467773438, "margin_dpo/margin_std": 106.9239273071289, "step": 410 }, { "KL/chosen_KL_mean": -190.93511962890625, "KL/mean": -231.67092895507812, "KL/rejected_KL_mean": -272.4067077636719, "KL/std": 99.30368041992188, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.00490330345928669, "fcm_dpo/delta": -0.0011177361011505127, "fcm_dpo/margin": 81.47159576416016, "fcm_dpo/q_t": 0.4086782932281494, "grad_norm": 9.048089027404785, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.282107353210449, "logits/rejected": -1.8727624416351318, "logps/chosen": -278.1014404296875, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -359.50274658203125, "loss": 1.0931, "margin_dpo/margin_mean": 81.47158813476562, "margin_dpo/margin_std": 107.64225006103516, "step": 411 }, { "KL/chosen_KL_mean": -181.34442138671875, "KL/mean": -221.64447021484375, "KL/rejected_KL_mean": -261.94451904296875, "KL/std": 100.02157592773438, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004941780120134354, "fcm_dpo/delta": 0.0014279000461101532, "fcm_dpo/margin": 80.60012817382812, "fcm_dpo/q_t": 0.4105263650417328, "grad_norm": 10.656685829162598, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.5061888694763184, "logits/rejected": -2.2947487831115723, "logps/chosen": -260.291748046875, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -368.0500793457031, "loss": 1.1212, "margin_dpo/margin_mean": 80.60012817382812, "margin_dpo/margin_std": 128.38589477539062, "step": 412 }, { "KL/chosen_KL_mean": -194.25924682617188, "KL/mean": -238.90493774414062, "KL/rejected_KL_mean": -283.5506591796875, "KL/std": 106.42913055419922, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004778787028044462, "fcm_dpo/delta": -0.03651279956102371, "fcm_dpo/margin": 89.29136657714844, "fcm_dpo/q_t": 0.3991602063179016, "grad_norm": 10.036735534667969, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.1789867877960205, "logits/rejected": -0.9200307130813599, "logps/chosen": -289.9539794921875, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -391.511474609375, "loss": 1.0912, "margin_dpo/margin_mean": 89.29136657714844, "margin_dpo/margin_std": 127.68019104003906, "step": 413 }, { "KL/chosen_KL_mean": -179.59365844726562, "KL/mean": -221.41680908203125, "KL/rejected_KL_mean": -263.2399597167969, "KL/std": 102.61119079589844, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0048412419855594635, "fcm_dpo/delta": -0.006429717876017094, "fcm_dpo/margin": 83.64633178710938, "fcm_dpo/q_t": 0.4077848196029663, "grad_norm": 12.581025123596191, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.4542040824890137, "logits/rejected": -1.0903745889663696, "logps/chosen": -267.8703308105469, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -356.1100158691406, "loss": 1.1157, "margin_dpo/margin_mean": 83.64633178710938, "margin_dpo/margin_std": 131.22421264648438, "step": 414 }, { "KL/chosen_KL_mean": -194.66937255859375, "KL/mean": -230.25827026367188, "KL/rejected_KL_mean": -265.84716796875, "KL/std": 93.3532485961914, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.005013193003833294, "fcm_dpo/delta": 0.05199574679136276, "fcm_dpo/margin": 71.17781829833984, "fcm_dpo/q_t": 0.4197695851325989, "grad_norm": 11.445807456970215, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.5714142322540283, "logits/rejected": -1.1897859573364258, "logps/chosen": -303.58526611328125, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -373.31854248046875, "loss": 1.1389, "margin_dpo/margin_mean": 71.17781066894531, "margin_dpo/margin_std": 114.18016052246094, "step": 415 }, { "KL/chosen_KL_mean": -194.76995849609375, "KL/mean": -243.41647338867188, "KL/rejected_KL_mean": -292.06298828125, "KL/std": 108.17674255371094, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.00489057507365942, "fcm_dpo/delta": -0.10240497440099716, "fcm_dpo/margin": 97.29300689697266, "fcm_dpo/q_t": 0.3867759704589844, "grad_norm": 9.775609016418457, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.6724520921707153, "logits/rejected": -1.1558361053466797, "logps/chosen": -288.1688537597656, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -389.7359619140625, "loss": 1.0477, "margin_dpo/margin_mean": 97.29301452636719, "margin_dpo/margin_std": 113.17356872558594, "step": 416 }, { "KL/chosen_KL_mean": -180.802001953125, "KL/mean": -225.87762451171875, "KL/rejected_KL_mean": -270.9532470703125, "KL/std": 100.23286437988281, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.004542744252830744, "fcm_dpo/delta": -0.01204155944287777, "fcm_dpo/margin": 90.15125274658203, "fcm_dpo/q_t": 0.40353959798812866, "grad_norm": 8.771530151367188, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.1298608779907227, "logits/rejected": -1.2028756141662598, "logps/chosen": -264.33734130859375, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -345.3951110839844, "loss": 1.0697, "margin_dpo/margin_mean": 90.15125274658203, "margin_dpo/margin_std": 102.09272766113281, "step": 417 }, { "KL/chosen_KL_mean": -172.66773986816406, "KL/mean": -229.79403686523438, "KL/rejected_KL_mean": -286.92034912109375, "KL/std": 107.87442016601562, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004312476143240929, "fcm_dpo/delta": -0.1262112259864807, "fcm_dpo/margin": 114.25260925292969, "fcm_dpo/q_t": 0.3795625567436218, "grad_norm": 10.387430191040039, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.306145429611206, "logits/rejected": -1.6656503677368164, "logps/chosen": -280.8892822265625, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -398.7850036621094, "loss": 1.0161, "margin_dpo/margin_mean": 114.25260925292969, "margin_dpo/margin_std": 106.28295135498047, "step": 418 }, { "KL/chosen_KL_mean": -180.50758361816406, "KL/mean": -230.32281494140625, "KL/rejected_KL_mean": -280.1380615234375, "KL/std": 95.87523651123047, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.004042868502438068, "fcm_dpo/delta": -0.00550253689289093, "fcm_dpo/margin": 99.63047790527344, "fcm_dpo/q_t": 0.40455931425094604, "grad_norm": 10.183452606201172, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.4140788316726685, "logits/rejected": -0.964925229549408, "logps/chosen": -277.69085693359375, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -378.3233642578125, "loss": 1.0689, "margin_dpo/margin_mean": 99.63047790527344, "margin_dpo/margin_std": 105.22881317138672, "step": 419 }, { "KL/chosen_KL_mean": -185.27598571777344, "KL/mean": -229.49407958984375, "KL/rejected_KL_mean": -273.712158203125, "KL/std": 101.43479919433594, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.004135258495807648, "fcm_dpo/delta": 0.0399765819311142, "fcm_dpo/margin": 88.4361801147461, "fcm_dpo/q_t": 0.41574931144714355, "grad_norm": 13.566866874694824, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.0671372413635254, "logits/rejected": -0.9336315393447876, "logps/chosen": -299.58447265625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -349.3957214355469, "loss": 1.1103, "margin_dpo/margin_mean": 88.43617248535156, "margin_dpo/margin_std": 117.27616119384766, "step": 420 }, { "KL/chosen_KL_mean": -176.27780151367188, "KL/mean": -225.52398681640625, "KL/rejected_KL_mean": -274.77020263671875, "KL/std": 104.53768920898438, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.004077053163200617, "fcm_dpo/delta": -0.002154218964278698, "fcm_dpo/margin": 98.49240112304688, "fcm_dpo/q_t": 0.4053928852081299, "grad_norm": 7.26201868057251, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.774345874786377, "logits/rejected": -1.3981082439422607, "logps/chosen": -262.15765380859375, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -371.106689453125, "loss": 1.0779, "margin_dpo/margin_mean": 98.49240112304688, "margin_dpo/margin_std": 115.37626647949219, "step": 421 }, { "KL/chosen_KL_mean": -174.26318359375, "KL/mean": -221.03480529785156, "KL/rejected_KL_mean": -267.8064270019531, "KL/std": 96.63790893554688, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004164643585681915, "fcm_dpo/delta": 0.012696895748376846, "fcm_dpo/margin": 93.54326629638672, "fcm_dpo/q_t": 0.4091499447822571, "grad_norm": 11.81709098815918, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.354222297668457, "logits/rejected": -1.9877787828445435, "logps/chosen": -274.7437744140625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -362.21466064453125, "loss": 1.0832, "margin_dpo/margin_mean": 93.54326629638672, "margin_dpo/margin_std": 109.40266418457031, "step": 422 }, { "KL/chosen_KL_mean": -161.64077758789062, "KL/mean": -209.74932861328125, "KL/rejected_KL_mean": -257.85791015625, "KL/std": 115.9759521484375, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.00421339925378561, "fcm_dpo/delta": -0.008434701710939407, "fcm_dpo/margin": 96.21711730957031, "fcm_dpo/q_t": 0.4066583216190338, "grad_norm": 9.362421989440918, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9903281927108765, "logits/rejected": -1.874161720275879, "logps/chosen": -240.09071350097656, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -349.9044189453125, "loss": 1.0952, "margin_dpo/margin_mean": 96.21711730957031, "margin_dpo/margin_std": 131.12876892089844, "step": 423 }, { "KL/chosen_KL_mean": -181.82192993164062, "KL/mean": -228.41619873046875, "KL/rejected_KL_mean": -275.01043701171875, "KL/std": 102.88063049316406, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0041581918485462666, "fcm_dpo/delta": 0.015522466041147709, "fcm_dpo/margin": 93.18852996826172, "fcm_dpo/q_t": 0.408974826335907, "grad_norm": 8.135769844055176, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.8989737033843994, "logits/rejected": -1.3189982175827026, "logps/chosen": -269.46429443359375, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -347.3760986328125, "loss": 1.0876, "margin_dpo/margin_mean": 93.18853759765625, "margin_dpo/margin_std": 111.62962341308594, "step": 424 }, { "KL/chosen_KL_mean": -189.65724182128906, "KL/mean": -221.40411376953125, "KL/rejected_KL_mean": -253.15098571777344, "KL/std": 106.3586196899414, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004575822502374649, "fcm_dpo/delta": 0.12859056890010834, "fcm_dpo/margin": 63.49373245239258, "fcm_dpo/q_t": 0.43871307373046875, "grad_norm": 10.344791412353516, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.7433452606201172, "logits/rejected": -1.680158019065857, "logps/chosen": -268.1770324707031, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -355.8996276855469, "loss": 1.2055, "margin_dpo/margin_mean": 63.49372863769531, "margin_dpo/margin_std": 133.62657165527344, "step": 425 }, { "KL/chosen_KL_mean": -161.49508666992188, "KL/mean": -208.01205444335938, "KL/rejected_KL_mean": -254.52903747558594, "KL/std": 98.043212890625, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.004668045789003372, "fcm_dpo/delta": -0.04395166411995888, "fcm_dpo/margin": 93.03394317626953, "fcm_dpo/q_t": 0.39758557081222534, "grad_norm": 11.590879440307617, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.6344958543777466, "logits/rejected": -0.8836744427680969, "logps/chosen": -270.00091552734375, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -342.8291015625, "loss": 1.0673, "margin_dpo/margin_mean": 93.03395080566406, "margin_dpo/margin_std": 114.79843139648438, "step": 426 }, { "KL/chosen_KL_mean": -133.31224060058594, "KL/mean": -176.33273315429688, "KL/rejected_KL_mean": -219.35321044921875, "KL/std": 93.21881103515625, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.004599003121256828, "fcm_dpo/delta": 0.005137770436704159, "fcm_dpo/margin": 86.04098510742188, "fcm_dpo/q_t": 0.40723955631256104, "grad_norm": 8.672361373901367, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.411729097366333, "logits/rejected": -1.7785676717758179, "logps/chosen": -232.43270874023438, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -305.07781982421875, "loss": 1.08, "margin_dpo/margin_mean": 86.04098510742188, "margin_dpo/margin_std": 100.27076721191406, "step": 427 }, { "KL/chosen_KL_mean": -189.97158813476562, "KL/mean": -226.59735107421875, "KL/rejected_KL_mean": -263.22314453125, "KL/std": 105.0072250366211, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.004701051861047745, "fcm_dpo/delta": 0.06650998443365097, "fcm_dpo/margin": 73.25154113769531, "fcm_dpo/q_t": 0.4227973222732544, "grad_norm": 10.52014446258545, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.574039101600647, "logits/rejected": -1.308484673500061, "logps/chosen": -295.94085693359375, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -372.3253173828125, "loss": 1.1476, "margin_dpo/margin_mean": 73.25153350830078, "margin_dpo/margin_std": 116.82107543945312, "step": 428 }, { "KL/chosen_KL_mean": -174.00875854492188, "KL/mean": -220.9925537109375, "KL/rejected_KL_mean": -267.97637939453125, "KL/std": 109.80433654785156, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.004827405791729689, "fcm_dpo/delta": -0.07309363782405853, "fcm_dpo/margin": 93.9676513671875, "fcm_dpo/q_t": 0.3944339454174042, "grad_norm": 8.785141944885254, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9824949502944946, "logits/rejected": -1.2712112665176392, "logps/chosen": -287.5536193847656, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -366.2183837890625, "loss": 1.0778, "margin_dpo/margin_mean": 93.96764373779297, "margin_dpo/margin_std": 125.07688903808594, "step": 429 }, { "KL/chosen_KL_mean": -165.680908203125, "KL/mean": -202.28976440429688, "KL/rejected_KL_mean": -238.89862060546875, "KL/std": 102.07193756103516, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0048201968893408775, "fcm_dpo/delta": 0.05452493578195572, "fcm_dpo/margin": 73.21769714355469, "fcm_dpo/q_t": 0.42087268829345703, "grad_norm": 9.603843688964844, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.3520021438598633, "logits/rejected": -1.317164421081543, "logps/chosen": -257.0002746582031, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -349.00823974609375, "loss": 1.1425, "margin_dpo/margin_mean": 73.21769714355469, "margin_dpo/margin_std": 119.04090881347656, "step": 430 }, { "KL/chosen_KL_mean": -167.1085662841797, "KL/mean": -210.1832275390625, "KL/rejected_KL_mean": -253.25787353515625, "KL/std": 104.34146118164062, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0047294339165091515, "fcm_dpo/delta": -0.00970328040421009, "fcm_dpo/margin": 86.14930725097656, "fcm_dpo/q_t": 0.40552496910095215, "grad_norm": 10.689348220825195, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8763704299926758, "logits/rejected": -1.4903032779693604, "logps/chosen": -260.289794921875, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -351.39013671875, "loss": 1.0954, "margin_dpo/margin_mean": 86.14930725097656, "margin_dpo/margin_std": 119.45526885986328, "step": 431 }, { "KL/chosen_KL_mean": -146.48623657226562, "KL/mean": -199.06298828125, "KL/rejected_KL_mean": -251.63973999023438, "KL/std": 108.39680480957031, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.004500912502408028, "fcm_dpo/delta": -0.09552829712629318, "fcm_dpo/margin": 105.15350341796875, "fcm_dpo/q_t": 0.385191947221756, "grad_norm": 9.114794731140137, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.4955390691757202, "logits/rejected": -0.9431396722793579, "logps/chosen": -250.92100524902344, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -356.72930908203125, "loss": 1.0233, "margin_dpo/margin_mean": 105.15351867675781, "margin_dpo/margin_std": 104.48580932617188, "step": 432 }, { "KL/chosen_KL_mean": -174.88900756835938, "KL/mean": -215.64187622070312, "KL/rejected_KL_mean": -256.394775390625, "KL/std": 102.32801055908203, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004537387751042843, "fcm_dpo/delta": 0.03354772925376892, "fcm_dpo/margin": 81.5057601928711, "fcm_dpo/q_t": 0.4151427149772644, "grad_norm": 8.943975448608398, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.5098967552185059, "logits/rejected": -1.184445858001709, "logps/chosen": -264.73223876953125, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -358.12823486328125, "loss": 1.1191, "margin_dpo/margin_mean": 81.5057601928711, "margin_dpo/margin_std": 116.47015380859375, "step": 433 }, { "KL/chosen_KL_mean": -150.51583862304688, "KL/mean": -197.19952392578125, "KL/rejected_KL_mean": -243.8831787109375, "KL/std": 113.97908020019531, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004431975074112415, "fcm_dpo/delta": -0.017410708591341972, "fcm_dpo/margin": 93.36735534667969, "fcm_dpo/q_t": 0.4067164361476898, "grad_norm": 11.946359634399414, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.4806783199310303, "logits/rejected": -1.8134434223175049, "logps/chosen": -248.20835876464844, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -327.9744873046875, "loss": 1.0925, "margin_dpo/margin_mean": 93.36734771728516, "margin_dpo/margin_std": 132.9771728515625, "step": 434 }, { "KL/chosen_KL_mean": -148.78521728515625, "KL/mean": -197.10113525390625, "KL/rejected_KL_mean": -245.41702270507812, "KL/std": 105.58357238769531, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.004340788349509239, "fcm_dpo/delta": -0.024639319628477097, "fcm_dpo/margin": 96.63182067871094, "fcm_dpo/q_t": 0.4023726284503937, "grad_norm": 9.715777397155762, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.306065559387207, "logits/rejected": -1.8516218662261963, "logps/chosen": -234.9571533203125, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -339.1687316894531, "loss": 1.0719, "margin_dpo/margin_mean": 96.63182067871094, "margin_dpo/margin_std": 118.41056823730469, "step": 435 }, { "KL/chosen_KL_mean": -124.25578308105469, "KL/mean": -174.0697021484375, "KL/rejected_KL_mean": -223.8836212158203, "KL/std": 89.47671508789062, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004190344829112291, "fcm_dpo/delta": -0.02238968014717102, "fcm_dpo/margin": 99.62784576416016, "fcm_dpo/q_t": 0.3997650742530823, "grad_norm": 9.754290580749512, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.8282690048217773, "logits/rejected": -0.9746068716049194, "logps/chosen": -230.04287719726562, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -312.5083312988281, "loss": 1.048, "margin_dpo/margin_mean": 99.62784576416016, "margin_dpo/margin_std": 92.94969177246094, "step": 436 }, { "KL/chosen_KL_mean": -145.0650177001953, "KL/mean": -185.87916564941406, "KL/rejected_KL_mean": -226.69329833984375, "KL/std": 95.86906433105469, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004383767489343882, "fcm_dpo/delta": 0.05058809742331505, "fcm_dpo/margin": 81.62826538085938, "fcm_dpo/q_t": 0.4161294102668762, "grad_norm": 10.377549171447754, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9345028400421143, "logits/rejected": -1.2734612226486206, "logps/chosen": -241.1270751953125, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -315.70550537109375, "loss": 1.1027, "margin_dpo/margin_mean": 81.62826538085938, "margin_dpo/margin_std": 95.74030303955078, "step": 437 }, { "KL/chosen_KL_mean": -148.28836059570312, "KL/mean": -189.84735107421875, "KL/rejected_KL_mean": -231.4063262939453, "KL/std": 96.94142150878906, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.004438676405698061, "fcm_dpo/delta": 0.03702712431550026, "fcm_dpo/margin": 83.1179428100586, "fcm_dpo/q_t": 0.41444602608680725, "grad_norm": 9.578204154968262, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.082916498184204, "logits/rejected": -1.264091968536377, "logps/chosen": -250.1437530517578, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -320.85400390625, "loss": 1.1055, "margin_dpo/margin_mean": 83.1179428100586, "margin_dpo/margin_std": 104.29715728759766, "step": 438 }, { "KL/chosen_KL_mean": -159.44351196289062, "KL/mean": -200.60992431640625, "KL/rejected_KL_mean": -241.77633666992188, "KL/std": 104.90416717529297, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004524773918092251, "fcm_dpo/delta": 0.02983338013291359, "fcm_dpo/margin": 82.33283996582031, "fcm_dpo/q_t": 0.4137023091316223, "grad_norm": 9.965533256530762, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.5409311056137085, "logits/rejected": -1.3040621280670166, "logps/chosen": -241.19912719726562, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -332.3627014160156, "loss": 1.107, "margin_dpo/margin_mean": 82.33283996582031, "margin_dpo/margin_std": 102.75505828857422, "step": 439 }, { "KL/chosen_KL_mean": -142.82180786132812, "KL/mean": -193.77041625976562, "KL/rejected_KL_mean": -244.7190399169922, "KL/std": 105.93240356445312, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0043622590601444244, "fcm_dpo/delta": -0.06370320916175842, "fcm_dpo/margin": 101.8972396850586, "fcm_dpo/q_t": 0.393854022026062, "grad_norm": 7.554465293884277, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.502855062484741, "logits/rejected": -2.0125017166137695, "logps/chosen": -248.46287536621094, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -348.12005615234375, "loss": 1.0543, "margin_dpo/margin_mean": 101.89723205566406, "margin_dpo/margin_std": 117.45439910888672, "step": 440 }, { "KL/chosen_KL_mean": -183.96746826171875, "KL/mean": -226.81033325195312, "KL/rejected_KL_mean": -269.6531677246094, "KL/std": 100.37661743164062, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0044663576409220695, "fcm_dpo/delta": 0.021376416087150574, "fcm_dpo/margin": 85.68572998046875, "fcm_dpo/q_t": 0.41281524300575256, "grad_norm": 8.818012237548828, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.8625195026397705, "logits/rejected": -1.510175347328186, "logps/chosen": -275.49700927734375, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -373.2725830078125, "loss": 1.0953, "margin_dpo/margin_mean": 85.68572235107422, "margin_dpo/margin_std": 110.04795837402344, "step": 441 }, { "KL/chosen_KL_mean": -178.11056518554688, "KL/mean": -233.2094268798828, "KL/rejected_KL_mean": -288.30828857421875, "KL/std": 106.51870727539062, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.004236767999827862, "fcm_dpo/delta": -0.08739327639341354, "fcm_dpo/margin": 110.19770050048828, "fcm_dpo/q_t": 0.3864463269710541, "grad_norm": 10.053476333618164, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.7396734952926636, "logits/rejected": -1.2747623920440674, "logps/chosen": -262.88812255859375, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -372.1324462890625, "loss": 1.0221, "margin_dpo/margin_mean": 110.19770812988281, "margin_dpo/margin_std": 107.79493713378906, "step": 442 }, { "KL/chosen_KL_mean": -174.3192901611328, "KL/mean": -208.06845092773438, "KL/rejected_KL_mean": -241.8175811767578, "KL/std": 103.80714416503906, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004457144066691399, "fcm_dpo/delta": 0.11748406291007996, "fcm_dpo/margin": 67.49827575683594, "fcm_dpo/q_t": 0.43304261565208435, "grad_norm": 10.211381912231445, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.7551958560943604, "logits/rejected": -1.1882660388946533, "logps/chosen": -276.96856689453125, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -334.85565185546875, "loss": 1.1613, "margin_dpo/margin_mean": 67.49828338623047, "margin_dpo/margin_std": 101.13074493408203, "step": 443 }, { "KL/chosen_KL_mean": -179.01644897460938, "KL/mean": -219.2322998046875, "KL/rejected_KL_mean": -259.4481506347656, "KL/std": 101.34861755371094, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0047531891614198685, "fcm_dpo/delta": 0.021632641553878784, "fcm_dpo/margin": 80.43168640136719, "fcm_dpo/q_t": 0.41293853521347046, "grad_norm": 11.190441131591797, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5717363357543945, "logits/rejected": -1.4038833379745483, "logps/chosen": -266.9361572265625, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -362.7716064453125, "loss": 1.1133, "margin_dpo/margin_mean": 80.43169403076172, "margin_dpo/margin_std": 117.4162826538086, "step": 444 }, { "KL/chosen_KL_mean": -157.08145141601562, "KL/mean": -203.96142578125, "KL/rejected_KL_mean": -250.84141540527344, "KL/std": 98.49146270751953, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.004538823384791613, "fcm_dpo/delta": -0.03530888259410858, "fcm_dpo/margin": 93.75994873046875, "fcm_dpo/q_t": 0.3985527455806732, "grad_norm": 8.740791320800781, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9047024250030518, "logits/rejected": -1.4872827529907227, "logps/chosen": -258.4823303222656, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -349.87933349609375, "loss": 1.0594, "margin_dpo/margin_mean": 93.75994873046875, "margin_dpo/margin_std": 104.12779235839844, "step": 445 }, { "KL/chosen_KL_mean": -150.4947052001953, "KL/mean": -192.52133178710938, "KL/rejected_KL_mean": -234.5479278564453, "KL/std": 103.90350341796875, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.004672561772167683, "fcm_dpo/delta": 0.008343033492565155, "fcm_dpo/margin": 84.05322265625, "fcm_dpo/q_t": 0.408476859331131, "grad_norm": 13.633797645568848, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9326446056365967, "logits/rejected": -1.7944533824920654, "logps/chosen": -237.91705322265625, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -341.2486877441406, "loss": 1.0835, "margin_dpo/margin_mean": 84.05322265625, "margin_dpo/margin_std": 99.71804809570312, "step": 446 }, { "KL/chosen_KL_mean": -148.468017578125, "KL/mean": -203.42764282226562, "KL/rejected_KL_mean": -258.3872985839844, "KL/std": 108.53922271728516, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004292653873562813, "fcm_dpo/delta": -0.09502758085727692, "fcm_dpo/margin": 109.91928100585938, "fcm_dpo/q_t": 0.38694506883621216, "grad_norm": 8.445388793945312, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.2395739555358887, "logits/rejected": -2.195986032485962, "logps/chosen": -232.08326721191406, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -363.2996826171875, "loss": 1.0376, "margin_dpo/margin_mean": 109.91927337646484, "margin_dpo/margin_std": 124.759033203125, "step": 447 }, { "KL/chosen_KL_mean": -160.82801818847656, "KL/mean": -214.08428955078125, "KL/rejected_KL_mean": -267.3405456542969, "KL/std": 113.25515747070312, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0041391910053789616, "fcm_dpo/delta": -0.05327794328331947, "fcm_dpo/margin": 106.51254272460938, "fcm_dpo/q_t": 0.3960916996002197, "grad_norm": 9.152627944946289, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.7358715534210205, "logits/rejected": -1.4677319526672363, "logps/chosen": -253.40380859375, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -375.03033447265625, "loss": 1.068, "margin_dpo/margin_mean": 106.51254272460938, "margin_dpo/margin_std": 132.10462951660156, "step": 448 }, { "KL/chosen_KL_mean": -158.02261352539062, "KL/mean": -212.85838317871094, "KL/rejected_KL_mean": -267.69415283203125, "KL/std": 105.88778686523438, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0038895336911082268, "fcm_dpo/delta": -0.03383919596672058, "fcm_dpo/margin": 109.67151641845703, "fcm_dpo/q_t": 0.3983582854270935, "grad_norm": 7.98623514175415, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.6605442762374878, "logits/rejected": -1.2283296585083008, "logps/chosen": -260.42156982421875, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -362.843017578125, "loss": 1.0571, "margin_dpo/margin_mean": 109.67152404785156, "margin_dpo/margin_std": 121.06404113769531, "step": 449 }, { "KL/chosen_KL_mean": -140.8233184814453, "KL/mean": -192.38009643554688, "KL/rejected_KL_mean": -243.93685913085938, "KL/std": 106.93052673339844, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0038526137359440327, "fcm_dpo/delta": 0.0033629857935011387, "fcm_dpo/margin": 103.11353302001953, "fcm_dpo/q_t": 0.4062155783176422, "grad_norm": 8.733736038208008, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3850913047790527, "logits/rejected": -2.1949970722198486, "logps/chosen": -227.81617736816406, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -352.4689025878906, "loss": 1.0764, "margin_dpo/margin_mean": 103.11353302001953, "margin_dpo/margin_std": 116.79083251953125, "step": 450 }, { "KL/chosen_KL_mean": -160.8050079345703, "KL/mean": -201.32327270507812, "KL/rejected_KL_mean": -241.841552734375, "KL/std": 106.51873779296875, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.004135802388191223, "fcm_dpo/delta": 0.07667016237974167, "fcm_dpo/margin": 81.03654479980469, "fcm_dpo/q_t": 0.4250057637691498, "grad_norm": 12.729406356811523, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.5209038257598877, "logits/rejected": -2.0272252559661865, "logps/chosen": -247.61630249023438, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -321.6971435546875, "loss": 1.1524, "margin_dpo/margin_mean": 81.03654479980469, "margin_dpo/margin_std": 134.23619079589844, "step": 451 }, { "KL/chosen_KL_mean": -140.3764190673828, "KL/mean": -185.95550537109375, "KL/rejected_KL_mean": -231.53460693359375, "KL/std": 101.13166809082031, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0041213491931557655, "fcm_dpo/delta": 0.025725658982992172, "fcm_dpo/margin": 91.15818786621094, "fcm_dpo/q_t": 0.4113754630088806, "grad_norm": 8.19970417022705, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7710109949111938, "logits/rejected": -1.7157502174377441, "logps/chosen": -219.7560577392578, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -335.25, "loss": 1.0884, "margin_dpo/margin_mean": 91.15818786621094, "margin_dpo/margin_std": 93.42237854003906, "step": 452 }, { "KL/chosen_KL_mean": -167.78671264648438, "KL/mean": -224.3004913330078, "KL/rejected_KL_mean": -280.81427001953125, "KL/std": 103.82917785644531, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.004067777656018734, "fcm_dpo/delta": -0.0774582177400589, "fcm_dpo/margin": 113.0275650024414, "fcm_dpo/q_t": 0.38939207792282104, "grad_norm": 10.339395523071289, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -2.0473880767822266, "logits/rejected": -1.584885597229004, "logps/chosen": -255.48190307617188, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -370.87249755859375, "loss": 1.0277, "margin_dpo/margin_mean": 113.02755737304688, "margin_dpo/margin_std": 110.46631622314453, "step": 453 }, { "KL/chosen_KL_mean": -177.093994140625, "KL/mean": -222.4930419921875, "KL/rejected_KL_mean": -267.89208984375, "KL/std": 106.5712661743164, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004148881882429123, "fcm_dpo/delta": 0.025061145424842834, "fcm_dpo/margin": 90.798095703125, "fcm_dpo/q_t": 0.4137468934059143, "grad_norm": 8.805424690246582, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.8701250553131104, "logits/rejected": -1.4103554487228394, "logps/chosen": -266.66021728515625, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -360.8131408691406, "loss": 1.1247, "margin_dpo/margin_mean": 90.79808807373047, "margin_dpo/margin_std": 137.8050079345703, "step": 454 }, { "KL/chosen_KL_mean": -166.859130859375, "KL/mean": -209.5973358154297, "KL/rejected_KL_mean": -252.33554077148438, "KL/std": 104.44961547851562, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.004269449971616268, "fcm_dpo/delta": 0.04066186398267746, "fcm_dpo/margin": 85.47642517089844, "fcm_dpo/q_t": 0.41606077551841736, "grad_norm": 9.27199935913086, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -2.0710363388061523, "logits/rejected": -2.0338056087493896, "logps/chosen": -244.48910522460938, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -371.3135070800781, "loss": 1.1181, "margin_dpo/margin_mean": 85.47642517089844, "margin_dpo/margin_std": 119.86483764648438, "step": 455 }, { "KL/chosen_KL_mean": -169.66302490234375, "KL/mean": -212.28829956054688, "KL/rejected_KL_mean": -254.91355895996094, "KL/std": 106.09620666503906, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0043830880895257, "fcm_dpo/delta": 0.03156711161136627, "fcm_dpo/margin": 85.25055694580078, "fcm_dpo/q_t": 0.41361093521118164, "grad_norm": 11.753689765930176, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.114284038543701, "logits/rejected": -1.9868475198745728, "logps/chosen": -259.2799072265625, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -364.4732666015625, "loss": 1.1007, "margin_dpo/margin_mean": 85.25054931640625, "margin_dpo/margin_std": 108.16165161132812, "step": 456 }, { "KL/chosen_KL_mean": -137.41519165039062, "KL/mean": -193.5865020751953, "KL/rejected_KL_mean": -249.7578125, "KL/std": 105.73872375488281, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.003981433808803558, "fcm_dpo/delta": -0.07225976884365082, "fcm_dpo/margin": 112.34262084960938, "fcm_dpo/q_t": 0.38890135288238525, "grad_norm": 9.90217113494873, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -1.9329041242599487, "logits/rejected": -1.6084647178649902, "logps/chosen": -224.73687744140625, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -340.5244140625, "loss": 1.0271, "margin_dpo/margin_mean": 112.34262084960938, "margin_dpo/margin_std": 94.18051147460938, "step": 457 }, { "KL/chosen_KL_mean": -159.91751098632812, "KL/mean": -201.1468505859375, "KL/rejected_KL_mean": -242.376220703125, "KL/std": 105.46524047851562, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004292857367545366, "fcm_dpo/delta": 0.05431023985147476, "fcm_dpo/margin": 82.45869445800781, "fcm_dpo/q_t": 0.42037999629974365, "grad_norm": 9.328666687011719, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.7003576755523682, "logits/rejected": -1.4668049812316895, "logps/chosen": -253.43795776367188, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -345.74517822265625, "loss": 1.1286, "margin_dpo/margin_mean": 82.45869445800781, "margin_dpo/margin_std": 123.0639877319336, "step": 458 }, { "KL/chosen_KL_mean": -152.59144592285156, "KL/mean": -204.4150848388672, "KL/rejected_KL_mean": -256.2387390136719, "KL/std": 110.7041015625, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004084193147718906, "fcm_dpo/delta": -0.03371460363268852, "fcm_dpo/margin": 103.64728546142578, "fcm_dpo/q_t": 0.39997541904449463, "grad_norm": 6.800149440765381, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.5392471551895142, "logits/rejected": -1.2865937948226929, "logps/chosen": -228.2796630859375, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -348.4092102050781, "loss": 1.0646, "margin_dpo/margin_mean": 103.64727783203125, "margin_dpo/margin_std": 118.68917846679688, "step": 459 }, { "KL/chosen_KL_mean": -155.90408325195312, "KL/mean": -203.7164764404297, "KL/rejected_KL_mean": -251.52890014648438, "KL/std": 107.17730712890625, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.004103401675820351, "fcm_dpo/delta": 0.008078858256340027, "fcm_dpo/margin": 95.62481689453125, "fcm_dpo/q_t": 0.40871763229370117, "grad_norm": 9.911456108093262, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0698177814483643, "logits/rejected": -2.0981292724609375, "logps/chosen": -237.02197265625, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -361.8412780761719, "loss": 1.0952, "margin_dpo/margin_mean": 95.62481689453125, "margin_dpo/margin_std": 123.4124755859375, "step": 460 }, { "KL/chosen_KL_mean": -192.91380310058594, "KL/mean": -231.76910400390625, "KL/rejected_KL_mean": -270.6243896484375, "KL/std": 94.09619140625, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0043737562373280525, "fcm_dpo/delta": 0.07270203530788422, "fcm_dpo/margin": 77.71058654785156, "fcm_dpo/q_t": 0.42356789112091064, "grad_norm": 8.996244430541992, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.3923935890197754, "logits/rejected": -1.1958699226379395, "logps/chosen": -274.497314453125, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -364.50152587890625, "loss": 1.1364, "margin_dpo/margin_mean": 77.71058654785156, "margin_dpo/margin_std": 113.68061828613281, "step": 461 }, { "KL/chosen_KL_mean": -147.03224182128906, "KL/mean": -196.3466033935547, "KL/rejected_KL_mean": -245.66094970703125, "KL/std": 101.13280487060547, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004395161755383015, "fcm_dpo/delta": -0.04266294091939926, "fcm_dpo/margin": 98.62871551513672, "fcm_dpo/q_t": 0.3973475992679596, "grad_norm": 8.887713432312012, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.3618640899658203, "logits/rejected": -1.8674726486206055, "logps/chosen": -241.23080444335938, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -331.2926025390625, "loss": 1.0536, "margin_dpo/margin_mean": 98.62872314453125, "margin_dpo/margin_std": 108.96040344238281, "step": 462 }, { "KL/chosen_KL_mean": -142.9713134765625, "KL/mean": -202.15509033203125, "KL/rejected_KL_mean": -261.3388671875, "KL/std": 108.89667510986328, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.004102802835404873, "fcm_dpo/delta": -0.11361393332481384, "fcm_dpo/margin": 118.36756134033203, "fcm_dpo/q_t": 0.38088417053222656, "grad_norm": 13.576370239257812, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.6695154905319214, "logits/rejected": -1.232237696647644, "logps/chosen": -228.89605712890625, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -358.24072265625, "loss": 1.024, "margin_dpo/margin_mean": 118.36756896972656, "margin_dpo/margin_std": 115.93345642089844, "step": 463 }, { "KL/chosen_KL_mean": -175.8203125, "KL/mean": -224.0398406982422, "KL/rejected_KL_mean": -272.2593688964844, "KL/std": 108.51643371582031, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0039004022255539894, "fcm_dpo/delta": 0.029271410778164864, "fcm_dpo/margin": 96.4390640258789, "fcm_dpo/q_t": 0.4129927158355713, "grad_norm": 11.510398864746094, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.839267611503601, "logits/rejected": -1.9201290607452393, "logps/chosen": -255.509521484375, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -379.55169677734375, "loss": 1.1006, "margin_dpo/margin_mean": 96.43907165527344, "margin_dpo/margin_std": 123.71908569335938, "step": 464 }, { "KL/chosen_KL_mean": -167.07476806640625, "KL/mean": -223.29644775390625, "KL/rejected_KL_mean": -279.51812744140625, "KL/std": 117.25180053710938, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0038283448666334152, "fcm_dpo/delta": -0.03935950621962547, "fcm_dpo/margin": 112.4433822631836, "fcm_dpo/q_t": 0.3965487778186798, "grad_norm": 8.137754440307617, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.5627102851867676, "logits/rejected": -1.2597496509552002, "logps/chosen": -258.93499755859375, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -398.2281494140625, "loss": 1.0445, "margin_dpo/margin_mean": 112.4433822631836, "margin_dpo/margin_std": 111.2726058959961, "step": 465 }, { "KL/chosen_KL_mean": -148.7098388671875, "KL/mean": -206.31321716308594, "KL/rejected_KL_mean": -263.9166259765625, "KL/std": 111.82376861572266, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.003700464963912964, "fcm_dpo/delta": -0.03380640968680382, "fcm_dpo/margin": 115.2067642211914, "fcm_dpo/q_t": 0.39800572395324707, "grad_norm": 6.42387580871582, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.6793692111968994, "logits/rejected": -1.3694219589233398, "logps/chosen": -233.41123962402344, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -359.96746826171875, "loss": 1.0497, "margin_dpo/margin_mean": 115.20675659179688, "margin_dpo/margin_std": 117.83970642089844, "step": 466 }, { "KL/chosen_KL_mean": -192.45555114746094, "KL/mean": -240.21981811523438, "KL/rejected_KL_mean": -287.9841003417969, "KL/std": 116.40983581542969, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0037497575394809246, "fcm_dpo/delta": 0.050113581120967865, "fcm_dpo/margin": 95.52854919433594, "fcm_dpo/q_t": 0.4176325500011444, "grad_norm": 11.864731788635254, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.5307505130767822, "logits/rejected": -1.9732885360717773, "logps/chosen": -301.7538757324219, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -396.8277587890625, "loss": 1.1226, "margin_dpo/margin_mean": 95.52854919433594, "margin_dpo/margin_std": 132.48223876953125, "step": 467 }, { "KL/chosen_KL_mean": -148.31137084960938, "KL/mean": -192.8290557861328, "KL/rejected_KL_mean": -237.34674072265625, "KL/std": 114.13443756103516, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.004017813131213188, "fcm_dpo/delta": 0.05155857652425766, "fcm_dpo/margin": 89.03536224365234, "fcm_dpo/q_t": 0.41969579458236694, "grad_norm": 8.982699394226074, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.930006742477417, "logits/rejected": -1.728805661201477, "logps/chosen": -246.6330108642578, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -344.0272216796875, "loss": 1.1268, "margin_dpo/margin_mean": 89.03535461425781, "margin_dpo/margin_std": 132.29644775390625, "step": 468 }, { "KL/chosen_KL_mean": -183.2869110107422, "KL/mean": -237.99691772460938, "KL/rejected_KL_mean": -292.7069396972656, "KL/std": 114.4707260131836, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.003870480228215456, "fcm_dpo/delta": -0.03449040651321411, "fcm_dpo/margin": 109.4200210571289, "fcm_dpo/q_t": 0.3991609513759613, "grad_norm": 10.035979270935059, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.27656626701355, "logits/rejected": -1.860854148864746, "logps/chosen": -280.0511169433594, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -402.30194091796875, "loss": 1.0636, "margin_dpo/margin_mean": 109.42002868652344, "margin_dpo/margin_std": 123.52103424072266, "step": 469 }, { "KL/chosen_KL_mean": -168.14144897460938, "KL/mean": -218.88327026367188, "KL/rejected_KL_mean": -269.62506103515625, "KL/std": 106.47032165527344, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.003936562221497297, "fcm_dpo/delta": 0.0006083110347390175, "fcm_dpo/margin": 101.48361206054688, "fcm_dpo/q_t": 0.40562596917152405, "grad_norm": 8.990194320678711, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.5900037288665771, "logits/rejected": -1.1659411191940308, "logps/chosen": -251.63812255859375, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -358.11083984375, "loss": 1.0684, "margin_dpo/margin_mean": 101.48361206054688, "margin_dpo/margin_std": 106.87571716308594, "step": 470 }, { "KL/chosen_KL_mean": -192.23399353027344, "KL/mean": -240.14114379882812, "KL/rejected_KL_mean": -288.0483093261719, "KL/std": 105.39311218261719, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004004901275038719, "fcm_dpo/delta": 0.02013130858540535, "fcm_dpo/margin": 95.81431579589844, "fcm_dpo/q_t": 0.4102010130882263, "grad_norm": 8.647626876831055, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6622886657714844, "logits/rejected": -1.2494151592254639, "logps/chosen": -280.70831298828125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -378.530029296875, "loss": 1.0821, "margin_dpo/margin_mean": 95.81430053710938, "margin_dpo/margin_std": 106.50276184082031, "step": 471 }, { "KL/chosen_KL_mean": -181.75262451171875, "KL/mean": -237.7877655029297, "KL/rejected_KL_mean": -293.8228759765625, "KL/std": 106.2997055053711, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.003849873784929514, "fcm_dpo/delta": -0.04171391949057579, "fcm_dpo/margin": 112.0702896118164, "fcm_dpo/q_t": 0.3972488045692444, "grad_norm": 7.814179420471191, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.1699891090393066, "logits/rejected": -1.4709644317626953, "logps/chosen": -306.986328125, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -414.87640380859375, "loss": 1.0465, "margin_dpo/margin_mean": 112.07028198242188, "margin_dpo/margin_std": 113.5272216796875, "step": 472 }, { "KL/chosen_KL_mean": -205.6837921142578, "KL/mean": -240.6844940185547, "KL/rejected_KL_mean": -275.6852111816406, "KL/std": 107.52933502197266, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0042533958330750465, "fcm_dpo/delta": 0.11854924261569977, "fcm_dpo/margin": 70.00141143798828, "fcm_dpo/q_t": 0.43486809730529785, "grad_norm": 10.238889694213867, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8478714227676392, "logits/rejected": -1.3584578037261963, "logps/chosen": -301.295166015625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -363.83636474609375, "loss": 1.1761, "margin_dpo/margin_mean": 70.00141143798828, "margin_dpo/margin_std": 124.87301635742188, "step": 473 }, { "KL/chosen_KL_mean": -190.67916870117188, "KL/mean": -227.19305419921875, "KL/rejected_KL_mean": -263.7069091796875, "KL/std": 101.93011474609375, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.00428810017183423, "fcm_dpo/delta": -0.0278239194303751, "fcm_dpo/margin": 73.02775573730469, "fcm_dpo/q_t": 0.42679375410079956, "grad_norm": 13.617505073547363, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.4910889863967896, "logits/rejected": -1.3012441396713257, "logps/chosen": -272.158935546875, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -360.17254638671875, "loss": 1.1729, "margin_dpo/margin_mean": 73.02775573730469, "margin_dpo/margin_std": 124.94854736328125, "step": 474 }, { "KL/chosen_KL_mean": -146.19815063476562, "KL/mean": -202.0678253173828, "KL/rejected_KL_mean": -257.9375, "KL/std": 109.16178894042969, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.004035063553601503, "fcm_dpo/delta": -0.06566924601793289, "fcm_dpo/margin": 111.73933410644531, "fcm_dpo/q_t": 0.3929583430290222, "grad_norm": 8.025890350341797, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.192681312561035, "logits/rejected": -2.14217209815979, "logps/chosen": -220.39413452148438, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -352.6299133300781, "loss": 1.0455, "margin_dpo/margin_mean": 111.73933410644531, "margin_dpo/margin_std": 125.58942413330078, "step": 475 }, { "KL/chosen_KL_mean": -170.07992553710938, "KL/mean": -219.94097900390625, "KL/rejected_KL_mean": -269.80206298828125, "KL/std": 111.20637512207031, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.003931241109967232, "fcm_dpo/delta": 0.009162843227386475, "fcm_dpo/margin": 99.72210693359375, "fcm_dpo/q_t": 0.409246027469635, "grad_norm": 9.20248794555664, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.034815549850464, "logits/rejected": -1.6624908447265625, "logps/chosen": -242.051025390625, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -350.06427001953125, "loss": 1.0861, "margin_dpo/margin_mean": 99.72210693359375, "margin_dpo/margin_std": 120.7479019165039, "step": 476 }, { "KL/chosen_KL_mean": -170.6642608642578, "KL/mean": -219.85513305664062, "KL/rejected_KL_mean": -269.0460205078125, "KL/std": 97.8788833618164, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004027211107313633, "fcm_dpo/delta": 0.004626287147402763, "fcm_dpo/margin": 98.38176727294922, "fcm_dpo/q_t": 0.40701961517333984, "grad_norm": 8.611215591430664, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.107069253921509, "logits/rejected": -1.173953652381897, "logps/chosen": -275.6697998046875, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -350.9244384765625, "loss": 1.0864, "margin_dpo/margin_mean": 98.38177490234375, "margin_dpo/margin_std": 122.05604553222656, "step": 477 }, { "KL/chosen_KL_mean": -164.94326782226562, "KL/mean": -216.14974975585938, "KL/rejected_KL_mean": -267.3562927246094, "KL/std": 105.94015502929688, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.003979393281042576, "fcm_dpo/delta": -0.009507795795798302, "fcm_dpo/margin": 102.41300964355469, "fcm_dpo/q_t": 0.4039463400840759, "grad_norm": 8.755995750427246, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.4797875881195068, "logits/rejected": -1.300398349761963, "logps/chosen": -241.73146057128906, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -357.79620361328125, "loss": 1.0717, "margin_dpo/margin_mean": 102.41300964355469, "margin_dpo/margin_std": 117.43950653076172, "step": 478 }, { "KL/chosen_KL_mean": -167.33551025390625, "KL/mean": -218.30801391601562, "KL/rejected_KL_mean": -269.2804870605469, "KL/std": 107.13488006591797, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004018645733594894, "fcm_dpo/delta": -0.013723667711019516, "fcm_dpo/margin": 101.94500732421875, "fcm_dpo/q_t": 0.4025961756706238, "grad_norm": 12.601256370544434, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.2994952201843262, "logits/rejected": -1.1555547714233398, "logps/chosen": -248.69781494140625, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -370.37164306640625, "loss": 1.0679, "margin_dpo/margin_mean": 101.94500732421875, "margin_dpo/margin_std": 111.07771301269531, "step": 479 }, { "KL/chosen_KL_mean": -195.28369140625, "KL/mean": -235.60623168945312, "KL/rejected_KL_mean": -275.9287414550781, "KL/std": 102.25660705566406, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004166172817349434, "fcm_dpo/delta": 0.0765077993273735, "fcm_dpo/margin": 80.64505767822266, "fcm_dpo/q_t": 0.4236886501312256, "grad_norm": 9.742987632751465, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.8034005165100098, "logits/rejected": -1.1288670301437378, "logps/chosen": -292.0576171875, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -362.33349609375, "loss": 1.1312, "margin_dpo/margin_mean": 80.64505004882812, "margin_dpo/margin_std": 112.79861450195312, "step": 480 }, { "KL/chosen_KL_mean": -189.96116638183594, "KL/mean": -237.68142700195312, "KL/rejected_KL_mean": -285.40167236328125, "KL/std": 106.7354965209961, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004211842082440853, "fcm_dpo/delta": -0.0025425544008612633, "fcm_dpo/margin": 95.44052124023438, "fcm_dpo/q_t": 0.40590736269950867, "grad_norm": 12.023763656616211, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.8979078531265259, "logits/rejected": -1.4595882892608643, "logps/chosen": -281.63134765625, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -384.0965881347656, "loss": 1.0829, "margin_dpo/margin_mean": 95.44052124023438, "margin_dpo/margin_std": 118.40924835205078, "step": 481 }, { "KL/chosen_KL_mean": -184.5528106689453, "KL/mean": -230.2271728515625, "KL/rejected_KL_mean": -275.9014892578125, "KL/std": 104.91099548339844, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004219825845211744, "fcm_dpo/delta": 0.01739475503563881, "fcm_dpo/margin": 91.34870147705078, "fcm_dpo/q_t": 0.4098677635192871, "grad_norm": 12.734659194946289, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.5278016328811646, "logits/rejected": -1.1686338186264038, "logps/chosen": -283.07293701171875, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -370.73095703125, "loss": 1.0975, "margin_dpo/margin_mean": 91.34870910644531, "margin_dpo/margin_std": 117.80315399169922, "step": 482 }, { "KL/chosen_KL_mean": -189.9127960205078, "KL/mean": -238.87062072753906, "KL/rejected_KL_mean": -287.82843017578125, "KL/std": 110.84757232666016, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004330779891461134, "fcm_dpo/delta": -0.03388409689068794, "fcm_dpo/margin": 97.9156494140625, "fcm_dpo/q_t": 0.39884495735168457, "grad_norm": 7.993433952331543, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.046731472015381, "logits/rejected": -1.4056227207183838, "logps/chosen": -297.0314025878906, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -388.9434509277344, "loss": 1.0665, "margin_dpo/margin_mean": 97.91565704345703, "margin_dpo/margin_std": 110.37515258789062, "step": 483 }, { "KL/chosen_KL_mean": -172.9492950439453, "KL/mean": -227.92584228515625, "KL/rejected_KL_mean": -282.90240478515625, "KL/std": 112.63650512695312, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004043051972985268, "fcm_dpo/delta": -0.05733608454465866, "fcm_dpo/margin": 109.9531021118164, "fcm_dpo/q_t": 0.3935399651527405, "grad_norm": 8.183160781860352, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.304008960723877, "logits/rejected": -1.8178056478500366, "logps/chosen": -259.92919921875, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -373.6260681152344, "loss": 1.0504, "margin_dpo/margin_mean": 109.95309448242188, "margin_dpo/margin_std": 122.92593383789062, "step": 484 }, { "KL/chosen_KL_mean": -164.94850158691406, "KL/mean": -224.50274658203125, "KL/rejected_KL_mean": -284.05694580078125, "KL/std": 117.53773498535156, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0037529785186052322, "fcm_dpo/delta": -0.06060013175010681, "fcm_dpo/margin": 119.10845947265625, "fcm_dpo/q_t": 0.39250531792640686, "grad_norm": 8.244450569152832, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.5987017154693604, "logits/rejected": -1.1734967231750488, "logps/chosen": -259.00726318359375, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -392.61993408203125, "loss": 1.0361, "margin_dpo/margin_mean": 119.10845947265625, "margin_dpo/margin_std": 119.64225769042969, "step": 485 }, { "KL/chosen_KL_mean": -189.08572387695312, "KL/mean": -241.26870727539062, "KL/rejected_KL_mean": -293.45166015625, "KL/std": 113.38517761230469, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0037648894358426332, "fcm_dpo/delta": 0.00785091146826744, "fcm_dpo/margin": 104.365966796875, "fcm_dpo/q_t": 0.40754181146621704, "grad_norm": 9.809331893920898, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.9336645603179932, "logits/rejected": -1.2181344032287598, "logps/chosen": -282.83160400390625, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -391.5223388671875, "loss": 1.0821, "margin_dpo/margin_mean": 104.365966796875, "margin_dpo/margin_std": 120.91163635253906, "step": 486 }, { "KL/chosen_KL_mean": -182.53012084960938, "KL/mean": -229.84848022460938, "KL/rejected_KL_mean": -277.1668701171875, "KL/std": 107.83840942382812, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0038454546593129635, "fcm_dpo/delta": 0.044067151844501495, "fcm_dpo/margin": 94.6367416381836, "fcm_dpo/q_t": 0.4157490134239197, "grad_norm": 9.017106056213379, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.02896785736084, "logits/rejected": -1.712674617767334, "logps/chosen": -270.5636901855469, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -375.6389465332031, "loss": 1.1019, "margin_dpo/margin_mean": 94.63673400878906, "margin_dpo/margin_std": 115.19402313232422, "step": 487 }, { "KL/chosen_KL_mean": -165.5919189453125, "KL/mean": -218.185302734375, "KL/rejected_KL_mean": -270.7786865234375, "KL/std": 113.87356567382812, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0038835126906633377, "fcm_dpo/delta": -0.01078521553426981, "fcm_dpo/margin": 105.1867446899414, "fcm_dpo/q_t": 0.4024002254009247, "grad_norm": 8.694355010986328, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.679408311843872, "logits/rejected": -1.4196105003356934, "logps/chosen": -250.38156127929688, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -361.0521240234375, "loss": 1.0579, "margin_dpo/margin_mean": 105.18673706054688, "margin_dpo/margin_std": 101.1312484741211, "step": 488 }, { "KL/chosen_KL_mean": -186.95716857910156, "KL/mean": -234.62844848632812, "KL/rejected_KL_mean": -282.29974365234375, "KL/std": 104.37632751464844, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.003943399526178837, "fcm_dpo/delta": 0.029441485181450844, "fcm_dpo/margin": 95.34255981445312, "fcm_dpo/q_t": 0.41292768716812134, "grad_norm": 9.580897331237793, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6564154624938965, "logits/rejected": -1.1676360368728638, "logps/chosen": -277.42645263671875, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -368.69732666015625, "loss": 1.095, "margin_dpo/margin_mean": 95.34255981445312, "margin_dpo/margin_std": 116.25650024414062, "step": 489 }, { "KL/chosen_KL_mean": -186.16368103027344, "KL/mean": -231.34458923339844, "KL/rejected_KL_mean": -276.52545166015625, "KL/std": 113.23566436767578, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004178388975560665, "fcm_dpo/delta": 0.0205252543091774, "fcm_dpo/margin": 90.36177825927734, "fcm_dpo/q_t": 0.4143442213535309, "grad_norm": 8.416526794433594, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1206247806549072, "logits/rejected": -2.2018582820892334, "logps/chosen": -271.48382568359375, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -392.5193176269531, "loss": 1.1427, "margin_dpo/margin_mean": 90.36177062988281, "margin_dpo/margin_std": 149.91238403320312, "step": 490 }, { "KL/chosen_KL_mean": -190.38973999023438, "KL/mean": -232.26031494140625, "KL/rejected_KL_mean": -274.130859375, "KL/std": 115.44818878173828, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004213862121105194, "fcm_dpo/delta": 0.057258449494838715, "fcm_dpo/margin": 83.74113464355469, "fcm_dpo/q_t": 0.4193074703216553, "grad_norm": 10.901538848876953, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8994166851043701, "logits/rejected": -1.4121054410934448, "logps/chosen": -276.53326416015625, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -354.810302734375, "loss": 1.1231, "margin_dpo/margin_mean": 83.74112701416016, "margin_dpo/margin_std": 117.56082153320312, "step": 491 }, { "KL/chosen_KL_mean": -179.54315185546875, "KL/mean": -212.6359405517578, "KL/rejected_KL_mean": -245.7287139892578, "KL/std": 113.1041259765625, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.004631312098354101, "fcm_dpo/delta": 0.11068399250507355, "fcm_dpo/margin": 66.185546875, "fcm_dpo/q_t": 0.43368858098983765, "grad_norm": 14.826434135437012, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.1616997718811035, "logits/rejected": -1.785988211631775, "logps/chosen": -260.6507568359375, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -326.480712890625, "loss": 1.204, "margin_dpo/margin_mean": 66.185546875, "margin_dpo/margin_std": 140.91004943847656, "step": 492 }, { "KL/chosen_KL_mean": -175.9051513671875, "KL/mean": -220.63882446289062, "KL/rejected_KL_mean": -265.37249755859375, "KL/std": 102.3276596069336, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0045919520780444145, "fcm_dpo/delta": -0.017785504460334778, "fcm_dpo/margin": 89.46736145019531, "fcm_dpo/q_t": 0.40286892652511597, "grad_norm": 17.409543991088867, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.472926139831543, "logits/rejected": -1.750483751296997, "logps/chosen": -288.11248779296875, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -358.9797058105469, "loss": 1.0799, "margin_dpo/margin_mean": 89.46736145019531, "margin_dpo/margin_std": 108.93269348144531, "step": 493 }, { "KL/chosen_KL_mean": -170.9441375732422, "KL/mean": -229.26998901367188, "KL/rejected_KL_mean": -287.5958557128906, "KL/std": 113.10713195800781, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004140090197324753, "fcm_dpo/delta": -0.12149029970169067, "fcm_dpo/margin": 116.65172576904297, "fcm_dpo/q_t": 0.3811013996601105, "grad_norm": 10.217585563659668, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.452221393585205, "logits/rejected": -1.3781497478485107, "logps/chosen": -244.05902099609375, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -379.75885009765625, "loss": 1.0235, "margin_dpo/margin_mean": 116.65172576904297, "margin_dpo/margin_std": 122.67697143554688, "step": 494 }, { "KL/chosen_KL_mean": -176.866455078125, "KL/mean": -227.14773559570312, "KL/rejected_KL_mean": -277.4289855957031, "KL/std": 107.31944274902344, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.00410061702132225, "fcm_dpo/delta": -0.015694664791226387, "fcm_dpo/margin": 100.56253051757812, "fcm_dpo/q_t": 0.4038406014442444, "grad_norm": 10.827166557312012, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.377007007598877, "logits/rejected": -1.923917531967163, "logps/chosen": -276.38677978515625, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -375.35986328125, "loss": 1.0814, "margin_dpo/margin_mean": 100.56253051757812, "margin_dpo/margin_std": 129.40432739257812, "step": 495 }, { "KL/chosen_KL_mean": -196.20697021484375, "KL/mean": -239.31588745117188, "KL/rejected_KL_mean": -282.4248046875, "KL/std": 107.97278594970703, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004274714272469282, "fcm_dpo/delta": 0.036878522485494614, "fcm_dpo/margin": 86.21782684326172, "fcm_dpo/q_t": 0.4142175614833832, "grad_norm": 9.565193176269531, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.1205685138702393, "logits/rejected": -1.4695580005645752, "logps/chosen": -304.063720703125, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -375.19537353515625, "loss": 1.1056, "margin_dpo/margin_mean": 86.21783447265625, "margin_dpo/margin_std": 109.3906021118164, "step": 496 }, { "KL/chosen_KL_mean": -208.99444580078125, "KL/mean": -237.22207641601562, "KL/rejected_KL_mean": -265.44970703125, "KL/std": 109.10059356689453, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004481859505176544, "fcm_dpo/delta": 0.052824873477220535, "fcm_dpo/margin": 56.45524978637695, "fcm_dpo/q_t": 0.443095862865448, "grad_norm": 9.307777404785156, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.8311712741851807, "logits/rejected": -1.3886873722076416, "logps/chosen": -301.0776672363281, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -347.2447509765625, "loss": 1.217, "margin_dpo/margin_mean": 56.45524978637695, "margin_dpo/margin_std": 117.29248046875, "step": 497 }, { "KL/chosen_KL_mean": -177.46533203125, "KL/mean": -223.8543701171875, "KL/rejected_KL_mean": -270.243408203125, "KL/std": 107.55472564697266, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.004422992467880249, "fcm_dpo/delta": -0.013142718002200127, "fcm_dpo/margin": 92.77811431884766, "fcm_dpo/q_t": 0.4036319851875305, "grad_norm": 12.881239891052246, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.8399391174316406, "logits/rejected": -1.3449983596801758, "logps/chosen": -275.6596984863281, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -360.9308776855469, "loss": 1.0825, "margin_dpo/margin_mean": 92.77811431884766, "margin_dpo/margin_std": 118.76289367675781, "step": 498 }, { "KL/chosen_KL_mean": -186.07382202148438, "KL/mean": -235.26194763183594, "KL/rejected_KL_mean": -284.4500732421875, "KL/std": 116.11099243164062, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0043342518620193005, "fcm_dpo/delta": -0.0334751158952713, "fcm_dpo/margin": 98.376220703125, "fcm_dpo/q_t": 0.4001666009426117, "grad_norm": 9.928204536437988, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.973004937171936, "logits/rejected": -1.9231486320495605, "logps/chosen": -267.1137390136719, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -391.37176513671875, "loss": 1.0721, "margin_dpo/margin_mean": 98.37621307373047, "margin_dpo/margin_std": 123.84742736816406, "step": 499 }, { "KL/chosen_KL_mean": -207.67031860351562, "KL/mean": -249.05398559570312, "KL/rejected_KL_mean": -290.4376525878906, "KL/std": 111.97634887695312, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004392310976982117, "fcm_dpo/delta": 0.04466567933559418, "fcm_dpo/margin": 82.76734924316406, "fcm_dpo/q_t": 0.4187754690647125, "grad_norm": 12.405267715454102, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.7243797779083252, "logits/rejected": -1.5557466745376587, "logps/chosen": -296.9183654785156, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -390.8478698730469, "loss": 1.1289, "margin_dpo/margin_mean": 82.76734924316406, "margin_dpo/margin_std": 127.0727310180664, "step": 500 }, { "KL/chosen_KL_mean": -180.59396362304688, "KL/mean": -227.3509063720703, "KL/rejected_KL_mean": -274.10784912109375, "KL/std": 121.18148040771484, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004541951231658459, "fcm_dpo/delta": -0.03745654225349426, "fcm_dpo/margin": 93.51385498046875, "fcm_dpo/q_t": 0.40188831090927124, "grad_norm": 10.185674667358398, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.570968747138977, "logits/rejected": -1.1326274871826172, "logps/chosen": -271.19091796875, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -359.40509033203125, "loss": 1.1055, "margin_dpo/margin_mean": 93.51385498046875, "margin_dpo/margin_std": 136.21078491210938, "step": 501 }, { "KL/chosen_KL_mean": -180.70953369140625, "KL/mean": -222.7574920654297, "KL/rejected_KL_mean": -264.805419921875, "KL/std": 101.07907104492188, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.004488520324230194, "fcm_dpo/delta": 0.02489810809493065, "fcm_dpo/margin": 84.09590911865234, "fcm_dpo/q_t": 0.41277021169662476, "grad_norm": 11.042770385742188, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.8486478328704834, "logits/rejected": -1.1064367294311523, "logps/chosen": -284.1802673339844, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -353.849609375, "loss": 1.1208, "margin_dpo/margin_mean": 84.09590911865234, "margin_dpo/margin_std": 124.51156616210938, "step": 502 }, { "KL/chosen_KL_mean": -179.888916015625, "KL/mean": -226.82615661621094, "KL/rejected_KL_mean": -273.7633972167969, "KL/std": 107.64358520507812, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004446156322956085, "fcm_dpo/delta": -0.02385556697845459, "fcm_dpo/margin": 93.87449645996094, "fcm_dpo/q_t": 0.40214404463768005, "grad_norm": 9.931525230407715, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.6443754434585571, "logits/rejected": -1.1365485191345215, "logps/chosen": -273.618896484375, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -361.37237548828125, "loss": 1.0811, "margin_dpo/margin_mean": 93.87449645996094, "margin_dpo/margin_std": 119.03880310058594, "step": 503 }, { "KL/chosen_KL_mean": -184.0254669189453, "KL/mean": -243.61758422851562, "KL/rejected_KL_mean": -303.2096862792969, "KL/std": 112.42001342773438, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.004008556716144085, "fcm_dpo/delta": -0.10220173746347427, "fcm_dpo/margin": 119.18424987792969, "fcm_dpo/q_t": 0.382771760225296, "grad_norm": 9.97011661529541, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.072629928588867, "logits/rejected": -1.7406232357025146, "logps/chosen": -276.31732177734375, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -412.2473449707031, "loss": 1.0136, "margin_dpo/margin_mean": 119.18424987792969, "margin_dpo/margin_std": 113.39259338378906, "step": 504 }, { "KL/chosen_KL_mean": -201.81146240234375, "KL/mean": -245.33914184570312, "KL/rejected_KL_mean": -288.8667907714844, "KL/std": 111.70893859863281, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.004056841135025024, "fcm_dpo/delta": 0.05689946189522743, "fcm_dpo/margin": 87.05530548095703, "fcm_dpo/q_t": 0.41872653365135193, "grad_norm": 8.614485740661621, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.322345733642578, "logits/rejected": -1.7596805095672607, "logps/chosen": -308.0884094238281, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -383.0729064941406, "loss": 1.1106, "margin_dpo/margin_mean": 87.05530548095703, "margin_dpo/margin_std": 107.56904602050781, "step": 505 }, { "KL/chosen_KL_mean": -195.37486267089844, "KL/mean": -244.34283447265625, "KL/rejected_KL_mean": -293.31085205078125, "KL/std": 105.14076232910156, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0040519945323467255, "fcm_dpo/delta": 0.0023591406643390656, "fcm_dpo/margin": 97.93598937988281, "fcm_dpo/q_t": 0.40552711486816406, "grad_norm": 8.938800811767578, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.918330430984497, "logits/rejected": -1.5553300380706787, "logps/chosen": -276.81573486328125, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -378.9752502441406, "loss": 1.0785, "margin_dpo/margin_mean": 97.93598937988281, "margin_dpo/margin_std": 110.76937866210938, "step": 506 }, { "KL/chosen_KL_mean": -202.72576904296875, "KL/mean": -252.12179565429688, "KL/rejected_KL_mean": -301.517822265625, "KL/std": 110.51676940917969, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004109387286007404, "fcm_dpo/delta": -0.007496925070881844, "fcm_dpo/margin": 98.79206848144531, "fcm_dpo/q_t": 0.4058946371078491, "grad_norm": 9.026779174804688, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.1303586959838867, "logits/rejected": -1.6862437725067139, "logps/chosen": -302.1527099609375, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -403.77081298828125, "loss": 1.0905, "margin_dpo/margin_mean": 98.79206848144531, "margin_dpo/margin_std": 130.46243286132812, "step": 507 }, { "KL/chosen_KL_mean": -196.7232666015625, "KL/mean": -238.46630859375, "KL/rejected_KL_mean": -280.2093505859375, "KL/std": 118.38835144042969, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.004275918938219547, "fcm_dpo/delta": 0.05209429934620857, "fcm_dpo/margin": 83.48609924316406, "fcm_dpo/q_t": 0.42019835114479065, "grad_norm": 16.01543426513672, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.0881407260894775, "logits/rejected": -1.6093202829360962, "logps/chosen": -294.638916015625, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -384.0072021484375, "loss": 1.1459, "margin_dpo/margin_mean": 83.48609924316406, "margin_dpo/margin_std": 138.5521697998047, "step": 508 }, { "KL/chosen_KL_mean": -184.04473876953125, "KL/mean": -222.1663818359375, "KL/rejected_KL_mean": -260.2880554199219, "KL/std": 92.48343658447266, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004566840827465057, "fcm_dpo/delta": 0.06155051663517952, "fcm_dpo/margin": 76.2433090209961, "fcm_dpo/q_t": 0.41984397172927856, "grad_norm": 15.245441436767578, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.8401517868041992, "logits/rejected": -1.1575584411621094, "logps/chosen": -284.77459716796875, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -347.23138427734375, "loss": 1.144, "margin_dpo/margin_mean": 76.2433090209961, "margin_dpo/margin_std": 122.85763549804688, "step": 509 }, { "KL/chosen_KL_mean": -215.96881103515625, "KL/mean": -263.01763916015625, "KL/rejected_KL_mean": -310.06646728515625, "KL/std": 111.68507385253906, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004521906841546297, "fcm_dpo/delta": -0.03239549323916435, "fcm_dpo/margin": 94.09764862060547, "fcm_dpo/q_t": 0.39980751276016235, "grad_norm": 12.527766227722168, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.5272164344787598, "logits/rejected": -1.3418081998825073, "logps/chosen": -305.13818359375, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -417.98583984375, "loss": 1.0691, "margin_dpo/margin_mean": 94.09765625, "margin_dpo/margin_std": 112.1168212890625, "step": 510 }, { "KL/chosen_KL_mean": -198.54161071777344, "KL/mean": -248.48558044433594, "KL/rejected_KL_mean": -298.4295654296875, "KL/std": 110.00723266601562, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004351949319243431, "fcm_dpo/delta": -0.04434714838862419, "fcm_dpo/margin": 99.88795471191406, "fcm_dpo/q_t": 0.3984013795852661, "grad_norm": 11.868995666503906, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.009999990463257, "logits/rejected": -1.4312794208526611, "logps/chosen": -288.2913818359375, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -396.16082763671875, "loss": 1.066, "margin_dpo/margin_mean": 99.88795471191406, "margin_dpo/margin_std": 123.59225463867188, "step": 511 }, { "KL/chosen_KL_mean": -188.88726806640625, "KL/mean": -237.19717407226562, "KL/rejected_KL_mean": -285.5071105957031, "KL/std": 107.33685302734375, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004302031360566616, "fcm_dpo/delta": -0.022358953952789307, "fcm_dpo/margin": 96.61984252929688, "fcm_dpo/q_t": 0.4023144841194153, "grad_norm": 13.822588920593262, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.6322684288024902, "logits/rejected": -1.1767578125, "logps/chosen": -274.6224060058594, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -373.1319885253906, "loss": 1.0895, "margin_dpo/margin_mean": 96.6198501586914, "margin_dpo/margin_std": 128.04885864257812, "step": 512 }, { "KL/chosen_KL_mean": -199.22348022460938, "KL/mean": -239.91012573242188, "KL/rejected_KL_mean": -280.5968017578125, "KL/std": 102.94220733642578, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004272971302270889, "fcm_dpo/delta": 0.0628068819642067, "fcm_dpo/margin": 81.37329864501953, "fcm_dpo/q_t": 0.419828861951828, "grad_norm": 11.659812927246094, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.2843985557556152, "logits/rejected": -1.583176612854004, "logps/chosen": -295.41400146484375, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -363.5145263671875, "loss": 1.1335, "margin_dpo/margin_mean": 81.37329864501953, "margin_dpo/margin_std": 116.58929443359375, "step": 513 }, { "KL/chosen_KL_mean": -193.14402770996094, "KL/mean": -237.41647338867188, "KL/rejected_KL_mean": -281.68896484375, "KL/std": 103.45222473144531, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004460887052118778, "fcm_dpo/delta": 0.006206745281815529, "fcm_dpo/margin": 88.54492950439453, "fcm_dpo/q_t": 0.40973925590515137, "grad_norm": 11.765347480773926, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.7550771236419678, "logits/rejected": -1.1885619163513184, "logps/chosen": -290.9676513671875, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -367.05877685546875, "loss": 1.1001, "margin_dpo/margin_mean": 88.544921875, "margin_dpo/margin_std": 122.73306274414062, "step": 514 }, { "KL/chosen_KL_mean": -185.42849731445312, "KL/mean": -229.0247802734375, "KL/rejected_KL_mean": -272.62109375, "KL/std": 107.41365051269531, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0044919708743691444, "fcm_dpo/delta": 0.010362658649682999, "fcm_dpo/margin": 87.1926040649414, "fcm_dpo/q_t": 0.4094597399234772, "grad_norm": 11.549036026000977, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.6700594425201416, "logits/rejected": -1.4352447986602783, "logps/chosen": -267.362060546875, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -366.1976623535156, "loss": 1.1003, "margin_dpo/margin_mean": 87.1926040649414, "margin_dpo/margin_std": 117.79281616210938, "step": 515 }, { "KL/chosen_KL_mean": -190.6689453125, "KL/mean": -232.67498779296875, "KL/rejected_KL_mean": -274.6810302734375, "KL/std": 123.53309631347656, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.004600757732987404, "fcm_dpo/delta": 0.016165729612112045, "fcm_dpo/margin": 84.01205444335938, "fcm_dpo/q_t": 0.41429927945137024, "grad_norm": 14.084328651428223, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.2635107040405273, "logits/rejected": -2.092216730117798, "logps/chosen": -287.7606201171875, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -393.2376708984375, "loss": 1.1419, "margin_dpo/margin_mean": 84.01205444335938, "margin_dpo/margin_std": 146.31146240234375, "step": 516 }, { "KL/chosen_KL_mean": -203.08294677734375, "KL/mean": -254.6483154296875, "KL/rejected_KL_mean": -306.21368408203125, "KL/std": 110.49296569824219, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004231208935379982, "fcm_dpo/delta": -0.055625155568122864, "fcm_dpo/margin": 103.13070678710938, "fcm_dpo/q_t": 0.39578747749328613, "grad_norm": 9.182027816772461, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.742663860321045, "logits/rejected": -1.3069212436676025, "logps/chosen": -296.26953125, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -401.80645751953125, "loss": 1.0695, "margin_dpo/margin_mean": 103.13070678710938, "margin_dpo/margin_std": 125.23605346679688, "step": 517 }, { "KL/chosen_KL_mean": -211.13165283203125, "KL/mean": -249.9853973388672, "KL/rejected_KL_mean": -288.839111328125, "KL/std": 104.248291015625, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004521133378148079, "fcm_dpo/delta": 0.059040945023298264, "fcm_dpo/margin": 77.70742797851562, "fcm_dpo/q_t": 0.42171451449394226, "grad_norm": 10.62437629699707, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.9598193168640137, "logits/rejected": -1.635427474975586, "logps/chosen": -298.15301513671875, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -380.08367919921875, "loss": 1.1467, "margin_dpo/margin_mean": 77.70741271972656, "margin_dpo/margin_std": 130.1714324951172, "step": 518 }, { "KL/chosen_KL_mean": -195.99508666992188, "KL/mean": -237.58984375, "KL/rejected_KL_mean": -279.1845397949219, "KL/std": 97.21993255615234, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.004611659795045853, "fcm_dpo/delta": 0.019941391423344612, "fcm_dpo/margin": 83.18944549560547, "fcm_dpo/q_t": 0.41153961420059204, "grad_norm": 11.319594383239746, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.170825719833374, "logits/rejected": -1.8138842582702637, "logps/chosen": -296.6826477050781, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -388.2149658203125, "loss": 1.0911, "margin_dpo/margin_mean": 83.18943786621094, "margin_dpo/margin_std": 101.48526000976562, "step": 519 }, { "KL/chosen_KL_mean": -183.04147338867188, "KL/mean": -233.17367553710938, "KL/rejected_KL_mean": -283.3058776855469, "KL/std": 109.75787353515625, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004435807000845671, "fcm_dpo/delta": -0.05901084840297699, "fcm_dpo/margin": 100.26441955566406, "fcm_dpo/q_t": 0.3931404948234558, "grad_norm": 9.482375144958496, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.6088190078735352, "logits/rejected": -1.173437476158142, "logps/chosen": -281.819091796875, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -390.2515869140625, "loss": 1.0444, "margin_dpo/margin_mean": 100.26441955566406, "margin_dpo/margin_std": 107.56095886230469, "step": 520 }, { "KL/chosen_KL_mean": -182.6173553466797, "KL/mean": -234.14501953125, "KL/rejected_KL_mean": -285.67266845703125, "KL/std": 102.90714263916016, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004328450653702021, "fcm_dpo/delta": -0.06017797440290451, "fcm_dpo/margin": 103.05530548095703, "fcm_dpo/q_t": 0.39295321702957153, "grad_norm": 10.526281356811523, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.818908452987671, "logits/rejected": -1.4743621349334717, "logps/chosen": -268.96844482421875, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -376.06585693359375, "loss": 1.0415, "margin_dpo/margin_mean": 103.05531311035156, "margin_dpo/margin_std": 105.0113525390625, "step": 521 }, { "KL/chosen_KL_mean": -191.94473266601562, "KL/mean": -237.34521484375, "KL/rejected_KL_mean": -282.7457275390625, "KL/std": 112.41240692138672, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.004256914369761944, "fcm_dpo/delta": 0.01591598615050316, "fcm_dpo/margin": 90.80098724365234, "fcm_dpo/q_t": 0.41031232476234436, "grad_norm": 10.254819869995117, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.6546640396118164, "logits/rejected": -1.2849856615066528, "logps/chosen": -267.9527587890625, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -357.18695068359375, "loss": 1.0972, "margin_dpo/margin_mean": 90.80098724365234, "margin_dpo/margin_std": 118.126953125, "step": 522 }, { "KL/chosen_KL_mean": -212.01632690429688, "KL/mean": -245.46231079101562, "KL/rejected_KL_mean": -278.90826416015625, "KL/std": 111.07551574707031, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.00457795662805438, "fcm_dpo/delta": 0.11087693274021149, "fcm_dpo/margin": 66.89192962646484, "fcm_dpo/q_t": 0.4335036873817444, "grad_norm": 11.684619903564453, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.0167059898376465, "logits/rejected": -1.3195887804031372, "logps/chosen": -308.5126037597656, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -369.9235534667969, "loss": 1.1869, "margin_dpo/margin_mean": 66.89193725585938, "margin_dpo/margin_std": 129.4554443359375, "step": 523 }, { "KL/chosen_KL_mean": -193.48545837402344, "KL/mean": -241.0328826904297, "KL/rejected_KL_mean": -288.580322265625, "KL/std": 111.08888244628906, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.004545300267636776, "fcm_dpo/delta": -0.04176174849271774, "fcm_dpo/margin": 95.09486389160156, "fcm_dpo/q_t": 0.3992077112197876, "grad_norm": 13.389994621276855, "learning_rate": 7.781828926091535e-08, "logits/chosen": -1.9810045957565308, "logits/rejected": -1.2601780891418457, "logps/chosen": -296.8909606933594, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -376.490478515625, "loss": 1.0668, "margin_dpo/margin_mean": 95.09486389160156, "margin_dpo/margin_std": 117.82820129394531, "step": 524 }, { "KL/chosen_KL_mean": -182.06393432617188, "KL/mean": -236.042724609375, "KL/rejected_KL_mean": -290.0215148925781, "KL/std": 110.91082763671875, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.00408831425011158, "fcm_dpo/delta": -0.06949687749147415, "fcm_dpo/margin": 107.95758819580078, "fcm_dpo/q_t": 0.39490848779678345, "grad_norm": 10.815958976745605, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.7110813856124878, "logits/rejected": -1.6259794235229492, "logps/chosen": -266.2704162597656, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -396.2831115722656, "loss": 1.0662, "margin_dpo/margin_mean": 107.95758056640625, "margin_dpo/margin_std": 128.749755859375, "step": 525 }, { "KL/chosen_KL_mean": -187.34388732910156, "KL/mean": -236.32496643066406, "KL/rejected_KL_mean": -285.3060302734375, "KL/std": 105.22526550292969, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.004189115017652512, "fcm_dpo/delta": -0.013044832274317741, "fcm_dpo/margin": 97.96211242675781, "fcm_dpo/q_t": 0.40409278869628906, "grad_norm": 8.543280601501465, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.8148391246795654, "logits/rejected": -1.1674784421920776, "logps/chosen": -290.197998046875, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -378.9574890136719, "loss": 1.0814, "margin_dpo/margin_mean": 97.96211242675781, "margin_dpo/margin_std": 123.94387817382812, "step": 526 }, { "KL/chosen_KL_mean": -189.80859375, "KL/mean": -239.47402954101562, "KL/rejected_KL_mean": -289.13946533203125, "KL/std": 110.06385803222656, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004126383922994137, "fcm_dpo/delta": -0.012454492971301079, "fcm_dpo/margin": 99.33087158203125, "fcm_dpo/q_t": 0.40202540159225464, "grad_norm": 10.117237091064453, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.205264091491699, "logits/rejected": -2.409168243408203, "logps/chosen": -286.57403564453125, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -374.53466796875, "loss": 1.0556, "margin_dpo/margin_mean": 99.33087158203125, "margin_dpo/margin_std": 96.45954895019531, "step": 527 }, { "KL/chosen_KL_mean": -176.65484619140625, "KL/mean": -225.84979248046875, "KL/rejected_KL_mean": -275.04473876953125, "KL/std": 111.92466735839844, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004126579500734806, "fcm_dpo/delta": -0.0077390652149915695, "fcm_dpo/margin": 98.38988494873047, "fcm_dpo/q_t": 0.40538328886032104, "grad_norm": 9.293272018432617, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.234043598175049, "logits/rejected": -1.727790117263794, "logps/chosen": -270.81976318359375, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -375.41705322265625, "loss": 1.0865, "margin_dpo/margin_mean": 98.389892578125, "margin_dpo/margin_std": 127.4913330078125, "step": 528 }, { "KL/chosen_KL_mean": -206.62451171875, "KL/mean": -250.62461853027344, "KL/rejected_KL_mean": -294.6247253417969, "KL/std": 110.61994171142578, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.004238904919475317, "fcm_dpo/delta": 0.031933028250932693, "fcm_dpo/margin": 88.0002212524414, "fcm_dpo/q_t": 0.4154202342033386, "grad_norm": 10.894309043884277, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8155776262283325, "logits/rejected": -1.6412382125854492, "logps/chosen": -287.40130615234375, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -391.8158874511719, "loss": 1.1207, "margin_dpo/margin_mean": 88.0002212524414, "margin_dpo/margin_std": 131.19284057617188, "step": 529 }, { "KL/chosen_KL_mean": -188.67886352539062, "KL/mean": -233.25741577148438, "KL/rejected_KL_mean": -277.83599853515625, "KL/std": 107.01472473144531, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.00423953402787447, "fcm_dpo/delta": 0.026565561071038246, "fcm_dpo/margin": 89.15716552734375, "fcm_dpo/q_t": 0.41244393587112427, "grad_norm": 9.93010139465332, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.9044246673583984, "logits/rejected": -1.600539207458496, "logps/chosen": -268.45458984375, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -369.889892578125, "loss": 1.0899, "margin_dpo/margin_mean": 89.15716552734375, "margin_dpo/margin_std": 103.22160339355469, "step": 530 }, { "KL/chosen_KL_mean": -182.50502014160156, "KL/mean": -238.20469665527344, "KL/rejected_KL_mean": -293.90435791015625, "KL/std": 113.31632995605469, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004077981226146221, "fcm_dpo/delta": -0.07133011519908905, "fcm_dpo/margin": 111.39933776855469, "fcm_dpo/q_t": 0.38953855633735657, "grad_norm": 11.416255950927734, "learning_rate": 7.141774982445147e-08, "logits/chosen": -2.0064780712127686, "logits/rejected": -1.421872854232788, "logps/chosen": -261.82421875, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -372.0126953125, "loss": 1.0328, "margin_dpo/margin_mean": 111.39933776855469, "margin_dpo/margin_std": 112.990234375, "step": 531 }, { "KL/chosen_KL_mean": -196.738037109375, "KL/mean": -256.9855041503906, "KL/rejected_KL_mean": -317.23297119140625, "KL/std": 108.19406127929688, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0038398748729377985, "fcm_dpo/delta": -0.08118928223848343, "fcm_dpo/margin": 120.4949722290039, "fcm_dpo/q_t": 0.3880921006202698, "grad_norm": 10.42966365814209, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.7015717029571533, "logits/rejected": -1.0773608684539795, "logps/chosen": -298.37493896484375, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -410.578369140625, "loss": 1.03, "margin_dpo/margin_mean": 120.49496459960938, "margin_dpo/margin_std": 120.7909164428711, "step": 532 }, { "KL/chosen_KL_mean": -190.49343872070312, "KL/mean": -225.28128051757812, "KL/rejected_KL_mean": -260.069091796875, "KL/std": 113.99247741699219, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004132966510951519, "fcm_dpo/delta": 0.12952706217765808, "fcm_dpo/margin": 69.57568359375, "fcm_dpo/q_t": 0.43749088048934937, "grad_norm": 11.413529396057129, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.491334915161133, "logits/rejected": -1.8979806900024414, "logps/chosen": -281.2400817871094, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -342.95758056640625, "loss": 1.1804, "margin_dpo/margin_mean": 69.57568359375, "margin_dpo/margin_std": 122.8413314819336, "step": 533 }, { "KL/chosen_KL_mean": -169.78988647460938, "KL/mean": -221.59530639648438, "KL/rejected_KL_mean": -273.4007568359375, "KL/std": 106.83055114746094, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.00414523808285594, "fcm_dpo/delta": -0.037487857043743134, "fcm_dpo/margin": 103.61085510253906, "fcm_dpo/q_t": 0.39708536863327026, "grad_norm": 9.572510719299316, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0532336235046387, "logits/rejected": -1.6615320444107056, "logps/chosen": -256.42071533203125, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -358.272705078125, "loss": 1.0481, "margin_dpo/margin_mean": 103.61085510253906, "margin_dpo/margin_std": 105.08069610595703, "step": 534 }, { "KL/chosen_KL_mean": -171.1964111328125, "KL/mean": -225.84744262695312, "KL/rejected_KL_mean": -280.49847412109375, "KL/std": 107.97726440429688, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.004018557257950306, "fcm_dpo/delta": -0.051255278289318085, "fcm_dpo/margin": 109.30207061767578, "fcm_dpo/q_t": 0.3958014249801636, "grad_norm": 7.738555908203125, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.9174203872680664, "logits/rejected": -1.735929250717163, "logps/chosen": -261.54180908203125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -381.2764587402344, "loss": 1.0555, "margin_dpo/margin_mean": 109.30206298828125, "margin_dpo/margin_std": 123.66868591308594, "step": 535 }, { "KL/chosen_KL_mean": -169.7835693359375, "KL/mean": -223.87155151367188, "KL/rejected_KL_mean": -277.95953369140625, "KL/std": 108.56980895996094, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0037977518513798714, "fcm_dpo/delta": -0.014207287691533566, "fcm_dpo/margin": 108.17593383789062, "fcm_dpo/q_t": 0.4020928740501404, "grad_norm": 8.509679794311523, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.869201421737671, "logits/rejected": -1.6185555458068848, "logps/chosen": -248.74314880371094, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -371.04730224609375, "loss": 1.0622, "margin_dpo/margin_mean": 108.17593383789062, "margin_dpo/margin_std": 113.86132049560547, "step": 536 }, { "KL/chosen_KL_mean": -148.52301025390625, "KL/mean": -205.3465576171875, "KL/rejected_KL_mean": -262.17010498046875, "KL/std": 112.01708984375, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0038091999012976885, "fcm_dpo/delta": -0.04434676468372345, "fcm_dpo/margin": 113.6470718383789, "fcm_dpo/q_t": 0.39545196294784546, "grad_norm": 9.038772583007812, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.8477423191070557, "logits/rejected": -1.190280556678772, "logps/chosen": -235.23239135742188, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -341.0777587890625, "loss": 1.0463, "margin_dpo/margin_mean": 113.64706420898438, "margin_dpo/margin_std": 109.00765991210938, "step": 537 }, { "KL/chosen_KL_mean": -204.20343017578125, "KL/mean": -250.4422607421875, "KL/rejected_KL_mean": -296.68115234375, "KL/std": 100.3648681640625, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0038188889157027006, "fcm_dpo/delta": 0.056692883372306824, "fcm_dpo/margin": 92.47769927978516, "fcm_dpo/q_t": 0.41828811168670654, "grad_norm": 8.365156173706055, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.7401854991912842, "logits/rejected": -1.1927344799041748, "logps/chosen": -302.64556884765625, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -387.7757873535156, "loss": 1.1044, "margin_dpo/margin_mean": 92.47769927978516, "margin_dpo/margin_std": 107.75445556640625, "step": 538 }, { "KL/chosen_KL_mean": -176.99952697753906, "KL/mean": -226.05471801757812, "KL/rejected_KL_mean": -275.10992431640625, "KL/std": 110.26033020019531, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0039301225915551186, "fcm_dpo/delta": 0.01775672473013401, "fcm_dpo/margin": 98.11040496826172, "fcm_dpo/q_t": 0.410300612449646, "grad_norm": 8.121596336364746, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.4960923194885254, "logits/rejected": -2.022033214569092, "logps/chosen": -267.931640625, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -372.555419921875, "loss": 1.0952, "margin_dpo/margin_mean": 98.11041259765625, "margin_dpo/margin_std": 125.67066955566406, "step": 539 }, { "KL/chosen_KL_mean": -190.4687042236328, "KL/mean": -249.8732147216797, "KL/rejected_KL_mean": -309.2777099609375, "KL/std": 120.63026428222656, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.00381926610134542, "fcm_dpo/delta": -0.06986113637685776, "fcm_dpo/margin": 118.80903625488281, "fcm_dpo/q_t": 0.3932064175605774, "grad_norm": 9.931330680847168, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9676454067230225, "logits/rejected": -1.731234073638916, "logps/chosen": -298.338623046875, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -430.67120361328125, "loss": 1.0533, "margin_dpo/margin_mean": 118.80902099609375, "margin_dpo/margin_std": 141.48289489746094, "step": 540 }, { "KL/chosen_KL_mean": -210.064208984375, "KL/mean": -247.1048583984375, "KL/rejected_KL_mean": -284.1455078125, "KL/std": 99.40155029296875, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0037059986498206854, "fcm_dpo/delta": 0.009373554959893227, "fcm_dpo/margin": 74.08131408691406, "fcm_dpo/q_t": 0.43541717529296875, "grad_norm": 8.738859176635742, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.3701493740081787, "logits/rejected": -1.9076457023620605, "logps/chosen": -316.6317138671875, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -393.21185302734375, "loss": 1.1732, "margin_dpo/margin_mean": 74.08131408691406, "margin_dpo/margin_std": 107.39910125732422, "step": 541 }, { "KL/chosen_KL_mean": -179.3540802001953, "KL/mean": -235.9091339111328, "KL/rejected_KL_mean": -292.46417236328125, "KL/std": 111.6932144165039, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0036551873199641705, "fcm_dpo/delta": -0.0169781856238842, "fcm_dpo/margin": 113.11009979248047, "fcm_dpo/q_t": 0.40142911672592163, "grad_norm": 9.084162712097168, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.3529162406921387, "logits/rejected": -2.23231840133667, "logps/chosen": -266.723388671875, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -395.2845458984375, "loss": 1.0621, "margin_dpo/margin_mean": 113.110107421875, "margin_dpo/margin_std": 121.28345489501953, "step": 542 }, { "KL/chosen_KL_mean": -204.50717163085938, "KL/mean": -235.25924682617188, "KL/rejected_KL_mean": -266.01129150390625, "KL/std": 118.56375122070312, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0038542316760867834, "fcm_dpo/delta": 0.05619710311293602, "fcm_dpo/margin": 61.504127502441406, "fcm_dpo/q_t": 0.4468112885951996, "grad_norm": 12.087319374084473, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.5959608554840088, "logits/rejected": -1.645727515220642, "logps/chosen": -291.6597595214844, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -377.54302978515625, "loss": 1.2378, "margin_dpo/margin_mean": 61.504127502441406, "margin_dpo/margin_std": 144.25534057617188, "step": 543 }, { "KL/chosen_KL_mean": -213.6728515625, "KL/mean": -252.23904418945312, "KL/rejected_KL_mean": -290.80523681640625, "KL/std": 111.09231567382812, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0041345409117639065, "fcm_dpo/delta": 0.0965152382850647, "fcm_dpo/margin": 77.13236999511719, "fcm_dpo/q_t": 0.4287955164909363, "grad_norm": 9.194199562072754, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.670285940170288, "logits/rejected": -1.5771570205688477, "logps/chosen": -299.81976318359375, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -397.19915771484375, "loss": 1.1471, "margin_dpo/margin_mean": 77.13236999511719, "margin_dpo/margin_std": 115.9404296875, "step": 544 }, { "KL/chosen_KL_mean": -189.48690795898438, "KL/mean": -236.909912109375, "KL/rejected_KL_mean": -284.3328857421875, "KL/std": 110.68553924560547, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0042663151398301125, "fcm_dpo/delta": -0.006164735183119774, "fcm_dpo/margin": 94.84600830078125, "fcm_dpo/q_t": 0.4064372181892395, "grad_norm": 9.429915428161621, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.5383062362670898, "logits/rejected": -1.1134296655654907, "logps/chosen": -285.9111328125, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -385.06695556640625, "loss": 1.1038, "margin_dpo/margin_mean": 94.84600830078125, "margin_dpo/margin_std": 137.39727783203125, "step": 545 }, { "KL/chosen_KL_mean": -196.01797485351562, "KL/mean": -238.37820434570312, "KL/rejected_KL_mean": -280.7384033203125, "KL/std": 111.9735107421875, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004315904341638088, "fcm_dpo/delta": 0.042093973606824875, "fcm_dpo/margin": 84.720458984375, "fcm_dpo/q_t": 0.4156394600868225, "grad_norm": 9.396034240722656, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.5199618339538574, "logits/rejected": -1.1660842895507812, "logps/chosen": -282.0390930175781, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -368.1110534667969, "loss": 1.121, "margin_dpo/margin_mean": 84.720458984375, "margin_dpo/margin_std": 122.87117004394531, "step": 546 }, { "KL/chosen_KL_mean": -188.789794921875, "KL/mean": -239.88662719726562, "KL/rejected_KL_mean": -290.98345947265625, "KL/std": 109.2253189086914, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004301467910408974, "fcm_dpo/delta": -0.050877414643764496, "fcm_dpo/margin": 102.19368743896484, "fcm_dpo/q_t": 0.39395058155059814, "grad_norm": 9.529462814331055, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.9377572536468506, "logits/rejected": -1.4802494049072266, "logps/chosen": -289.4407958984375, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -388.89520263671875, "loss": 1.0375, "margin_dpo/margin_mean": 102.19369506835938, "margin_dpo/margin_std": 97.36422729492188, "step": 547 }, { "KL/chosen_KL_mean": -185.44662475585938, "KL/mean": -227.90811157226562, "KL/rejected_KL_mean": -270.36956787109375, "KL/std": 111.56069946289062, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0043181562796235085, "fcm_dpo/delta": 0.040627092123031616, "fcm_dpo/margin": 84.92294311523438, "fcm_dpo/q_t": 0.41698479652404785, "grad_norm": 10.380844116210938, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.2765321731567383, "logits/rejected": -1.9200258255004883, "logps/chosen": -281.47674560546875, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -368.7828063964844, "loss": 1.1408, "margin_dpo/margin_mean": 84.9229507446289, "margin_dpo/margin_std": 140.64474487304688, "step": 548 }, { "KL/chosen_KL_mean": -175.83407592773438, "KL/mean": -218.42575073242188, "KL/rejected_KL_mean": -261.01739501953125, "KL/std": 106.35212707519531, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0045236144214868546, "fcm_dpo/delta": 0.014449838548898697, "fcm_dpo/margin": 85.1833267211914, "fcm_dpo/q_t": 0.410690575838089, "grad_norm": 11.812987327575684, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8661720752716064, "logits/rejected": -1.4512462615966797, "logps/chosen": -259.2850341796875, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -342.39764404296875, "loss": 1.1267, "margin_dpo/margin_mean": 85.18333435058594, "margin_dpo/margin_std": 130.54217529296875, "step": 549 }, { "KL/chosen_KL_mean": -194.06210327148438, "KL/mean": -236.4482421875, "KL/rejected_KL_mean": -278.8343505859375, "KL/std": 100.49607849121094, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004530435893684626, "fcm_dpo/delta": 0.01895136944949627, "fcm_dpo/margin": 84.7722396850586, "fcm_dpo/q_t": 0.4114938974380493, "grad_norm": 9.803827285766602, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.5152158737182617, "logits/rejected": -1.3000779151916504, "logps/chosen": -283.65643310546875, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -380.14984130859375, "loss": 1.0996, "margin_dpo/margin_mean": 84.77224731445312, "margin_dpo/margin_std": 109.75442504882812, "step": 550 }, { "KL/chosen_KL_mean": -173.33200073242188, "KL/mean": -225.23492431640625, "KL/rejected_KL_mean": -277.13787841796875, "KL/std": 115.2762451171875, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.00421195849776268, "fcm_dpo/delta": -0.05294880270957947, "fcm_dpo/margin": 103.80587005615234, "fcm_dpo/q_t": 0.3967006802558899, "grad_norm": 11.912663459777832, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.5145747661590576, "logits/rejected": -1.372016429901123, "logps/chosen": -268.4141540527344, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -394.85711669921875, "loss": 1.0549, "margin_dpo/margin_mean": 103.80587005615234, "margin_dpo/margin_std": 117.71893310546875, "step": 551 }, { "KL/chosen_KL_mean": -166.958740234375, "KL/mean": -239.98623657226562, "KL/rejected_KL_mean": -313.01373291015625, "KL/std": 124.44613647460938, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0035577097442001104, "fcm_dpo/delta": -0.1835552603006363, "fcm_dpo/margin": 146.05496215820312, "fcm_dpo/q_t": 0.3663980960845947, "grad_norm": 10.805037498474121, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.1827666759490967, "logits/rejected": -1.8287346363067627, "logps/chosen": -268.367919921875, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -423.8521728515625, "loss": 0.9855, "margin_dpo/margin_mean": 146.05496215820312, "margin_dpo/margin_std": 129.2575225830078, "step": 552 }, { "KL/chosen_KL_mean": -179.8019561767578, "KL/mean": -223.43923950195312, "KL/rejected_KL_mean": -267.0765075683594, "KL/std": 115.21827697753906, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.003806656925007701, "fcm_dpo/delta": 0.07999402284622192, "fcm_dpo/margin": 87.2745361328125, "fcm_dpo/q_t": 0.42378297448158264, "grad_norm": 8.876866340637207, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.4489043951034546, "logits/rejected": -1.1158946752548218, "logps/chosen": -277.7658386230469, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -370.6536865234375, "loss": 1.1347, "margin_dpo/margin_mean": 87.2745361328125, "margin_dpo/margin_std": 123.2732162475586, "step": 553 }, { "KL/chosen_KL_mean": -188.33950805664062, "KL/mean": -236.68453979492188, "KL/rejected_KL_mean": -285.02960205078125, "KL/std": 116.08642578125, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.003979367204010487, "fcm_dpo/delta": 0.016594700515270233, "fcm_dpo/margin": 96.6900634765625, "fcm_dpo/q_t": 0.4115562438964844, "grad_norm": 8.655234336853027, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.85090172290802, "logits/rejected": -1.0696371793746948, "logps/chosen": -292.02630615234375, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -378.0216979980469, "loss": 1.1238, "margin_dpo/margin_mean": 96.69007110595703, "margin_dpo/margin_std": 149.8068389892578, "step": 554 }, { "KL/chosen_KL_mean": -195.94937133789062, "KL/mean": -240.57176208496094, "KL/rejected_KL_mean": -285.19415283203125, "KL/std": 119.2501220703125, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.004054956138134003, "fcm_dpo/delta": 0.046466849744319916, "fcm_dpo/margin": 89.2447738647461, "fcm_dpo/q_t": 0.41858410835266113, "grad_norm": 9.15817928314209, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.6810362339019775, "logits/rejected": -1.5911612510681152, "logps/chosen": -282.15301513671875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -399.7413635253906, "loss": 1.1311, "margin_dpo/margin_mean": 89.2447738647461, "margin_dpo/margin_std": 138.661376953125, "step": 555 }, { "KL/chosen_KL_mean": -177.32774353027344, "KL/mean": -225.7073974609375, "KL/rejected_KL_mean": -274.0870666503906, "KL/std": 106.21105194091797, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004059879574924707, "fcm_dpo/delta": 0.007892880588769913, "fcm_dpo/margin": 96.75930786132812, "fcm_dpo/q_t": 0.40836095809936523, "grad_norm": 10.733738899230957, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.6827600002288818, "logits/rejected": -1.3865995407104492, "logps/chosen": -265.7312316894531, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -380.7261962890625, "loss": 1.0899, "margin_dpo/margin_mean": 96.75930786132812, "margin_dpo/margin_std": 120.78567504882812, "step": 556 }, { "KL/chosen_KL_mean": -190.53179931640625, "KL/mean": -246.15054321289062, "KL/rejected_KL_mean": -301.769287109375, "KL/std": 122.62556457519531, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.003899186383932829, "fcm_dpo/delta": -0.04700414836406708, "fcm_dpo/margin": 111.23750305175781, "fcm_dpo/q_t": 0.3976070284843445, "grad_norm": 7.981445789337158, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -2.027235507965088, "logits/rejected": -1.855471134185791, "logps/chosen": -289.3670349121094, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -417.4181823730469, "loss": 1.0647, "margin_dpo/margin_mean": 111.23750305175781, "margin_dpo/margin_std": 134.1967010498047, "step": 557 }, { "KL/chosen_KL_mean": -187.78338623046875, "KL/mean": -247.28897094726562, "KL/rejected_KL_mean": -306.7945556640625, "KL/std": 105.86164855957031, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.003729649353772402, "fcm_dpo/delta": -0.05811203643679619, "fcm_dpo/margin": 119.01118469238281, "fcm_dpo/q_t": 0.3922663927078247, "grad_norm": 10.313725471496582, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.527472972869873, "logits/rejected": -1.9346539974212646, "logps/chosen": -293.79522705078125, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -415.1858825683594, "loss": 1.0315, "margin_dpo/margin_mean": 119.01118469238281, "margin_dpo/margin_std": 109.18175506591797, "step": 558 }, { "KL/chosen_KL_mean": -205.09068298339844, "KL/mean": -251.6517333984375, "KL/rejected_KL_mean": -298.2127685546875, "KL/std": 108.59555053710938, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.003875882364809513, "fcm_dpo/delta": 0.047184575349092484, "fcm_dpo/margin": 93.1220703125, "fcm_dpo/q_t": 0.4161943197250366, "grad_norm": 10.004549980163574, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.4138479232788086, "logits/rejected": -1.9449197053909302, "logps/chosen": -305.51544189453125, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -398.62548828125, "loss": 1.1121, "margin_dpo/margin_mean": 93.1220703125, "margin_dpo/margin_std": 122.61740112304688, "step": 559 }, { "KL/chosen_KL_mean": -193.34327697753906, "KL/mean": -239.6318817138672, "KL/rejected_KL_mean": -285.92047119140625, "KL/std": 106.23603820800781, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0039976127445697784, "fcm_dpo/delta": 0.03676475211977959, "fcm_dpo/margin": 92.57717895507812, "fcm_dpo/q_t": 0.41472727060317993, "grad_norm": 11.115269660949707, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.5629104375839233, "logits/rejected": -1.1307933330535889, "logps/chosen": -280.9787902832031, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -373.3003234863281, "loss": 1.1074, "margin_dpo/margin_mean": 92.57718658447266, "margin_dpo/margin_std": 122.63211822509766, "step": 560 }, { "KL/chosen_KL_mean": -202.09681701660156, "KL/mean": -245.21005249023438, "KL/rejected_KL_mean": -288.32330322265625, "KL/std": 104.766357421875, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004225855227559805, "fcm_dpo/delta": 0.04233109951019287, "fcm_dpo/margin": 86.22648620605469, "fcm_dpo/q_t": 0.4154870808124542, "grad_norm": 8.939573287963867, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.9314839839935303, "logits/rejected": -1.9047929048538208, "logps/chosen": -276.97821044921875, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -394.37127685546875, "loss": 1.0979, "margin_dpo/margin_mean": 86.22648620605469, "margin_dpo/margin_std": 100.73314666748047, "step": 561 }, { "KL/chosen_KL_mean": -183.59701538085938, "KL/mean": -227.78411865234375, "KL/rejected_KL_mean": -271.97119140625, "KL/std": 105.93780517578125, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.004241475835442543, "fcm_dpo/delta": 0.0301115270704031, "fcm_dpo/margin": 88.37420654296875, "fcm_dpo/q_t": 0.41453397274017334, "grad_norm": 10.962593078613281, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.1768453121185303, "logits/rejected": -1.7285135984420776, "logps/chosen": -284.26025390625, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -374.21209716796875, "loss": 1.1195, "margin_dpo/margin_mean": 88.37420654296875, "margin_dpo/margin_std": 130.53775024414062, "step": 562 }, { "KL/chosen_KL_mean": -195.2608184814453, "KL/mean": -231.14773559570312, "KL/rejected_KL_mean": -267.03466796875, "KL/std": 106.2956314086914, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004690384026616812, "fcm_dpo/delta": 0.07232832163572311, "fcm_dpo/margin": 71.77384948730469, "fcm_dpo/q_t": 0.42372503876686096, "grad_norm": 11.77003002166748, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.6012787818908691, "logits/rejected": -1.2616195678710938, "logps/chosen": -291.52001953125, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -368.2397155761719, "loss": 1.1465, "margin_dpo/margin_mean": 71.77384948730469, "margin_dpo/margin_std": 113.68309020996094, "step": 563 }, { "KL/chosen_KL_mean": -190.52479553222656, "KL/mean": -230.7515869140625, "KL/rejected_KL_mean": -270.9783935546875, "KL/std": 107.18028259277344, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.004796772729605436, "fcm_dpo/delta": 0.01645272597670555, "fcm_dpo/margin": 80.4535903930664, "fcm_dpo/q_t": 0.40993914008140564, "grad_norm": 13.301226615905762, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.4120516777038574, "logits/rejected": -1.8483572006225586, "logps/chosen": -303.4851989746094, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -370.50640869140625, "loss": 1.0901, "margin_dpo/margin_mean": 80.45358276367188, "margin_dpo/margin_std": 97.2706527709961, "step": 564 }, { "KL/chosen_KL_mean": -186.1470947265625, "KL/mean": -236.64028930664062, "KL/rejected_KL_mean": -287.133544921875, "KL/std": 116.79981231689453, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.004476791247725487, "fcm_dpo/delta": -0.06863445043563843, "fcm_dpo/margin": 100.98644256591797, "fcm_dpo/q_t": 0.39353320002555847, "grad_norm": 9.422685623168945, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.8262708187103271, "logits/rejected": -1.3311853408813477, "logps/chosen": -281.5126647949219, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -379.35089111328125, "loss": 1.0603, "margin_dpo/margin_mean": 100.98644256591797, "margin_dpo/margin_std": 127.39874267578125, "step": 565 }, { "KL/chosen_KL_mean": -195.31668090820312, "KL/mean": -242.88067626953125, "KL/rejected_KL_mean": -290.4446716308594, "KL/std": 118.585693359375, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004413786809891462, "fcm_dpo/delta": -0.02552678808569908, "fcm_dpo/margin": 95.12799835205078, "fcm_dpo/q_t": 0.400856614112854, "grad_norm": 9.358752250671387, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9880223274230957, "logits/rejected": -1.5616940259933472, "logps/chosen": -294.2898864746094, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -401.26556396484375, "loss": 1.0766, "margin_dpo/margin_mean": 95.12800598144531, "margin_dpo/margin_std": 118.27320861816406, "step": 566 }, { "KL/chosen_KL_mean": -189.912109375, "KL/mean": -231.7052764892578, "KL/rejected_KL_mean": -273.4984436035156, "KL/std": 107.1867904663086, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.004358864389359951, "fcm_dpo/delta": 0.042042460292577744, "fcm_dpo/margin": 83.58634185791016, "fcm_dpo/q_t": 0.4153832197189331, "grad_norm": 10.283785820007324, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.47320556640625, "logits/rejected": -1.8271918296813965, "logps/chosen": -282.6156921386719, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -364.7227783203125, "loss": 1.1176, "margin_dpo/margin_mean": 83.58634185791016, "margin_dpo/margin_std": 113.42523193359375, "step": 567 }, { "KL/chosen_KL_mean": -180.1405029296875, "KL/mean": -235.78829956054688, "KL/rejected_KL_mean": -291.4360656738281, "KL/std": 112.91004943847656, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.00414569117128849, "fcm_dpo/delta": -0.08459100127220154, "fcm_dpo/margin": 111.29556274414062, "fcm_dpo/q_t": 0.38812631368637085, "grad_norm": 8.730255126953125, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.496009111404419, "logits/rejected": -1.3699193000793457, "logps/chosen": -253.27276611328125, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -386.38873291015625, "loss": 1.0309, "margin_dpo/margin_mean": 111.29556274414062, "margin_dpo/margin_std": 114.66712951660156, "step": 568 }, { "KL/chosen_KL_mean": -190.9644775390625, "KL/mean": -236.14071655273438, "KL/rejected_KL_mean": -281.31695556640625, "KL/std": 113.94610595703125, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004245729185640812, "fcm_dpo/delta": 0.01972942426800728, "fcm_dpo/margin": 90.35244750976562, "fcm_dpo/q_t": 0.4114586114883423, "grad_norm": 14.76627254486084, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.956578016281128, "logits/rejected": -1.5804533958435059, "logps/chosen": -280.18157958984375, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -367.7279968261719, "loss": 1.121, "margin_dpo/margin_mean": 90.35244750976562, "margin_dpo/margin_std": 137.39065551757812, "step": 569 }, { "KL/chosen_KL_mean": -170.7084197998047, "KL/mean": -229.48855590820312, "KL/rejected_KL_mean": -288.2687072753906, "KL/std": 108.62493896484375, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0039512645453214645, "fcm_dpo/delta": -0.08491760492324829, "fcm_dpo/margin": 117.56031036376953, "fcm_dpo/q_t": 0.38636642694473267, "grad_norm": 9.014178276062012, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.2770864963531494, "logits/rejected": -1.5911725759506226, "logps/chosen": -265.47119140625, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -377.1156311035156, "loss": 1.0222, "margin_dpo/margin_mean": 117.56031036376953, "margin_dpo/margin_std": 113.32740783691406, "step": 570 }, { "KL/chosen_KL_mean": -176.9221954345703, "KL/mean": -228.87454223632812, "KL/rejected_KL_mean": -280.8269348144531, "KL/std": 107.00721740722656, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.003909130115061998, "fcm_dpo/delta": -0.00828348845243454, "fcm_dpo/margin": 103.90473175048828, "fcm_dpo/q_t": 0.40498384833335876, "grad_norm": 12.096136093139648, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.820430040359497, "logits/rejected": -1.6674396991729736, "logps/chosen": -257.74346923828125, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -377.84222412109375, "loss": 1.0822, "margin_dpo/margin_mean": 103.90473175048828, "margin_dpo/margin_std": 129.22872924804688, "step": 571 }, { "KL/chosen_KL_mean": -182.782470703125, "KL/mean": -234.978271484375, "KL/rejected_KL_mean": -287.174072265625, "KL/std": 113.7223892211914, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0038328419905155897, "fcm_dpo/delta": -0.00027335528284311295, "fcm_dpo/margin": 104.3916015625, "fcm_dpo/q_t": 0.4069628119468689, "grad_norm": 8.225194931030273, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.8123067617416382, "logits/rejected": -1.6455323696136475, "logps/chosen": -260.1623840332031, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -377.74005126953125, "loss": 1.0777, "margin_dpo/margin_mean": 104.39160919189453, "margin_dpo/margin_std": 122.50469970703125, "step": 572 }, { "KL/chosen_KL_mean": -177.3428955078125, "KL/mean": -226.02633666992188, "KL/rejected_KL_mean": -274.70977783203125, "KL/std": 113.15019226074219, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.004002134781330824, "fcm_dpo/delta": 0.0080435611307621, "fcm_dpo/margin": 97.36686706542969, "fcm_dpo/q_t": 0.4079618453979492, "grad_norm": 8.870624542236328, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1087613105773926, "logits/rejected": -1.878624439239502, "logps/chosen": -261.0489501953125, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -365.4072265625, "loss": 1.0945, "margin_dpo/margin_mean": 97.36686706542969, "margin_dpo/margin_std": 120.51199340820312, "step": 573 }, { "KL/chosen_KL_mean": -185.91970825195312, "KL/mean": -239.4563751220703, "KL/rejected_KL_mean": -292.9930419921875, "KL/std": 112.05694580078125, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.003887261264026165, "fcm_dpo/delta": -0.021296825259923935, "fcm_dpo/margin": 107.0733413696289, "fcm_dpo/q_t": 0.401614248752594, "grad_norm": 9.143852233886719, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.125391960144043, "logits/rejected": -1.798872947692871, "logps/chosen": -271.0491943359375, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -392.04705810546875, "loss": 1.0712, "margin_dpo/margin_mean": 107.07334899902344, "margin_dpo/margin_std": 125.93998718261719, "step": 574 }, { "KL/chosen_KL_mean": -193.12460327148438, "KL/mean": -241.726806640625, "KL/rejected_KL_mean": -290.3289794921875, "KL/std": 115.21127319335938, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0038832882419228554, "fcm_dpo/delta": 0.0277350302785635, "fcm_dpo/margin": 97.20440673828125, "fcm_dpo/q_t": 0.4131816625595093, "grad_norm": 10.753693580627441, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.4087352752685547, "logits/rejected": -0.9442593455314636, "logps/chosen": -298.89544677734375, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -391.2784423828125, "loss": 1.1085, "margin_dpo/margin_mean": 97.20440673828125, "margin_dpo/margin_std": 134.5465087890625, "step": 575 }, { "KL/chosen_KL_mean": -191.45001220703125, "KL/mean": -238.90328979492188, "KL/rejected_KL_mean": -286.3565673828125, "KL/std": 106.45429992675781, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.00401077838614583, "fcm_dpo/delta": 0.02334871143102646, "fcm_dpo/margin": 94.9065170288086, "fcm_dpo/q_t": 0.4116271734237671, "grad_norm": 10.949911117553711, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.9023017883300781, "logits/rejected": -1.8441388607025146, "logps/chosen": -272.654541015625, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -391.7659606933594, "loss": 1.1036, "margin_dpo/margin_mean": 94.9065170288086, "margin_dpo/margin_std": 126.71326446533203, "step": 576 }, { "KL/chosen_KL_mean": -171.556396484375, "KL/mean": -230.5654296875, "KL/rejected_KL_mean": -289.5744323730469, "KL/std": 115.80413818359375, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0037578102201223373, "fcm_dpo/delta": -0.058981649577617645, "fcm_dpo/margin": 118.01802062988281, "fcm_dpo/q_t": 0.39339113235473633, "grad_norm": 7.562312126159668, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.9576408863067627, "logits/rejected": -1.2548158168792725, "logps/chosen": -273.0205993652344, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -378.90081787109375, "loss": 1.0504, "margin_dpo/margin_mean": 118.01802062988281, "margin_dpo/margin_std": 132.09739685058594, "step": 577 }, { "KL/chosen_KL_mean": -204.68649291992188, "KL/mean": -247.83502197265625, "KL/rejected_KL_mean": -290.9835205078125, "KL/std": 110.45543670654297, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.003968789707869291, "fcm_dpo/delta": 0.06937086582183838, "fcm_dpo/margin": 86.29702758789062, "fcm_dpo/q_t": 0.42215654253959656, "grad_norm": 12.940128326416016, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.1704639196395874, "logits/rejected": -0.9251707196235657, "logps/chosen": -281.9789123535156, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -379.23052978515625, "loss": 1.133, "margin_dpo/margin_mean": 86.29702758789062, "margin_dpo/margin_std": 126.06503295898438, "step": 578 }, { "KL/chosen_KL_mean": -182.72462463378906, "KL/mean": -226.29354858398438, "KL/rejected_KL_mean": -269.86248779296875, "KL/std": 109.7479019165039, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004111485555768013, "fcm_dpo/delta": 0.050045885145664215, "fcm_dpo/margin": 87.13787841796875, "fcm_dpo/q_t": 0.4181913733482361, "grad_norm": 8.351226806640625, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.6601219177246094, "logits/rejected": -1.3649230003356934, "logps/chosen": -270.06768798828125, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -371.9835205078125, "loss": 1.1153, "margin_dpo/margin_mean": 87.13787841796875, "margin_dpo/margin_std": 112.15208435058594, "step": 579 }, { "KL/chosen_KL_mean": -175.22711181640625, "KL/mean": -228.91363525390625, "KL/rejected_KL_mean": -282.60015869140625, "KL/std": 109.42263793945312, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.004019011743366718, "fcm_dpo/delta": -0.043996501713991165, "fcm_dpo/margin": 107.3730239868164, "fcm_dpo/q_t": 0.39861851930618286, "grad_norm": 9.745208740234375, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.5959017276763916, "logits/rejected": -1.285590410232544, "logps/chosen": -264.1559143066406, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -371.2967529296875, "loss": 1.0682, "margin_dpo/margin_mean": 107.37303161621094, "margin_dpo/margin_std": 132.42739868164062, "step": 580 }, { "KL/chosen_KL_mean": -185.06842041015625, "KL/mean": -233.3929443359375, "KL/rejected_KL_mean": -281.71746826171875, "KL/std": 110.44488525390625, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004099488258361816, "fcm_dpo/delta": 0.004615187179297209, "fcm_dpo/margin": 96.6490478515625, "fcm_dpo/q_t": 0.4081599712371826, "grad_norm": 11.389532089233398, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.0976383686065674, "logits/rejected": -1.7584354877471924, "logps/chosen": -277.2916564941406, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -381.10693359375, "loss": 1.1054, "margin_dpo/margin_mean": 96.6490478515625, "margin_dpo/margin_std": 138.30250549316406, "step": 581 }, { "KL/chosen_KL_mean": -164.0067596435547, "KL/mean": -208.38397216796875, "KL/rejected_KL_mean": -252.7611846923828, "KL/std": 115.88804626464844, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.003949675243347883, "fcm_dpo/delta": -0.06933049857616425, "fcm_dpo/margin": 88.75440979003906, "fcm_dpo/q_t": 0.4172694683074951, "grad_norm": 9.15951156616211, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.153769016265869, "logits/rejected": -2.145334005355835, "logps/chosen": -251.34237670898438, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -367.91558837890625, "loss": 1.1357, "margin_dpo/margin_mean": 88.75440979003906, "margin_dpo/margin_std": 127.68058776855469, "step": 582 }, { "KL/chosen_KL_mean": -197.207275390625, "KL/mean": -246.26710510253906, "KL/rejected_KL_mean": -295.32696533203125, "KL/std": 108.8316650390625, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.003828838001936674, "fcm_dpo/delta": 0.02911684848368168, "fcm_dpo/margin": 98.11968994140625, "fcm_dpo/q_t": 0.4135994017124176, "grad_norm": 10.230498313903809, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4590063095092773, "logits/rejected": -1.2228822708129883, "logps/chosen": -280.92431640625, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -389.91162109375, "loss": 1.105, "margin_dpo/margin_mean": 98.11969757080078, "margin_dpo/margin_std": 128.8970947265625, "step": 583 }, { "KL/chosen_KL_mean": -175.81919860839844, "KL/mean": -227.7564697265625, "KL/rejected_KL_mean": -279.6937561035156, "KL/std": 112.84162902832031, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.003999840933829546, "fcm_dpo/delta": -0.0243501216173172, "fcm_dpo/margin": 103.87452697753906, "fcm_dpo/q_t": 0.40122729539871216, "grad_norm": 7.917211055755615, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.8282233476638794, "logits/rejected": -1.3636149168014526, "logps/chosen": -270.0787658691406, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -384.17645263671875, "loss": 1.0824, "margin_dpo/margin_mean": 103.87454223632812, "margin_dpo/margin_std": 127.54959106445312, "step": 584 }, { "KL/chosen_KL_mean": -187.14633178710938, "KL/mean": -245.1075439453125, "KL/rejected_KL_mean": -303.0687561035156, "KL/std": 129.0911407470703, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0037594614550471306, "fcm_dpo/delta": -0.04601842537522316, "fcm_dpo/margin": 115.92243194580078, "fcm_dpo/q_t": 0.3981257677078247, "grad_norm": 8.944036483764648, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.861903429031372, "logits/rejected": -1.5853607654571533, "logps/chosen": -272.7830810546875, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -393.354248046875, "loss": 1.0751, "margin_dpo/margin_mean": 115.92243957519531, "margin_dpo/margin_std": 151.68585205078125, "step": 585 }, { "KL/chosen_KL_mean": -206.81826782226562, "KL/mean": -245.75973510742188, "KL/rejected_KL_mean": -284.70123291015625, "KL/std": 112.6707763671875, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.003966891672462225, "fcm_dpo/delta": 0.1074204370379448, "fcm_dpo/margin": 77.88296508789062, "fcm_dpo/q_t": 0.43015584349632263, "grad_norm": 10.49301815032959, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.919288158416748, "logits/rejected": -1.658668875694275, "logps/chosen": -290.4993896484375, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -383.06036376953125, "loss": 1.1459, "margin_dpo/margin_mean": 77.88296508789062, "margin_dpo/margin_std": 106.9440689086914, "step": 586 }, { "KL/chosen_KL_mean": -168.79898071289062, "KL/mean": -211.0465545654297, "KL/rejected_KL_mean": -253.29409790039062, "KL/std": 108.76683807373047, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.004254631698131561, "fcm_dpo/delta": 0.04868852719664574, "fcm_dpo/margin": 84.49512481689453, "fcm_dpo/q_t": 0.41767388582229614, "grad_norm": 11.459514617919922, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.3201582431793213, "logits/rejected": -2.035637855529785, "logps/chosen": -246.37649536132812, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -342.61444091796875, "loss": 1.1142, "margin_dpo/margin_mean": 84.4951171875, "margin_dpo/margin_std": 113.68359375, "step": 587 }, { "KL/chosen_KL_mean": -183.42970275878906, "KL/mean": -222.69842529296875, "KL/rejected_KL_mean": -261.9671630859375, "KL/std": 111.39553833007812, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004442101344466209, "fcm_dpo/delta": 0.062147121876478195, "fcm_dpo/margin": 78.53741455078125, "fcm_dpo/q_t": 0.4230227470397949, "grad_norm": 15.030350685119629, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.8100414276123047, "logits/rejected": -1.7749302387237549, "logps/chosen": -267.23394775390625, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -375.0865783691406, "loss": 1.1526, "margin_dpo/margin_mean": 78.53742218017578, "margin_dpo/margin_std": 134.36380004882812, "step": 588 }, { "KL/chosen_KL_mean": -194.4432373046875, "KL/mean": -247.24388122558594, "KL/rejected_KL_mean": -300.04449462890625, "KL/std": 109.34538269042969, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004312071017920971, "fcm_dpo/delta": -0.07188694179058075, "fcm_dpo/margin": 105.60128021240234, "fcm_dpo/q_t": 0.38996413350105286, "grad_norm": 8.921002388000488, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.4974005222320557, "logits/rejected": -1.096717119216919, "logps/chosen": -290.90826416015625, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -416.08172607421875, "loss": 1.0311, "margin_dpo/margin_mean": 105.60128021240234, "margin_dpo/margin_std": 105.37413024902344, "step": 589 }, { "KL/chosen_KL_mean": -178.83346557617188, "KL/mean": -231.03274536132812, "KL/rejected_KL_mean": -283.2320556640625, "KL/std": 102.13998413085938, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004180076532065868, "fcm_dpo/delta": -0.04747690260410309, "fcm_dpo/margin": 104.39857482910156, "fcm_dpo/q_t": 0.39601701498031616, "grad_norm": 11.388259887695312, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.9363112449645996, "logits/rejected": -1.7939822673797607, "logps/chosen": -267.2613220214844, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -393.4308166503906, "loss": 1.0456, "margin_dpo/margin_mean": 104.39857482910156, "margin_dpo/margin_std": 106.64580535888672, "step": 590 }, { "KL/chosen_KL_mean": -170.48922729492188, "KL/mean": -225.77151489257812, "KL/rejected_KL_mean": -281.0538024902344, "KL/std": 109.4065170288086, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0038784565404057503, "fcm_dpo/delta": -0.03798164799809456, "fcm_dpo/margin": 110.56460571289062, "fcm_dpo/q_t": 0.396883100271225, "grad_norm": 7.080513000488281, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9342072010040283, "logits/rejected": -1.3198884725570679, "logps/chosen": -258.3388977050781, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -370.63299560546875, "loss": 1.0439, "margin_dpo/margin_mean": 110.56460571289062, "margin_dpo/margin_std": 106.34786987304688, "step": 591 }, { "KL/chosen_KL_mean": -203.6820831298828, "KL/mean": -246.32786560058594, "KL/rejected_KL_mean": -288.9736328125, "KL/std": 106.7988510131836, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004088940564543009, "fcm_dpo/delta": 0.06160195544362068, "fcm_dpo/margin": 85.29154968261719, "fcm_dpo/q_t": 0.4204869866371155, "grad_norm": 11.2477388381958, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7485032081604004, "logits/rejected": -1.4182438850402832, "logps/chosen": -278.6578369140625, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -370.5262756347656, "loss": 1.1396, "margin_dpo/margin_mean": 85.29154968261719, "margin_dpo/margin_std": 134.32858276367188, "step": 592 }, { "KL/chosen_KL_mean": -186.58702087402344, "KL/mean": -222.41665649414062, "KL/rejected_KL_mean": -258.246337890625, "KL/std": 111.2916030883789, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004399223253130913, "fcm_dpo/delta": 0.10118479281663895, "fcm_dpo/margin": 71.6593017578125, "fcm_dpo/q_t": 0.4307076930999756, "grad_norm": 15.633330345153809, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8987032175064087, "logits/rejected": -0.5988609790802002, "logps/chosen": -258.8260498046875, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -328.34283447265625, "loss": 1.1694, "margin_dpo/margin_mean": 71.6593017578125, "margin_dpo/margin_std": 124.26176452636719, "step": 593 }, { "KL/chosen_KL_mean": -165.21910095214844, "KL/mean": -221.832275390625, "KL/rejected_KL_mean": -278.4454650878906, "KL/std": 108.50535583496094, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.00429191067814827, "fcm_dpo/delta": -0.11289025843143463, "fcm_dpo/margin": 113.22634887695312, "fcm_dpo/q_t": 0.3814272880554199, "grad_norm": 9.776017189025879, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9055101871490479, "logits/rejected": -1.377734899520874, "logps/chosen": -255.6568145751953, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -370.7130432128906, "loss": 1.01, "margin_dpo/margin_mean": 113.22634887695312, "margin_dpo/margin_std": 104.90364074707031, "step": 594 }, { "KL/chosen_KL_mean": -178.665283203125, "KL/mean": -224.57638549804688, "KL/rejected_KL_mean": -270.48748779296875, "KL/std": 108.58784484863281, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0041984920389950275, "fcm_dpo/delta": 0.016737869009375572, "fcm_dpo/margin": 91.82220458984375, "fcm_dpo/q_t": 0.41156378388404846, "grad_norm": 9.676054954528809, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.7353259325027466, "logits/rejected": -1.3244211673736572, "logps/chosen": -260.1800842285156, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -357.0079650878906, "loss": 1.1029, "margin_dpo/margin_mean": 91.82220458984375, "margin_dpo/margin_std": 124.55223083496094, "step": 595 }, { "KL/chosen_KL_mean": -185.54888916015625, "KL/mean": -227.34774780273438, "KL/rejected_KL_mean": -269.1466064453125, "KL/std": 107.7799072265625, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.00432699266821146, "fcm_dpo/delta": 0.04621616005897522, "fcm_dpo/margin": 83.59773254394531, "fcm_dpo/q_t": 0.4186498522758484, "grad_norm": 13.536149024963379, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.3702822923660278, "logits/rejected": -0.761730432510376, "logps/chosen": -273.7569580078125, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -355.2403564453125, "loss": 1.1389, "margin_dpo/margin_mean": 83.59772491455078, "margin_dpo/margin_std": 135.8723602294922, "step": 596 }, { "KL/chosen_KL_mean": -179.35340881347656, "KL/mean": -234.5875244140625, "KL/rejected_KL_mean": -289.8216552734375, "KL/std": 115.89057922363281, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.004081946797668934, "fcm_dpo/delta": -0.06790487468242645, "fcm_dpo/margin": 110.46826171875, "fcm_dpo/q_t": 0.3926623463630676, "grad_norm": 9.030370712280273, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.4170469045639038, "logits/rejected": -1.3022372722625732, "logps/chosen": -262.9433898925781, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -396.76605224609375, "loss": 1.0467, "margin_dpo/margin_mean": 110.46826171875, "margin_dpo/margin_std": 125.30593872070312, "step": 597 }, { "KL/chosen_KL_mean": -171.83316040039062, "KL/mean": -218.16253662109375, "KL/rejected_KL_mean": -264.4919128417969, "KL/std": 108.0069808959961, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0040701148100197315, "fcm_dpo/delta": 0.02704760432243347, "fcm_dpo/margin": 92.65876770019531, "fcm_dpo/q_t": 0.4119781255722046, "grad_norm": 9.677848815917969, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -2.06377911567688, "logits/rejected": -1.757104754447937, "logps/chosen": -262.0157775878906, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -359.29046630859375, "loss": 1.1078, "margin_dpo/margin_mean": 92.65876770019531, "margin_dpo/margin_std": 122.48324584960938, "step": 598 }, { "KL/chosen_KL_mean": -174.50347900390625, "KL/mean": -215.53561401367188, "KL/rejected_KL_mean": -256.5677490234375, "KL/std": 112.43232727050781, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0042659817263484, "fcm_dpo/delta": 0.05947946012020111, "fcm_dpo/margin": 82.06429290771484, "fcm_dpo/q_t": 0.4201176166534424, "grad_norm": 9.322607040405273, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.6233630180358887, "logits/rejected": -1.5014293193817139, "logps/chosen": -252.3457489013672, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -345.765380859375, "loss": 1.1388, "margin_dpo/margin_mean": 82.06429290771484, "margin_dpo/margin_std": 124.67076110839844, "step": 599 }, { "KL/chosen_KL_mean": -195.08737182617188, "KL/mean": -241.5779571533203, "KL/rejected_KL_mean": -288.06854248046875, "KL/std": 106.41990661621094, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004411207512021065, "fcm_dpo/delta": -0.012797925621271133, "fcm_dpo/margin": 92.9811782836914, "fcm_dpo/q_t": 0.40466082096099854, "grad_norm": 9.498991966247559, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.1530280113220215, "logits/rejected": -1.7921597957611084, "logps/chosen": -287.27520751953125, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -381.0228271484375, "loss": 1.0807, "margin_dpo/margin_mean": 92.9811782836914, "margin_dpo/margin_std": 115.85541534423828, "step": 600 }, { "KL/chosen_KL_mean": -196.66270446777344, "KL/mean": -246.177734375, "KL/rejected_KL_mean": -295.69281005859375, "KL/std": 108.30654907226562, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004220297560095787, "fcm_dpo/delta": -0.02477908879518509, "fcm_dpo/margin": 99.03010559082031, "fcm_dpo/q_t": 0.4006590247154236, "grad_norm": 9.440180778503418, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7232615947723389, "logits/rejected": -1.3968720436096191, "logps/chosen": -285.87884521484375, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -395.86334228515625, "loss": 1.0575, "margin_dpo/margin_mean": 99.03010559082031, "margin_dpo/margin_std": 103.40138244628906, "step": 601 }, { "KL/chosen_KL_mean": -156.29562377929688, "KL/mean": -220.64752197265625, "KL/rejected_KL_mean": -284.9994201660156, "KL/std": 118.40528869628906, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0038689197972416878, "fcm_dpo/delta": -0.1312892585992813, "fcm_dpo/margin": 128.70379638671875, "fcm_dpo/q_t": 0.3786994218826294, "grad_norm": 8.73089599609375, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.06764554977417, "logits/rejected": -1.8054910898208618, "logps/chosen": -236.3532257080078, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -378.1969299316406, "loss": 1.0128, "margin_dpo/margin_mean": 128.70379638671875, "margin_dpo/margin_std": 135.51797485351562, "step": 602 }, { "KL/chosen_KL_mean": -166.4561767578125, "KL/mean": -219.6211395263672, "KL/rejected_KL_mean": -272.7861328125, "KL/std": 106.99681854248047, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0037628458812832832, "fcm_dpo/delta": -0.00022720824927091599, "fcm_dpo/margin": 106.3299560546875, "fcm_dpo/q_t": 0.4061580300331116, "grad_norm": 7.664916515350342, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.151101589202881, "logits/rejected": -1.5744508504867554, "logps/chosen": -268.7657470703125, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -380.85498046875, "loss": 1.0761, "margin_dpo/margin_mean": 106.3299560546875, "margin_dpo/margin_std": 123.28419494628906, "step": 603 }, { "KL/chosen_KL_mean": -173.93922424316406, "KL/mean": -228.7691650390625, "KL/rejected_KL_mean": -283.59912109375, "KL/std": 112.32112121582031, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.003674921113997698, "fcm_dpo/delta": -0.004895247519016266, "fcm_dpo/margin": 109.65989685058594, "fcm_dpo/q_t": 0.40347927808761597, "grad_norm": 8.060246467590332, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.7026365995407104, "logits/rejected": -1.5508904457092285, "logps/chosen": -252.11331176757812, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -383.09521484375, "loss": 1.0675, "margin_dpo/margin_mean": 109.65989685058594, "margin_dpo/margin_std": 115.06400299072266, "step": 604 }, { "KL/chosen_KL_mean": -190.24703979492188, "KL/mean": -235.107666015625, "KL/rejected_KL_mean": -279.9682922363281, "KL/std": 99.81083679199219, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0038959209341555834, "fcm_dpo/delta": 0.06114455312490463, "fcm_dpo/margin": 89.72125244140625, "fcm_dpo/q_t": 0.4198763370513916, "grad_norm": 9.095611572265625, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8481495380401611, "logits/rejected": -1.3553366661071777, "logps/chosen": -285.0203552246094, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -387.273193359375, "loss": 1.1133, "margin_dpo/margin_mean": 89.72125244140625, "margin_dpo/margin_std": 112.92613983154297, "step": 605 }, { "KL/chosen_KL_mean": -183.04666137695312, "KL/mean": -229.88568115234375, "KL/rejected_KL_mean": -276.72467041015625, "KL/std": 109.65816497802734, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004116829950362444, "fcm_dpo/delta": 0.013837818056344986, "fcm_dpo/margin": 93.67800903320312, "fcm_dpo/q_t": 0.409184992313385, "grad_norm": 8.682604789733887, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.8666157722473145, "logits/rejected": -1.482033371925354, "logps/chosen": -270.800048828125, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -380.9668884277344, "loss": 1.0965, "margin_dpo/margin_mean": 93.67800903320312, "margin_dpo/margin_std": 117.37605285644531, "step": 606 }, { "KL/chosen_KL_mean": -177.29498291015625, "KL/mean": -230.63784790039062, "KL/rejected_KL_mean": -283.980712890625, "KL/std": 117.51255798339844, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0039460668340325356, "fcm_dpo/delta": -0.026557695120573044, "fcm_dpo/margin": 106.68572998046875, "fcm_dpo/q_t": 0.4017018675804138, "grad_norm": 11.707988739013672, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.4765821695327759, "logits/rejected": -0.9246842265129089, "logps/chosen": -265.6240234375, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -370.7488098144531, "loss": 1.0849, "margin_dpo/margin_mean": 106.68572235107422, "margin_dpo/margin_std": 143.82960510253906, "step": 607 }, { "KL/chosen_KL_mean": -162.1451416015625, "KL/mean": -217.11334228515625, "KL/rejected_KL_mean": -272.08154296875, "KL/std": 115.51004028320312, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0037355171516537666, "fcm_dpo/delta": -0.018896538764238358, "fcm_dpo/margin": 109.93641662597656, "fcm_dpo/q_t": 0.40140485763549805, "grad_norm": 8.350780487060547, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.9920448064804077, "logits/rejected": -1.70081627368927, "logps/chosen": -247.91452026367188, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -372.31439208984375, "loss": 1.0755, "margin_dpo/margin_mean": 109.93641662597656, "margin_dpo/margin_std": 126.38310241699219, "step": 608 }, { "KL/chosen_KL_mean": -180.6356658935547, "KL/mean": -232.78399658203125, "KL/rejected_KL_mean": -284.93231201171875, "KL/std": 116.54350280761719, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.003774629207327962, "fcm_dpo/delta": 0.00595618411898613, "fcm_dpo/margin": 104.29660034179688, "fcm_dpo/q_t": 0.40791359543800354, "grad_norm": 10.141780853271484, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.064966917037964, "logits/rejected": -1.6305575370788574, "logps/chosen": -273.60223388671875, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -380.8504638671875, "loss": 1.0889, "margin_dpo/margin_mean": 104.29660034179688, "margin_dpo/margin_std": 123.11428833007812, "step": 609 }, { "KL/chosen_KL_mean": -165.69210815429688, "KL/mean": -220.9575958251953, "KL/rejected_KL_mean": -276.22308349609375, "KL/std": 110.83015441894531, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0037607711274176836, "fcm_dpo/delta": -0.020632430911064148, "fcm_dpo/margin": 110.53097534179688, "fcm_dpo/q_t": 0.40036845207214355, "grad_norm": 8.114296913146973, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.7294440269470215, "logits/rejected": -1.184781551361084, "logps/chosen": -260.39239501953125, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -365.91046142578125, "loss": 1.06, "margin_dpo/margin_mean": 110.53097534179688, "margin_dpo/margin_std": 117.20445251464844, "step": 610 }, { "KL/chosen_KL_mean": -174.9722442626953, "KL/mean": -229.29006958007812, "KL/rejected_KL_mean": -283.6078796386719, "KL/std": 115.04771423339844, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.003816768527030945, "fcm_dpo/delta": -0.020454049110412598, "fcm_dpo/margin": 108.63562774658203, "fcm_dpo/q_t": 0.4019621014595032, "grad_norm": 9.767206192016602, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.176180839538574, "logits/rejected": -1.8389451503753662, "logps/chosen": -261.7555847167969, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -373.4509582519531, "loss": 1.0642, "margin_dpo/margin_mean": 108.63562774658203, "margin_dpo/margin_std": 117.04779815673828, "step": 611 }, { "KL/chosen_KL_mean": -192.77597045898438, "KL/mean": -247.89263916015625, "KL/rejected_KL_mean": -303.00927734375, "KL/std": 108.60649108886719, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0036667552776634693, "fcm_dpo/delta": -0.005497426725924015, "fcm_dpo/margin": 110.23329162597656, "fcm_dpo/q_t": 0.40423545241355896, "grad_norm": 8.42858600616455, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.9850564002990723, "logits/rejected": -1.130592703819275, "logps/chosen": -293.796142578125, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -396.79229736328125, "loss": 1.0639, "margin_dpo/margin_mean": 110.23329162597656, "margin_dpo/margin_std": 114.10325622558594, "step": 612 }, { "KL/chosen_KL_mean": -154.61459350585938, "KL/mean": -217.0794677734375, "KL/rejected_KL_mean": -279.5443420410156, "KL/std": 111.27737426757812, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0035167327150702477, "fcm_dpo/delta": -0.051379963755607605, "fcm_dpo/margin": 124.92976379394531, "fcm_dpo/q_t": 0.3931143879890442, "grad_norm": 7.625861167907715, "learning_rate": 1.551886292185553e-08, "logits/chosen": -2.040809154510498, "logits/rejected": -1.8472626209259033, "logps/chosen": -243.60321044921875, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -389.53985595703125, "loss": 1.0295, "margin_dpo/margin_mean": 124.92974853515625, "margin_dpo/margin_std": 109.30656433105469, "step": 613 }, { "KL/chosen_KL_mean": -193.5235595703125, "KL/mean": -243.984375, "KL/rejected_KL_mean": -294.4451904296875, "KL/std": 105.37423706054688, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.003604646772146225, "fcm_dpo/delta": 0.04435862973332405, "fcm_dpo/margin": 100.9216079711914, "fcm_dpo/q_t": 0.41553831100463867, "grad_norm": 9.88770580291748, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.3460655212402344, "logits/rejected": -1.339262843132019, "logps/chosen": -273.7236328125, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -404.3075866699219, "loss": 1.1085, "margin_dpo/margin_mean": 100.92160034179688, "margin_dpo/margin_std": 129.11671447753906, "step": 614 }, { "KL/chosen_KL_mean": -179.91796875, "KL/mean": -221.37899780273438, "KL/rejected_KL_mean": -262.84002685546875, "KL/std": 106.55502319335938, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0036438931711018085, "fcm_dpo/delta": -0.008691830560564995, "fcm_dpo/margin": 82.92204284667969, "fcm_dpo/q_t": 0.427216112613678, "grad_norm": 8.394569396972656, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.8370261192321777, "logits/rejected": -1.4437079429626465, "logps/chosen": -280.3532409667969, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -364.02008056640625, "loss": 1.1437, "margin_dpo/margin_mean": 82.92203521728516, "margin_dpo/margin_std": 102.05447387695312, "step": 615 }, { "KL/chosen_KL_mean": -183.62240600585938, "KL/mean": -240.22479248046875, "KL/rejected_KL_mean": -296.8271484375, "KL/std": 106.3695068359375, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.003650183556601405, "fcm_dpo/delta": -0.017906587570905685, "fcm_dpo/margin": 113.20475769042969, "fcm_dpo/q_t": 0.4018813371658325, "grad_norm": 7.842961311340332, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.9686646461486816, "logits/rejected": -1.4860084056854248, "logps/chosen": -276.1153564453125, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -378.8878173828125, "loss": 1.0663, "margin_dpo/margin_mean": 113.20475769042969, "margin_dpo/margin_std": 125.05500793457031, "step": 616 }, { "KL/chosen_KL_mean": -190.85256958007812, "KL/mean": -235.85400390625, "KL/rejected_KL_mean": -280.85546875, "KL/std": 104.0780029296875, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0037637026980519295, "fcm_dpo/delta": 0.07367709279060364, "fcm_dpo/margin": 90.00287628173828, "fcm_dpo/q_t": 0.4221077263355255, "grad_norm": 8.967413902282715, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.5459469556808472, "logits/rejected": -0.9002398252487183, "logps/chosen": -281.9495849609375, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -351.2655029296875, "loss": 1.1227, "margin_dpo/margin_mean": 90.00288391113281, "margin_dpo/margin_std": 116.38085174560547, "step": 617 }, { "KL/chosen_KL_mean": -192.59060668945312, "KL/mean": -243.29434204101562, "KL/rejected_KL_mean": -293.9980773925781, "KL/std": 115.40835571289062, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0038488677237182856, "fcm_dpo/delta": 0.011981149204075336, "fcm_dpo/margin": 101.40750122070312, "fcm_dpo/q_t": 0.40803104639053345, "grad_norm": 8.09562873840332, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.017791271209717, "logits/rejected": -1.6545506715774536, "logps/chosen": -294.6112060546875, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -393.79925537109375, "loss": 1.092, "margin_dpo/margin_mean": 101.4074935913086, "margin_dpo/margin_std": 127.61875915527344, "step": 618 }, { "KL/chosen_KL_mean": -187.60089111328125, "KL/mean": -232.1138916015625, "KL/rejected_KL_mean": -276.62689208984375, "KL/std": 113.59451293945312, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004028439521789551, "fcm_dpo/delta": 0.05038651451468468, "fcm_dpo/margin": 89.02603149414062, "fcm_dpo/q_t": 0.4196791350841522, "grad_norm": 9.19076919555664, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.5153785943984985, "logits/rejected": -1.1529409885406494, "logps/chosen": -277.3422546386719, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -376.5282897949219, "loss": 1.1299, "margin_dpo/margin_mean": 89.02603149414062, "margin_dpo/margin_std": 136.4460906982422, "step": 619 }, { "KL/chosen_KL_mean": -178.0783233642578, "KL/mean": -227.55520629882812, "KL/rejected_KL_mean": -277.0321044921875, "KL/std": 119.09371948242188, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.003717872779816389, "fcm_dpo/delta": -0.09368382394313812, "fcm_dpo/margin": 98.95375061035156, "fcm_dpo/q_t": 0.4086172878742218, "grad_norm": 8.851722717285156, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.081454277038574, "logits/rejected": -1.547012209892273, "logps/chosen": -263.20263671875, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -362.44464111328125, "loss": 1.1127, "margin_dpo/margin_mean": 98.95375061035156, "margin_dpo/margin_std": 128.03948974609375, "step": 620 }, { "KL/chosen_KL_mean": -182.5325469970703, "KL/mean": -230.86085510253906, "KL/rejected_KL_mean": -279.18914794921875, "KL/std": 114.67340087890625, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0036109895445406437, "fcm_dpo/delta": -0.05921613797545433, "fcm_dpo/margin": 96.65660858154297, "fcm_dpo/q_t": 0.4160424470901489, "grad_norm": 8.411218643188477, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.9587397575378418, "logits/rejected": -1.696195363998413, "logps/chosen": -271.7809753417969, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -374.65380859375, "loss": 1.1329, "margin_dpo/margin_mean": 96.6566162109375, "margin_dpo/margin_std": 134.06405639648438, "step": 621 }, { "KL/chosen_KL_mean": -179.16458129882812, "KL/mean": -230.34756469726562, "KL/rejected_KL_mean": -281.530517578125, "KL/std": 102.33671569824219, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0035848021507263184, "fcm_dpo/delta": 0.04053201526403427, "fcm_dpo/margin": 102.36595153808594, "fcm_dpo/q_t": 0.41414734721183777, "grad_norm": 9.29975414276123, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.5279839038848877, "logits/rejected": -0.876506507396698, "logps/chosen": -278.9587097167969, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -372.3587646484375, "loss": 1.0916, "margin_dpo/margin_mean": 102.36595153808594, "margin_dpo/margin_std": 110.90505981445312, "step": 622 }, { "KL/chosen_KL_mean": -187.873046875, "KL/mean": -224.1385498046875, "KL/rejected_KL_mean": -260.404052734375, "KL/std": 111.29364776611328, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0037934817373752594, "fcm_dpo/delta": 0.03881670907139778, "fcm_dpo/margin": 72.531005859375, "fcm_dpo/q_t": 0.437284380197525, "grad_norm": 9.216378211975098, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.2542662620544434, "logits/rejected": -1.8190817832946777, "logps/chosen": -278.3285827636719, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -351.726806640625, "loss": 1.1956, "margin_dpo/margin_mean": 72.531005859375, "margin_dpo/margin_std": 137.27642822265625, "step": 623 }, { "KL/chosen_KL_mean": -193.28582763671875, "KL/mean": -251.74313354492188, "KL/rejected_KL_mean": -310.2004699707031, "KL/std": 110.03108215332031, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0036795325577259064, "fcm_dpo/delta": -0.038604382425546646, "fcm_dpo/margin": 116.91461944580078, "fcm_dpo/q_t": 0.39747557044029236, "grad_norm": 9.757772445678711, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.5461580753326416, "logits/rejected": -1.6346063613891602, "logps/chosen": -303.16107177734375, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -414.97369384765625, "loss": 1.0528, "margin_dpo/margin_mean": 116.91461944580078, "margin_dpo/margin_std": 126.3757553100586, "step": 624 }, { "KL/chosen_KL_mean": -200.03021240234375, "KL/mean": -241.8419189453125, "KL/rejected_KL_mean": -283.65362548828125, "KL/std": 108.85662841796875, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0039311484433710575, "fcm_dpo/delta": 0.08333711326122284, "fcm_dpo/margin": 83.6234130859375, "fcm_dpo/q_t": 0.4252508282661438, "grad_norm": 8.675471305847168, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.759660005569458, "logits/rejected": -1.3879314661026, "logps/chosen": -287.1983642578125, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -375.5151062011719, "loss": 1.1341, "margin_dpo/margin_mean": 83.62340545654297, "margin_dpo/margin_std": 116.25682067871094, "step": 625 }, { "KL/chosen_KL_mean": -189.37950134277344, "KL/mean": -231.24563598632812, "KL/rejected_KL_mean": -273.11175537109375, "KL/std": 120.64944458007812, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.004196333698928356, "fcm_dpo/delta": 0.05720204859972, "fcm_dpo/margin": 83.73226928710938, "fcm_dpo/q_t": 0.42099034786224365, "grad_norm": 11.4525785446167, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.340951681137085, "logits/rejected": -0.901903510093689, "logps/chosen": -293.60369873046875, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -377.28924560546875, "loss": 1.1372, "margin_dpo/margin_mean": 83.73226928710938, "margin_dpo/margin_std": 131.2925262451172, "step": 626 }, { "KL/chosen_KL_mean": -168.24758911132812, "KL/mean": -221.35076904296875, "KL/rejected_KL_mean": -274.45391845703125, "KL/std": 123.96116638183594, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004074117634445429, "fcm_dpo/delta": -0.041749387979507446, "fcm_dpo/margin": 106.20633697509766, "fcm_dpo/q_t": 0.3996013104915619, "grad_norm": 7.538987636566162, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.5892834663391113, "logits/rejected": -1.7373359203338623, "logps/chosen": -243.1822052001953, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -387.02679443359375, "loss": 1.0708, "margin_dpo/margin_mean": 106.20634460449219, "margin_dpo/margin_std": 136.67178344726562, "step": 627 }, { "KL/chosen_KL_mean": -178.50091552734375, "KL/mean": -232.77456665039062, "KL/rejected_KL_mean": -287.0482177734375, "KL/std": 111.74896240234375, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.003927373327314854, "fcm_dpo/delta": -0.03348799794912338, "fcm_dpo/margin": 108.54729461669922, "fcm_dpo/q_t": 0.3993317484855652, "grad_norm": 9.689674377441406, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.059016704559326, "logits/rejected": -1.8010170459747314, "logps/chosen": -272.1920166015625, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -394.39215087890625, "loss": 1.0627, "margin_dpo/margin_mean": 108.54728698730469, "margin_dpo/margin_std": 125.42695617675781, "step": 628 }, { "KL/chosen_KL_mean": -191.4917755126953, "KL/mean": -241.22293090820312, "KL/rejected_KL_mean": -290.9541015625, "KL/std": 112.29240417480469, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0038855532184243202, "fcm_dpo/delta": 0.016064152121543884, "fcm_dpo/margin": 99.46231079101562, "fcm_dpo/q_t": 0.40909695625305176, "grad_norm": 9.078751564025879, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.0830211639404297, "logits/rejected": -1.6512298583984375, "logps/chosen": -294.72216796875, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -388.12249755859375, "loss": 1.0897, "margin_dpo/margin_mean": 99.46231079101562, "margin_dpo/margin_std": 119.22362518310547, "step": 629 }, { "KL/chosen_KL_mean": -174.38153076171875, "KL/mean": -206.2703399658203, "KL/rejected_KL_mean": -238.1591796875, "KL/std": 104.59029388427734, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0040490999817848206, "fcm_dpo/delta": 0.044404536485672, "fcm_dpo/margin": 63.77765655517578, "fcm_dpo/q_t": 0.4400923550128937, "grad_norm": 11.104445457458496, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.172636032104492, "logits/rejected": -1.6638758182525635, "logps/chosen": -268.279052734375, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -327.5335693359375, "loss": 1.2007, "margin_dpo/margin_mean": 63.77765655517578, "margin_dpo/margin_std": 118.41985321044922, "step": 630 }, { "KL/chosen_KL_mean": -172.2218780517578, "KL/mean": -220.09315490722656, "KL/rejected_KL_mean": -267.96441650390625, "KL/std": 104.09918212890625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.004133149981498718, "fcm_dpo/delta": 0.0052426280453801155, "fcm_dpo/margin": 95.7425537109375, "fcm_dpo/q_t": 0.40726393461227417, "grad_norm": 8.122526168823242, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.541438579559326, "logits/rejected": -2.1196670532226562, "logps/chosen": -259.9927062988281, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -356.6468505859375, "loss": 1.0879, "margin_dpo/margin_mean": 95.7425537109375, "margin_dpo/margin_std": 119.77947998046875, "step": 631 }, { "KL/chosen_KL_mean": -169.8888702392578, "KL/mean": -222.41021728515625, "KL/rejected_KL_mean": -274.9315490722656, "KL/std": 108.78047943115234, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0039759124629199505, "fcm_dpo/delta": -0.02622455358505249, "fcm_dpo/margin": 105.04267883300781, "fcm_dpo/q_t": 0.40170058608055115, "grad_norm": 10.80182933807373, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.9632394313812256, "logits/rejected": -1.5323657989501953, "logps/chosen": -258.515380859375, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -366.3824462890625, "loss": 1.0809, "margin_dpo/margin_mean": 105.04267883300781, "margin_dpo/margin_std": 133.74151611328125, "step": 632 }, { "KL/chosen_KL_mean": -210.071044921875, "KL/mean": -240.69473266601562, "KL/rejected_KL_mean": -271.31842041015625, "KL/std": 110.54777526855469, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.004107899032533169, "fcm_dpo/delta": 0.026708204299211502, "fcm_dpo/margin": 61.247413635253906, "fcm_dpo/q_t": 0.4437352418899536, "grad_norm": 10.903070449829102, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.111607313156128, "logits/rejected": -1.9197615385055542, "logps/chosen": -291.4454650878906, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -369.94415283203125, "loss": 1.2391, "margin_dpo/margin_mean": 61.24741744995117, "margin_dpo/margin_std": 148.76986694335938, "step": 633 }, { "KL/chosen_KL_mean": -193.47607421875, "KL/mean": -233.3980255126953, "KL/rejected_KL_mean": -273.3199462890625, "KL/std": 100.50273895263672, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.004083740524947643, "fcm_dpo/delta": -0.03870100900530815, "fcm_dpo/margin": 79.84388732910156, "fcm_dpo/q_t": 0.42173707485198975, "grad_norm": 10.498061180114746, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.3660235404968262, "logits/rejected": -1.194472312927246, "logps/chosen": -289.59893798828125, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -386.16778564453125, "loss": 1.1351, "margin_dpo/margin_mean": 79.84388732910156, "margin_dpo/margin_std": 102.61000061035156, "step": 634 }, { "KL/chosen_KL_mean": -202.178466796875, "KL/mean": -233.14056396484375, "KL/rejected_KL_mean": -264.1026306152344, "KL/std": 110.21914672851562, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0040747979655861855, "fcm_dpo/delta": 0.03439530357718468, "fcm_dpo/margin": 61.924190521240234, "fcm_dpo/q_t": 0.4424029290676117, "grad_norm": 11.974666595458984, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.8775907754898071, "logits/rejected": -1.3463542461395264, "logps/chosen": -300.8625793457031, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -354.0018310546875, "loss": 1.2197, "margin_dpo/margin_mean": 61.92418670654297, "margin_dpo/margin_std": 132.546630859375, "step": 635 }, { "KL/chosen_KL_mean": -179.7556610107422, "KL/mean": -232.82977294921875, "KL/rejected_KL_mean": -285.90386962890625, "KL/std": 112.44625854492188, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004053937271237373, "fcm_dpo/delta": -0.03855369612574577, "fcm_dpo/margin": 106.14818572998047, "fcm_dpo/q_t": 0.39769119024276733, "grad_norm": 9.280352592468262, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.4339473247528076, "logits/rejected": -0.9476470351219177, "logps/chosen": -270.1736145019531, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -373.6107482910156, "loss": 1.0515, "margin_dpo/margin_mean": 106.14818572998047, "margin_dpo/margin_std": 112.72525024414062, "step": 636 }, { "KL/chosen_KL_mean": -181.00131225585938, "KL/mean": -237.64810180664062, "KL/rejected_KL_mean": -294.2948913574219, "KL/std": 125.03131103515625, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0038698329590260983, "fcm_dpo/delta": -0.04908343404531479, "fcm_dpo/margin": 113.2936019897461, "fcm_dpo/q_t": 0.3948747515678406, "grad_norm": 7.89529275894165, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.9948451519012451, "logits/rejected": -1.893082618713379, "logps/chosen": -263.4510192871094, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -398.323486328125, "loss": 1.0468, "margin_dpo/margin_mean": 113.2936019897461, "margin_dpo/margin_std": 118.64928436279297, "step": 637 }, { "KL/chosen_KL_mean": -169.70547485351562, "KL/mean": -224.96200561523438, "KL/rejected_KL_mean": -280.2185363769531, "KL/std": 111.0159912109375, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.003812754061073065, "fcm_dpo/delta": -0.029070258140563965, "fcm_dpo/margin": 110.5130615234375, "fcm_dpo/q_t": 0.3985944986343384, "grad_norm": 8.200310707092285, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.6119685173034668, "logits/rejected": -0.9354720115661621, "logps/chosen": -261.6304626464844, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -366.50555419921875, "loss": 1.0497, "margin_dpo/margin_mean": 110.51306915283203, "margin_dpo/margin_std": 103.50019073486328, "step": 638 }, { "KL/chosen_KL_mean": -224.64617919921875, "KL/mean": -250.35079956054688, "KL/rejected_KL_mean": -276.055419921875, "KL/std": 103.21662902832031, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0038381130434572697, "fcm_dpo/delta": 0.07810951769351959, "fcm_dpo/margin": 51.40925598144531, "fcm_dpo/q_t": 0.454649418592453, "grad_norm": 8.877825736999512, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.4186136722564697, "logits/rejected": -0.6594133377075195, "logps/chosen": -329.1737060546875, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -357.5357666015625, "loss": 1.2564, "margin_dpo/margin_mean": 51.40925979614258, "margin_dpo/margin_std": 128.70448303222656, "step": 639 }, { "KL/chosen_KL_mean": -191.51063537597656, "KL/mean": -241.23934936523438, "KL/rejected_KL_mean": -290.968017578125, "KL/std": 111.6656494140625, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.004119779448956251, "fcm_dpo/delta": -0.02093707025051117, "fcm_dpo/margin": 99.45740509033203, "fcm_dpo/q_t": 0.40178513526916504, "grad_norm": 13.306367874145508, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.843888282775879, "logits/rejected": -1.3291130065917969, "logps/chosen": -289.3959045410156, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -403.6730651855469, "loss": 1.0829, "margin_dpo/margin_mean": 99.45741271972656, "margin_dpo/margin_std": 112.31251525878906, "step": 640 }, { "KL/chosen_KL_mean": -174.7601776123047, "KL/mean": -226.59732055664062, "KL/rejected_KL_mean": -278.43450927734375, "KL/std": 129.5594482421875, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0039021826814860106, "fcm_dpo/delta": -0.005742413457483053, "fcm_dpo/margin": 103.67433166503906, "fcm_dpo/q_t": 0.4084070920944214, "grad_norm": 8.369839668273926, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.2399630546569824, "logits/rejected": -1.6922959089279175, "logps/chosen": -271.2058410644531, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -373.5701904296875, "loss": 1.0933, "margin_dpo/margin_mean": 103.67433166503906, "margin_dpo/margin_std": 144.27401733398438, "step": 641 }, { "KL/chosen_KL_mean": -193.90155029296875, "KL/mean": -242.21580505371094, "KL/rejected_KL_mean": -290.5300598144531, "KL/std": 111.63656616210938, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.003986832685768604, "fcm_dpo/delta": 0.01716899685561657, "fcm_dpo/margin": 96.62850952148438, "fcm_dpo/q_t": 0.4126904010772705, "grad_norm": 10.790578842163086, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.52507483959198, "logits/rejected": -1.192151427268982, "logps/chosen": -294.661376953125, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -405.2377014160156, "loss": 1.1091, "margin_dpo/margin_mean": 96.62850952148438, "margin_dpo/margin_std": 139.56434631347656, "step": 642 }, { "KL/chosen_KL_mean": -183.42486572265625, "KL/mean": -229.60354614257812, "KL/rejected_KL_mean": -275.7822265625, "KL/std": 103.52198791503906, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.003944868221879005, "fcm_dpo/delta": 0.04044123739004135, "fcm_dpo/margin": 92.3573226928711, "fcm_dpo/q_t": 0.41507256031036377, "grad_norm": 11.705669403076172, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.1813697814941406, "logits/rejected": -1.8098371028900146, "logps/chosen": -268.16851806640625, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -370.10064697265625, "loss": 1.1152, "margin_dpo/margin_mean": 92.3573226928711, "margin_dpo/margin_std": 119.23394775390625, "step": 643 }, { "KL/chosen_KL_mean": -190.00430297851562, "KL/mean": -227.37820434570312, "KL/rejected_KL_mean": -264.7521057128906, "KL/std": 104.83724975585938, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004394426476210356, "fcm_dpo/delta": 0.08517163246870041, "fcm_dpo/margin": 74.74778747558594, "fcm_dpo/q_t": 0.4259120523929596, "grad_norm": 11.558446884155273, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.88118314743042, "logits/rejected": -1.6167457103729248, "logps/chosen": -275.3270568847656, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -359.3607177734375, "loss": 1.1439, "margin_dpo/margin_mean": 74.74778747558594, "margin_dpo/margin_std": 113.71812438964844, "step": 644 }, { "KL/chosen_KL_mean": -199.62762451171875, "KL/mean": -224.5467529296875, "KL/rejected_KL_mean": -249.46588134765625, "KL/std": 113.48255920410156, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.00463274959474802, "fcm_dpo/delta": 0.06644649058580399, "fcm_dpo/margin": 49.83824157714844, "fcm_dpo/q_t": 0.44848859310150146, "grad_norm": 11.248617172241211, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.408837914466858, "logits/rejected": -1.2418994903564453, "logps/chosen": -282.2178955078125, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -352.4696350097656, "loss": 1.2535, "margin_dpo/margin_mean": 49.83824157714844, "margin_dpo/margin_std": 129.0828857421875, "step": 645 }, { "KL/chosen_KL_mean": -177.1444091796875, "KL/mean": -226.27700805664062, "KL/rejected_KL_mean": -275.4096374511719, "KL/std": 98.77959442138672, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.004469218663871288, "fcm_dpo/delta": -0.05554269999265671, "fcm_dpo/margin": 98.2652359008789, "fcm_dpo/q_t": 0.39431098103523254, "grad_norm": 9.642194747924805, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.7415103912353516, "logits/rejected": -1.1314187049865723, "logps/chosen": -268.33642578125, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -359.1879577636719, "loss": 1.0519, "margin_dpo/margin_mean": 98.2652359008789, "margin_dpo/margin_std": 107.87950897216797, "step": 646 }, { "KL/chosen_KL_mean": -191.4246063232422, "KL/mean": -234.71511840820312, "KL/rejected_KL_mean": -278.005615234375, "KL/std": 119.36912536621094, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004551664460450411, "fcm_dpo/delta": 0.007363822776824236, "fcm_dpo/margin": 86.58102416992188, "fcm_dpo/q_t": 0.4110236167907715, "grad_norm": 11.356980323791504, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.4603995084762573, "logits/rejected": -1.3231842517852783, "logps/chosen": -280.26904296875, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -377.50396728515625, "loss": 1.1154, "margin_dpo/margin_mean": 86.58102416992188, "margin_dpo/margin_std": 132.5184326171875, "step": 647 }, { "KL/chosen_KL_mean": -179.94142150878906, "KL/mean": -239.11666870117188, "KL/rejected_KL_mean": -298.2919006347656, "KL/std": 122.26051330566406, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.00419621029868722, "fcm_dpo/delta": -0.12785784900188446, "fcm_dpo/margin": 118.3504867553711, "fcm_dpo/q_t": 0.3796168267726898, "grad_norm": 10.65594482421875, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.9941062927246094, "logits/rejected": -1.9229565858840942, "logps/chosen": -267.7895202636719, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -402.96197509765625, "loss": 1.0222, "margin_dpo/margin_mean": 118.35047912597656, "margin_dpo/margin_std": 129.99365234375, "step": 648 }, { "KL/chosen_KL_mean": -163.90948486328125, "KL/mean": -214.38482666015625, "KL/rejected_KL_mean": -264.860107421875, "KL/std": 106.59906005859375, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.003924950025975704, "fcm_dpo/delta": 0.002111189067363739, "fcm_dpo/margin": 100.95064544677734, "fcm_dpo/q_t": 0.4058684706687927, "grad_norm": 10.419607162475586, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.4783401489257812, "logits/rejected": -2.081491708755493, "logps/chosen": -253.60198974609375, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -353.56671142578125, "loss": 1.0722, "margin_dpo/margin_mean": 100.95064544677734, "margin_dpo/margin_std": 103.39350891113281, "step": 649 }, { "KL/chosen_KL_mean": -194.28549194335938, "KL/mean": -241.24166870117188, "KL/rejected_KL_mean": -288.1978759765625, "KL/std": 115.1053695678711, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004094364121556282, "fcm_dpo/delta": 0.019030271098017693, "fcm_dpo/margin": 93.912353515625, "fcm_dpo/q_t": 0.4115172326564789, "grad_norm": 9.639708518981934, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.2926201820373535, "logits/rejected": -1.0931750535964966, "logps/chosen": -284.2160949707031, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -390.8106689453125, "loss": 1.1106, "margin_dpo/margin_mean": 93.912353515625, "margin_dpo/margin_std": 136.22308349609375, "step": 650 }, { "KL/chosen_KL_mean": -161.333251953125, "KL/mean": -218.95286560058594, "KL/rejected_KL_mean": -276.57244873046875, "KL/std": 111.23741149902344, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0038903618697077036, "fcm_dpo/delta": -0.0629769042134285, "fcm_dpo/margin": 115.23920440673828, "fcm_dpo/q_t": 0.3908272385597229, "grad_norm": 8.80312728881836, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.0383224487304688, "logits/rejected": -1.8626933097839355, "logps/chosen": -240.5205841064453, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -382.50579833984375, "loss": 1.0304, "margin_dpo/margin_mean": 115.23919677734375, "margin_dpo/margin_std": 108.9111557006836, "step": 651 }, { "KL/chosen_KL_mean": -168.91648864746094, "KL/mean": -217.26036071777344, "KL/rejected_KL_mean": -265.6042175292969, "KL/std": 103.2699966430664, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.003957123029977083, "fcm_dpo/delta": 0.020528223365545273, "fcm_dpo/margin": 96.68772888183594, "fcm_dpo/q_t": 0.4098827540874481, "grad_norm": 9.074678421020508, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.498349189758301, "logits/rejected": -2.2057290077209473, "logps/chosen": -270.70672607421875, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -381.9288024902344, "loss": 1.0764, "margin_dpo/margin_mean": 96.68772888183594, "margin_dpo/margin_std": 98.55806732177734, "step": 652 }, { "KL/chosen_KL_mean": -193.4945831298828, "KL/mean": -240.3189697265625, "KL/rejected_KL_mean": -287.1433410644531, "KL/std": 113.72066497802734, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004038790240883827, "fcm_dpo/delta": 0.026311412453651428, "fcm_dpo/margin": 93.64875793457031, "fcm_dpo/q_t": 0.4107356369495392, "grad_norm": 9.433415412902832, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.119779109954834, "logits/rejected": -1.8654866218566895, "logps/chosen": -286.8481750488281, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -396.2666015625, "loss": 1.0925, "margin_dpo/margin_mean": 93.64875793457031, "margin_dpo/margin_std": 111.11227416992188, "step": 653 }, { "KL/chosen_KL_mean": -193.86370849609375, "KL/mean": -234.43515014648438, "KL/rejected_KL_mean": -275.006591796875, "KL/std": 108.48920440673828, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.004242418333888054, "fcm_dpo/delta": 0.06725054234266281, "fcm_dpo/margin": 81.14292907714844, "fcm_dpo/q_t": 0.4242001473903656, "grad_norm": 12.206831932067871, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.4023430347442627, "logits/rejected": -2.2262144088745117, "logps/chosen": -282.979248046875, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -379.92657470703125, "loss": 1.1514, "margin_dpo/margin_mean": 81.14292907714844, "margin_dpo/margin_std": 137.83480834960938, "step": 654 }, { "KL/chosen_KL_mean": -176.72406005859375, "KL/mean": -225.15277099609375, "KL/rejected_KL_mean": -273.58148193359375, "KL/std": 118.09814453125, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.004281155299395323, "fcm_dpo/delta": -0.018530821427702904, "fcm_dpo/margin": 96.85740661621094, "fcm_dpo/q_t": 0.4026387929916382, "grad_norm": 9.076825141906738, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.037400484085083, "logits/rejected": -2.041269302368164, "logps/chosen": -257.7601623535156, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -376.3838195800781, "loss": 1.0852, "margin_dpo/margin_mean": 96.85740661621094, "margin_dpo/margin_std": 126.9530029296875, "step": 655 }, { "KL/chosen_KL_mean": -177.8463592529297, "KL/mean": -239.445556640625, "KL/rejected_KL_mean": -301.04473876953125, "KL/std": 118.54623413085938, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0038832942955195904, "fcm_dpo/delta": -0.10495135188102722, "fcm_dpo/margin": 123.19837188720703, "fcm_dpo/q_t": 0.3839613199234009, "grad_norm": 9.364027976989746, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.5245699882507324, "logits/rejected": -1.5951318740844727, "logps/chosen": -263.17169189453125, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -419.3833923339844, "loss": 1.0167, "margin_dpo/margin_mean": 123.19837188720703, "margin_dpo/margin_std": 122.74725341796875, "step": 656 }, { "KL/chosen_KL_mean": -193.88511657714844, "KL/mean": -239.5493621826172, "KL/rejected_KL_mean": -285.21356201171875, "KL/std": 109.5688247680664, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0038881292566657066, "fcm_dpo/delta": 0.05385424569249153, "fcm_dpo/margin": 91.32847595214844, "fcm_dpo/q_t": 0.41954126954078674, "grad_norm": 10.092354774475098, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9185328483581543, "logits/rejected": -1.7520803213119507, "logps/chosen": -274.08282470703125, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -387.7948913574219, "loss": 1.1283, "margin_dpo/margin_mean": 91.32848358154297, "margin_dpo/margin_std": 132.05799865722656, "step": 657 }, { "KL/chosen_KL_mean": -191.52586364746094, "KL/mean": -234.04798889160156, "KL/rejected_KL_mean": -276.57012939453125, "KL/std": 103.22021484375, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.004235205706208944, "fcm_dpo/delta": 0.04642435163259506, "fcm_dpo/margin": 85.04422760009766, "fcm_dpo/q_t": 0.4168456196784973, "grad_norm": 9.020868301391602, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.9510843753814697, "logits/rejected": -1.4778974056243896, "logps/chosen": -286.160400390625, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -378.20172119140625, "loss": 1.1119, "margin_dpo/margin_mean": 85.04422760009766, "margin_dpo/margin_std": 110.81825256347656, "step": 658 }, { "KL/chosen_KL_mean": -173.00537109375, "KL/mean": -224.76473999023438, "KL/rejected_KL_mean": -276.52410888671875, "KL/std": 111.13883972167969, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0040440307930111885, "fcm_dpo/delta": -0.026479197666049004, "fcm_dpo/margin": 103.51874542236328, "fcm_dpo/q_t": 0.402265727519989, "grad_norm": 8.627798080444336, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.146304130554199, "logits/rejected": -1.1837315559387207, "logps/chosen": -285.5612487792969, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -370.44573974609375, "loss": 1.0949, "margin_dpo/margin_mean": 103.51874542236328, "margin_dpo/margin_std": 143.7548370361328, "step": 659 }, { "KL/chosen_KL_mean": -182.53799438476562, "KL/mean": -235.00827026367188, "KL/rejected_KL_mean": -287.4785461425781, "KL/std": 107.92465209960938, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.004019572399556637, "fcm_dpo/delta": -0.02768884226679802, "fcm_dpo/margin": 104.9405517578125, "fcm_dpo/q_t": 0.40025514364242554, "grad_norm": 11.498600006103516, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8496956825256348, "logits/rejected": -1.2236132621765137, "logps/chosen": -275.9154357910156, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -368.87335205078125, "loss": 1.0612, "margin_dpo/margin_mean": 104.9405517578125, "margin_dpo/margin_std": 118.27317810058594, "step": 660 }, { "KL/chosen_KL_mean": -179.8583984375, "KL/mean": -238.87274169921875, "KL/rejected_KL_mean": -297.8870849609375, "KL/std": 121.55139923095703, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0038358806632459164, "fcm_dpo/delta": -0.06792447715997696, "fcm_dpo/margin": 118.02873229980469, "fcm_dpo/q_t": 0.39251708984375, "grad_norm": 8.80207633972168, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.7833678722381592, "logits/rejected": -1.2674695253372192, "logps/chosen": -267.71356201171875, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -390.2904052734375, "loss": 1.0478, "margin_dpo/margin_mean": 118.02873229980469, "margin_dpo/margin_std": 134.9127655029297, "step": 661 }, { "KL/chosen_KL_mean": -175.9539794921875, "KL/mean": -230.25424194335938, "KL/rejected_KL_mean": -284.55450439453125, "KL/std": 105.75666809082031, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.003742070170119405, "fcm_dpo/delta": -0.008298039436340332, "fcm_dpo/margin": 108.60050964355469, "fcm_dpo/q_t": 0.40354645252227783, "grad_norm": 9.85058879852295, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.8102635145187378, "logits/rejected": -1.2452540397644043, "logps/chosen": -278.7337646484375, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -379.7798156738281, "loss": 1.0638, "margin_dpo/margin_mean": 108.60050964355469, "margin_dpo/margin_std": 111.99691772460938, "step": 662 }, { "KL/chosen_KL_mean": -178.27735900878906, "KL/mean": -227.74539184570312, "KL/rejected_KL_mean": -277.2134094238281, "KL/std": 109.75566101074219, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.003815930336713791, "fcm_dpo/delta": 0.02681881934404373, "fcm_dpo/margin": 98.93606567382812, "fcm_dpo/q_t": 0.4124357998371124, "grad_norm": 10.062081336975098, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.0878844261169434, "logits/rejected": -1.867976188659668, "logps/chosen": -274.62396240234375, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -397.7398681640625, "loss": 1.1007, "margin_dpo/margin_mean": 98.93606567382812, "margin_dpo/margin_std": 127.12557983398438, "step": 663 }, { "KL/chosen_KL_mean": -197.17193603515625, "KL/mean": -241.5465087890625, "KL/rejected_KL_mean": -285.92108154296875, "KL/std": 112.59728240966797, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.004002304282039404, "fcm_dpo/delta": 0.05300997942686081, "fcm_dpo/margin": 88.74915313720703, "fcm_dpo/q_t": 0.41805464029312134, "grad_norm": 8.663013458251953, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.6554685831069946, "logits/rejected": -1.4834306240081787, "logps/chosen": -289.01434326171875, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -397.7578125, "loss": 1.1114, "margin_dpo/margin_mean": 88.74916076660156, "margin_dpo/margin_std": 111.20690155029297, "step": 664 }, { "KL/chosen_KL_mean": -169.34512329101562, "KL/mean": -217.69509887695312, "KL/rejected_KL_mean": -266.0450744628906, "KL/std": 107.68728637695312, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004072671756148338, "fcm_dpo/delta": 0.0070788320153951645, "fcm_dpo/margin": 96.69996643066406, "fcm_dpo/q_t": 0.407009482383728, "grad_norm": 8.200661659240723, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.4228625297546387, "logits/rejected": -1.9965994358062744, "logps/chosen": -257.53131103515625, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -357.95709228515625, "loss": 1.0806, "margin_dpo/margin_mean": 96.69996643066406, "margin_dpo/margin_std": 111.74192810058594, "step": 665 }, { "KL/chosen_KL_mean": -185.30508422851562, "KL/mean": -235.10159301757812, "KL/rejected_KL_mean": -284.8981018066406, "KL/std": 106.27835083007812, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.003993149381130934, "fcm_dpo/delta": 0.0022132322192192078, "fcm_dpo/margin": 99.593017578125, "fcm_dpo/q_t": 0.4076208472251892, "grad_norm": 9.832144737243652, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.586835503578186, "logits/rejected": -1.2059011459350586, "logps/chosen": -286.2796630859375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -386.14801025390625, "loss": 1.0898, "margin_dpo/margin_mean": 99.593017578125, "margin_dpo/margin_std": 125.78850555419922, "step": 666 }, { "KL/chosen_KL_mean": -201.88131713867188, "KL/mean": -252.78021240234375, "KL/rejected_KL_mean": -303.67913818359375, "KL/std": 110.84483337402344, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.00399154145270586, "fcm_dpo/delta": -0.008255456574261189, "fcm_dpo/margin": 101.79780578613281, "fcm_dpo/q_t": 0.4034695029258728, "grad_norm": 9.32685661315918, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.5764670372009277, "logits/rejected": -0.930832028388977, "logps/chosen": -299.4524841308594, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -397.2638854980469, "loss": 1.0648, "margin_dpo/margin_mean": 101.79780578613281, "margin_dpo/margin_std": 107.47825622558594, "step": 667 }, { "KL/chosen_KL_mean": -179.4757080078125, "KL/mean": -238.95538330078125, "KL/rejected_KL_mean": -298.43505859375, "KL/std": 118.08253479003906, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0038198656402528286, "fcm_dpo/delta": -0.07032635807991028, "fcm_dpo/margin": 118.95933532714844, "fcm_dpo/q_t": 0.39200258255004883, "grad_norm": 9.66501235961914, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.2733278274536133, "logits/rejected": -1.751124382019043, "logps/chosen": -282.05078125, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -407.25274658203125, "loss": 1.0389, "margin_dpo/margin_mean": 118.95933532714844, "margin_dpo/margin_std": 128.84530639648438, "step": 668 }, { "KL/chosen_KL_mean": -162.86099243164062, "KL/mean": -219.8414764404297, "KL/rejected_KL_mean": -276.8219909667969, "KL/std": 114.9860610961914, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.003677038475871086, "fcm_dpo/delta": -0.024075452238321304, "fcm_dpo/margin": 113.96102142333984, "fcm_dpo/q_t": 0.40050822496414185, "grad_norm": 8.869400024414062, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.7032012939453125, "logits/rejected": -1.3088483810424805, "logps/chosen": -252.3557586669922, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -359.34149169921875, "loss": 1.0592, "margin_dpo/margin_mean": 113.96102905273438, "margin_dpo/margin_std": 123.23050689697266, "step": 669 }, { "KL/chosen_KL_mean": -163.99136352539062, "KL/mean": -226.77130126953125, "KL/rejected_KL_mean": -289.55126953125, "KL/std": 117.44845581054688, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0035272622480988503, "fcm_dpo/delta": -0.054918497800827026, "fcm_dpo/margin": 125.55986785888672, "fcm_dpo/q_t": 0.39230599999427795, "grad_norm": 7.062045097351074, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7462191581726074, "logits/rejected": -1.211168885231018, "logps/chosen": -259.44598388671875, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -391.0841979980469, "loss": 1.0376, "margin_dpo/margin_mean": 125.55986785888672, "margin_dpo/margin_std": 123.17598724365234, "step": 670 }, { "KL/chosen_KL_mean": -200.0113525390625, "KL/mean": -242.36036682128906, "KL/rejected_KL_mean": -284.70941162109375, "KL/std": 108.4527359008789, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0034706704318523407, "fcm_dpo/delta": 0.008694111369550228, "fcm_dpo/margin": 84.69805908203125, "fcm_dpo/q_t": 0.43071454763412476, "grad_norm": 8.572225570678711, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6634113788604736, "logits/rejected": -1.4397910833358765, "logps/chosen": -282.13446044921875, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -374.9290771484375, "loss": 1.158, "margin_dpo/margin_mean": 84.69805145263672, "margin_dpo/margin_std": 120.9536361694336, "step": 671 }, { "KL/chosen_KL_mean": -199.8853759765625, "KL/mean": -236.63079833984375, "KL/rejected_KL_mean": -273.376220703125, "KL/std": 104.28819274902344, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.003845369443297386, "fcm_dpo/delta": 0.13705827295780182, "fcm_dpo/margin": 73.49085235595703, "fcm_dpo/q_t": 0.43729957938194275, "grad_norm": 11.843745231628418, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.0440523624420166, "logits/rejected": -1.7184014320373535, "logps/chosen": -289.947265625, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -373.8294677734375, "loss": 1.1722, "margin_dpo/margin_mean": 73.49085235595703, "margin_dpo/margin_std": 110.37739562988281, "step": 672 }, { "KL/chosen_KL_mean": -178.74862670898438, "KL/mean": -220.3880615234375, "KL/rejected_KL_mean": -262.02752685546875, "KL/std": 107.22188568115234, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004227695055305958, "fcm_dpo/delta": 0.05630987137556076, "fcm_dpo/margin": 83.2789077758789, "fcm_dpo/q_t": 0.41917288303375244, "grad_norm": 11.934531211853027, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.8926316499710083, "logits/rejected": -1.525010585784912, "logps/chosen": -258.01177978515625, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -353.37677001953125, "loss": 1.1166, "margin_dpo/margin_mean": 83.2789077758789, "margin_dpo/margin_std": 110.01065063476562, "step": 673 }, { "KL/chosen_KL_mean": -167.3341827392578, "KL/mean": -226.89703369140625, "KL/rejected_KL_mean": -286.45989990234375, "KL/std": 107.56410217285156, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.003930181264877319, "fcm_dpo/delta": -0.09018109738826752, "fcm_dpo/margin": 119.125732421875, "fcm_dpo/q_t": 0.38703328371047974, "grad_norm": 7.566953182220459, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.607371211051941, "logits/rejected": -1.205336570739746, "logps/chosen": -242.79249572753906, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -362.66351318359375, "loss": 1.0225, "margin_dpo/margin_mean": 119.125732421875, "margin_dpo/margin_std": 119.02410888671875, "step": 674 }, { "KL/chosen_KL_mean": -182.99386596679688, "KL/mean": -235.67391967773438, "KL/rejected_KL_mean": -288.3539733886719, "KL/std": 116.75942993164062, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0037516290321946144, "fcm_dpo/delta": 0.0015406571328639984, "fcm_dpo/margin": 105.3600845336914, "fcm_dpo/q_t": 0.4070886969566345, "grad_norm": 9.67018985748291, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.547683835029602, "logits/rejected": -1.2098263502120972, "logps/chosen": -262.90338134765625, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -369.57220458984375, "loss": 1.1036, "margin_dpo/margin_mean": 105.36009216308594, "margin_dpo/margin_std": 143.00711059570312, "step": 675 }, { "KL/chosen_KL_mean": -197.1929931640625, "KL/mean": -248.6881561279297, "KL/rejected_KL_mean": -300.183349609375, "KL/std": 111.93295288085938, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0038422499783337116, "fcm_dpo/delta": 0.004857627674937248, "fcm_dpo/margin": 102.9903335571289, "fcm_dpo/q_t": 0.4069933295249939, "grad_norm": 8.297662734985352, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -2.0438857078552246, "logits/rejected": -1.4660178422927856, "logps/chosen": -295.3641052246094, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -395.2080078125, "loss": 1.0866, "margin_dpo/margin_mean": 102.9903335571289, "margin_dpo/margin_std": 126.00289916992188, "step": 676 }, { "KL/chosen_KL_mean": -176.8895263671875, "KL/mean": -228.70379638671875, "KL/rejected_KL_mean": -280.5180969238281, "KL/std": 110.3225326538086, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.003846679348498583, "fcm_dpo/delta": 0.001089809462428093, "fcm_dpo/margin": 103.62857055664062, "fcm_dpo/q_t": 0.40575695037841797, "grad_norm": 8.170503616333008, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.6920377016067505, "logits/rejected": -1.0173145532608032, "logps/chosen": -268.268798828125, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -363.3958740234375, "loss": 1.0696, "margin_dpo/margin_mean": 103.62857055664062, "margin_dpo/margin_std": 108.09207916259766, "step": 677 }, { "KL/chosen_KL_mean": -168.1561279296875, "KL/mean": -229.11553955078125, "KL/rejected_KL_mean": -290.074951171875, "KL/std": 117.71171569824219, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0037596174515783787, "fcm_dpo/delta": -0.07588943094015121, "fcm_dpo/margin": 121.91881561279297, "fcm_dpo/q_t": 0.3889780044555664, "grad_norm": 9.549760818481445, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.6692875623703003, "logits/rejected": -1.3438408374786377, "logps/chosen": -243.802001953125, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -377.04107666015625, "loss": 1.0314, "margin_dpo/margin_mean": 121.9188232421875, "margin_dpo/margin_std": 120.77877807617188, "step": 678 }, { "KL/chosen_KL_mean": -194.36534118652344, "KL/mean": -244.77426147460938, "KL/rejected_KL_mean": -295.18316650390625, "KL/std": 98.39960479736328, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.003691274207085371, "fcm_dpo/delta": 0.03427767753601074, "fcm_dpo/margin": 100.8178482055664, "fcm_dpo/q_t": 0.4132958948612213, "grad_norm": 8.27920913696289, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.1921324729919434, "logits/rejected": -1.6749025583267212, "logps/chosen": -275.1387939453125, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -378.0616760253906, "loss": 1.0843, "margin_dpo/margin_mean": 100.8178482055664, "margin_dpo/margin_std": 105.69536590576172, "step": 679 }, { "KL/chosen_KL_mean": -193.8516845703125, "KL/mean": -249.17422485351562, "KL/rejected_KL_mean": -304.49676513671875, "KL/std": 117.10699462890625, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0037879503797739744, "fcm_dpo/delta": -0.028351671993732452, "fcm_dpo/margin": 110.64509582519531, "fcm_dpo/q_t": 0.40094757080078125, "grad_norm": 6.961892127990723, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9003690481185913, "logits/rejected": -1.567399501800537, "logps/chosen": -301.53460693359375, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -420.59161376953125, "loss": 1.0719, "margin_dpo/margin_mean": 110.64508056640625, "margin_dpo/margin_std": 127.26298522949219, "step": 680 }, { "KL/chosen_KL_mean": -177.51193237304688, "KL/mean": -225.78309631347656, "KL/rejected_KL_mean": -274.05426025390625, "KL/std": 110.65043640136719, "epoch": 1.0, "fcm_dpo/beta": 0.0037827298510819674, "fcm_dpo/delta": 0.04159975424408913, "fcm_dpo/margin": 96.54232788085938, "fcm_dpo/q_t": 0.4157445430755615, "grad_norm": 9.110100746154785, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.287074565887451, "logits/rejected": -1.9757800102233887, "logps/chosen": -270.52301025390625, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -368.87640380859375, "loss": 1.1044, "margin_dpo/margin_mean": 96.54232788085938, "margin_dpo/margin_std": 120.92645263671875, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.114833921237839, "train_runtime": 1971.8939, "train_samples_per_second": 22.11, "train_steps_per_second": 0.345 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }