{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.81816482543945, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.75609588623047, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.036800384521484375, "KL/mean": 0.02181517332792282, "KL/rejected_KL_mean": 0.006832122802734375, "KL/std": 0.29881206154823303, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.029974564909934998, "fcm_dpo/q_t": 0.49925094842910767, "grad_norm": 25.990829467773438, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4273410439491272, "logits/rejected": 0.6504765152931213, "logps/chosen": -93.77418518066406, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.22267150878906, "loss": 1.3839, "margin_dpo/margin_mean": 0.02997477352619171, "margin_dpo/margin_std": 0.46543633937835693, "step": 3 }, { "KL/chosen_KL_mean": -0.0356597900390625, "KL/mean": -0.006167158484458923, "KL/rejected_KL_mean": 0.023326873779296875, "KL/std": 0.367583692073822, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.058989495038986206, "fcm_dpo/q_t": 0.501470148563385, "grad_norm": 32.708370208740234, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6463060975074768, "logits/rejected": 0.7783059477806091, "logps/chosen": -87.328125, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.80093383789062, "loss": 1.3929, "margin_dpo/margin_mean": -0.05898940563201904, "margin_dpo/margin_std": 0.539353609085083, "step": 4 }, { "KL/chosen_KL_mean": -0.01611328125, "KL/mean": -0.004562385380268097, "KL/rejected_KL_mean": 0.006984710693359375, "KL/std": 0.30425113439559937, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.023090139031410217, "fcm_dpo/q_t": 0.5005767345428467, "grad_norm": 35.2536735534668, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.3846132457256317, "logits/rejected": 0.5906330347061157, "logps/chosen": -89.35287475585938, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.7408447265625, "loss": 1.3891, "margin_dpo/margin_mean": -0.023089826107025146, "margin_dpo/margin_std": 0.4508253335952759, "step": 5 }, { "KL/chosen_KL_mean": -0.01535797119140625, "KL/mean": -0.027334511280059814, "KL/rejected_KL_mean": -0.039306640625, "KL/std": 0.2799898386001587, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02394247055053711, "fcm_dpo/q_t": 0.4994018077850342, "grad_norm": 35.98002243041992, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.5741077065467834, "logits/rejected": 0.8980114459991455, "logps/chosen": -97.33683776855469, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.92276000976562, "loss": 1.3844, "margin_dpo/margin_mean": 0.023942381143569946, "margin_dpo/margin_std": 0.4234223961830139, "step": 6 }, { "KL/chosen_KL_mean": -0.034698486328125, "KL/mean": -0.019638612866401672, "KL/rejected_KL_mean": -0.00458526611328125, "KL/std": 0.29589569568634033, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.030112534761428833, "fcm_dpo/q_t": 0.5007528066635132, "grad_norm": 38.46144104003906, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.48946601152420044, "logits/rejected": 0.5366263389587402, "logps/chosen": -86.6832275390625, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.62076568603516, "loss": 1.3897, "margin_dpo/margin_mean": -0.03011268377304077, "margin_dpo/margin_std": 0.3839470446109772, "step": 7 }, { "KL/chosen_KL_mean": -0.031841278076171875, "KL/mean": 0.0018782317638397217, "KL/rejected_KL_mean": 0.035602569580078125, "KL/std": 0.2913409471511841, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06744384765625, "fcm_dpo/q_t": 0.5016841888427734, "grad_norm": 32.81808853149414, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.41760268807411194, "logits/rejected": 0.7302404642105103, "logps/chosen": -89.95552062988281, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.19243621826172, "loss": 1.3936, "margin_dpo/margin_mean": -0.06744343042373657, "margin_dpo/margin_std": 0.45135021209716797, "step": 8 }, { "KL/chosen_KL_mean": 0.021244049072265625, "KL/mean": 0.013112969696521759, "KL/rejected_KL_mean": 0.00498199462890625, "KL/std": 0.3151127099990845, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016258880496025085, "fcm_dpo/q_t": 0.4995930790901184, "grad_norm": 36.3211669921875, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.720187246799469, "logits/rejected": 0.903660774230957, "logps/chosen": -103.83589172363281, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.31433868408203, "loss": 1.3852, "margin_dpo/margin_mean": 0.016258180141448975, "margin_dpo/margin_std": 0.44475120306015015, "step": 9 }, { "KL/chosen_KL_mean": 0.025463104248046875, "KL/mean": 0.0919063538312912, "KL/rejected_KL_mean": 0.15834426879882812, "KL/std": 0.2605481743812561, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.13288173079490662, "fcm_dpo/q_t": 0.503320038318634, "grad_norm": 33.83922576904297, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.7664551734924316, "logits/rejected": 0.9906111359596252, "logps/chosen": -76.17890167236328, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -86.99375915527344, "loss": 1.4, "margin_dpo/margin_mean": -0.1328815519809723, "margin_dpo/margin_std": 0.39378607273101807, "step": 10 }, { "KL/chosen_KL_mean": 0.028110504150390625, "KL/mean": -0.006145946681499481, "KL/rejected_KL_mean": -0.0404052734375, "KL/std": 0.29041725397109985, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0685100108385086, "fcm_dpo/q_t": 0.49828821420669556, "grad_norm": 36.954307556152344, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.2833446264266968, "logits/rejected": 0.4991028308868408, "logps/chosen": -82.33838653564453, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.30502319335938, "loss": 1.3799, "margin_dpo/margin_mean": 0.06850995123386383, "margin_dpo/margin_std": 0.3976043462753296, "step": 11 }, { "KL/chosen_KL_mean": 0.0064239501953125, "KL/mean": -0.022876247763633728, "KL/rejected_KL_mean": -0.05217742919921875, "KL/std": 0.37394559383392334, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05860540270805359, "fcm_dpo/q_t": 0.4985356330871582, "grad_norm": 39.91716766357422, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.41215819120407104, "logits/rejected": 0.5410385727882385, "logps/chosen": -99.09907531738281, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.32359313964844, "loss": 1.3811, "margin_dpo/margin_mean": 0.058605700731277466, "margin_dpo/margin_std": 0.49439164996147156, "step": 12 }, { "KL/chosen_KL_mean": 0.0016326904296875, "KL/mean": 0.010229602456092834, "KL/rejected_KL_mean": 0.018833160400390625, "KL/std": 0.3227289319038391, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01719525456428528, "fcm_dpo/q_t": 0.5004311800003052, "grad_norm": 42.14787673950195, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.33703529834747314, "logits/rejected": 0.5813360214233398, "logps/chosen": -90.55809020996094, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.6722640991211, "loss": 1.3886, "margin_dpo/margin_mean": -0.0171949565410614, "margin_dpo/margin_std": 0.48364514112472534, "step": 13 }, { "KL/chosen_KL_mean": -0.084320068359375, "KL/mean": -0.04265942424535751, "KL/rejected_KL_mean": -0.00099945068359375, "KL/std": 0.3363949656486511, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08332423865795135, "fcm_dpo/q_t": 0.5020800828933716, "grad_norm": 35.9296760559082, "learning_rate": 9.420289855072464e-08, "logits/chosen": 1.0389986038208008, "logits/rejected": 1.1321544647216797, "logps/chosen": -99.9114990234375, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.94300079345703, "loss": 1.3953, "margin_dpo/margin_mean": -0.08332401514053345, "margin_dpo/margin_std": 0.4918796718120575, "step": 14 }, { "KL/chosen_KL_mean": -0.007213592529296875, "KL/mean": -0.0027718544006347656, "KL/rejected_KL_mean": 0.00167083740234375, "KL/std": 0.30277401208877563, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.008890509605407715, "fcm_dpo/q_t": 0.5002241134643555, "grad_norm": 31.74273109436035, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.45355454087257385, "logits/rejected": 0.625147819519043, "logps/chosen": -78.91718292236328, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.0606689453125, "loss": 1.3876, "margin_dpo/margin_mean": -0.008890360593795776, "margin_dpo/margin_std": 0.42160820960998535, "step": 15 }, { "KL/chosen_KL_mean": 0.008602142333984375, "KL/mean": -0.0029915273189544678, "KL/rejected_KL_mean": -0.01458740234375, "KL/std": 0.3337206542491913, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.023186147212982178, "fcm_dpo/q_t": 0.4994204640388489, "grad_norm": 34.50189971923828, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.4964686632156372, "logits/rejected": 0.7473509311676025, "logps/chosen": -97.4146728515625, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.61404418945312, "loss": 1.3845, "margin_dpo/margin_mean": 0.0231858491897583, "margin_dpo/margin_std": 0.4618992805480957, "step": 16 }, { "KL/chosen_KL_mean": 0.031551361083984375, "KL/mean": 0.0026029273867607117, "KL/rejected_KL_mean": -0.026348114013671875, "KL/std": 0.28192365169525146, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05790342390537262, "fcm_dpo/q_t": 0.4985527992248535, "grad_norm": 35.619327545166016, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5685204267501831, "logits/rejected": 0.8081611394882202, "logps/chosen": -104.33275604248047, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.49407196044922, "loss": 1.3809, "margin_dpo/margin_mean": 0.05790260434150696, "margin_dpo/margin_std": 0.3960239291191101, "step": 17 }, { "KL/chosen_KL_mean": 0.0307769775390625, "KL/mean": 0.039263881742954254, "KL/rejected_KL_mean": 0.047748565673828125, "KL/std": 0.33492666482925415, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.016970619559288025, "fcm_dpo/q_t": 0.5004260540008545, "grad_norm": 41.609825134277344, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.386380672454834, "logits/rejected": 0.7304573059082031, "logps/chosen": -87.0611801147461, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.80297088623047, "loss": 1.3887, "margin_dpo/margin_mean": -0.016970351338386536, "margin_dpo/margin_std": 0.5241757035255432, "step": 18 }, { "KL/chosen_KL_mean": 0.00543975830078125, "KL/mean": -0.016459211707115173, "KL/rejected_KL_mean": -0.038360595703125, "KL/std": 0.30041781067848206, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04379662871360779, "fcm_dpo/q_t": 0.4989047646522522, "grad_norm": 31.158756256103516, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.5527125597000122, "logits/rejected": 0.8271359205245972, "logps/chosen": -105.86810302734375, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.96859741210938, "loss": 1.3824, "margin_dpo/margin_mean": 0.04379647970199585, "margin_dpo/margin_std": 0.4359992742538452, "step": 19 }, { "KL/chosen_KL_mean": 0.02628326416015625, "KL/mean": 0.008456870913505554, "KL/rejected_KL_mean": -0.00937652587890625, "KL/std": 0.3380834460258484, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.035659998655319214, "fcm_dpo/q_t": 0.4991084933280945, "grad_norm": 32.720279693603516, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.3497970402240753, "logits/rejected": 0.6359144449234009, "logps/chosen": -90.73184204101562, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.92169189453125, "loss": 1.3833, "margin_dpo/margin_mean": 0.035660237073898315, "margin_dpo/margin_std": 0.46866458654403687, "step": 20 }, { "KL/chosen_KL_mean": 0.052135467529296875, "KL/mean": 0.017426058650016785, "KL/rejected_KL_mean": -0.017284393310546875, "KL/std": 0.275062620639801, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06941965222358704, "fcm_dpo/q_t": 0.49826449155807495, "grad_norm": 31.512779235839844, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.41625073552131653, "logits/rejected": 0.5840957164764404, "logps/chosen": -80.28132629394531, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.95101928710938, "loss": 1.3797, "margin_dpo/margin_mean": 0.06942012906074524, "margin_dpo/margin_std": 0.3377068340778351, "step": 21 }, { "KL/chosen_KL_mean": 0.056121826171875, "KL/mean": 0.02898523211479187, "KL/rejected_KL_mean": 0.001850128173828125, "KL/std": 0.32471421360969543, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05426383018493652, "fcm_dpo/q_t": 0.49864423274993896, "grad_norm": 42.70122528076172, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.6878045201301575, "logits/rejected": 0.809969425201416, "logps/chosen": -95.33917999267578, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.47166442871094, "loss": 1.3813, "margin_dpo/margin_mean": 0.054263561964035034, "margin_dpo/margin_std": 0.4099903106689453, "step": 22 }, { "KL/chosen_KL_mean": -0.0449066162109375, "KL/mean": -0.011196993291378021, "KL/rejected_KL_mean": 0.022518157958984375, "KL/std": 0.30727264285087585, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0674227923154831, "fcm_dpo/q_t": 0.5016839504241943, "grad_norm": 31.6392765045166, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.640638530254364, "logits/rejected": 0.8146892189979553, "logps/chosen": -90.68241882324219, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.57173919677734, "loss": 1.3935, "margin_dpo/margin_mean": -0.06742346286773682, "margin_dpo/margin_std": 0.41370058059692383, "step": 23 }, { "KL/chosen_KL_mean": 0.022386550903320312, "KL/mean": -0.013950265944004059, "KL/rejected_KL_mean": -0.05028533935546875, "KL/std": 0.27053341269493103, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07267145812511444, "fcm_dpo/q_t": 0.49818432331085205, "grad_norm": 44.23604965209961, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.6689919829368591, "logits/rejected": 0.6986947059631348, "logps/chosen": -69.89490509033203, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.66018676757812, "loss": 1.3794, "margin_dpo/margin_mean": 0.07267236709594727, "margin_dpo/margin_std": 0.3896770477294922, "step": 24 }, { "KL/chosen_KL_mean": 0.023502349853515625, "KL/mean": -0.022247523069381714, "KL/rejected_KL_mean": -0.06799697875976562, "KL/std": 0.3245248794555664, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09148669242858887, "fcm_dpo/q_t": 0.497714638710022, "grad_norm": 36.90292739868164, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.4752264618873596, "logits/rejected": 0.6418728232383728, "logps/chosen": -80.80198669433594, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.02510070800781, "loss": 1.3777, "margin_dpo/margin_mean": 0.09148728847503662, "margin_dpo/margin_std": 0.4696190357208252, "step": 25 }, { "KL/chosen_KL_mean": -0.008998870849609375, "KL/mean": -0.020681306719779968, "KL/rejected_KL_mean": -0.0323638916015625, "KL/std": 0.3125627934932709, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.023365527391433716, "fcm_dpo/q_t": 0.49941515922546387, "grad_norm": 41.07014846801758, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.23564063012599945, "logits/rejected": 0.4415525794029236, "logps/chosen": -88.91017150878906, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.86055755615234, "loss": 1.3845, "margin_dpo/margin_mean": 0.02336600422859192, "margin_dpo/margin_std": 0.44299352169036865, "step": 26 }, { "KL/chosen_KL_mean": -0.0501708984375, "KL/mean": -0.0889873281121254, "KL/rejected_KL_mean": -0.12780380249023438, "KL/std": 0.31280991435050964, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07763789594173431, "fcm_dpo/q_t": 0.4980607330799103, "grad_norm": 44.2216911315918, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.17439910769462585, "logits/rejected": 0.3622664213180542, "logps/chosen": -77.64617919921875, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.06631469726562, "loss": 1.379, "margin_dpo/margin_mean": 0.07763791084289551, "margin_dpo/margin_std": 0.40198463201522827, "step": 27 }, { "KL/chosen_KL_mean": 0.08165359497070312, "KL/mean": 0.017925187945365906, "KL/rejected_KL_mean": -0.045795440673828125, "KL/std": 0.3445565700531006, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1274522840976715, "fcm_dpo/q_t": 0.49681541323661804, "grad_norm": 36.75702667236328, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.6362975835800171, "logits/rejected": 0.8584526181221008, "logps/chosen": -102.14691162109375, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.00529479980469, "loss": 1.3741, "margin_dpo/margin_mean": 0.12745356559753418, "margin_dpo/margin_std": 0.4592340886592865, "step": 28 }, { "KL/chosen_KL_mean": 0.062053680419921875, "KL/mean": -0.013778947293758392, "KL/rejected_KL_mean": -0.08961105346679688, "KL/std": 0.3661460876464844, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1516689509153366, "fcm_dpo/q_t": 0.49621111154556274, "grad_norm": 42.29579544067383, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.24060426652431488, "logits/rejected": 0.49444904923439026, "logps/chosen": -88.58499908447266, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.04972839355469, "loss": 1.3717, "margin_dpo/margin_mean": 0.15166985988616943, "margin_dpo/margin_std": 0.4639388918876648, "step": 29 }, { "KL/chosen_KL_mean": 0.019504547119140625, "KL/mean": -0.03419766575098038, "KL/rejected_KL_mean": -0.0879058837890625, "KL/std": 0.34171295166015625, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10740886628627777, "fcm_dpo/q_t": 0.4973176419734955, "grad_norm": 39.06724166870117, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.6574506759643555, "logits/rejected": 0.7762874364852905, "logps/chosen": -88.36888122558594, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.40679931640625, "loss": 1.3762, "margin_dpo/margin_mean": 0.1074095070362091, "margin_dpo/margin_std": 0.4783516526222229, "step": 30 }, { "KL/chosen_KL_mean": -0.026531219482421875, "KL/mean": -0.08629752695560455, "KL/rejected_KL_mean": -0.14606475830078125, "KL/std": 0.34380581974983215, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1195308268070221, "fcm_dpo/q_t": 0.49701517820358276, "grad_norm": 31.148284912109375, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.8406588435173035, "logits/rejected": 1.1009732484817505, "logps/chosen": -101.15219116210938, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -80.00448608398438, "loss": 1.3749, "margin_dpo/margin_mean": 0.11953100562095642, "margin_dpo/margin_std": 0.4522687792778015, "step": 31 }, { "KL/chosen_KL_mean": -0.042377471923828125, "KL/mean": -0.07454848289489746, "KL/rejected_KL_mean": -0.106719970703125, "KL/std": 0.35065943002700806, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06434893608093262, "fcm_dpo/q_t": 0.4983883500099182, "grad_norm": 35.34646987915039, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.5705538988113403, "logits/rejected": 0.7210696935653687, "logps/chosen": -96.65940856933594, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.33108520507812, "loss": 1.3805, "margin_dpo/margin_mean": 0.06434932351112366, "margin_dpo/margin_std": 0.5026516914367676, "step": 32 }, { "KL/chosen_KL_mean": 0.030651092529296875, "KL/mean": -0.018570266664028168, "KL/rejected_KL_mean": -0.06779861450195312, "KL/std": 0.33048465847969055, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09845055639743805, "fcm_dpo/q_t": 0.4975391626358032, "grad_norm": 33.72600173950195, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5893163681030273, "logits/rejected": 0.7692603468894958, "logps/chosen": -81.49036407470703, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.87374877929688, "loss": 1.3769, "margin_dpo/margin_mean": 0.098451167345047, "margin_dpo/margin_std": 0.42850440740585327, "step": 33 }, { "KL/chosen_KL_mean": 0.05431365966796875, "KL/mean": -0.06270265579223633, "KL/rejected_KL_mean": -0.179718017578125, "KL/std": 0.3409511148929596, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2340332269668579, "fcm_dpo/q_t": 0.4941551089286804, "grad_norm": 41.035491943359375, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.7188920974731445, "logits/rejected": 0.8907482624053955, "logps/chosen": -77.14772033691406, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.89848327636719, "loss": 1.3635, "margin_dpo/margin_mean": 0.23403361439704895, "margin_dpo/margin_std": 0.39520418643951416, "step": 34 }, { "KL/chosen_KL_mean": -0.01303863525390625, "KL/mean": -0.07201364636421204, "KL/rejected_KL_mean": -0.1309967041015625, "KL/std": 0.36781972646713257, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1179613471031189, "fcm_dpo/q_t": 0.49705395102500916, "grad_norm": 42.871360778808594, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.13439635932445526, "logits/rejected": 0.29126274585723877, "logps/chosen": -77.58338928222656, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.31954956054688, "loss": 1.3752, "margin_dpo/margin_mean": 0.11796194314956665, "margin_dpo/margin_std": 0.5193536281585693, "step": 35 }, { "KL/chosen_KL_mean": -0.044219970703125, "KL/mean": -0.1466740071773529, "KL/rejected_KL_mean": -0.2491302490234375, "KL/std": 0.4095802307128906, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20490311086177826, "fcm_dpo/q_t": 0.49489250779151917, "grad_norm": 32.26992416381836, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.38015902042388916, "logits/rejected": 0.5680924654006958, "logps/chosen": -83.33245849609375, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -92.00654602050781, "loss": 1.3669, "margin_dpo/margin_mean": 0.2049029916524887, "margin_dpo/margin_std": 0.6056284308433533, "step": 36 }, { "KL/chosen_KL_mean": -0.0431976318359375, "KL/mean": -0.1611766666173935, "KL/rejected_KL_mean": -0.2791595458984375, "KL/std": 0.40961721539497375, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2359502613544464, "fcm_dpo/q_t": 0.49410897493362427, "grad_norm": 36.20394515991211, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.5003249049186707, "logits/rejected": 0.6472585797309875, "logps/chosen": -94.81428527832031, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -85.99642944335938, "loss": 1.3637, "margin_dpo/margin_mean": 0.23595041036605835, "margin_dpo/margin_std": 0.5791332125663757, "step": 37 }, { "KL/chosen_KL_mean": 0.060016632080078125, "KL/mean": -0.07544571906328201, "KL/rejected_KL_mean": -0.21090316772460938, "KL/std": 0.43366146087646484, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.27092409133911133, "fcm_dpo/q_t": 0.49323758482933044, "grad_norm": 36.84627914428711, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.4961952269077301, "logits/rejected": 0.6347057819366455, "logps/chosen": -75.91848754882812, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.31492614746094, "loss": 1.3603, "margin_dpo/margin_mean": 0.2709242105484009, "margin_dpo/margin_std": 0.5866086483001709, "step": 38 }, { "KL/chosen_KL_mean": -0.027729034423828125, "KL/mean": -0.17838197946548462, "KL/rejected_KL_mean": -0.3290367126464844, "KL/std": 0.4137577414512634, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.30130690336227417, "fcm_dpo/q_t": 0.4924755096435547, "grad_norm": 36.1956901550293, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.3576347827911377, "logits/rejected": 0.5378552675247192, "logps/chosen": -81.21350860595703, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.52495574951172, "loss": 1.3572, "margin_dpo/margin_mean": 0.30130746960639954, "margin_dpo/margin_std": 0.5835903286933899, "step": 39 }, { "KL/chosen_KL_mean": 0.06023406982421875, "KL/mean": -0.08938847482204437, "KL/rejected_KL_mean": -0.23900604248046875, "KL/std": 0.3598046898841858, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2992418706417084, "fcm_dpo/q_t": 0.4925246238708496, "grad_norm": 29.0565128326416, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.6989736557006836, "logits/rejected": 0.9015105962753296, "logps/chosen": -83.2723388671875, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.49491882324219, "loss": 1.3572, "margin_dpo/margin_mean": 0.299241304397583, "margin_dpo/margin_std": 0.4735555648803711, "step": 40 }, { "KL/chosen_KL_mean": -0.046054840087890625, "KL/mean": -0.1908104568719864, "KL/rejected_KL_mean": -0.3355751037597656, "KL/std": 0.4774826169013977, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2895239293575287, "fcm_dpo/q_t": 0.4927750825881958, "grad_norm": 33.29792785644531, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.5843805074691772, "logits/rejected": 0.7682211399078369, "logps/chosen": -93.19471740722656, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.41477966308594, "loss": 1.3589, "margin_dpo/margin_mean": 0.28952381014823914, "margin_dpo/margin_std": 0.7069178819656372, "step": 41 }, { "KL/chosen_KL_mean": 0.02227783203125, "KL/mean": -0.15322038531303406, "KL/rejected_KL_mean": -0.3287200927734375, "KL/std": 0.4862113893032074, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3509939908981323, "fcm_dpo/q_t": 0.49123477935791016, "grad_norm": 40.9924201965332, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.49761223793029785, "logits/rejected": 0.6407053470611572, "logps/chosen": -90.6793441772461, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.19336700439453, "loss": 1.3524, "margin_dpo/margin_mean": 0.3509941101074219, "margin_dpo/margin_std": 0.601630449295044, "step": 42 }, { "KL/chosen_KL_mean": 0.002086639404296875, "KL/mean": -0.1864456832408905, "KL/rejected_KL_mean": -0.3749809265136719, "KL/std": 0.49178388714790344, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.37706559896469116, "fcm_dpo/q_t": 0.4905906915664673, "grad_norm": 38.125343322753906, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.561970591545105, "logits/rejected": 0.7507692575454712, "logps/chosen": -89.6419448852539, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.32683563232422, "loss": 1.35, "margin_dpo/margin_mean": 0.3770659565925598, "margin_dpo/margin_std": 0.6495331525802612, "step": 43 }, { "KL/chosen_KL_mean": -0.053157806396484375, "KL/mean": -0.23000340163707733, "KL/rejected_KL_mean": -0.4068489074707031, "KL/std": 0.45470303297042847, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.35368964076042175, "fcm_dpo/q_t": 0.49116647243499756, "grad_norm": 34.782806396484375, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8172358274459839, "logits/rejected": 0.9388797283172607, "logps/chosen": -81.93147277832031, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.82847595214844, "loss": 1.3519, "margin_dpo/margin_mean": 0.35369038581848145, "margin_dpo/margin_std": 0.5031374096870422, "step": 44 }, { "KL/chosen_KL_mean": -0.09437942504882812, "KL/mean": -0.20421305298805237, "KL/rejected_KL_mean": -0.3140411376953125, "KL/std": 0.4009099006652832, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2196604162454605, "fcm_dpo/q_t": 0.49451276659965515, "grad_norm": 24.588449478149414, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.7413744926452637, "logits/rejected": 0.8563113212585449, "logps/chosen": -77.43898010253906, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.56178283691406, "loss": 1.3651, "margin_dpo/margin_mean": 0.21966062486171722, "margin_dpo/margin_std": 0.5006591081619263, "step": 45 }, { "KL/chosen_KL_mean": 0.03072357177734375, "KL/mean": -0.20954471826553345, "KL/rejected_KL_mean": -0.4498176574707031, "KL/std": 0.5726507306098938, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.48054489493370056, "fcm_dpo/q_t": 0.48801061511039734, "grad_norm": 32.74445343017578, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.3083534240722656, "logits/rejected": 0.5420459508895874, "logps/chosen": -90.31010437011719, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.12633514404297, "loss": 1.3399, "margin_dpo/margin_mean": 0.4805442988872528, "margin_dpo/margin_std": 0.6589239835739136, "step": 46 }, { "KL/chosen_KL_mean": -0.024677276611328125, "KL/mean": -0.21249380707740784, "KL/rejected_KL_mean": -0.40030670166015625, "KL/std": 0.47989171743392944, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3756275773048401, "fcm_dpo/q_t": 0.4906248450279236, "grad_norm": 34.102970123291016, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.5263398289680481, "logits/rejected": 0.7131489515304565, "logps/chosen": -104.43597412109375, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.65520477294922, "loss": 1.3502, "margin_dpo/margin_mean": 0.37562689185142517, "margin_dpo/margin_std": 0.6325584650039673, "step": 47 }, { "KL/chosen_KL_mean": 0.010265350341796875, "KL/mean": -0.27038508653640747, "KL/rejected_KL_mean": -0.5510330200195312, "KL/std": 0.7065243721008301, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5612926483154297, "fcm_dpo/q_t": 0.4860079288482666, "grad_norm": 38.79480743408203, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.1903664767742157, "logits/rejected": 0.3882509768009186, "logps/chosen": -91.09001159667969, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.63160705566406, "loss": 1.3331, "margin_dpo/margin_mean": 0.5612931847572327, "margin_dpo/margin_std": 0.9257842302322388, "step": 48 }, { "KL/chosen_KL_mean": 0.072052001953125, "KL/mean": -0.3168909549713135, "KL/rejected_KL_mean": -0.7058296203613281, "KL/std": 0.8207674026489258, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7778768539428711, "fcm_dpo/q_t": 0.48062747716903687, "grad_norm": 39.404754638671875, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.14548581838607788, "logits/rejected": 0.6228343844413757, "logps/chosen": -92.93162536621094, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.45482635498047, "loss": 1.3124, "margin_dpo/margin_mean": 0.7778770923614502, "margin_dpo/margin_std": 0.9855620861053467, "step": 49 }, { "KL/chosen_KL_mean": -0.16970443725585938, "KL/mean": -0.4590950310230255, "KL/rejected_KL_mean": -0.7484779357910156, "KL/std": 0.7413825988769531, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5787715315818787, "fcm_dpo/q_t": 0.48558372259140015, "grad_norm": 33.3923225402832, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.7593181729316711, "logits/rejected": 0.9374700784683228, "logps/chosen": -94.79651641845703, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.32283782958984, "loss": 1.3315, "margin_dpo/margin_mean": 0.578771710395813, "margin_dpo/margin_std": 0.9519371390342712, "step": 50 }, { "KL/chosen_KL_mean": -0.0363616943359375, "KL/mean": -0.39341437816619873, "KL/rejected_KL_mean": -0.7504653930664062, "KL/std": 0.8432061672210693, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7141003608703613, "fcm_dpo/q_t": 0.4822392165660858, "grad_norm": 29.83407211303711, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5468622446060181, "logits/rejected": 0.6844440698623657, "logps/chosen": -87.54364013671875, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.22282409667969, "loss": 1.3191, "margin_dpo/margin_mean": 0.7141010165214539, "margin_dpo/margin_std": 1.0649070739746094, "step": 51 }, { "KL/chosen_KL_mean": 0.06545639038085938, "KL/mean": -0.40916281938552856, "KL/rejected_KL_mean": -0.8837890625, "KL/std": 0.8294536471366882, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9492437839508057, "fcm_dpo/q_t": 0.47635534405708313, "grad_norm": 31.928457260131836, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.31253260374069214, "logits/rejected": 0.5183150172233582, "logps/chosen": -90.56480407714844, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.9228515625, "loss": 1.2957, "margin_dpo/margin_mean": 0.9492431879043579, "margin_dpo/margin_std": 0.9134783148765564, "step": 52 }, { "KL/chosen_KL_mean": -0.004390716552734375, "KL/mean": -0.5401964783668518, "KL/rejected_KL_mean": -1.0760078430175781, "KL/std": 1.099616527557373, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.071617603302002, "fcm_dpo/q_t": 0.4734228253364563, "grad_norm": 39.50225067138672, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.11315110325813293, "logits/rejected": 0.22718656063079834, "logps/chosen": -81.58746337890625, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.73753356933594, "loss": 1.2865, "margin_dpo/margin_mean": 1.0716174840927124, "margin_dpo/margin_std": 1.2796818017959595, "step": 53 }, { "KL/chosen_KL_mean": -0.017459869384765625, "KL/mean": -0.5744718909263611, "KL/rejected_KL_mean": -1.1314849853515625, "KL/std": 1.3422296047210693, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1140236854553223, "fcm_dpo/q_t": 0.47237810492515564, "grad_norm": 39.77252197265625, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.2819768786430359, "logits/rejected": 0.45697668194770813, "logps/chosen": -88.92762756347656, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.24903869628906, "loss": 1.2828, "margin_dpo/margin_mean": 1.1140236854553223, "margin_dpo/margin_std": 1.3781472444534302, "step": 54 }, { "KL/chosen_KL_mean": 0.08527374267578125, "KL/mean": -0.4903573989868164, "KL/rejected_KL_mean": -1.0659828186035156, "KL/std": 1.2253904342651367, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1512532234191895, "fcm_dpo/q_t": 0.47155553102493286, "grad_norm": 35.18937301635742, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.3454848527908325, "logits/rejected": 0.6755592823028564, "logps/chosen": -92.37065124511719, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.03691864013672, "loss": 1.2811, "margin_dpo/margin_mean": 1.1512532234191895, "margin_dpo/margin_std": 1.5947515964508057, "step": 55 }, { "KL/chosen_KL_mean": -0.07695770263671875, "KL/mean": -0.5974669456481934, "KL/rejected_KL_mean": -1.1179695129394531, "KL/std": 1.1016244888305664, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0410125255584717, "fcm_dpo/q_t": 0.4742281436920166, "grad_norm": 33.36632537841797, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.12713563442230225, "logits/rejected": 0.4612199366092682, "logps/chosen": -87.31361389160156, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.76351165771484, "loss": 1.2903, "margin_dpo/margin_mean": 1.0410122871398926, "margin_dpo/margin_std": 1.468178391456604, "step": 56 }, { "KL/chosen_KL_mean": -0.09600448608398438, "KL/mean": -0.6127393245697021, "KL/rejected_KL_mean": -1.1294746398925781, "KL/std": 1.1381404399871826, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.033482551574707, "fcm_dpo/q_t": 0.47436362504959106, "grad_norm": 33.99971389770508, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.24552205204963684, "logits/rejected": 0.4567967653274536, "logps/chosen": -98.24674987792969, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.18231201171875, "loss": 1.2906, "margin_dpo/margin_mean": 1.0334827899932861, "margin_dpo/margin_std": 1.4181424379348755, "step": 57 }, { "KL/chosen_KL_mean": 0.2312164306640625, "KL/mean": -0.5277173519134521, "KL/rejected_KL_mean": -1.2866439819335938, "KL/std": 1.3929860591888428, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5178608894348145, "fcm_dpo/q_t": 0.46262702345848083, "grad_norm": 38.152400970458984, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.6012738347053528, "logits/rejected": 0.9653952121734619, "logps/chosen": -99.3497543334961, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.50891876220703, "loss": 1.249, "margin_dpo/margin_mean": 1.5178605318069458, "margin_dpo/margin_std": 1.801656723022461, "step": 58 }, { "KL/chosen_KL_mean": 0.0190277099609375, "KL/mean": -0.5759836435317993, "KL/rejected_KL_mean": -1.1709976196289062, "KL/std": 1.1743402481079102, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1900187730789185, "fcm_dpo/q_t": 0.4704796075820923, "grad_norm": 34.497215270996094, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.05628311634063721, "logits/rejected": 0.24238519370555878, "logps/chosen": -89.78329467773438, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.44767761230469, "loss": 1.2757, "margin_dpo/margin_mean": 1.190018653869629, "margin_dpo/margin_std": 1.3906362056732178, "step": 59 }, { "KL/chosen_KL_mean": 0.062015533447265625, "KL/mean": -0.534925639629364, "KL/rejected_KL_mean": -1.1318626403808594, "KL/std": 1.25186288356781, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.193880558013916, "fcm_dpo/q_t": 0.47036224603652954, "grad_norm": 28.86550521850586, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.4519467353820801, "logits/rejected": 0.7258943915367126, "logps/chosen": -95.09369659423828, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.50430297851562, "loss": 1.2751, "margin_dpo/margin_mean": 1.1938806772232056, "margin_dpo/margin_std": 1.368309497833252, "step": 60 }, { "KL/chosen_KL_mean": -0.21264266967773438, "KL/mean": -0.8184325098991394, "KL/rejected_KL_mean": -1.4242210388183594, "KL/std": 1.4343489408493042, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2115778923034668, "fcm_dpo/q_t": 0.470175564289093, "grad_norm": 28.343278884887695, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.13284355401992798, "logits/rejected": 0.27287614345550537, "logps/chosen": -85.9449462890625, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.38998413085938, "loss": 1.2774, "margin_dpo/margin_mean": 1.2115782499313354, "margin_dpo/margin_std": 1.8137283325195312, "step": 61 }, { "KL/chosen_KL_mean": -0.21696853637695312, "KL/mean": -0.8274495601654053, "KL/rejected_KL_mean": -1.4379310607910156, "KL/std": 1.4318726062774658, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2209663391113281, "fcm_dpo/q_t": 0.46986156702041626, "grad_norm": 28.55064582824707, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.2784372568130493, "logits/rejected": 0.4804416298866272, "logps/chosen": -81.85235595703125, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.47624206542969, "loss": 1.2774, "margin_dpo/margin_mean": 1.220966100692749, "margin_dpo/margin_std": 1.8640551567077637, "step": 62 }, { "KL/chosen_KL_mean": -0.016605377197265625, "KL/mean": -0.8077281713485718, "KL/rejected_KL_mean": -1.5988540649414062, "KL/std": 1.3883479833602905, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5822484493255615, "fcm_dpo/q_t": 0.4608187973499298, "grad_norm": 34.929203033447266, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.5167672038078308, "logits/rejected": 0.7374684810638428, "logps/chosen": -103.64066314697266, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -105.00188446044922, "loss": 1.2404, "margin_dpo/margin_mean": 1.5822486877441406, "margin_dpo/margin_std": 1.5414446592330933, "step": 63 }, { "KL/chosen_KL_mean": -0.11608123779296875, "KL/mean": -1.0210025310516357, "KL/rejected_KL_mean": -1.925933837890625, "KL/std": 1.88437819480896, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8098537921905518, "fcm_dpo/q_t": 0.4556460380554199, "grad_norm": 34.6662712097168, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.584801435470581, "logits/rejected": 0.8066180944442749, "logps/chosen": -87.11766815185547, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.51140594482422, "loss": 1.2255, "margin_dpo/margin_mean": 1.8098540306091309, "margin_dpo/margin_std": 2.14643931388855, "step": 64 }, { "KL/chosen_KL_mean": -0.0353851318359375, "KL/mean": -0.9857560396194458, "KL/rejected_KL_mean": -1.936126708984375, "KL/std": 1.7333950996398926, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9007368087768555, "fcm_dpo/q_t": 0.453359991312027, "grad_norm": 37.97596740722656, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.10751180350780487, "logits/rejected": 0.2527211010456085, "logps/chosen": -91.2573013305664, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.27165985107422, "loss": 1.2164, "margin_dpo/margin_mean": 1.9007365703582764, "margin_dpo/margin_std": 2.0984323024749756, "step": 65 }, { "KL/chosen_KL_mean": -0.2741737365722656, "KL/mean": -0.9159947037696838, "KL/rejected_KL_mean": -1.5578155517578125, "KL/std": 1.604280948638916, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2836424112319946, "fcm_dpo/q_t": 0.4684101641178131, "grad_norm": 25.564075469970703, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.03958917781710625, "logits/rejected": 0.2371593713760376, "logps/chosen": -84.05839538574219, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.3099136352539, "loss": 1.2712, "margin_dpo/margin_mean": 1.2836428880691528, "margin_dpo/margin_std": 1.8423571586608887, "step": 66 }, { "KL/chosen_KL_mean": -0.862060546875, "KL/mean": -1.4640653133392334, "KL/rejected_KL_mean": -2.0660629272460938, "KL/std": 2.09283185005188, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2040014266967773, "fcm_dpo/q_t": 0.4708825945854187, "grad_norm": 24.268627166748047, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.23121094703674316, "logits/rejected": 0.5018829107284546, "logps/chosen": -88.53501892089844, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.15281677246094, "loss": 1.289, "margin_dpo/margin_mean": 1.2040016651153564, "margin_dpo/margin_std": 2.8043532371520996, "step": 67 }, { "KL/chosen_KL_mean": -0.2660789489746094, "KL/mean": -1.323544979095459, "KL/rejected_KL_mean": -2.381000518798828, "KL/std": 2.2348899841308594, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.1149191856384277, "fcm_dpo/q_t": 0.4489330053329468, "grad_norm": 28.46953010559082, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.13828939199447632, "logits/rejected": 0.1428043693304062, "logps/chosen": -97.34492492675781, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.49972534179688, "loss": 1.2046, "margin_dpo/margin_mean": 2.1149191856384277, "margin_dpo/margin_std": 2.7578306198120117, "step": 68 }, { "KL/chosen_KL_mean": -0.2566108703613281, "KL/mean": -1.3858494758605957, "KL/rejected_KL_mean": -2.515087127685547, "KL/std": 2.400379180908203, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.258471965789795, "fcm_dpo/q_t": 0.44565168023109436, "grad_norm": 30.757658004760742, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.3174981474876404, "logits/rejected": 0.425376832485199, "logps/chosen": -85.976318359375, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.99529266357422, "loss": 1.1943, "margin_dpo/margin_mean": 2.258472442626953, "margin_dpo/margin_std": 2.945460796356201, "step": 69 }, { "KL/chosen_KL_mean": -0.5695610046386719, "KL/mean": -1.846235990524292, "KL/rejected_KL_mean": -3.1229095458984375, "KL/std": 2.72996187210083, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.5533456802368164, "fcm_dpo/q_t": 0.43917179107666016, "grad_norm": 29.4632511138916, "learning_rate": 5e-07, "logits/chosen": 0.4226685166358948, "logits/rejected": 0.6090021133422852, "logps/chosen": -95.57951354980469, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.33563232421875, "loss": 1.1743, "margin_dpo/margin_mean": 2.5533461570739746, "margin_dpo/margin_std": 3.291403293609619, "step": 70 }, { "KL/chosen_KL_mean": -0.05511474609375, "KL/mean": -1.892472267150879, "KL/rejected_KL_mean": -3.729827880859375, "KL/std": 3.5702404975891113, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.674713373184204, "fcm_dpo/q_t": 0.41432809829711914, "grad_norm": 34.28628921508789, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.015051089227199554, "logits/rejected": 0.36013656854629517, "logps/chosen": -91.032470703125, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.32086181640625, "loss": 1.0907, "margin_dpo/margin_mean": 3.674713611602783, "margin_dpo/margin_std": 4.111626148223877, "step": 71 }, { "KL/chosen_KL_mean": -0.5602607727050781, "KL/mean": -2.0173287391662598, "KL/rejected_KL_mean": -3.4743995666503906, "KL/std": 2.922941207885742, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.914137363433838, "fcm_dpo/q_t": 0.4300009608268738, "grad_norm": 29.64742660522461, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.15872156620025635, "logits/rejected": 0.1673320233821869, "logps/chosen": -98.4517822265625, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.6661148071289, "loss": 1.1421, "margin_dpo/margin_mean": 2.914137363433838, "margin_dpo/margin_std": 3.2901432514190674, "step": 72 }, { "KL/chosen_KL_mean": -0.7892036437988281, "KL/mean": -2.358266592025757, "KL/rejected_KL_mean": -3.9273300170898438, "KL/std": 3.8605570793151855, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.1381263732910156, "fcm_dpo/q_t": 0.42862066626548767, "grad_norm": 27.93511199951172, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.4713694453239441, "logits/rejected": -0.17588545382022858, "logps/chosen": -96.5582275390625, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.85977172851562, "loss": 1.1517, "margin_dpo/margin_mean": 3.1381258964538574, "margin_dpo/margin_std": 4.895217418670654, "step": 73 }, { "KL/chosen_KL_mean": -1.0672454833984375, "KL/mean": -2.7722110748291016, "KL/rejected_KL_mean": -4.4771728515625, "KL/std": 3.7395336627960205, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.409931182861328, "fcm_dpo/q_t": 0.4205431044101715, "grad_norm": 31.315034866333008, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.09020809829235077, "logits/rejected": 0.052455835044384, "logps/chosen": -79.87564086914062, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.12394714355469, "loss": 1.1205, "margin_dpo/margin_mean": 3.40993070602417, "margin_dpo/margin_std": 4.431780815124512, "step": 74 }, { "KL/chosen_KL_mean": -1.0992622375488281, "KL/mean": -3.373971939086914, "KL/rejected_KL_mean": -5.648681640625, "KL/std": 4.5876359939575195, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.09933328628540039, "fcm_dpo/delta": -0.06711969524621964, "fcm_dpo/margin": 4.54942512512207, "fcm_dpo/q_t": 0.39791277050971985, "grad_norm": 32.256263732910156, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.1517581343650818, "logits/rejected": 0.04852603003382683, "logps/chosen": -79.38111877441406, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.05179595947266, "loss": 1.0491, "margin_dpo/margin_mean": 4.549424648284912, "margin_dpo/margin_std": 5.396270751953125, "step": 75 }, { "KL/chosen_KL_mean": -1.4258346557617188, "KL/mean": -3.014122724533081, "KL/rejected_KL_mean": -4.602413177490234, "KL/std": 3.865478992462158, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.09866657853126526, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.176578998565674, "fcm_dpo/q_t": 0.4274882376194, "grad_norm": 22.54557228088379, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.5843378901481628, "logits/rejected": -0.2021692842245102, "logps/chosen": -89.302978515625, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.32209777832031, "loss": 1.145, "margin_dpo/margin_mean": 3.1765787601470947, "margin_dpo/margin_std": 4.5940680503845215, "step": 76 }, { "KL/chosen_KL_mean": -1.6254348754882812, "KL/mean": -3.3220784664154053, "KL/rejected_KL_mean": -5.0187225341796875, "KL/std": 4.433165550231934, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.09866657853126526, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.393289089202881, "fcm_dpo/q_t": 0.42219918966293335, "grad_norm": 26.920801162719727, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.2383897453546524, "logits/rejected": 0.1270105242729187, "logps/chosen": -101.3257827758789, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.2244873046875, "loss": 1.1353, "margin_dpo/margin_mean": 3.3932883739471436, "margin_dpo/margin_std": 4.919417858123779, "step": 77 }, { "KL/chosen_KL_mean": -0.6964759826660156, "KL/mean": -2.5159130096435547, "KL/rejected_KL_mean": -4.335346221923828, "KL/std": 3.7234981060028076, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.09866657853126526, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.638871192932129, "fcm_dpo/q_t": 0.4161747097969055, "grad_norm": 25.15743637084961, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.09597080200910568, "logits/rejected": 0.12996482849121094, "logps/chosen": -90.9931869506836, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.47306823730469, "loss": 1.1023, "margin_dpo/margin_mean": 3.638871669769287, "margin_dpo/margin_std": 4.392696380615234, "step": 78 }, { "KL/chosen_KL_mean": -0.933258056640625, "KL/mean": -3.3430418968200684, "KL/rejected_KL_mean": -5.752830505371094, "KL/std": 5.458436965942383, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.09770765155553818, "fcm_dpo/delta": -0.07435141503810883, "fcm_dpo/margin": 4.819570541381836, "fcm_dpo/q_t": 0.3938406705856323, "grad_norm": 26.37002944946289, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.7128242254257202, "logits/rejected": -0.3353617489337921, "logps/chosen": -87.31158447265625, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.860595703125, "loss": 1.0459, "margin_dpo/margin_mean": 4.819570064544678, "margin_dpo/margin_std": 6.016210556030273, "step": 79 }, { "KL/chosen_KL_mean": -1.6091575622558594, "KL/mean": -3.5220985412597656, "KL/rejected_KL_mean": -5.435039520263672, "KL/std": 4.782077789306641, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.09721022844314575, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.825878620147705, "fcm_dpo/q_t": 0.4159051477909088, "grad_norm": 24.883785247802734, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.694533109664917, "logits/rejected": -0.3355705440044403, "logps/chosen": -95.57947540283203, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -98.00946044921875, "loss": 1.1085, "margin_dpo/margin_mean": 3.825878381729126, "margin_dpo/margin_std": 5.280166149139404, "step": 80 }, { "KL/chosen_KL_mean": -1.0774002075195312, "KL/mean": -3.783722162246704, "KL/rejected_KL_mean": -6.490039825439453, "KL/std": 5.954261779785156, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.09506790339946747, "fcm_dpo/delta": -0.11142244935035706, "fcm_dpo/margin": 5.4126458168029785, "fcm_dpo/q_t": 0.38907331228256226, "grad_norm": 26.655338287353516, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6559590101242065, "logits/rejected": -0.23608830571174622, "logps/chosen": -100.90752410888672, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.16250610351562, "loss": 1.0318, "margin_dpo/margin_mean": 5.412646293640137, "margin_dpo/margin_std": 7.018255233764648, "step": 81 }, { "KL/chosen_KL_mean": -1.9857330322265625, "KL/mean": -3.7995853424072266, "KL/rejected_KL_mean": -5.613437652587891, "KL/std": 4.549775123596191, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.0948246568441391, "fcm_dpo/delta": -0.025652676820755005, "fcm_dpo/margin": 3.6277072429656982, "fcm_dpo/q_t": 0.4204840660095215, "grad_norm": 25.26444435119629, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.2619868218898773, "logits/rejected": -0.0464022196829319, "logps/chosen": -85.0317153930664, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.13938903808594, "loss": 1.1263, "margin_dpo/margin_mean": 3.6277074813842773, "margin_dpo/margin_std": 4.987654685974121, "step": 82 }, { "KL/chosen_KL_mean": -2.43548583984375, "KL/mean": -4.534881591796875, "KL/rejected_KL_mean": -6.634269714355469, "KL/std": 5.262630462646484, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.09458140283823013, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.198792457580566, "fcm_dpo/q_t": 0.41214898228645325, "grad_norm": 25.716339111328125, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.49118122458457947, "logits/rejected": -0.06322473287582397, "logps/chosen": -94.61170196533203, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.37891387939453, "loss": 1.1184, "margin_dpo/margin_mean": 4.198792934417725, "margin_dpo/margin_std": 6.681699752807617, "step": 83 }, { "KL/chosen_KL_mean": -3.0947036743164062, "KL/mean": -4.399570465087891, "KL/rejected_KL_mean": -5.704444885253906, "KL/std": 4.704437255859375, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.09458140283823013, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.6097395420074463, "fcm_dpo/q_t": 0.44363635778427124, "grad_norm": 27.139984130859375, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.269260048866272, "logits/rejected": 0.1387065052986145, "logps/chosen": -104.62120056152344, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -98.13053131103516, "loss": 1.2131, "margin_dpo/margin_mean": 2.6097402572631836, "margin_dpo/margin_std": 4.994235515594482, "step": 84 }, { "KL/chosen_KL_mean": -2.708049774169922, "KL/mean": -4.452479839324951, "KL/rejected_KL_mean": -6.196907043457031, "KL/std": 5.190540313720703, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.09458140283823013, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.488859176635742, "fcm_dpo/q_t": 0.42533332109451294, "grad_norm": 25.58100700378418, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.34033992886543274, "logits/rejected": -0.0032431259751319885, "logps/chosen": -98.8354263305664, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.25209045410156, "loss": 1.1649, "margin_dpo/margin_mean": 3.488859176635742, "margin_dpo/margin_std": 6.216080665588379, "step": 85 }, { "KL/chosen_KL_mean": -2.1392478942871094, "KL/mean": -4.37795877456665, "KL/rejected_KL_mean": -6.616672515869141, "KL/std": 5.980083465576172, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.09458140283823013, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.477426528930664, "fcm_dpo/q_t": 0.4109261631965637, "grad_norm": 21.370189666748047, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6570388078689575, "logits/rejected": -0.415638267993927, "logps/chosen": -83.843505859375, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.27217102050781, "loss": 1.111, "margin_dpo/margin_mean": 4.477426528930664, "margin_dpo/margin_std": 7.459688186645508, "step": 86 }, { "KL/chosen_KL_mean": -2.544567108154297, "KL/mean": -4.993801593780518, "KL/rejected_KL_mean": -7.443035125732422, "KL/std": 6.638210296630859, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.09350112825632095, "fcm_dpo/delta": -0.057436924427747726, "fcm_dpo/margin": 4.898465633392334, "fcm_dpo/q_t": 0.39723488688468933, "grad_norm": 25.484420776367188, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.9348252415657043, "logits/rejected": -0.5627981424331665, "logps/chosen": -95.23052978515625, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.36121368408203, "loss": 1.0977, "margin_dpo/margin_mean": 4.898465633392334, "margin_dpo/margin_std": 7.770233154296875, "step": 87 }, { "KL/chosen_KL_mean": -3.0133323669433594, "KL/mean": -4.957260608673096, "KL/rejected_KL_mean": -6.90118408203125, "KL/std": 6.351088047027588, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.09262136369943619, "fcm_dpo/delta": -0.04726821556687355, "fcm_dpo/margin": 3.8878514766693115, "fcm_dpo/q_t": 0.4198633134365082, "grad_norm": 22.98442840576172, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.39207613468170166, "logits/rejected": -0.20508261024951935, "logps/chosen": -103.77631378173828, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -120.05155944824219, "loss": 1.1474, "margin_dpo/margin_mean": 3.8878509998321533, "margin_dpo/margin_std": 6.631938457489014, "step": 88 }, { "KL/chosen_KL_mean": -2.478809356689453, "KL/mean": -4.6209001541137695, "KL/rejected_KL_mean": -6.762989044189453, "KL/std": 5.48379373550415, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.09262136369943619, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.284178256988525, "fcm_dpo/q_t": 0.4153037965297699, "grad_norm": 22.603946685791016, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7442423105239868, "logits/rejected": -0.407105416059494, "logps/chosen": -92.16989135742188, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.3313217163086, "loss": 1.1212, "margin_dpo/margin_mean": 4.284177780151367, "margin_dpo/margin_std": 7.230534553527832, "step": 89 }, { "KL/chosen_KL_mean": -2.1734619140625, "KL/mean": -4.930908679962158, "KL/rejected_KL_mean": -7.688358306884766, "KL/std": 6.349458694458008, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.09175005555152893, "fcm_dpo/delta": -0.0949685201048851, "fcm_dpo/margin": 5.514894962310791, "fcm_dpo/q_t": 0.39062580466270447, "grad_norm": 25.23427391052246, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.3678678572177887, "logits/rejected": -0.24826332926750183, "logps/chosen": -83.55601501464844, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.46549987792969, "loss": 1.0474, "margin_dpo/margin_mean": 5.514893531799316, "margin_dpo/margin_std": 7.5883989334106445, "step": 90 }, { "KL/chosen_KL_mean": -2.514057159423828, "KL/mean": -4.842683792114258, "KL/rejected_KL_mean": -7.1713104248046875, "KL/std": 6.2337799072265625, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.09087873995304108, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.657258033752441, "fcm_dpo/q_t": 0.4088599979877472, "grad_norm": 24.120492935180664, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.4771268963813782, "logits/rejected": -0.06662188470363617, "logps/chosen": -99.7359390258789, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.14805603027344, "loss": 1.1013, "margin_dpo/margin_mean": 4.657257556915283, "margin_dpo/margin_std": 7.250188827514648, "step": 91 }, { "KL/chosen_KL_mean": -2.8020858764648438, "KL/mean": -4.821112155914307, "KL/rejected_KL_mean": -6.8401336669921875, "KL/std": 6.10978889465332, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.09087873995304108, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.038049697875977, "fcm_dpo/q_t": 0.4184359312057495, "grad_norm": 22.051654815673828, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8975330591201782, "logits/rejected": -0.5339422225952148, "logps/chosen": -87.33236694335938, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.3414535522461, "loss": 1.1309, "margin_dpo/margin_mean": 4.038049697875977, "margin_dpo/margin_std": 6.48369026184082, "step": 92 }, { "KL/chosen_KL_mean": -2.9955368041992188, "KL/mean": -4.915168762207031, "KL/rejected_KL_mean": -6.834804534912109, "KL/std": 5.893106937408447, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.09087873995304108, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.8392677307128906, "fcm_dpo/q_t": 0.42271336913108826, "grad_norm": 24.856229782104492, "learning_rate": 4.982595640958425e-07, "logits/chosen": -1.068142056465149, "logits/rejected": -0.5115662813186646, "logps/chosen": -93.2459716796875, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.92901611328125, "loss": 1.1649, "margin_dpo/margin_mean": 3.8392677307128906, "margin_dpo/margin_std": 7.141761779785156, "step": 93 }, { "KL/chosen_KL_mean": -3.1853561401367188, "KL/mean": -6.139699459075928, "KL/rejected_KL_mean": -9.094039916992188, "KL/std": 6.912477016448975, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.08829281479120255, "fcm_dpo/delta": -0.14433667063713074, "fcm_dpo/margin": 5.908690452575684, "fcm_dpo/q_t": 0.3839508891105652, "grad_norm": 25.180158615112305, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.695547342300415, "logits/rejected": -0.3893578350543976, "logps/chosen": -95.63957214355469, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.3413314819336, "loss": 1.0434, "margin_dpo/margin_mean": 5.908689975738525, "margin_dpo/margin_std": 7.935752868652344, "step": 94 }, { "KL/chosen_KL_mean": -2.5814590454101562, "KL/mean": -4.914426803588867, "KL/rejected_KL_mean": -7.2473907470703125, "KL/std": 5.77323055267334, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.08786122500896454, "fcm_dpo/delta": -0.04912182316184044, "fcm_dpo/margin": 4.665931701660156, "fcm_dpo/q_t": 0.41028860211372375, "grad_norm": 20.626972198486328, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.9603093862533569, "logits/rejected": -0.6262775659561157, "logps/chosen": -81.45516967773438, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.83860778808594, "loss": 1.1072, "margin_dpo/margin_mean": 4.665931701660156, "margin_dpo/margin_std": 7.103507041931152, "step": 95 }, { "KL/chosen_KL_mean": -2.3505096435546875, "KL/mean": -5.233642578125, "KL/rejected_KL_mean": -8.116771697998047, "KL/std": 6.99694299697876, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.08590106666088104, "fcm_dpo/delta": -0.10070987045764923, "fcm_dpo/margin": 5.766265392303467, "fcm_dpo/q_t": 0.39157330989837646, "grad_norm": 21.25580406188965, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.6541320085525513, "logits/rejected": -0.193709135055542, "logps/chosen": -108.94267272949219, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.88479614257812, "loss": 1.0483, "margin_dpo/margin_mean": 5.766265869140625, "margin_dpo/margin_std": 7.692651748657227, "step": 96 }, { "KL/chosen_KL_mean": -3.376434326171875, "KL/mean": -6.000686168670654, "KL/rejected_KL_mean": -8.62493896484375, "KL/std": 6.112391471862793, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.08484761416912079, "fcm_dpo/delta": -0.09884320199489594, "fcm_dpo/margin": 5.24851131439209, "fcm_dpo/q_t": 0.3987596929073334, "grad_norm": 23.878461837768555, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.45077234506607056, "logits/rejected": -0.038176506757736206, "logps/chosen": -92.05632019042969, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.4372329711914, "loss": 1.0642, "margin_dpo/margin_mean": 5.248510360717773, "margin_dpo/margin_std": 6.503949165344238, "step": 97 }, { "KL/chosen_KL_mean": -2.5479469299316406, "KL/mean": -5.492012977600098, "KL/rejected_KL_mean": -8.436077117919922, "KL/std": 5.818595886230469, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.08247773349285126, "fcm_dpo/delta": -0.0919768214225769, "fcm_dpo/margin": 5.888130187988281, "fcm_dpo/q_t": 0.38975873589515686, "grad_norm": 21.860132217407227, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6643891334533691, "logits/rejected": -0.3965454697608948, "logps/chosen": -94.79044342041016, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.95556640625, "loss": 1.0272, "margin_dpo/margin_mean": 5.888130187988281, "margin_dpo/margin_std": 6.594422340393066, "step": 98 }, { "KL/chosen_KL_mean": -1.9422111511230469, "KL/mean": -5.216865539550781, "KL/rejected_KL_mean": -8.491527557373047, "KL/std": 7.034444808959961, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.08075308054685593, "fcm_dpo/delta": -0.13621443510055542, "fcm_dpo/margin": 6.549323081970215, "fcm_dpo/q_t": 0.388407826423645, "grad_norm": 21.545473098754883, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.683335542678833, "logits/rejected": -0.46955689787864685, "logps/chosen": -74.12686157226562, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.37284851074219, "loss": 1.0313, "margin_dpo/margin_mean": 6.549323081970215, "margin_dpo/margin_std": 8.998091697692871, "step": 99 }, { "KL/chosen_KL_mean": -2.2657508850097656, "KL/mean": -4.6339521408081055, "KL/rejected_KL_mean": -7.0021514892578125, "KL/std": 5.324350357055664, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.07989377528429031, "fcm_dpo/delta": -0.022937653586268425, "fcm_dpo/margin": 4.736395835876465, "fcm_dpo/q_t": 0.4120832681655884, "grad_norm": 18.775609970092773, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.580803394317627, "logits/rejected": -0.08000028878450394, "logps/chosen": -91.77957916259766, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -88.21928405761719, "loss": 1.0916, "margin_dpo/margin_mean": 4.736395835876465, "margin_dpo/margin_std": 5.7029218673706055, "step": 100 }, { "KL/chosen_KL_mean": -3.630359649658203, "KL/mean": -5.743829727172852, "KL/rejected_KL_mean": -7.8572998046875, "KL/std": 6.871623992919922, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.07989377528429031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.22693395614624, "fcm_dpo/q_t": 0.4226118326187134, "grad_norm": 22.630651473999023, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.8240365386009216, "logits/rejected": -0.552397608757019, "logps/chosen": -98.03355407714844, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.76731872558594, "loss": 1.159, "margin_dpo/margin_mean": 4.226934432983398, "margin_dpo/margin_std": 7.36295223236084, "step": 101 }, { "KL/chosen_KL_mean": -4.242183685302734, "KL/mean": -6.0341339111328125, "KL/rejected_KL_mean": -7.826084136962891, "KL/std": 6.606969356536865, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.07989377528429031, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5839052200317383, "fcm_dpo/q_t": 0.4350793659687042, "grad_norm": 27.295082092285156, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.9114592671394348, "logits/rejected": -0.5431338548660278, "logps/chosen": -95.85160827636719, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.59500122070312, "loss": 1.2061, "margin_dpo/margin_mean": 3.5839054584503174, "margin_dpo/margin_std": 7.465989589691162, "step": 102 }, { "KL/chosen_KL_mean": -3.286590576171875, "KL/mean": -6.327729225158691, "KL/rejected_KL_mean": -9.368873596191406, "KL/std": 6.912601470947266, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.07879278063774109, "fcm_dpo/delta": -0.08339595794677734, "fcm_dpo/margin": 6.082282543182373, "fcm_dpo/q_t": 0.391263872385025, "grad_norm": 21.917448043823242, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.8168344497680664, "logits/rejected": -0.42187339067459106, "logps/chosen": -100.68898010253906, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.35055541992188, "loss": 1.0508, "margin_dpo/margin_mean": 6.082283020019531, "margin_dpo/margin_std": 7.792575836181641, "step": 103 }, { "KL/chosen_KL_mean": -2.5164642333984375, "KL/mean": -5.848998069763184, "KL/rejected_KL_mean": -9.181529998779297, "KL/std": 7.54154634475708, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.07705871015787125, "fcm_dpo/delta": -0.12000001966953278, "fcm_dpo/margin": 6.665066242218018, "fcm_dpo/q_t": 0.38898754119873047, "grad_norm": 18.27594757080078, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.16951584815979, "logits/rejected": -0.8305656909942627, "logps/chosen": -97.3728256225586, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -106.20822143554688, "loss": 1.033, "margin_dpo/margin_mean": 6.665067195892334, "margin_dpo/margin_std": 8.904952049255371, "step": 104 }, { "KL/chosen_KL_mean": -3.391803741455078, "KL/mean": -6.665005683898926, "KL/rejected_KL_mean": -9.938209533691406, "KL/std": 6.894161224365234, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.07594871520996094, "fcm_dpo/delta": -0.10010599344968796, "fcm_dpo/margin": 6.54640531539917, "fcm_dpo/q_t": 0.3891414701938629, "grad_norm": 21.640180587768555, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.0288052558898926, "logits/rejected": -0.5525459051132202, "logps/chosen": -100.14970397949219, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.51570129394531, "loss": 1.0477, "margin_dpo/margin_mean": 6.546405792236328, "margin_dpo/margin_std": 8.666999816894531, "step": 105 }, { "KL/chosen_KL_mean": -3.3176841735839844, "KL/mean": -6.479623794555664, "KL/rejected_KL_mean": -9.641563415527344, "KL/std": 7.176762580871582, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.07465924322605133, "fcm_dpo/delta": -0.07088378816843033, "fcm_dpo/margin": 6.323873519897461, "fcm_dpo/q_t": 0.39682602882385254, "grad_norm": 19.307086944580078, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.7973189949989319, "logits/rejected": -0.3633381724357605, "logps/chosen": -95.7555923461914, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.65126037597656, "loss": 1.0613, "margin_dpo/margin_mean": 6.323873043060303, "margin_dpo/margin_std": 8.697924613952637, "step": 106 }, { "KL/chosen_KL_mean": -2.517681121826172, "KL/mean": -6.188137054443359, "KL/rejected_KL_mean": -9.858604431152344, "KL/std": 9.163302421569824, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.07312531024217606, "fcm_dpo/delta": -0.0682314783334732, "fcm_dpo/margin": 7.3409271240234375, "fcm_dpo/q_t": 0.3917413353919983, "grad_norm": 19.720603942871094, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.8825316429138184, "logits/rejected": -0.36801016330718994, "logps/chosen": -107.64268493652344, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -111.04110717773438, "loss": 1.0495, "margin_dpo/margin_mean": 7.3409271240234375, "margin_dpo/margin_std": 11.423040390014648, "step": 107 }, { "KL/chosen_KL_mean": -3.5868759155273438, "KL/mean": -7.094350814819336, "KL/rejected_KL_mean": -10.601825714111328, "KL/std": 8.462604522705078, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.07165085524320602, "fcm_dpo/delta": -0.10184749215841293, "fcm_dpo/margin": 7.014951705932617, "fcm_dpo/q_t": 0.39027124643325806, "grad_norm": 19.532438278198242, "learning_rate": 4.952587095041881e-07, "logits/chosen": -1.1337040662765503, "logits/rejected": -0.6370459794998169, "logps/chosen": -92.79623413085938, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -112.4396743774414, "loss": 1.0426, "margin_dpo/margin_mean": 7.014951705932617, "margin_dpo/margin_std": 9.186939239501953, "step": 108 }, { "KL/chosen_KL_mean": -2.8776016235351562, "KL/mean": -7.1740827560424805, "KL/rejected_KL_mean": -11.470569610595703, "KL/std": 8.444448471069336, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.06927010416984558, "fcm_dpo/delta": -0.20836535096168518, "fcm_dpo/margin": 8.59296703338623, "fcm_dpo/q_t": 0.3701675832271576, "grad_norm": 18.486238479614258, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.6630968451499939, "logits/rejected": -0.4302247166633606, "logps/chosen": -103.06468963623047, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -115.00796508789062, "loss": 0.9861, "margin_dpo/margin_mean": 8.592967987060547, "margin_dpo/margin_std": 10.425989151000977, "step": 109 }, { "KL/chosen_KL_mean": -3.7486534118652344, "KL/mean": -6.28520393371582, "KL/rejected_KL_mean": -8.821754455566406, "KL/std": 6.999473571777344, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.06831652671098709, "fcm_dpo/delta": -0.029901064932346344, "fcm_dpo/margin": 5.073103427886963, "fcm_dpo/q_t": 0.42112576961517334, "grad_norm": 20.221851348876953, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.4351181089878082, "logits/rejected": -0.030291110277175903, "logps/chosen": -110.64437103271484, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.6329574584961, "loss": 1.1403, "margin_dpo/margin_mean": 5.073103904724121, "margin_dpo/margin_std": 7.989859104156494, "step": 110 }, { "KL/chosen_KL_mean": -3.348278045654297, "KL/mean": -8.355974197387695, "KL/rejected_KL_mean": -13.363670349121094, "KL/std": 9.830649375915527, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.06523895263671875, "fcm_dpo/delta": -0.27328556776046753, "fcm_dpo/margin": 10.015396118164062, "fcm_dpo/q_t": 0.3562661409378052, "grad_norm": 16.94731903076172, "learning_rate": 4.944834074412042e-07, "logits/chosen": -1.1406621932983398, "logits/rejected": -0.8625224828720093, "logps/chosen": -96.67764282226562, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -117.54115295410156, "loss": 0.9481, "margin_dpo/margin_mean": 10.015397071838379, "margin_dpo/margin_std": 10.94532585144043, "step": 111 }, { "KL/chosen_KL_mean": -3.8059654235839844, "KL/mean": -6.352234363555908, "KL/rejected_KL_mean": -8.898502349853516, "KL/std": 5.701839923858643, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.06468275934457779, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.092533111572266, "fcm_dpo/q_t": 0.42389750480651855, "grad_norm": 17.039443969726562, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.287933349609375, "logits/rejected": -0.7846507430076599, "logps/chosen": -86.55300903320312, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -80.45695495605469, "loss": 1.1434, "margin_dpo/margin_mean": 5.092533111572266, "margin_dpo/margin_std": 7.763899803161621, "step": 112 }, { "KL/chosen_KL_mean": -4.8219757080078125, "KL/mean": -7.550571441650391, "KL/rejected_KL_mean": -10.279167175292969, "KL/std": 7.640338897705078, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.06468275934457779, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.457193374633789, "fcm_dpo/q_t": 0.4191613495349884, "grad_norm": 20.537086486816406, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.9956755042076111, "logits/rejected": -0.45875298976898193, "logps/chosen": -102.72146606445312, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -94.69876098632812, "loss": 1.1221, "margin_dpo/margin_mean": 5.457192897796631, "margin_dpo/margin_std": 7.401350021362305, "step": 113 }, { "KL/chosen_KL_mean": -4.467662811279297, "KL/mean": -8.323104858398438, "KL/rejected_KL_mean": -12.178546905517578, "KL/std": 8.815366744995117, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.06366957724094391, "fcm_dpo/delta": -0.07893900573253632, "fcm_dpo/margin": 7.710888385772705, "fcm_dpo/q_t": 0.3946099281311035, "grad_norm": 19.951396942138672, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.1893324851989746, "logits/rejected": -0.6149265766143799, "logps/chosen": -100.96278381347656, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -104.19390869140625, "loss": 1.0724, "margin_dpo/margin_mean": 7.7108893394470215, "margin_dpo/margin_std": 11.497998237609863, "step": 114 }, { "KL/chosen_KL_mean": -4.577579498291016, "KL/mean": -8.42097282409668, "KL/rejected_KL_mean": -12.264362335205078, "KL/std": 9.177568435668945, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.06287835538387299, "fcm_dpo/delta": -0.08752565830945969, "fcm_dpo/margin": 7.686782360076904, "fcm_dpo/q_t": 0.39274534583091736, "grad_norm": 17.738801956176758, "learning_rate": 4.933595135901732e-07, "logits/chosen": -1.006196141242981, "logits/rejected": -0.595270037651062, "logps/chosen": -107.75770568847656, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -117.98031616210938, "loss": 1.0516, "margin_dpo/margin_mean": 7.6867828369140625, "margin_dpo/margin_std": 10.050143241882324, "step": 115 }, { "KL/chosen_KL_mean": -4.0863037109375, "KL/mean": -7.332365989685059, "KL/rejected_KL_mean": -10.578426361083984, "KL/std": 7.313093185424805, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.062317103147506714, "fcm_dpo/delta": -0.03973669558763504, "fcm_dpo/margin": 6.492128372192383, "fcm_dpo/q_t": 0.40741050243377686, "grad_norm": 16.700048446655273, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.0309791564941406, "logits/rejected": -0.595293402671814, "logps/chosen": -85.24467468261719, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -96.88428497314453, "loss": 1.0856, "margin_dpo/margin_mean": 6.492128849029541, "margin_dpo/margin_std": 8.306146621704102, "step": 116 }, { "KL/chosen_KL_mean": -5.0340118408203125, "KL/mean": -7.632197380065918, "KL/rejected_KL_mean": -10.230388641357422, "KL/std": 7.498805046081543, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.06206947937607765, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.196374893188477, "fcm_dpo/q_t": 0.42628949880599976, "grad_norm": 24.371545791625977, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.8127318024635315, "logits/rejected": -0.39380383491516113, "logps/chosen": -102.1149673461914, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -85.37030029296875, "loss": 1.169, "margin_dpo/margin_mean": 5.196374893188477, "margin_dpo/margin_std": 9.28470230102539, "step": 117 }, { "KL/chosen_KL_mean": -3.638507843017578, "KL/mean": -7.3475470542907715, "KL/rejected_KL_mean": -11.056583404541016, "KL/std": 8.082574844360352, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.06158997118473053, "fcm_dpo/delta": -0.03877658769488335, "fcm_dpo/margin": 7.418074131011963, "fcm_dpo/q_t": 0.39512962102890015, "grad_norm": 16.011783599853516, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.427377462387085, "logits/rejected": -1.1111563444137573, "logps/chosen": -90.17474365234375, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -93.99222564697266, "loss": 1.0448, "margin_dpo/margin_mean": 7.418074131011963, "margin_dpo/margin_std": 8.619097709655762, "step": 118 }, { "KL/chosen_KL_mean": -5.3523712158203125, "KL/mean": -9.298324584960938, "KL/rejected_KL_mean": -13.244274139404297, "KL/std": 9.694939613342285, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.06077214330434799, "fcm_dpo/delta": -0.06683772802352905, "fcm_dpo/margin": 7.891900062561035, "fcm_dpo/q_t": 0.392980694770813, "grad_norm": 18.439542770385742, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.1314113140106201, "logits/rejected": -0.8000422716140747, "logps/chosen": -109.96145629882812, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -122.40208435058594, "loss": 1.087, "margin_dpo/margin_mean": 7.891901016235352, "margin_dpo/margin_std": 12.34011173248291, "step": 119 }, { "KL/chosen_KL_mean": -4.330284118652344, "KL/mean": -9.522979736328125, "KL/rejected_KL_mean": -14.71567153930664, "KL/std": 11.591257095336914, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.058743223547935486, "fcm_dpo/delta": -0.22390630841255188, "fcm_dpo/margin": 10.38538932800293, "fcm_dpo/q_t": 0.3713318705558777, "grad_norm": 21.191116333007812, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.1767940521240234, "logits/rejected": -0.6085222363471985, "logps/chosen": -119.01771545410156, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -125.90153503417969, "loss": 1.0207, "margin_dpo/margin_mean": 10.38538932800293, "margin_dpo/margin_std": 14.67840576171875, "step": 120 }, { "KL/chosen_KL_mean": -4.004894256591797, "KL/mean": -8.678792953491211, "KL/rejected_KL_mean": -13.35269546508789, "KL/std": 8.953929901123047, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.05671404302120209, "fcm_dpo/delta": -0.13825546205043793, "fcm_dpo/margin": 9.347806930541992, "fcm_dpo/q_t": 0.37923043966293335, "grad_norm": 18.084074020385742, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.9611396789550781, "logits/rejected": -0.626045823097229, "logps/chosen": -96.12155151367188, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -128.32559204101562, "loss": 0.9964, "margin_dpo/margin_mean": 9.347806930541992, "margin_dpo/margin_std": 9.987987518310547, "step": 121 }, { "KL/chosen_KL_mean": -4.217292785644531, "KL/mean": -9.6202392578125, "KL/rejected_KL_mean": -15.023178100585938, "KL/std": 9.76900863647461, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.05504893511533737, "fcm_dpo/delta": -0.2066752016544342, "fcm_dpo/margin": 10.805885314941406, "fcm_dpo/q_t": 0.3654572069644928, "grad_norm": 18.022781372070312, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.7885080575942993, "logits/rejected": -0.6948219537734985, "logps/chosen": -89.09486389160156, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -126.10616302490234, "loss": 0.9681, "margin_dpo/margin_mean": 10.80588436126709, "margin_dpo/margin_std": 11.441054344177246, "step": 122 }, { "KL/chosen_KL_mean": -4.734001159667969, "KL/mean": -8.639993667602539, "KL/rejected_KL_mean": -12.54598617553711, "KL/std": 8.978158950805664, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.05344327911734581, "fcm_dpo/delta": -0.07371605932712555, "fcm_dpo/margin": 7.811985969543457, "fcm_dpo/q_t": 0.40421897172927856, "grad_norm": 15.539241790771484, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.1246767044067383, "logits/rejected": -0.8017904758453369, "logps/chosen": -103.21275329589844, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -106.40016174316406, "loss": 1.0843, "margin_dpo/margin_mean": 7.811985492706299, "margin_dpo/margin_std": 10.28927993774414, "step": 123 }, { "KL/chosen_KL_mean": -4.476387023925781, "KL/mean": -9.307952880859375, "KL/rejected_KL_mean": -14.139518737792969, "KL/std": 11.263481140136719, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.052555106580257416, "fcm_dpo/delta": -0.11360366642475128, "fcm_dpo/margin": 9.663127899169922, "fcm_dpo/q_t": 0.3938758373260498, "grad_norm": 15.880965232849121, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.0805766582489014, "logits/rejected": -0.8153550624847412, "logps/chosen": -107.23928833007812, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -137.1451416015625, "loss": 1.0643, "margin_dpo/margin_mean": 9.663127899169922, "margin_dpo/margin_std": 14.64742660522461, "step": 124 }, { "KL/chosen_KL_mean": -5.996479034423828, "KL/mean": -9.428139686584473, "KL/rejected_KL_mean": -12.859806060791016, "KL/std": 9.142630577087402, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.0522427000105381, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.863320827484131, "fcm_dpo/q_t": 0.4191066026687622, "grad_norm": 16.40275001525879, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.938865065574646, "logits/rejected": -0.7199312448501587, "logps/chosen": -96.42276000976562, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -102.10882568359375, "loss": 1.1404, "margin_dpo/margin_mean": 6.863320350646973, "margin_dpo/margin_std": 11.256654739379883, "step": 125 }, { "KL/chosen_KL_mean": -4.998634338378906, "KL/mean": -9.9326810836792, "KL/rejected_KL_mean": -14.866737365722656, "KL/std": 9.739662170410156, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.051387492567300797, "fcm_dpo/delta": -0.11279116570949554, "fcm_dpo/margin": 9.868102073669434, "fcm_dpo/q_t": 0.38241010904312134, "grad_norm": 15.908402442932129, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.9480875730514526, "logits/rejected": -0.6641227006912231, "logps/chosen": -99.42151641845703, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -118.748291015625, "loss": 1.0052, "margin_dpo/margin_mean": 9.868102073669434, "margin_dpo/margin_std": 10.221273422241211, "step": 126 }, { "KL/chosen_KL_mean": -5.000720977783203, "KL/mean": -8.728801727294922, "KL/rejected_KL_mean": -12.45688247680664, "KL/std": 9.083657264709473, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.05107739195227623, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.456164360046387, "fcm_dpo/q_t": 0.4144880175590515, "grad_norm": 15.962535858154297, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.8345644474029541, "logits/rejected": -0.4829590320587158, "logps/chosen": -89.75177001953125, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -96.00212097167969, "loss": 1.1177, "margin_dpo/margin_mean": 7.45616340637207, "margin_dpo/margin_std": 11.30911636352539, "step": 127 }, { "KL/chosen_KL_mean": -4.9147796630859375, "KL/mean": -8.803231239318848, "KL/rejected_KL_mean": -12.691679000854492, "KL/std": 11.215261459350586, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.0508519671857357, "fcm_dpo/delta": -0.04432990774512291, "fcm_dpo/margin": 7.77690315246582, "fcm_dpo/q_t": 0.41065800189971924, "grad_norm": 17.821998596191406, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7033657431602478, "logits/rejected": -0.21893227100372314, "logps/chosen": -95.534423828125, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -95.79515075683594, "loss": 1.1355, "margin_dpo/margin_mean": 7.776902675628662, "margin_dpo/margin_std": 13.915573120117188, "step": 128 }, { "KL/chosen_KL_mean": -5.46490478515625, "KL/mean": -9.565896987915039, "KL/rejected_KL_mean": -13.666889190673828, "KL/std": 9.182836532592773, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.050626542419195175, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.201982498168945, "fcm_dpo/q_t": 0.4096764922142029, "grad_norm": 14.573949813842773, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.8725214600563049, "logits/rejected": -0.5648083686828613, "logps/chosen": -104.38361358642578, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -116.95108032226562, "loss": 1.1074, "margin_dpo/margin_mean": 8.201982498168945, "margin_dpo/margin_std": 12.508721351623535, "step": 129 }, { "KL/chosen_KL_mean": -4.298320770263672, "KL/mean": -9.568132400512695, "KL/rejected_KL_mean": -14.837947845458984, "KL/std": 10.596726417541504, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.049902915954589844, "fcm_dpo/delta": -0.14501740038394928, "fcm_dpo/margin": 10.539631843566895, "fcm_dpo/q_t": 0.3838665783405304, "grad_norm": 13.868953704833984, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.3415653705596924, "logits/rejected": -1.1024945974349976, "logps/chosen": -77.48200988769531, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -102.85948944091797, "loss": 1.0214, "margin_dpo/margin_mean": 10.539631843566895, "margin_dpo/margin_std": 12.580199241638184, "step": 130 }, { "KL/chosen_KL_mean": -3.745616912841797, "KL/mean": -8.537296295166016, "KL/rejected_KL_mean": -13.328975677490234, "KL/std": 10.591804504394531, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.048716336488723755, "fcm_dpo/delta": -0.09503188729286194, "fcm_dpo/margin": 9.583351135253906, "fcm_dpo/q_t": 0.39458730816841125, "grad_norm": 16.28065299987793, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.1018061637878418, "logits/rejected": -0.8890005350112915, "logps/chosen": -77.70205688476562, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -107.78480529785156, "loss": 1.055, "margin_dpo/margin_mean": 9.58335018157959, "margin_dpo/margin_std": 12.076282501220703, "step": 131 }, { "KL/chosen_KL_mean": -5.416557312011719, "KL/mean": -10.159845352172852, "KL/rejected_KL_mean": -14.903129577636719, "KL/std": 11.069058418273926, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.04763271287083626, "fcm_dpo/delta": -0.06473176926374435, "fcm_dpo/margin": 9.486564636230469, "fcm_dpo/q_t": 0.3971814215183258, "grad_norm": 16.285139083862305, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.4297378063201904, "logits/rejected": -1.0191959142684937, "logps/chosen": -97.85604095458984, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -109.03564453125, "loss": 1.0683, "margin_dpo/margin_mean": 9.486564636230469, "margin_dpo/margin_std": 12.715907096862793, "step": 132 }, { "KL/chosen_KL_mean": -4.457523345947266, "KL/mean": -9.427650451660156, "KL/rejected_KL_mean": -14.397781372070312, "KL/std": 11.091157913208008, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.04717353731393814, "fcm_dpo/delta": -0.07225474715232849, "fcm_dpo/margin": 9.940265655517578, "fcm_dpo/q_t": 0.39410215616226196, "grad_norm": 14.838846206665039, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.6731667518615723, "logits/rejected": -1.2887438535690308, "logps/chosen": -97.57418823242188, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -116.49269104003906, "loss": 1.041, "margin_dpo/margin_mean": 9.940265655517578, "margin_dpo/margin_std": 11.829200744628906, "step": 133 }, { "KL/chosen_KL_mean": -4.7056732177734375, "KL/mean": -10.084417343139648, "KL/rejected_KL_mean": -15.46316146850586, "KL/std": 10.490726470947266, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.04636751115322113, "fcm_dpo/delta": -0.12548519670963287, "fcm_dpo/margin": 10.757482528686523, "fcm_dpo/q_t": 0.3853466510772705, "grad_norm": 14.27737808227539, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.7040252685546875, "logits/rejected": -1.3741722106933594, "logps/chosen": -88.25579833984375, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -109.3447265625, "loss": 1.0176, "margin_dpo/margin_mean": 10.757482528686523, "margin_dpo/margin_std": 11.360851287841797, "step": 134 }, { "KL/chosen_KL_mean": -4.398399353027344, "KL/mean": -9.787229537963867, "KL/rejected_KL_mean": -15.176063537597656, "KL/std": 11.693634033203125, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.045278601348400116, "fcm_dpo/delta": -0.09241008013486862, "fcm_dpo/margin": 10.77766227722168, "fcm_dpo/q_t": 0.3918192386627197, "grad_norm": 18.502573013305664, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.36971116065979, "logits/rejected": -1.2465593814849854, "logps/chosen": -81.98102569580078, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -136.2325439453125, "loss": 1.0371, "margin_dpo/margin_mean": 10.77766227722168, "margin_dpo/margin_std": 13.108964920043945, "step": 135 }, { "KL/chosen_KL_mean": -5.095481872558594, "KL/mean": -10.361509323120117, "KL/rejected_KL_mean": -15.627532958984375, "KL/std": 11.768016815185547, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.044433578848838806, "fcm_dpo/delta": -0.07139217853546143, "fcm_dpo/margin": 10.532051086425781, "fcm_dpo/q_t": 0.3928908407688141, "grad_norm": 14.051191329956055, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.5767922401428223, "logits/rejected": -1.1483311653137207, "logps/chosen": -99.26821899414062, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -118.8918228149414, "loss": 1.0336, "margin_dpo/margin_mean": 10.532051086425781, "margin_dpo/margin_std": 11.826497077941895, "step": 136 }, { "KL/chosen_KL_mean": -4.045673370361328, "KL/mean": -12.327224731445312, "KL/rejected_KL_mean": -20.608776092529297, "KL/std": 15.169339179992676, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.041995421051979065, "fcm_dpo/delta": -0.32045978307724, "fcm_dpo/margin": 16.563108444213867, "fcm_dpo/q_t": 0.34719356894493103, "grad_norm": 16.534019470214844, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.5331034660339355, "logits/rejected": -1.3159593343734741, "logps/chosen": -82.16441345214844, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -138.53114318847656, "loss": 0.9064, "margin_dpo/margin_mean": 16.563108444213867, "margin_dpo/margin_std": 16.618528366088867, "step": 137 }, { "KL/chosen_KL_mean": -6.839290618896484, "KL/mean": -11.939729690551758, "KL/rejected_KL_mean": -17.040164947509766, "KL/std": 11.220688819885254, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.041321028023958206, "fcm_dpo/delta": -0.02873852103948593, "fcm_dpo/margin": 10.200878143310547, "fcm_dpo/q_t": 0.40346652269363403, "grad_norm": 12.946439743041992, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.9364591240882874, "logits/rejected": -0.6707330346107483, "logps/chosen": -99.66923522949219, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -115.68856811523438, "loss": 1.0708, "margin_dpo/margin_mean": 10.200879096984863, "margin_dpo/margin_std": 12.60525131225586, "step": 138 }, { "KL/chosen_KL_mean": -4.635814666748047, "KL/mean": -10.579815864562988, "KL/rejected_KL_mean": -16.523815155029297, "KL/std": 12.15534782409668, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.040701352059841156, "fcm_dpo/delta": -0.08837475627660751, "fcm_dpo/margin": 11.8880033493042, "fcm_dpo/q_t": 0.38738855719566345, "grad_norm": 13.228471755981445, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.2835688591003418, "logits/rejected": -0.9537800550460815, "logps/chosen": -83.63882446289062, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -102.35115051269531, "loss": 1.0371, "margin_dpo/margin_mean": 11.888002395629883, "margin_dpo/margin_std": 14.422991752624512, "step": 139 }, { "KL/chosen_KL_mean": -5.138938903808594, "KL/mean": -10.540010452270508, "KL/rejected_KL_mean": -15.941082000732422, "KL/std": 11.524022102355957, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.04049248248338699, "fcm_dpo/delta": -0.025836758315563202, "fcm_dpo/margin": 10.802145004272461, "fcm_dpo/q_t": 0.39876601099967957, "grad_norm": 12.525565147399902, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.7493927478790283, "logits/rejected": -1.348315954208374, "logps/chosen": -93.49138641357422, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -106.51585388183594, "loss": 1.0532, "margin_dpo/margin_mean": 10.802145004272461, "margin_dpo/margin_std": 12.263673782348633, "step": 140 }, { "KL/chosen_KL_mean": -7.002662658691406, "KL/mean": -13.104984283447266, "KL/rejected_KL_mean": -19.20730209350586, "KL/std": 15.115472793579102, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.03986864537000656, "fcm_dpo/delta": -0.09096501767635345, "fcm_dpo/margin": 12.204638481140137, "fcm_dpo/q_t": 0.38849306106567383, "grad_norm": 15.797204971313477, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.3024959564208984, "logits/rejected": -0.9077330231666565, "logps/chosen": -114.22282409667969, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -123.9195327758789, "loss": 1.0422, "margin_dpo/margin_mean": 12.20463752746582, "margin_dpo/margin_std": 15.34821605682373, "step": 141 }, { "KL/chosen_KL_mean": -7.206478118896484, "KL/mean": -13.212976455688477, "KL/rejected_KL_mean": -19.21947479248047, "KL/std": 14.038681030273438, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.039226606488227844, "fcm_dpo/delta": -0.054905593395233154, "fcm_dpo/margin": 12.012996673583984, "fcm_dpo/q_t": 0.3946160078048706, "grad_norm": 12.59189510345459, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.4158157110214233, "logits/rejected": -1.1645511388778687, "logps/chosen": -95.9354248046875, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -118.94985961914062, "loss": 1.0412, "margin_dpo/margin_mean": 12.0129976272583, "margin_dpo/margin_std": 14.659996032714844, "step": 142 }, { "KL/chosen_KL_mean": -5.284786224365234, "KL/mean": -10.21754264831543, "KL/rejected_KL_mean": -15.15029525756836, "KL/std": 11.668998718261719, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.03896590694785118, "fcm_dpo/delta": -0.03334076702594757, "fcm_dpo/margin": 9.865499496459961, "fcm_dpo/q_t": 0.41083699464797974, "grad_norm": 12.686075210571289, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.6093742847442627, "logits/rejected": -1.1548908948898315, "logps/chosen": -90.78936767578125, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -100.9559555053711, "loss": 1.097, "margin_dpo/margin_mean": 9.865498542785645, "margin_dpo/margin_std": 12.701343536376953, "step": 143 }, { "KL/chosen_KL_mean": -6.5080718994140625, "KL/mean": -11.372692108154297, "KL/rejected_KL_mean": -16.237316131591797, "KL/std": 12.327178955078125, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.038793809711933136, "fcm_dpo/delta": -0.04436177760362625, "fcm_dpo/margin": 9.729241371154785, "fcm_dpo/q_t": 0.41397175192832947, "grad_norm": 16.087512969970703, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.4030654430389404, "logits/rejected": -0.852055013179779, "logps/chosen": -111.19515991210938, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -103.9563980102539, "loss": 1.1263, "margin_dpo/margin_mean": 9.729241371154785, "margin_dpo/margin_std": 15.207479476928711, "step": 144 }, { "KL/chosen_KL_mean": -5.439601898193359, "KL/mean": -11.379199981689453, "KL/rejected_KL_mean": -17.318801879882812, "KL/std": 11.930824279785156, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.03828851133584976, "fcm_dpo/delta": -0.05745704472064972, "fcm_dpo/margin": 11.879202842712402, "fcm_dpo/q_t": 0.394794762134552, "grad_norm": 12.943273544311523, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.8453783988952637, "logits/rejected": -0.23572726547718048, "logps/chosen": -93.95948791503906, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -101.08393859863281, "loss": 1.0426, "margin_dpo/margin_mean": 11.879203796386719, "margin_dpo/margin_std": 13.366875648498535, "step": 145 }, { "KL/chosen_KL_mean": -5.9093017578125, "KL/mean": -12.391645431518555, "KL/rejected_KL_mean": -18.87398910522461, "KL/std": 12.88823127746582, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.037484802305698395, "fcm_dpo/delta": -0.09193861484527588, "fcm_dpo/margin": 12.964683532714844, "fcm_dpo/q_t": 0.3875201344490051, "grad_norm": 12.755082130432129, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.6843762397766113, "logits/rejected": -1.046419382095337, "logps/chosen": -108.81517028808594, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -107.82334899902344, "loss": 1.0245, "margin_dpo/margin_mean": 12.964683532714844, "margin_dpo/margin_std": 14.194541931152344, "step": 146 }, { "KL/chosen_KL_mean": -5.680316925048828, "KL/mean": -12.381059646606445, "KL/rejected_KL_mean": -19.081806182861328, "KL/std": 12.347671508789062, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.03695180267095566, "fcm_dpo/delta": -0.10012054443359375, "fcm_dpo/margin": 13.401491165161133, "fcm_dpo/q_t": 0.38499823212623596, "grad_norm": 12.047464370727539, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.3339593410491943, "logits/rejected": -0.7316333055496216, "logps/chosen": -94.9425048828125, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -95.58192443847656, "loss": 1.0058, "margin_dpo/margin_mean": 13.401491165161133, "margin_dpo/margin_std": 13.294912338256836, "step": 147 }, { "KL/chosen_KL_mean": -4.000236511230469, "KL/mean": -10.284065246582031, "KL/rejected_KL_mean": -16.567893981933594, "KL/std": 12.97103500366211, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.03635237365961075, "fcm_dpo/delta": -0.10709331929683685, "fcm_dpo/margin": 12.567655563354492, "fcm_dpo/q_t": 0.3973866403102875, "grad_norm": 12.61980152130127, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.70595121383667, "logits/rejected": -1.3145498037338257, "logps/chosen": -95.76312255859375, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -126.95342254638672, "loss": 1.0545, "margin_dpo/margin_mean": 12.567655563354492, "margin_dpo/margin_std": 14.780699729919434, "step": 148 }, { "KL/chosen_KL_mean": -7.202705383300781, "KL/mean": -13.170902252197266, "KL/rejected_KL_mean": -19.13909149169922, "KL/std": 13.75649642944336, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.03559616953134537, "fcm_dpo/delta": -0.051274027675390244, "fcm_dpo/margin": 11.936376571655273, "fcm_dpo/q_t": 0.406826376914978, "grad_norm": 12.422693252563477, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.2077505588531494, "logits/rejected": -0.8719635605812073, "logps/chosen": -100.68672180175781, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -124.78279113769531, "loss": 1.0899, "margin_dpo/margin_mean": 11.936378479003906, "margin_dpo/margin_std": 17.20027732849121, "step": 149 }, { "KL/chosen_KL_mean": -5.873744964599609, "KL/mean": -12.158742904663086, "KL/rejected_KL_mean": -18.443740844726562, "KL/std": 14.696922302246094, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.035273633897304535, "fcm_dpo/delta": -0.04551144316792488, "fcm_dpo/margin": 12.569992065429688, "fcm_dpo/q_t": 0.40014541149139404, "grad_norm": 11.103093147277832, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.2018685340881348, "logits/rejected": -0.9345606565475464, "logps/chosen": -81.65243530273438, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -108.47050476074219, "loss": 1.0658, "margin_dpo/margin_mean": 12.569992065429688, "margin_dpo/margin_std": 16.47577667236328, "step": 150 }, { "KL/chosen_KL_mean": -7.696537017822266, "KL/mean": -13.772958755493164, "KL/rejected_KL_mean": -19.849376678466797, "KL/std": 14.226795196533203, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.035273633897304535, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.152836799621582, "fcm_dpo/q_t": 0.4032111167907715, "grad_norm": 13.550172805786133, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.9508569240570068, "logits/rejected": -1.3468466997146606, "logps/chosen": -110.43075561523438, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -125.88719177246094, "loss": 1.0749, "margin_dpo/margin_mean": 12.152836799621582, "margin_dpo/margin_std": 16.163352966308594, "step": 151 }, { "KL/chosen_KL_mean": -4.882720947265625, "KL/mean": -12.274900436401367, "KL/rejected_KL_mean": -19.66708755493164, "KL/std": 14.159213066101074, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.034601084887981415, "fcm_dpo/delta": -0.11783605068922043, "fcm_dpo/margin": 14.784357070922852, "fcm_dpo/q_t": 0.3825019896030426, "grad_norm": 12.291085243225098, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.6210277080535889, "logits/rejected": -1.2385753393173218, "logps/chosen": -102.47346496582031, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -130.1662139892578, "loss": 1.0116, "margin_dpo/margin_mean": 14.784357070922852, "margin_dpo/margin_std": 16.491104125976562, "step": 152 }, { "KL/chosen_KL_mean": -7.411235809326172, "KL/mean": -12.635819435119629, "KL/rejected_KL_mean": -17.86040496826172, "KL/std": 13.727727890014648, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.034452054649591446, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.44917106628418, "fcm_dpo/q_t": 0.4172041416168213, "grad_norm": 12.22294807434082, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.8873589038848877, "logits/rejected": -1.5744202136993408, "logps/chosen": -94.43683624267578, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -108.92893981933594, "loss": 1.1293, "margin_dpo/margin_mean": 10.449170112609863, "margin_dpo/margin_std": 15.787732124328613, "step": 153 }, { "KL/chosen_KL_mean": -5.724079132080078, "KL/mean": -14.247114181518555, "KL/rejected_KL_mean": -22.77014923095703, "KL/std": 15.056798934936523, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.0337500274181366, "fcm_dpo/delta": -0.18570157885551453, "fcm_dpo/margin": 17.046077728271484, "fcm_dpo/q_t": 0.3681698739528656, "grad_norm": 11.956178665161133, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.303554892539978, "logits/rejected": -0.9750627875328064, "logps/chosen": -96.82974243164062, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -134.1425323486328, "loss": 0.9617, "margin_dpo/margin_mean": 17.046077728271484, "margin_dpo/margin_std": 15.960840225219727, "step": 154 }, { "KL/chosen_KL_mean": -7.848949432373047, "KL/mean": -14.182060241699219, "KL/rejected_KL_mean": -20.515167236328125, "KL/std": 15.474185943603516, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.03282182291150093, "fcm_dpo/delta": -0.05667358636856079, "fcm_dpo/margin": 12.666220664978027, "fcm_dpo/q_t": 0.402273952960968, "grad_norm": 14.35477352142334, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.3230957984924316, "logits/rejected": -0.9305977821350098, "logps/chosen": -102.40992736816406, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -105.28875732421875, "loss": 1.1041, "margin_dpo/margin_mean": 12.666221618652344, "margin_dpo/margin_std": 19.145790100097656, "step": 155 }, { "KL/chosen_KL_mean": -9.482612609863281, "KL/mean": -15.20271110534668, "KL/rejected_KL_mean": -20.922813415527344, "KL/std": 13.834479331970215, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.032601747661828995, "fcm_dpo/delta": -0.03363886848092079, "fcm_dpo/margin": 11.440195083618164, "fcm_dpo/q_t": 0.41580700874328613, "grad_norm": 12.684883117675781, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.6773122549057007, "logits/rejected": -1.1783679723739624, "logps/chosen": -104.13800048828125, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -105.34215545654297, "loss": 1.1205, "margin_dpo/margin_mean": 11.440195083618164, "margin_dpo/margin_std": 16.899335861206055, "step": 156 }, { "KL/chosen_KL_mean": -8.8172607421875, "KL/mean": -16.886798858642578, "KL/rejected_KL_mean": -24.956336975097656, "KL/std": 14.879756927490234, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.03204510360956192, "fcm_dpo/delta": -0.12345730513334274, "fcm_dpo/margin": 16.13906478881836, "fcm_dpo/q_t": 0.37973982095718384, "grad_norm": 12.848160743713379, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -2.0880422592163086, "logits/rejected": -1.63654625415802, "logps/chosen": -105.82965087890625, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -136.65768432617188, "loss": 0.9925, "margin_dpo/margin_mean": 16.139066696166992, "margin_dpo/margin_std": 15.600648880004883, "step": 157 }, { "KL/chosen_KL_mean": -8.15750503540039, "KL/mean": -16.530437469482422, "KL/rejected_KL_mean": -24.903366088867188, "KL/std": 16.062965393066406, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.03125136345624924, "fcm_dpo/delta": -0.12998166680335999, "fcm_dpo/margin": 16.745866775512695, "fcm_dpo/q_t": 0.3802149295806885, "grad_norm": 11.851430892944336, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.5688194036483765, "logits/rejected": -1.306272029876709, "logps/chosen": -102.87046813964844, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -145.8406982421875, "loss": 0.9957, "margin_dpo/margin_mean": 16.745866775512695, "margin_dpo/margin_std": 17.371625900268555, "step": 158 }, { "KL/chosen_KL_mean": -7.671909332275391, "KL/mean": -15.098465919494629, "KL/rejected_KL_mean": -22.525020599365234, "KL/std": 15.611011505126953, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.030712001025676727, "fcm_dpo/delta": -0.05887080729007721, "fcm_dpo/margin": 14.853116989135742, "fcm_dpo/q_t": 0.39532470703125, "grad_norm": 10.854537963867188, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.9624030590057373, "logits/rejected": -1.4633495807647705, "logps/chosen": -115.89015197753906, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -117.36404418945312, "loss": 1.0497, "margin_dpo/margin_mean": 14.853116989135742, "margin_dpo/margin_std": 17.84347915649414, "step": 159 }, { "KL/chosen_KL_mean": -6.4754180908203125, "KL/mean": -13.829362869262695, "KL/rejected_KL_mean": -21.183319091796875, "KL/std": 14.391960144042969, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.0303662046790123, "fcm_dpo/delta": -0.04286792501807213, "fcm_dpo/margin": 14.707895278930664, "fcm_dpo/q_t": 0.3954135775566101, "grad_norm": 11.17697811126709, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.6473910808563232, "logits/rejected": -1.097322940826416, "logps/chosen": -104.63882446289062, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -123.46429443359375, "loss": 1.0402, "margin_dpo/margin_mean": 14.707895278930664, "margin_dpo/margin_std": 15.488279342651367, "step": 160 }, { "KL/chosen_KL_mean": -8.6473388671875, "KL/mean": -15.280235290527344, "KL/rejected_KL_mean": -21.913131713867188, "KL/std": 13.213111877441406, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.0301434937864542, "fcm_dpo/delta": -0.03680583834648132, "fcm_dpo/margin": 13.265800476074219, "fcm_dpo/q_t": 0.4072236716747284, "grad_norm": 11.060824394226074, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.8893542289733887, "logits/rejected": -1.4002642631530762, "logps/chosen": -95.23921203613281, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -110.9295654296875, "loss": 1.0896, "margin_dpo/margin_mean": 13.265800476074219, "margin_dpo/margin_std": 17.097854614257812, "step": 161 }, { "KL/chosen_KL_mean": -7.714481353759766, "KL/mean": -15.09288501739502, "KL/rejected_KL_mean": -22.47128677368164, "KL/std": 15.402703285217285, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.02986234799027443, "fcm_dpo/delta": -0.04685326665639877, "fcm_dpo/margin": 14.75680160522461, "fcm_dpo/q_t": 0.3969874978065491, "grad_norm": 10.722710609436035, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.3560696840286255, "logits/rejected": -0.9583055377006531, "logps/chosen": -105.11737060546875, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -118.34051513671875, "loss": 1.0497, "margin_dpo/margin_mean": 14.756800651550293, "margin_dpo/margin_std": 16.47856330871582, "step": 162 }, { "KL/chosen_KL_mean": -7.362133026123047, "KL/mean": -16.139331817626953, "KL/rejected_KL_mean": -24.91653823852539, "KL/std": 17.488971710205078, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.029265735298395157, "fcm_dpo/delta": -0.12028864026069641, "fcm_dpo/margin": 17.55440902709961, "fcm_dpo/q_t": 0.38297730684280396, "grad_norm": 9.634289741516113, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.4584856033325195, "logits/rejected": -1.0437427759170532, "logps/chosen": -89.53752136230469, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -107.78128051757812, "loss": 1.0035, "margin_dpo/margin_mean": 17.55440902709961, "margin_dpo/margin_std": 18.79764175415039, "step": 163 }, { "KL/chosen_KL_mean": -11.27200698852539, "KL/mean": -17.26695442199707, "KL/rejected_KL_mean": -23.26189422607422, "KL/std": 13.830659866333008, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.02892138995230198, "fcm_dpo/delta": -0.03979634866118431, "fcm_dpo/margin": 11.989883422851562, "fcm_dpo/q_t": 0.41801515221595764, "grad_norm": 11.302268028259277, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.725064992904663, "logits/rejected": -1.2782957553863525, "logps/chosen": -101.74627685546875, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -110.28998565673828, "loss": 1.1201, "margin_dpo/margin_mean": 11.989883422851562, "margin_dpo/margin_std": 15.38986587524414, "step": 164 }, { "KL/chosen_KL_mean": -7.741268157958984, "KL/mean": -16.36927604675293, "KL/rejected_KL_mean": -24.997276306152344, "KL/std": 16.384065628051758, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.02871411293745041, "fcm_dpo/delta": -0.07218816876411438, "fcm_dpo/margin": 17.256010055541992, "fcm_dpo/q_t": 0.3877648413181305, "grad_norm": 9.31027889251709, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.825324535369873, "logits/rejected": -1.4027917385101318, "logps/chosen": -95.05449676513672, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -106.89985656738281, "loss": 1.0134, "margin_dpo/margin_mean": 17.256010055541992, "margin_dpo/margin_std": 18.67444610595703, "step": 165 }, { "KL/chosen_KL_mean": -12.444908142089844, "KL/mean": -20.918582916259766, "KL/rejected_KL_mean": -29.392257690429688, "KL/std": 17.931068420410156, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.028157442808151245, "fcm_dpo/delta": -0.08108597993850708, "fcm_dpo/margin": 16.947343826293945, "fcm_dpo/q_t": 0.3947068750858307, "grad_norm": 10.671768188476562, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.8059653043746948, "logits/rejected": -1.6063941717147827, "logps/chosen": -94.03097534179688, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -132.55743408203125, "loss": 1.0432, "margin_dpo/margin_mean": 16.947345733642578, "margin_dpo/margin_std": 21.656051635742188, "step": 166 }, { "KL/chosen_KL_mean": -11.04953384399414, "KL/mean": -20.187631607055664, "KL/rejected_KL_mean": -29.325725555419922, "KL/std": 16.577194213867188, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.027474835515022278, "fcm_dpo/delta": -0.10328071564435959, "fcm_dpo/margin": 18.276193618774414, "fcm_dpo/q_t": 0.3831254243850708, "grad_norm": 9.851593971252441, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.0503933429718018, "logits/rejected": -0.8773993849754333, "logps/chosen": -94.6112060546875, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -128.3787841796875, "loss": 0.9979, "margin_dpo/margin_mean": 18.276193618774414, "margin_dpo/margin_std": 17.161945343017578, "step": 167 }, { "KL/chosen_KL_mean": -10.606239318847656, "KL/mean": -18.478729248046875, "KL/rejected_KL_mean": -26.35122299194336, "KL/std": 16.47789764404297, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.02738286182284355, "fcm_dpo/delta": -0.0335882194340229, "fcm_dpo/margin": 15.744975090026855, "fcm_dpo/q_t": 0.39930278062820435, "grad_norm": 10.66128158569336, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.5977544784545898, "logits/rejected": -1.2542513608932495, "logps/chosen": -90.5374755859375, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -106.45338439941406, "loss": 1.0496, "margin_dpo/margin_mean": 15.744975090026855, "margin_dpo/margin_std": 16.61350440979004, "step": 168 }, { "KL/chosen_KL_mean": -9.219734191894531, "KL/mean": -18.233787536621094, "KL/rejected_KL_mean": -27.24783706665039, "KL/std": 17.75716781616211, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.02691551297903061, "fcm_dpo/delta": -0.08963513374328613, "fcm_dpo/margin": 18.02810287475586, "fcm_dpo/q_t": 0.38849133253097534, "grad_norm": 10.403179168701172, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.5234253406524658, "logits/rejected": -1.2967262268066406, "logps/chosen": -92.12344360351562, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -117.3697509765625, "loss": 1.0182, "margin_dpo/margin_mean": 18.02810287475586, "margin_dpo/margin_std": 19.036895751953125, "step": 169 }, { "KL/chosen_KL_mean": -10.807022094726562, "KL/mean": -18.60367202758789, "KL/rejected_KL_mean": -26.400325775146484, "KL/std": 14.700691223144531, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.026526974514126778, "fcm_dpo/delta": -0.0523189902305603, "fcm_dpo/margin": 15.593304634094238, "fcm_dpo/q_t": 0.40140897035598755, "grad_norm": 10.714163780212402, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.512270450592041, "logits/rejected": -1.297321081161499, "logps/chosen": -85.78899383544922, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -108.56593322753906, "loss": 1.0524, "margin_dpo/margin_mean": 15.593304634094238, "margin_dpo/margin_std": 14.680818557739258, "step": 170 }, { "KL/chosen_KL_mean": -13.889385223388672, "KL/mean": -22.627288818359375, "KL/rejected_KL_mean": -31.36520004272461, "KL/std": 21.29543685913086, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.026415839791297913, "fcm_dpo/delta": -0.0420711413025856, "fcm_dpo/margin": 17.475814819335938, "fcm_dpo/q_t": 0.4001199007034302, "grad_norm": 12.37187671661377, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.894243836402893, "logits/rejected": -1.3642196655273438, "logps/chosen": -132.867919921875, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -147.13450622558594, "loss": 1.0729, "margin_dpo/margin_mean": 17.475814819335938, "margin_dpo/margin_std": 25.546125411987305, "step": 171 }, { "KL/chosen_KL_mean": -14.673095703125, "KL/mean": -22.436616897583008, "KL/rejected_KL_mean": -30.20013427734375, "KL/std": 15.346516609191895, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.02615191973745823, "fcm_dpo/delta": -0.029126416891813278, "fcm_dpo/margin": 15.527044296264648, "fcm_dpo/q_t": 0.40231645107269287, "grad_norm": 10.095913887023926, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.3809781074523926, "logits/rejected": -1.6274173259735107, "logps/chosen": -115.82688903808594, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -121.20581817626953, "loss": 1.0474, "margin_dpo/margin_mean": 15.527044296264648, "margin_dpo/margin_std": 12.513900756835938, "step": 172 }, { "KL/chosen_KL_mean": -13.195404052734375, "KL/mean": -20.006118774414062, "KL/rejected_KL_mean": -26.81684112548828, "KL/std": 14.444755554199219, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.02615191973745823, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.621437072753906, "fcm_dpo/q_t": 0.41543108224868774, "grad_norm": 9.708830833435059, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.5293452739715576, "logits/rejected": -1.042283535003662, "logps/chosen": -112.25209045410156, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -110.66757202148438, "loss": 1.0997, "margin_dpo/margin_mean": 13.621437072753906, "margin_dpo/margin_std": 15.281864166259766, "step": 173 }, { "KL/chosen_KL_mean": -12.901369094848633, "KL/mean": -22.30677032470703, "KL/rejected_KL_mean": -31.712173461914062, "KL/std": 19.813518524169922, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.025836342945694923, "fcm_dpo/delta": -0.09031384438276291, "fcm_dpo/margin": 18.810808181762695, "fcm_dpo/q_t": 0.3875962495803833, "grad_norm": 11.290393829345703, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.8626902103424072, "logits/rejected": -1.8220134973526, "logps/chosen": -93.31027221679688, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -139.91387939453125, "loss": 1.0192, "margin_dpo/margin_mean": 18.810808181762695, "margin_dpo/margin_std": 20.07048797607422, "step": 174 }, { "KL/chosen_KL_mean": -16.27322769165039, "KL/mean": -25.332111358642578, "KL/rejected_KL_mean": -34.39100646972656, "KL/std": 17.594398498535156, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.025539778172969818, "fcm_dpo/delta": -0.05638523027300835, "fcm_dpo/margin": 18.11777114868164, "fcm_dpo/q_t": 0.3908485472202301, "grad_norm": 11.044235229492188, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.0275006294250488, "logits/rejected": -0.7489303350448608, "logps/chosen": -98.98971557617188, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -131.05813598632812, "loss": 1.0209, "margin_dpo/margin_mean": 18.11777114868164, "margin_dpo/margin_std": 16.79754638671875, "step": 175 }, { "KL/chosen_KL_mean": -13.516059875488281, "KL/mean": -22.8359375, "KL/rejected_KL_mean": -32.155818939208984, "KL/std": 19.200759887695312, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.025120196864008904, "fcm_dpo/delta": -0.07160620391368866, "fcm_dpo/margin": 18.639751434326172, "fcm_dpo/q_t": 0.3930055499076843, "grad_norm": 10.807122230529785, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.6670327186584473, "logits/rejected": -1.5074999332427979, "logps/chosen": -93.1453628540039, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -134.84442138671875, "loss": 1.0428, "margin_dpo/margin_mean": 18.639751434326172, "margin_dpo/margin_std": 22.160873413085938, "step": 176 }, { "KL/chosen_KL_mean": -15.089218139648438, "KL/mean": -25.98273468017578, "KL/rejected_KL_mean": -36.87623977661133, "KL/std": 22.02425193786621, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.024607334285974503, "fcm_dpo/delta": -0.1435873806476593, "fcm_dpo/margin": 21.78702163696289, "fcm_dpo/q_t": 0.3796229362487793, "grad_norm": 9.756264686584473, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.6794366836547852, "logits/rejected": -1.3426331281661987, "logps/chosen": -107.65031433105469, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -140.21463012695312, "loss": 0.9901, "margin_dpo/margin_mean": 21.78702163696289, "margin_dpo/margin_std": 22.888599395751953, "step": 177 }, { "KL/chosen_KL_mean": -14.00545883178711, "KL/mean": -26.929828643798828, "KL/rejected_KL_mean": -39.85418701171875, "KL/std": 21.462936401367188, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.02351992204785347, "fcm_dpo/delta": -0.22162586450576782, "fcm_dpo/margin": 25.848731994628906, "fcm_dpo/q_t": 0.36077815294265747, "grad_norm": 10.738805770874023, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.6188769340515137, "logits/rejected": -1.402688980102539, "logps/chosen": -98.27125549316406, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -150.6514892578125, "loss": 0.9327, "margin_dpo/margin_mean": 25.848731994628906, "margin_dpo/margin_std": 23.32689094543457, "step": 178 }, { "KL/chosen_KL_mean": -17.8377685546875, "KL/mean": -26.071609497070312, "KL/rejected_KL_mean": -34.30545425415039, "KL/std": 20.685653686523438, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.023159923031926155, "fcm_dpo/delta": -0.02397700771689415, "fcm_dpo/margin": 16.467681884765625, "fcm_dpo/q_t": 0.4129234552383423, "grad_norm": 10.340534210205078, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.3789422512054443, "logits/rejected": -0.9386358261108398, "logps/chosen": -111.2857437133789, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -128.5608673095703, "loss": 1.1141, "margin_dpo/margin_mean": 16.467681884765625, "margin_dpo/margin_std": 24.0804443359375, "step": 179 }, { "KL/chosen_KL_mean": -14.984962463378906, "KL/mean": -25.6824951171875, "KL/rejected_KL_mean": -36.380027770996094, "KL/std": 19.940820693969727, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.02278139814734459, "fcm_dpo/delta": -0.09229715913534164, "fcm_dpo/margin": 21.39505386352539, "fcm_dpo/q_t": 0.3865565359592438, "grad_norm": 12.307478904724121, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.1561471223831177, "logits/rejected": -0.9105365872383118, "logps/chosen": -103.82050323486328, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -140.2869873046875, "loss": 1.0056, "margin_dpo/margin_mean": 21.395055770874023, "margin_dpo/margin_std": 20.519596099853516, "step": 180 }, { "KL/chosen_KL_mean": -14.901191711425781, "KL/mean": -23.032230377197266, "KL/rejected_KL_mean": -31.16326141357422, "KL/std": 19.137805938720703, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.0227363258600235, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.262067794799805, "fcm_dpo/q_t": 0.4145600497722626, "grad_norm": 9.392159461975098, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.6266851425170898, "logits/rejected": -1.1644341945648193, "logps/chosen": -94.1351547241211, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -100.33768463134766, "loss": 1.0999, "margin_dpo/margin_mean": 16.262067794799805, "margin_dpo/margin_std": 20.46678924560547, "step": 181 }, { "KL/chosen_KL_mean": -18.684349060058594, "KL/mean": -29.320972442626953, "KL/rejected_KL_mean": -39.95759582519531, "KL/std": 22.128334045410156, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.02241043746471405, "fcm_dpo/delta": -0.07218550890684128, "fcm_dpo/margin": 21.273242950439453, "fcm_dpo/q_t": 0.39086461067199707, "grad_norm": 10.014082908630371, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.4264249801635742, "logits/rejected": -1.2132376432418823, "logps/chosen": -94.76653289794922, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -134.92620849609375, "loss": 1.0334, "margin_dpo/margin_mean": 21.273242950439453, "margin_dpo/margin_std": 24.390094757080078, "step": 182 }, { "KL/chosen_KL_mean": -19.58672332763672, "KL/mean": -29.895694732666016, "KL/rejected_KL_mean": -40.20466995239258, "KL/std": 20.120588302612305, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.022118957713246346, "fcm_dpo/delta": -0.06545867025852203, "fcm_dpo/margin": 20.61794090270996, "fcm_dpo/q_t": 0.3931583762168884, "grad_norm": 10.484023094177246, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.5172655582427979, "logits/rejected": -0.9027112126350403, "logps/chosen": -113.49462890625, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -123.76528930664062, "loss": 1.031, "margin_dpo/margin_mean": 20.61794090270996, "margin_dpo/margin_std": 20.638778686523438, "step": 183 }, { "KL/chosen_KL_mean": -17.270435333251953, "KL/mean": -24.365909576416016, "KL/rejected_KL_mean": -31.46137237548828, "KL/std": 20.92058563232422, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.022118957713246346, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.190932273864746, "fcm_dpo/q_t": 0.4258190989494324, "grad_norm": 9.476871490478516, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.5224368572235107, "logits/rejected": -1.2369121313095093, "logps/chosen": -104.13834381103516, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -115.74592590332031, "loss": 1.1564, "margin_dpo/margin_mean": 14.190933227539062, "margin_dpo/margin_std": 22.61478042602539, "step": 184 }, { "KL/chosen_KL_mean": -18.44916534423828, "KL/mean": -26.84320068359375, "KL/rejected_KL_mean": -35.23722839355469, "KL/std": 20.280242919921875, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.022118957713246346, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.788070678710938, "fcm_dpo/q_t": 0.4146394431591034, "grad_norm": 10.539263725280762, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.1481313705444336, "logits/rejected": -1.527503490447998, "logps/chosen": -112.47058868408203, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -116.69390869140625, "loss": 1.1108, "margin_dpo/margin_mean": 16.788070678710938, "margin_dpo/margin_std": 23.261137008666992, "step": 185 }, { "KL/chosen_KL_mean": -23.069805145263672, "KL/mean": -32.24045181274414, "KL/rejected_KL_mean": -41.41110610961914, "KL/std": 19.881616592407227, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.02197815105319023, "fcm_dpo/delta": -0.03193129226565361, "fcm_dpo/margin": 18.341291427612305, "fcm_dpo/q_t": 0.4055716395378113, "grad_norm": 11.27127742767334, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.3209800720214844, "logits/rejected": -1.4111700057983398, "logps/chosen": -137.26148986816406, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -134.75022888183594, "loss": 1.0772, "margin_dpo/margin_mean": 18.341291427612305, "margin_dpo/margin_std": 21.670860290527344, "step": 186 }, { "KL/chosen_KL_mean": -19.570362091064453, "KL/mean": -28.399642944335938, "KL/rejected_KL_mean": -37.228919982910156, "KL/std": 22.194194793701172, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.021930847316980362, "fcm_dpo/delta": -0.02156929299235344, "fcm_dpo/margin": 17.658559799194336, "fcm_dpo/q_t": 0.4108586013317108, "grad_norm": 9.927408218383789, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.8999593257904053, "logits/rejected": -1.6667678356170654, "logps/chosen": -96.43991088867188, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -125.39385986328125, "loss": 1.102, "margin_dpo/margin_mean": 17.658559799194336, "margin_dpo/margin_std": 24.213088989257812, "step": 187 }, { "KL/chosen_KL_mean": -15.143726348876953, "KL/mean": -25.274307250976562, "KL/rejected_KL_mean": -35.40489196777344, "KL/std": 21.8590087890625, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.02164667658507824, "fcm_dpo/delta": -0.054415229707956314, "fcm_dpo/margin": 20.261165618896484, "fcm_dpo/q_t": 0.3961760699748993, "grad_norm": 10.226794242858887, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.260589599609375, "logits/rejected": -2.0269412994384766, "logps/chosen": -94.33853149414062, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -132.23379516601562, "loss": 1.0454, "margin_dpo/margin_mean": 20.261165618896484, "margin_dpo/margin_std": 20.766544342041016, "step": 188 }, { "KL/chosen_KL_mean": -19.98076629638672, "KL/mean": -30.359134674072266, "KL/rejected_KL_mean": -40.737510681152344, "KL/std": 21.883520126342773, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.02148013934493065, "fcm_dpo/delta": -0.07753288745880127, "fcm_dpo/margin": 20.75674819946289, "fcm_dpo/q_t": 0.3983674943447113, "grad_norm": 9.241652488708496, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.9551546573638916, "logits/rejected": -1.3334847688674927, "logps/chosen": -103.87356567382812, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -120.20340728759766, "loss": 1.0532, "margin_dpo/margin_mean": 20.75674819946289, "margin_dpo/margin_std": 24.446044921875, "step": 189 }, { "KL/chosen_KL_mean": -15.928657531738281, "KL/mean": -28.407791137695312, "KL/rejected_KL_mean": -40.88692855834961, "KL/std": 24.16010284423828, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.020902208983898163, "fcm_dpo/delta": -0.1284371018409729, "fcm_dpo/margin": 24.958263397216797, "fcm_dpo/q_t": 0.37806451320648193, "grad_norm": 9.948405265808105, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.8647632598876953, "logits/rejected": -1.2676379680633545, "logps/chosen": -119.37012481689453, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -150.10195922851562, "loss": 0.9836, "margin_dpo/margin_mean": 24.958263397216797, "margin_dpo/margin_std": 22.83268928527832, "step": 190 }, { "KL/chosen_KL_mean": -15.150222778320312, "KL/mean": -29.26351547241211, "KL/rejected_KL_mean": -43.37682342529297, "KL/std": 26.32284164428711, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.020249437540769577, "fcm_dpo/delta": -0.18175840377807617, "fcm_dpo/margin": 28.226593017578125, "fcm_dpo/q_t": 0.369060218334198, "grad_norm": 10.933730125427246, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.5983753204345703, "logits/rejected": -1.289866328239441, "logps/chosen": -114.93573760986328, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -159.37710571289062, "loss": 0.9602, "margin_dpo/margin_mean": 28.226593017578125, "margin_dpo/margin_std": 26.908191680908203, "step": 191 }, { "KL/chosen_KL_mean": -24.491458892822266, "KL/mean": -34.3617057800293, "KL/rejected_KL_mean": -44.2319450378418, "KL/std": 20.997669219970703, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.020031504333019257, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.740489959716797, "fcm_dpo/q_t": 0.4085342288017273, "grad_norm": 11.052075386047363, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.6176681518554688, "logits/rejected": -1.3676519393920898, "logps/chosen": -125.3517837524414, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -153.67694091796875, "loss": 1.0848, "margin_dpo/margin_mean": 19.740489959716797, "margin_dpo/margin_std": 24.39140510559082, "step": 192 }, { "KL/chosen_KL_mean": -20.46947479248047, "KL/mean": -28.474037170410156, "KL/rejected_KL_mean": -36.478599548339844, "KL/std": 22.454416275024414, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.020031504333019257, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 16.009117126464844, "fcm_dpo/q_t": 0.42553651332855225, "grad_norm": 10.42332649230957, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.1036696434020996, "logits/rejected": -1.5211818218231201, "logps/chosen": -113.40779113769531, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -119.58778381347656, "loss": 1.1634, "margin_dpo/margin_mean": 16.009119033813477, "margin_dpo/margin_std": 27.268314361572266, "step": 193 }, { "KL/chosen_KL_mean": -20.759841918945312, "KL/mean": -30.139968872070312, "KL/rejected_KL_mean": -39.52009582519531, "KL/std": 23.01276206970215, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.019996438175439835, "fcm_dpo/delta": -0.017535997554659843, "fcm_dpo/margin": 18.760257720947266, "fcm_dpo/q_t": 0.4130212664604187, "grad_norm": 8.751951217651367, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.2868900299072266, "logits/rejected": -1.899591326713562, "logps/chosen": -84.5514144897461, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -108.24674987792969, "loss": 1.1023, "margin_dpo/margin_mean": 18.760257720947266, "margin_dpo/margin_std": 24.43203353881836, "step": 194 }, { "KL/chosen_KL_mean": -19.884803771972656, "KL/mean": -32.46330261230469, "KL/rejected_KL_mean": -45.041805267333984, "KL/std": 23.40045738220215, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0197166558355093, "fcm_dpo/delta": -0.10089940577745438, "fcm_dpo/margin": 25.157005310058594, "fcm_dpo/q_t": 0.38462767004966736, "grad_norm": 9.157100677490234, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.3765931129455566, "logits/rejected": -1.2605276107788086, "logps/chosen": -103.770263671875, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -149.04306030273438, "loss": 1.0157, "margin_dpo/margin_mean": 25.157005310058594, "margin_dpo/margin_std": 26.74881362915039, "step": 195 }, { "KL/chosen_KL_mean": -23.29226303100586, "KL/mean": -35.03050231933594, "KL/rejected_KL_mean": -46.76873779296875, "KL/std": 25.003662109375, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.01939188688993454, "fcm_dpo/delta": -0.05789773166179657, "fcm_dpo/margin": 23.47647476196289, "fcm_dpo/q_t": 0.3921882212162018, "grad_norm": 8.930368423461914, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.2890312671661377, "logits/rejected": -0.8334081172943115, "logps/chosen": -112.6936264038086, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -130.1160888671875, "loss": 1.0268, "margin_dpo/margin_mean": 23.476472854614258, "margin_dpo/margin_std": 22.423782348632812, "step": 196 }, { "KL/chosen_KL_mean": -23.51174545288086, "KL/mean": -30.70380401611328, "KL/rejected_KL_mean": -37.8958625793457, "KL/std": 20.233854293823242, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.019337372854351997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.384117126464844, "fcm_dpo/q_t": 0.43356990814208984, "grad_norm": 9.049604415893555, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.4218149185180664, "logits/rejected": -1.0900018215179443, "logps/chosen": -110.35244750976562, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -120.68875885009766, "loss": 1.1688, "margin_dpo/margin_mean": 14.384117126464844, "margin_dpo/margin_std": 21.570735931396484, "step": 197 }, { "KL/chosen_KL_mean": -26.931995391845703, "KL/mean": -38.78173828125, "KL/rejected_KL_mean": -50.63147735595703, "KL/std": 23.94927406311035, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.019183345139026642, "fcm_dpo/delta": -0.05721375346183777, "fcm_dpo/margin": 23.699487686157227, "fcm_dpo/q_t": 0.39427173137664795, "grad_norm": 10.704930305480957, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7081890106201172, "logits/rejected": -1.5364720821380615, "logps/chosen": -111.98023986816406, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -154.73513793945312, "loss": 1.034, "margin_dpo/margin_mean": 23.699487686157227, "margin_dpo/margin_std": 24.77570343017578, "step": 198 }, { "KL/chosen_KL_mean": -26.485694885253906, "KL/mean": -36.15583419799805, "KL/rejected_KL_mean": -45.82597732543945, "KL/std": 23.345836639404297, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.019117360934615135, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.340286254882812, "fcm_dpo/q_t": 0.41347378492355347, "grad_norm": 9.361525535583496, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.8157496452331543, "logits/rejected": -1.562117576599121, "logps/chosen": -108.27239227294922, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -136.61676025390625, "loss": 1.1022, "margin_dpo/margin_mean": 19.340286254882812, "margin_dpo/margin_std": 24.414854049682617, "step": 199 }, { "KL/chosen_KL_mean": -26.117855072021484, "KL/mean": -37.55307388305664, "KL/rejected_KL_mean": -48.9882926940918, "KL/std": 24.97366714477539, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.01889294758439064, "fcm_dpo/delta": -0.059040993452072144, "fcm_dpo/margin": 22.870437622070312, "fcm_dpo/q_t": 0.40030092000961304, "grad_norm": 9.385558128356934, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.7772903442382812, "logits/rejected": -1.243633508682251, "logps/chosen": -110.38594818115234, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -135.95616149902344, "loss": 1.071, "margin_dpo/margin_mean": 22.870437622070312, "margin_dpo/margin_std": 28.85576057434082, "step": 200 }, { "KL/chosen_KL_mean": -27.92850112915039, "KL/mean": -39.05589294433594, "KL/rejected_KL_mean": -50.18328094482422, "KL/std": 26.992494583129883, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.018826236948370934, "fcm_dpo/delta": -0.0354352630674839, "fcm_dpo/margin": 22.254777908325195, "fcm_dpo/q_t": 0.40364688634872437, "grad_norm": 9.570076942443848, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.4072399139404297, "logits/rejected": -1.0212725400924683, "logps/chosen": -135.70098876953125, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -169.97576904296875, "loss": 1.0819, "margin_dpo/margin_mean": 22.254779815673828, "margin_dpo/margin_std": 29.438425064086914, "step": 201 }, { "KL/chosen_KL_mean": -25.957561492919922, "KL/mean": -36.313472747802734, "KL/rejected_KL_mean": -46.66938781738281, "KL/std": 22.399188995361328, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.018759526312351227, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.71181869506836, "fcm_dpo/q_t": 0.40998154878616333, "grad_norm": 9.774576187133789, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.7439677715301514, "logits/rejected": -1.6190273761749268, "logps/chosen": -101.93001556396484, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -141.12936401367188, "loss": 1.0901, "margin_dpo/margin_mean": 20.71181869506836, "margin_dpo/margin_std": 25.393144607543945, "step": 202 }, { "KL/chosen_KL_mean": -23.20511245727539, "KL/mean": -34.93291473388672, "KL/rejected_KL_mean": -46.66072082519531, "KL/std": 23.407787322998047, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.018691878765821457, "fcm_dpo/delta": -0.03619145601987839, "fcm_dpo/margin": 23.455617904663086, "fcm_dpo/q_t": 0.3971450924873352, "grad_norm": 8.828924179077148, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.9745625257492065, "logits/rejected": -1.5262441635131836, "logps/chosen": -118.17227172851562, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -139.54840087890625, "loss": 1.0446, "margin_dpo/margin_mean": 23.45561981201172, "margin_dpo/margin_std": 24.514080047607422, "step": 203 }, { "KL/chosen_KL_mean": -29.36083984375, "KL/mean": -39.46080780029297, "KL/rejected_KL_mean": -49.56077575683594, "KL/std": 23.47601318359375, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.01860107108950615, "fcm_dpo/delta": -0.012449702247977257, "fcm_dpo/margin": 20.1999454498291, "fcm_dpo/q_t": 0.41303396224975586, "grad_norm": 9.622632026672363, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.839796543121338, "logits/rejected": -1.1714508533477783, "logps/chosen": -145.71803283691406, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -142.9583740234375, "loss": 1.108, "margin_dpo/margin_mean": 20.199947357177734, "margin_dpo/margin_std": 27.263164520263672, "step": 204 }, { "KL/chosen_KL_mean": -27.428409576416016, "KL/mean": -40.46244812011719, "KL/rejected_KL_mean": -53.49647521972656, "KL/std": 25.916481018066406, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.01831725798547268, "fcm_dpo/delta": -0.08162471652030945, "fcm_dpo/margin": 26.068065643310547, "fcm_dpo/q_t": 0.38635745644569397, "grad_norm": 8.790822982788086, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.1478800773620605, "logits/rejected": -1.7038779258728027, "logps/chosen": -116.28775787353516, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -145.35096740722656, "loss": 1.0035, "margin_dpo/margin_mean": 26.068065643310547, "margin_dpo/margin_std": 22.11089324951172, "step": 205 }, { "KL/chosen_KL_mean": -27.927993774414062, "KL/mean": -39.998260498046875, "KL/rejected_KL_mean": -52.06853103637695, "KL/std": 25.179162979125977, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.018086768686771393, "fcm_dpo/delta": -0.05233947932720184, "fcm_dpo/margin": 24.140539169311523, "fcm_dpo/q_t": 0.39578622579574585, "grad_norm": 10.07776927947998, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.5779359340667725, "logits/rejected": -1.4325804710388184, "logps/chosen": -104.4743423461914, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -155.02166748046875, "loss": 1.0333, "margin_dpo/margin_mean": 24.140541076660156, "margin_dpo/margin_std": 21.4137020111084, "step": 206 }, { "KL/chosen_KL_mean": -29.433483123779297, "KL/mean": -42.8851318359375, "KL/rejected_KL_mean": -56.33677291870117, "KL/std": 31.006515502929688, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.01786675490438938, "fcm_dpo/delta": -0.08472082018852234, "fcm_dpo/margin": 26.903297424316406, "fcm_dpo/q_t": 0.39274871349334717, "grad_norm": 10.068941116333008, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.123220920562744, "logits/rejected": -1.7041255235671997, "logps/chosen": -115.66513061523438, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -146.99188232421875, "loss": 1.0517, "margin_dpo/margin_mean": 26.903297424316406, "margin_dpo/margin_std": 35.565582275390625, "step": 207 }, { "KL/chosen_KL_mean": -30.521190643310547, "KL/mean": -44.26188659667969, "KL/rejected_KL_mean": -58.002586364746094, "KL/std": 28.09922218322754, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.017632437869906425, "fcm_dpo/delta": -0.08532623201608658, "fcm_dpo/margin": 27.48138427734375, "fcm_dpo/q_t": 0.38549184799194336, "grad_norm": 10.019658088684082, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.6502221822738647, "logits/rejected": -1.2892166376113892, "logps/chosen": -124.68074035644531, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -158.9649200439453, "loss": 1.0085, "margin_dpo/margin_mean": 27.481386184692383, "margin_dpo/margin_std": 25.543088912963867, "step": 208 }, { "KL/chosen_KL_mean": -26.065933227539062, "KL/mean": -38.21156311035156, "KL/rejected_KL_mean": -50.3571891784668, "KL/std": 25.0701847076416, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.017319614067673683, "fcm_dpo/delta": -0.04665808752179146, "fcm_dpo/margin": 24.29125213623047, "fcm_dpo/q_t": 0.4021304249763489, "grad_norm": 8.902782440185547, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.2279834747314453, "logits/rejected": -1.7620651721954346, "logps/chosen": -110.23649597167969, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -137.97674560546875, "loss": 1.0674, "margin_dpo/margin_mean": 24.29125213623047, "margin_dpo/margin_std": 28.511917114257812, "step": 209 }, { "KL/chosen_KL_mean": -29.69204330444336, "KL/mean": -41.26403045654297, "KL/rejected_KL_mean": -52.83601760864258, "KL/std": 26.602697372436523, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.017269153147935867, "fcm_dpo/delta": -0.014589082449674606, "fcm_dpo/margin": 23.14397621154785, "fcm_dpo/q_t": 0.4068108797073364, "grad_norm": 8.375253677368164, "learning_rate": 4.381713366536311e-07, "logits/chosen": -2.0546035766601562, "logits/rejected": -1.6732302904129028, "logps/chosen": -110.86322021484375, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -137.01080322265625, "loss": 1.0767, "margin_dpo/margin_mean": 23.143978118896484, "margin_dpo/margin_std": 26.898284912109375, "step": 210 }, { "KL/chosen_KL_mean": -33.29568099975586, "KL/mean": -44.4815559387207, "KL/rejected_KL_mean": -55.66743087768555, "KL/std": 28.187870025634766, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.017269153147935867, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.37175750732422, "fcm_dpo/q_t": 0.41065722703933716, "grad_norm": 10.1694974899292, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.121279239654541, "logits/rejected": -1.4990158081054688, "logps/chosen": -142.00839233398438, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -149.22308349609375, "loss": 1.0937, "margin_dpo/margin_mean": 22.37175750732422, "margin_dpo/margin_std": 28.66611671447754, "step": 211 }, { "KL/chosen_KL_mean": -23.44382095336914, "KL/mean": -39.97626495361328, "KL/rejected_KL_mean": -56.508705139160156, "KL/std": 32.06361770629883, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.016813460737466812, "fcm_dpo/delta": -0.1655687391757965, "fcm_dpo/margin": 33.064884185791016, "fcm_dpo/q_t": 0.37187084555625916, "grad_norm": 8.86347770690918, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.8784894943237305, "logits/rejected": -1.4652349948883057, "logps/chosen": -121.80577087402344, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -166.39871215820312, "loss": 0.9703, "margin_dpo/margin_mean": 33.064884185791016, "margin_dpo/margin_std": 31.931957244873047, "step": 212 }, { "KL/chosen_KL_mean": -27.61355209350586, "KL/mean": -40.08576202392578, "KL/rejected_KL_mean": -52.55796813964844, "KL/std": 29.30805206298828, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.01660960540175438, "fcm_dpo/delta": -0.02913464792072773, "fcm_dpo/margin": 24.944412231445312, "fcm_dpo/q_t": 0.4037206768989563, "grad_norm": 11.066473960876465, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.9975203275680542, "logits/rejected": -1.2116813659667969, "logps/chosen": -135.66885375976562, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -152.70211791992188, "loss": 1.0811, "margin_dpo/margin_mean": 24.94441032409668, "margin_dpo/margin_std": 32.148006439208984, "step": 213 }, { "KL/chosen_KL_mean": -29.427356719970703, "KL/mean": -41.1382942199707, "KL/rejected_KL_mean": -52.84923553466797, "KL/std": 31.38196563720703, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.01660960540175438, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.421878814697266, "fcm_dpo/q_t": 0.411690354347229, "grad_norm": 8.68384838104248, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.759763240814209, "logits/rejected": -1.636683464050293, "logps/chosen": -107.23209381103516, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -137.12503051757812, "loss": 1.1109, "margin_dpo/margin_mean": 23.421878814697266, "margin_dpo/margin_std": 34.579612731933594, "step": 214 }, { "KL/chosen_KL_mean": -25.382671356201172, "KL/mean": -41.342430114746094, "KL/rejected_KL_mean": -57.302188873291016, "KL/std": 30.8649845123291, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.0163031667470932, "fcm_dpo/delta": -0.12696954607963562, "fcm_dpo/margin": 31.919509887695312, "fcm_dpo/q_t": 0.3793559670448303, "grad_norm": 8.559704780578613, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.6803730726242065, "logits/rejected": -1.201277732849121, "logps/chosen": -110.49656677246094, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -154.1636962890625, "loss": 0.9881, "margin_dpo/margin_mean": 31.919509887695312, "margin_dpo/margin_std": 30.555484771728516, "step": 215 }, { "KL/chosen_KL_mean": -27.577133178710938, "KL/mean": -41.491416931152344, "KL/rejected_KL_mean": -55.40570068359375, "KL/std": 27.730878829956055, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.016085781157016754, "fcm_dpo/delta": -0.06673884391784668, "fcm_dpo/margin": 27.828567504882812, "fcm_dpo/q_t": 0.39560553431510925, "grad_norm": 8.142143249511719, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.6838860511779785, "logits/rejected": -1.2654565572738647, "logps/chosen": -108.16943359375, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -136.82553100585938, "loss": 1.0388, "margin_dpo/margin_mean": 27.82856559753418, "margin_dpo/margin_std": 28.643287658691406, "step": 216 }, { "KL/chosen_KL_mean": -33.744850158691406, "KL/mean": -45.13068389892578, "KL/rejected_KL_mean": -56.51652145385742, "KL/std": 28.67714500427246, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.015978427603840828, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.77166748046875, "fcm_dpo/q_t": 0.41663694381713867, "grad_norm": 9.89979076385498, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.2240395545959473, "logits/rejected": -1.9249663352966309, "logps/chosen": -127.21743774414062, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -160.00503540039062, "loss": 1.1189, "margin_dpo/margin_mean": 22.771665573120117, "margin_dpo/margin_std": 32.84123611450195, "step": 217 }, { "KL/chosen_KL_mean": -32.7946662902832, "KL/mean": -46.499908447265625, "KL/rejected_KL_mean": -60.205169677734375, "KL/std": 28.979339599609375, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.015842361375689507, "fcm_dpo/delta": -0.04276038333773613, "fcm_dpo/margin": 27.410493850708008, "fcm_dpo/q_t": 0.40005767345428467, "grad_norm": 8.663224220275879, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.6143229007720947, "logits/rejected": -1.241370677947998, "logps/chosen": -121.85350036621094, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -154.51197814941406, "loss": 1.0699, "margin_dpo/margin_mean": 27.410493850708008, "margin_dpo/margin_std": 34.8592529296875, "step": 218 }, { "KL/chosen_KL_mean": -29.503684997558594, "KL/mean": -41.87945556640625, "KL/rejected_KL_mean": -54.25524139404297, "KL/std": 33.71786880493164, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.015766892582178116, "fcm_dpo/delta": -0.0478665791451931, "fcm_dpo/margin": 24.751556396484375, "fcm_dpo/q_t": 0.41008254885673523, "grad_norm": 9.564081192016602, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.9678373336791992, "logits/rejected": -1.5243706703186035, "logps/chosen": -130.51101684570312, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -146.72317504882812, "loss": 1.1094, "margin_dpo/margin_mean": 24.751554489135742, "margin_dpo/margin_std": 35.8717041015625, "step": 219 }, { "KL/chosen_KL_mean": -30.39773178100586, "KL/mean": -41.88014221191406, "KL/rejected_KL_mean": -53.36255645751953, "KL/std": 28.028125762939453, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.015691421926021576, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.964834213256836, "fcm_dpo/q_t": 0.4150242209434509, "grad_norm": 9.9650239944458, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5608274936676025, "logits/rejected": -1.2149956226348877, "logps/chosen": -125.93495178222656, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -147.66958618164062, "loss": 1.1037, "margin_dpo/margin_mean": 22.96483612060547, "margin_dpo/margin_std": 27.618053436279297, "step": 220 }, { "KL/chosen_KL_mean": -28.557613372802734, "KL/mean": -46.28312683105469, "KL/rejected_KL_mean": -64.00863647460938, "KL/std": 34.91450500488281, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.015430586412549019, "fcm_dpo/delta": -0.15536801517009735, "fcm_dpo/margin": 35.45102310180664, "fcm_dpo/q_t": 0.37310490012168884, "grad_norm": 9.726601600646973, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.2585285902023315, "logits/rejected": -1.122518539428711, "logps/chosen": -111.40248107910156, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -174.8204345703125, "loss": 0.969, "margin_dpo/margin_mean": 35.451026916503906, "margin_dpo/margin_std": 30.65349578857422, "step": 221 }, { "KL/chosen_KL_mean": -36.694210052490234, "KL/mean": -51.1397819519043, "KL/rejected_KL_mean": -65.58535766601562, "KL/std": 33.43290710449219, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.015117788687348366, "fcm_dpo/delta": -0.06187653914093971, "fcm_dpo/margin": 28.89113998413086, "fcm_dpo/q_t": 0.399762898683548, "grad_norm": 9.70200252532959, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.8452446460723877, "logits/rejected": -1.4080854654312134, "logps/chosen": -131.836181640625, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -172.3897705078125, "loss": 1.0797, "margin_dpo/margin_mean": 28.89113998413086, "margin_dpo/margin_std": 38.96831512451172, "step": 222 }, { "KL/chosen_KL_mean": -35.69712829589844, "KL/mean": -45.998207092285156, "KL/rejected_KL_mean": -56.29928970336914, "KL/std": 29.66242218017578, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.015024246647953987, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.602161407470703, "fcm_dpo/q_t": 0.42753151059150696, "grad_norm": 9.430747032165527, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.434075117111206, "logits/rejected": -2.0612635612487793, "logps/chosen": -121.2722396850586, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -142.75167846679688, "loss": 1.1574, "margin_dpo/margin_mean": 20.602161407470703, "margin_dpo/margin_std": 32.441505432128906, "step": 223 }, { "KL/chosen_KL_mean": -32.531673431396484, "KL/mean": -47.218963623046875, "KL/rejected_KL_mean": -61.90625, "KL/std": 32.68992233276367, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.014941703528165817, "fcm_dpo/delta": -0.02754564769566059, "fcm_dpo/margin": 29.374574661254883, "fcm_dpo/q_t": 0.3985831141471863, "grad_norm": 8.990771293640137, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.9715614318847656, "logits/rejected": -1.526955485343933, "logps/chosen": -115.25547790527344, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -144.5016326904297, "loss": 1.056, "margin_dpo/margin_mean": 29.374574661254883, "margin_dpo/margin_std": 34.31231689453125, "step": 224 }, { "KL/chosen_KL_mean": -36.60541915893555, "KL/mean": -51.86333465576172, "KL/rejected_KL_mean": -67.12125396728516, "KL/std": 34.50746536254883, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.01484585553407669, "fcm_dpo/delta": -0.06456275284290314, "fcm_dpo/margin": 30.515830993652344, "fcm_dpo/q_t": 0.3966881036758423, "grad_norm": 9.873493194580078, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.010681629180908, "logits/rejected": -1.5350074768066406, "logps/chosen": -132.28311157226562, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -172.2181396484375, "loss": 1.0591, "margin_dpo/margin_mean": 30.515830993652344, "margin_dpo/margin_std": 38.72557067871094, "step": 225 }, { "KL/chosen_KL_mean": -30.30035400390625, "KL/mean": -44.854774475097656, "KL/rejected_KL_mean": -59.409202575683594, "KL/std": 33.775474548339844, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.014563683420419693, "fcm_dpo/delta": -0.06356345117092133, "fcm_dpo/margin": 29.108837127685547, "fcm_dpo/q_t": 0.4002900719642639, "grad_norm": 8.141740798950195, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.740525245666504, "logits/rejected": -1.1921453475952148, "logps/chosen": -123.76127624511719, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -146.1109619140625, "loss": 1.0535, "margin_dpo/margin_mean": 29.108837127685547, "margin_dpo/margin_std": 30.042036056518555, "step": 226 }, { "KL/chosen_KL_mean": -40.950679779052734, "KL/mean": -56.071876525878906, "KL/rejected_KL_mean": -71.19307708740234, "KL/std": 32.73542785644531, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.014525892212986946, "fcm_dpo/delta": -0.026016484946012497, "fcm_dpo/margin": 30.242395401000977, "fcm_dpo/q_t": 0.3994138836860657, "grad_norm": 9.084327697753906, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.7653049230575562, "logits/rejected": -1.4278146028518677, "logps/chosen": -133.76495361328125, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -175.9300079345703, "loss": 1.0685, "margin_dpo/margin_mean": 30.242395401000977, "margin_dpo/margin_std": 38.70856475830078, "step": 227 }, { "KL/chosen_KL_mean": -38.48236846923828, "KL/mean": -49.890846252441406, "KL/rejected_KL_mean": -61.29933547973633, "KL/std": 30.52364730834961, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.0144881010055542, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.816965103149414, "fcm_dpo/q_t": 0.42229217290878296, "grad_norm": 9.228846549987793, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.968273639678955, "logits/rejected": -1.5406962633132935, "logps/chosen": -132.51950073242188, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -157.32086181640625, "loss": 1.1428, "margin_dpo/margin_mean": 22.816965103149414, "margin_dpo/margin_std": 34.500877380371094, "step": 228 }, { "KL/chosen_KL_mean": -34.5913200378418, "KL/mean": -53.28887939453125, "KL/rejected_KL_mean": -71.98643493652344, "KL/std": 37.01482391357422, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.014203216880559921, "fcm_dpo/delta": -0.13840141892433167, "fcm_dpo/margin": 37.39511489868164, "fcm_dpo/q_t": 0.37920820713043213, "grad_norm": 9.021657943725586, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.7789816856384277, "logits/rejected": -1.137805461883545, "logps/chosen": -130.1689910888672, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -161.1602325439453, "loss": 0.998, "margin_dpo/margin_mean": 37.395118713378906, "margin_dpo/margin_std": 40.51954650878906, "step": 229 }, { "KL/chosen_KL_mean": -35.977481842041016, "KL/mean": -55.37608337402344, "KL/rejected_KL_mean": -74.77468872070312, "KL/std": 36.623939514160156, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.013794184662401676, "fcm_dpo/delta": -0.14284750819206238, "fcm_dpo/margin": 38.79719543457031, "fcm_dpo/q_t": 0.37507760524749756, "grad_norm": 10.155854225158691, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.6680927276611328, "logits/rejected": -1.6080806255340576, "logps/chosen": -99.69110870361328, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -166.6833953857422, "loss": 0.9788, "margin_dpo/margin_mean": 38.79719543457031, "margin_dpo/margin_std": 36.36500930786133, "step": 230 }, { "KL/chosen_KL_mean": -39.87543487548828, "KL/mean": -56.61247253417969, "KL/rejected_KL_mean": -73.34950256347656, "KL/std": 35.37971115112305, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.013500538654625416, "fcm_dpo/delta": -0.07174161076545715, "fcm_dpo/margin": 33.47406768798828, "fcm_dpo/q_t": 0.3930647373199463, "grad_norm": 8.103151321411133, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.6662116050720215, "logits/rejected": -1.0895342826843262, "logps/chosen": -135.33111572265625, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -154.30517578125, "loss": 1.0264, "margin_dpo/margin_mean": 33.474063873291016, "margin_dpo/margin_std": 30.69268035888672, "step": 231 }, { "KL/chosen_KL_mean": -42.159332275390625, "KL/mean": -56.17972946166992, "KL/rejected_KL_mean": -70.20013427734375, "KL/std": 32.85322952270508, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.013500538654625416, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.04080581665039, "fcm_dpo/q_t": 0.4108729958534241, "grad_norm": 8.840381622314453, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.7747501134872437, "logits/rejected": -1.3068337440490723, "logps/chosen": -139.0565948486328, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -159.96475219726562, "loss": 1.0904, "margin_dpo/margin_mean": 28.04080581665039, "margin_dpo/margin_std": 32.800350189208984, "step": 232 }, { "KL/chosen_KL_mean": -43.984920501708984, "KL/mean": -63.520362854003906, "KL/rejected_KL_mean": -83.05579376220703, "KL/std": 41.37113952636719, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.01323840580880642, "fcm_dpo/delta": -0.12381123006343842, "fcm_dpo/margin": 39.070884704589844, "fcm_dpo/q_t": 0.38021862506866455, "grad_norm": 9.440388679504395, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.7187304496765137, "logits/rejected": -1.518168568611145, "logps/chosen": -133.03524780273438, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -195.81497192382812, "loss": 0.9999, "margin_dpo/margin_mean": 39.070884704589844, "margin_dpo/margin_std": 40.254859924316406, "step": 233 }, { "KL/chosen_KL_mean": -43.19142150878906, "KL/mean": -63.87306213378906, "KL/rejected_KL_mean": -84.55470275878906, "KL/std": 39.56639099121094, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.012814173474907875, "fcm_dpo/delta": -0.1370774805545807, "fcm_dpo/margin": 41.36328125, "fcm_dpo/q_t": 0.3785781264305115, "grad_norm": 8.503283500671387, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.484082579612732, "logits/rejected": -1.4156904220581055, "logps/chosen": -117.50978088378906, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -190.94229125976562, "loss": 0.9982, "margin_dpo/margin_mean": 41.36328125, "margin_dpo/margin_std": 43.563804626464844, "step": 234 }, { "KL/chosen_KL_mean": -51.8078727722168, "KL/mean": -67.50657653808594, "KL/rejected_KL_mean": -83.20528411865234, "KL/std": 35.69585418701172, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.012795321643352509, "fcm_dpo/delta": -0.014732970856130123, "fcm_dpo/margin": 31.397411346435547, "fcm_dpo/q_t": 0.4053681492805481, "grad_norm": 8.54539680480957, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.625724196434021, "logits/rejected": -1.0695563554763794, "logps/chosen": -150.02491760253906, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -180.45205688476562, "loss": 1.0687, "margin_dpo/margin_mean": 31.397411346435547, "margin_dpo/margin_std": 33.80141830444336, "step": 235 }, { "KL/chosen_KL_mean": -43.478763580322266, "KL/mean": -63.925193786621094, "KL/rejected_KL_mean": -84.37162780761719, "KL/std": 44.864784240722656, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.012573162093758583, "fcm_dpo/delta": -0.1201900988817215, "fcm_dpo/margin": 40.892860412597656, "fcm_dpo/q_t": 0.3864075839519501, "grad_norm": 9.274273872375488, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.496977686882019, "logits/rejected": -1.5031020641326904, "logps/chosen": -122.31649780273438, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -193.43505859375, "loss": 1.0159, "margin_dpo/margin_mean": 40.892860412597656, "margin_dpo/margin_std": 48.56379318237305, "step": 236 }, { "KL/chosen_KL_mean": -51.76095199584961, "KL/mean": -66.50702667236328, "KL/rejected_KL_mean": -81.25310516357422, "KL/std": 39.63652420043945, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.012473011389374733, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.492143630981445, "fcm_dpo/q_t": 0.41860416531562805, "grad_norm": 8.742682456970215, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.389622688293457, "logits/rejected": -1.0331757068634033, "logps/chosen": -136.9722442626953, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -181.16310119628906, "loss": 1.1459, "margin_dpo/margin_mean": 29.492143630981445, "margin_dpo/margin_std": 51.506187438964844, "step": 237 }, { "KL/chosen_KL_mean": -56.613590240478516, "KL/mean": -74.01359558105469, "KL/rejected_KL_mean": -91.41358947753906, "KL/std": 41.13188552856445, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.012404928915202618, "fcm_dpo/delta": -0.027366815134882927, "fcm_dpo/margin": 34.80000305175781, "fcm_dpo/q_t": 0.40118274092674255, "grad_norm": 8.064826011657715, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.606259822845459, "logits/rejected": -1.4619371891021729, "logps/chosen": -135.34481811523438, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -186.83200073242188, "loss": 1.065, "margin_dpo/margin_mean": 34.80000305175781, "margin_dpo/margin_std": 42.831199645996094, "step": 238 }, { "KL/chosen_KL_mean": -51.2583122253418, "KL/mean": -67.450439453125, "KL/rejected_KL_mean": -83.6425552368164, "KL/std": 36.67399597167969, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.012253775261342525, "fcm_dpo/delta": -0.06129899621009827, "fcm_dpo/margin": 32.38423156738281, "fcm_dpo/q_t": 0.40765923261642456, "grad_norm": 8.851716995239258, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.813656210899353, "logits/rejected": -1.3356380462646484, "logps/chosen": -143.8592529296875, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -184.16024780273438, "loss": 1.091, "margin_dpo/margin_mean": 32.38423156738281, "margin_dpo/margin_std": 40.899375915527344, "step": 239 }, { "KL/chosen_KL_mean": -53.49985885620117, "KL/mean": -67.83880615234375, "KL/rejected_KL_mean": -82.17776489257812, "KL/std": 36.968421936035156, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.012253775261342525, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.67790985107422, "fcm_dpo/q_t": 0.417873740196228, "grad_norm": 9.727497100830078, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.1291685104370117, "logits/rejected": -1.7706754207611084, "logps/chosen": -139.65963745117188, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -162.63343811035156, "loss": 1.1226, "margin_dpo/margin_mean": 28.67790985107422, "margin_dpo/margin_std": 39.78876495361328, "step": 240 }, { "KL/chosen_KL_mean": -49.19717025756836, "KL/mean": -74.63330841064453, "KL/rejected_KL_mean": -100.06944274902344, "KL/std": 42.194427490234375, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.011918624863028526, "fcm_dpo/delta": -0.21907201409339905, "fcm_dpo/margin": 50.872291564941406, "fcm_dpo/q_t": 0.3613222539424896, "grad_norm": 8.40471076965332, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.7071913480758667, "logits/rejected": -1.0197988748550415, "logps/chosen": -150.24310302734375, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -194.11880493164062, "loss": 0.9421, "margin_dpo/margin_mean": 50.872291564941406, "margin_dpo/margin_std": 47.28986358642578, "step": 241 }, { "KL/chosen_KL_mean": -58.02065658569336, "KL/mean": -76.44148254394531, "KL/rejected_KL_mean": -94.86229705810547, "KL/std": 37.965980529785156, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.011675633490085602, "fcm_dpo/delta": -0.045258380472660065, "fcm_dpo/margin": 36.841636657714844, "fcm_dpo/q_t": 0.3991265296936035, "grad_norm": 8.87813949584961, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.93534255027771, "logits/rejected": -1.2771207094192505, "logps/chosen": -153.69332885742188, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -185.516845703125, "loss": 1.0598, "margin_dpo/margin_mean": 36.841636657714844, "margin_dpo/margin_std": 42.13862609863281, "step": 242 }, { "KL/chosen_KL_mean": -57.43193054199219, "KL/mean": -75.78082275390625, "KL/rejected_KL_mean": -94.12972259521484, "KL/std": 40.72606658935547, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.011522211134433746, "fcm_dpo/delta": -0.08729466050863266, "fcm_dpo/margin": 36.69779968261719, "fcm_dpo/q_t": 0.40286481380462646, "grad_norm": 8.504070281982422, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.7973403930664062, "logits/rejected": -0.9543045163154602, "logps/chosen": -155.4633331298828, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -177.31777954101562, "loss": 1.0856, "margin_dpo/margin_mean": 36.69779968261719, "margin_dpo/margin_std": 48.49281311035156, "step": 243 }, { "KL/chosen_KL_mean": -60.145545959472656, "KL/mean": -79.38227081298828, "KL/rejected_KL_mean": -98.61900329589844, "KL/std": 44.64064025878906, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.011343654245138168, "fcm_dpo/delta": -0.03425247222185135, "fcm_dpo/margin": 38.473445892333984, "fcm_dpo/q_t": 0.3987772464752197, "grad_norm": 8.326695442199707, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.143329620361328, "logits/rejected": -1.8130483627319336, "logps/chosen": -149.03945922851562, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -201.1917724609375, "loss": 1.0554, "margin_dpo/margin_mean": 38.47344970703125, "margin_dpo/margin_std": 44.325504302978516, "step": 244 }, { "KL/chosen_KL_mean": -51.478519439697266, "KL/mean": -71.76177978515625, "KL/rejected_KL_mean": -92.04503631591797, "KL/std": 41.154842376708984, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.011290188878774643, "fcm_dpo/delta": -0.047356173396110535, "fcm_dpo/margin": 40.56651306152344, "fcm_dpo/q_t": 0.39330077171325684, "grad_norm": 7.858860969543457, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.191114902496338, "logits/rejected": -1.3378942012786865, "logps/chosen": -155.67251586914062, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -184.7014923095703, "loss": 1.0332, "margin_dpo/margin_mean": 40.56651306152344, "margin_dpo/margin_std": 42.27428436279297, "step": 245 }, { "KL/chosen_KL_mean": -66.6472396850586, "KL/mean": -82.73460388183594, "KL/rejected_KL_mean": -98.82199096679688, "KL/std": 40.677730560302734, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.011206718161702156, "fcm_dpo/delta": -0.013369091786444187, "fcm_dpo/margin": 32.17475128173828, "fcm_dpo/q_t": 0.4163565933704376, "grad_norm": 8.031668663024902, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.9104095697402954, "logits/rejected": -1.3698757886886597, "logps/chosen": -169.8662872314453, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -189.814208984375, "loss": 1.116, "margin_dpo/margin_mean": 32.174747467041016, "margin_dpo/margin_std": 44.20770263671875, "step": 246 }, { "KL/chosen_KL_mean": -60.36969757080078, "KL/mean": -86.88036346435547, "KL/rejected_KL_mean": -113.39100646972656, "KL/std": 45.52782440185547, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.010851811617612839, "fcm_dpo/delta": -0.1872985064983368, "fcm_dpo/margin": 53.02130889892578, "fcm_dpo/q_t": 0.36848264932632446, "grad_norm": 8.23104476928711, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.225315570831299, "logits/rejected": -2.051692008972168, "logps/chosen": -147.36407470703125, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -224.72903442382812, "loss": 0.9629, "margin_dpo/margin_mean": 53.02130889892578, "margin_dpo/margin_std": 51.41064453125, "step": 247 }, { "KL/chosen_KL_mean": -69.56263732910156, "KL/mean": -87.07566833496094, "KL/rejected_KL_mean": -104.58869934082031, "KL/std": 40.099143981933594, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.010775953531265259, "fcm_dpo/delta": -0.017380600795149803, "fcm_dpo/margin": 35.02606201171875, "fcm_dpo/q_t": 0.41126227378845215, "grad_norm": 8.67447566986084, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.7851436138153076, "logits/rejected": -1.7607920169830322, "logps/chosen": -144.34814453125, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -202.8656005859375, "loss": 1.0913, "margin_dpo/margin_mean": 35.02606201171875, "margin_dpo/margin_std": 41.163787841796875, "step": 248 }, { "KL/chosen_KL_mean": -77.35226440429688, "KL/mean": -93.23014068603516, "KL/rejected_KL_mean": -109.10801696777344, "KL/std": 43.12325668334961, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.010757224634289742, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.755752563476562, "fcm_dpo/q_t": 0.42096278071403503, "grad_norm": 9.668108940124512, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.201794147491455, "logits/rejected": -1.660149335861206, "logps/chosen": -173.05606079101562, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -196.25448608398438, "loss": 1.1414, "margin_dpo/margin_mean": 31.755748748779297, "margin_dpo/margin_std": 49.300933837890625, "step": 249 }, { "KL/chosen_KL_mean": -68.45155334472656, "KL/mean": -86.73603820800781, "KL/rejected_KL_mean": -105.02052307128906, "KL/std": 43.671241760253906, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.010669252835214138, "fcm_dpo/delta": -0.04105791077017784, "fcm_dpo/margin": 36.56897735595703, "fcm_dpo/q_t": 0.408291220664978, "grad_norm": 8.744075775146484, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9338915348052979, "logits/rejected": -0.9890934824943542, "logps/chosen": -182.50376892089844, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -186.10821533203125, "loss": 1.0974, "margin_dpo/margin_mean": 36.56897735595703, "margin_dpo/margin_std": 46.83900833129883, "step": 250 }, { "KL/chosen_KL_mean": -68.72640228271484, "KL/mean": -84.63517761230469, "KL/rejected_KL_mean": -100.5439453125, "KL/std": 40.83884811401367, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.010620731860399246, "fcm_dpo/delta": -0.02279057539999485, "fcm_dpo/margin": 31.81754493713379, "fcm_dpo/q_t": 0.4221749007701874, "grad_norm": 9.524369239807129, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.8133560419082642, "logits/rejected": -1.1033942699432373, "logps/chosen": -166.43768310546875, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -184.0713653564453, "loss": 1.1441, "margin_dpo/margin_mean": 31.81754493713379, "margin_dpo/margin_std": 49.280799865722656, "step": 251 }, { "KL/chosen_KL_mean": -68.9948501586914, "KL/mean": -85.1153564453125, "KL/rejected_KL_mean": -101.23585510253906, "KL/std": 43.66602325439453, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.010620731860399246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.24101257324219, "fcm_dpo/q_t": 0.4195539355278015, "grad_norm": 9.216890335083008, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.055116653442383, "logits/rejected": -1.5120069980621338, "logps/chosen": -177.98095703125, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -192.80010986328125, "loss": 1.1322, "margin_dpo/margin_mean": 32.24101257324219, "margin_dpo/margin_std": 46.31875991821289, "step": 252 }, { "KL/chosen_KL_mean": -75.73011779785156, "KL/mean": -90.70814514160156, "KL/rejected_KL_mean": -105.68616485595703, "KL/std": 47.14838409423828, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.010620731860399246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.9560546875, "fcm_dpo/q_t": 0.42445021867752075, "grad_norm": 9.219466209411621, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.046051502227783, "logits/rejected": -1.8378751277923584, "logps/chosen": -175.9464111328125, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -211.3628692626953, "loss": 1.1402, "margin_dpo/margin_mean": 29.956058502197266, "margin_dpo/margin_std": 40.652366638183594, "step": 253 }, { "KL/chosen_KL_mean": -73.55949401855469, "KL/mean": -89.14334106445312, "KL/rejected_KL_mean": -104.7271957397461, "KL/std": 44.55557632446289, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.010620731860399246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.167705535888672, "fcm_dpo/q_t": 0.42186063528060913, "grad_norm": 8.911653518676758, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.9695714712142944, "logits/rejected": -1.4364848136901855, "logps/chosen": -172.294677734375, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -198.4654541015625, "loss": 1.1449, "margin_dpo/margin_mean": 31.167705535888672, "margin_dpo/margin_std": 47.546836853027344, "step": 254 }, { "KL/chosen_KL_mean": -74.10322570800781, "KL/mean": -97.99256896972656, "KL/rejected_KL_mean": -121.88191223144531, "KL/std": 50.240203857421875, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.010466604493558407, "fcm_dpo/delta": -0.10526317358016968, "fcm_dpo/margin": 47.77869415283203, "fcm_dpo/q_t": 0.3840145766735077, "grad_norm": 8.962532043457031, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.032167911529541, "logits/rejected": -1.6986751556396484, "logps/chosen": -164.4857940673828, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -218.95816040039062, "loss": 1.0139, "margin_dpo/margin_mean": 47.778690338134766, "margin_dpo/margin_std": 51.10535430908203, "step": 255 }, { "KL/chosen_KL_mean": -72.45974731445312, "KL/mean": -93.60669708251953, "KL/rejected_KL_mean": -114.75364685058594, "KL/std": 52.21851348876953, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.010353711433708668, "fcm_dpo/delta": -0.044200170785188675, "fcm_dpo/margin": 42.293907165527344, "fcm_dpo/q_t": 0.4007514715194702, "grad_norm": 9.395307540893555, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.9729305505752563, "logits/rejected": -1.6399288177490234, "logps/chosen": -161.21255493164062, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -213.2474822998047, "loss": 1.0888, "margin_dpo/margin_mean": 42.29390335083008, "margin_dpo/margin_std": 61.24385070800781, "step": 256 }, { "KL/chosen_KL_mean": -74.18669891357422, "KL/mean": -95.71031188964844, "KL/rejected_KL_mean": -117.23391723632812, "KL/std": 53.71492004394531, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.010096490383148193, "fcm_dpo/delta": -0.10363708436489105, "fcm_dpo/margin": 43.04722595214844, "fcm_dpo/q_t": 0.3990858197212219, "grad_norm": 8.417766571044922, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.0459794998168945, "logits/rejected": -1.6538000106811523, "logps/chosen": -156.99021911621094, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -203.10171508789062, "loss": 1.0679, "margin_dpo/margin_mean": 43.04722213745117, "margin_dpo/margin_std": 51.75374984741211, "step": 257 }, { "KL/chosen_KL_mean": -77.14776611328125, "KL/mean": -96.53451538085938, "KL/rejected_KL_mean": -115.92124938964844, "KL/std": 51.037654876708984, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.010043701156973839, "fcm_dpo/delta": -0.05256004258990288, "fcm_dpo/margin": 38.77349090576172, "fcm_dpo/q_t": 0.40851378440856934, "grad_norm": 9.30900764465332, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.244677782058716, "logits/rejected": -1.8930213451385498, "logps/chosen": -167.33551025390625, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -219.39193725585938, "loss": 1.0946, "margin_dpo/margin_mean": 38.77349090576172, "margin_dpo/margin_std": 48.762977600097656, "step": 258 }, { "KL/chosen_KL_mean": -83.28414916992188, "KL/mean": -100.00907135009766, "KL/rejected_KL_mean": -116.73399353027344, "KL/std": 51.55987548828125, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.009946268051862717, "fcm_dpo/delta": -0.022392423823475838, "fcm_dpo/margin": 33.44984436035156, "fcm_dpo/q_t": 0.4221954047679901, "grad_norm": 8.882987976074219, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.1505887508392334, "logits/rejected": -0.8880763053894043, "logps/chosen": -174.05670166015625, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -211.32215881347656, "loss": 1.1617, "margin_dpo/margin_mean": 33.44984436035156, "margin_dpo/margin_std": 58.06321716308594, "step": 259 }, { "KL/chosen_KL_mean": -85.28182220458984, "KL/mean": -99.54391479492188, "KL/rejected_KL_mean": -113.80598449707031, "KL/std": 52.31573486328125, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.009946268051862717, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.524166107177734, "fcm_dpo/q_t": 0.43363040685653687, "grad_norm": 10.191603660583496, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.5985629558563232, "logits/rejected": -1.6323063373565674, "logps/chosen": -160.87451171875, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -214.65151977539062, "loss": 1.1988, "margin_dpo/margin_mean": 28.524166107177734, "margin_dpo/margin_std": 56.83106994628906, "step": 260 }, { "KL/chosen_KL_mean": -65.47834777832031, "KL/mean": -90.43338012695312, "KL/rejected_KL_mean": -115.3884048461914, "KL/std": 52.436073303222656, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.00975704938173294, "fcm_dpo/delta": -0.09603700041770935, "fcm_dpo/margin": 49.91006851196289, "fcm_dpo/q_t": 0.38675636053085327, "grad_norm": 8.258631706237793, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.973351240158081, "logits/rejected": -1.603840947151184, "logps/chosen": -159.73324584960938, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -209.04539489746094, "loss": 1.016, "margin_dpo/margin_mean": 49.91006851196289, "margin_dpo/margin_std": 50.96443176269531, "step": 261 }, { "KL/chosen_KL_mean": -71.91462707519531, "KL/mean": -95.10321044921875, "KL/rejected_KL_mean": -118.29180908203125, "KL/std": 55.37150192260742, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.00964161567389965, "fcm_dpo/delta": -0.059506606310606, "fcm_dpo/margin": 46.37718200683594, "fcm_dpo/q_t": 0.3963865041732788, "grad_norm": 8.255736351013184, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.6767419576644897, "logits/rejected": -1.1335854530334473, "logps/chosen": -157.1819305419922, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -209.11788940429688, "loss": 1.0557, "margin_dpo/margin_mean": 46.37718200683594, "margin_dpo/margin_std": 55.45066833496094, "step": 262 }, { "KL/chosen_KL_mean": -57.97856140136719, "KL/mean": -89.41752624511719, "KL/rejected_KL_mean": -120.85649108886719, "KL/std": 60.14796447753906, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.009375881403684616, "fcm_dpo/delta": -0.20110929012298584, "fcm_dpo/margin": 62.877925872802734, "fcm_dpo/q_t": 0.3646092116832733, "grad_norm": 8.061029434204102, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.9442676305770874, "logits/rejected": -1.370525598526001, "logps/chosen": -153.73646545410156, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -222.69024658203125, "loss": 0.9641, "margin_dpo/margin_mean": 62.877925872802734, "margin_dpo/margin_std": 64.16009521484375, "step": 263 }, { "KL/chosen_KL_mean": -67.4524917602539, "KL/mean": -94.15245056152344, "KL/rejected_KL_mean": -120.85240173339844, "KL/std": 56.85414123535156, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.009121667593717575, "fcm_dpo/delta": -0.09175059199333191, "fcm_dpo/margin": 53.399906158447266, "fcm_dpo/q_t": 0.3869553208351135, "grad_norm": 8.004034996032715, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.1619534492492676, "logits/rejected": -1.8795039653778076, "logps/chosen": -158.0751495361328, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -221.17794799804688, "loss": 1.023, "margin_dpo/margin_mean": 53.399906158447266, "margin_dpo/margin_std": 58.050437927246094, "step": 264 }, { "KL/chosen_KL_mean": -71.47785186767578, "KL/mean": -94.71150207519531, "KL/rejected_KL_mean": -117.94515991210938, "KL/std": 58.039058685302734, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0090035330504179, "fcm_dpo/delta": -0.04949821159243584, "fcm_dpo/margin": 46.467308044433594, "fcm_dpo/q_t": 0.40423065423965454, "grad_norm": 8.478856086730957, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.069333076477051, "logits/rejected": -1.6625170707702637, "logps/chosen": -163.85018920898438, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -212.57272338867188, "loss": 1.0865, "margin_dpo/margin_mean": 46.467308044433594, "margin_dpo/margin_std": 62.592384338378906, "step": 265 }, { "KL/chosen_KL_mean": -83.45158386230469, "KL/mean": -104.64128112792969, "KL/rejected_KL_mean": -125.83097839355469, "KL/std": 56.140228271484375, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0090035330504179, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.37940216064453, "fcm_dpo/q_t": 0.41225337982177734, "grad_norm": 9.853046417236328, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.335521936416626, "logits/rejected": -1.5173250436782837, "logps/chosen": -185.651611328125, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -200.19740295410156, "loss": 1.1105, "margin_dpo/margin_mean": 42.37940216064453, "margin_dpo/margin_std": 59.65619659423828, "step": 266 }, { "KL/chosen_KL_mean": -73.69149780273438, "KL/mean": -98.49076843261719, "KL/rejected_KL_mean": -123.29003143310547, "KL/std": 56.370479583740234, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.00889680813997984, "fcm_dpo/delta": -0.05962253361940384, "fcm_dpo/margin": 49.59852600097656, "fcm_dpo/q_t": 0.3972480595111847, "grad_norm": 8.628073692321777, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.807843804359436, "logits/rejected": -1.1817494630813599, "logps/chosen": -177.97750854492188, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -219.2772216796875, "loss": 1.049, "margin_dpo/margin_mean": 49.59852600097656, "margin_dpo/margin_std": 54.28278732299805, "step": 267 }, { "KL/chosen_KL_mean": -73.09322357177734, "KL/mean": -102.35946655273438, "KL/rejected_KL_mean": -131.62570190429688, "KL/std": 56.763404846191406, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.008718919940292835, "fcm_dpo/delta": -0.10098595917224884, "fcm_dpo/margin": 58.53248596191406, "fcm_dpo/q_t": 0.3806135356426239, "grad_norm": 8.814906120300293, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.5419634580612183, "logits/rejected": -1.388629674911499, "logps/chosen": -159.78944396972656, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -240.81753540039062, "loss": 0.9965, "margin_dpo/margin_mean": 58.5324821472168, "margin_dpo/margin_std": 55.96549987792969, "step": 268 }, { "KL/chosen_KL_mean": -78.02372741699219, "KL/mean": -106.55096435546875, "KL/rejected_KL_mean": -135.07818603515625, "KL/std": 61.005714416503906, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.008615316823124886, "fcm_dpo/delta": -0.09616195410490036, "fcm_dpo/margin": 57.054473876953125, "fcm_dpo/q_t": 0.38567501306533813, "grad_norm": 7.986194610595703, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.9286860227584839, "logits/rejected": -1.4817708730697632, "logps/chosen": -182.99554443359375, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -247.55465698242188, "loss": 1.013, "margin_dpo/margin_mean": 57.054473876953125, "margin_dpo/margin_std": 58.244667053222656, "step": 269 }, { "KL/chosen_KL_mean": -87.40202331542969, "KL/mean": -111.02715301513672, "KL/rejected_KL_mean": -134.65228271484375, "KL/std": 59.94041442871094, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.008523860946297646, "fcm_dpo/delta": -0.033993981778621674, "fcm_dpo/margin": 47.250247955322266, "fcm_dpo/q_t": 0.4067686200141907, "grad_norm": 8.870279312133789, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.3512754440307617, "logits/rejected": -1.9338464736938477, "logps/chosen": -188.47586059570312, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -252.40516662597656, "loss": 1.1047, "margin_dpo/margin_mean": 47.250247955322266, "margin_dpo/margin_std": 68.19851684570312, "step": 270 }, { "KL/chosen_KL_mean": -72.41447448730469, "KL/mean": -101.97602844238281, "KL/rejected_KL_mean": -131.53756713867188, "KL/std": 56.046234130859375, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.008359793573617935, "fcm_dpo/delta": -0.09931644797325134, "fcm_dpo/margin": 59.12310028076172, "fcm_dpo/q_t": 0.3839338421821594, "grad_norm": 8.863029479980469, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.3047429323196411, "logits/rejected": -0.8346935510635376, "logps/chosen": -169.13906860351562, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -230.06198120117188, "loss": 1.0204, "margin_dpo/margin_mean": 59.12310028076172, "margin_dpo/margin_std": 64.1628189086914, "step": 271 }, { "KL/chosen_KL_mean": -62.54093933105469, "KL/mean": -90.94066619873047, "KL/rejected_KL_mean": -119.34039306640625, "KL/std": 62.41492462158203, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.008238209411501884, "fcm_dpo/delta": -0.07127663493156433, "fcm_dpo/margin": 56.79944610595703, "fcm_dpo/q_t": 0.39254501461982727, "grad_norm": 8.10979175567627, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.6103856563568115, "logits/rejected": -1.5034111738204956, "logps/chosen": -143.51815795898438, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -227.895751953125, "loss": 1.045, "margin_dpo/margin_mean": 56.7994499206543, "margin_dpo/margin_std": 67.7859878540039, "step": 272 }, { "KL/chosen_KL_mean": -85.21701049804688, "KL/mean": -114.32984924316406, "KL/rejected_KL_mean": -143.4427032470703, "KL/std": 63.96059799194336, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.008034183643758297, "fcm_dpo/delta": -0.10820163041353226, "fcm_dpo/margin": 58.225685119628906, "fcm_dpo/q_t": 0.3915891945362091, "grad_norm": 7.999573230743408, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.6182548999786377, "logits/rejected": -1.2065818309783936, "logps/chosen": -177.44161987304688, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -240.80575561523438, "loss": 1.0556, "margin_dpo/margin_mean": 58.225685119628906, "margin_dpo/margin_std": 72.6716537475586, "step": 273 }, { "KL/chosen_KL_mean": -87.37579345703125, "KL/mean": -110.03181457519531, "KL/rejected_KL_mean": -132.68785095214844, "KL/std": 62.39153289794922, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.008034183643758297, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 45.31206512451172, "fcm_dpo/q_t": 0.41487959027290344, "grad_norm": 12.414081573486328, "learning_rate": 3.75e-07, "logits/chosen": -2.240809202194214, "logits/rejected": -1.8644057512283325, "logps/chosen": -169.63668823242188, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -216.56484985351562, "loss": 1.1064, "margin_dpo/margin_mean": 45.31206512451172, "margin_dpo/margin_std": 57.7353515625, "step": 274 }, { "KL/chosen_KL_mean": -74.13685607910156, "KL/mean": -99.96736145019531, "KL/rejected_KL_mean": -125.7978744506836, "KL/std": 61.81304931640625, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.007987311109900475, "fcm_dpo/delta": -0.058685120195150375, "fcm_dpo/margin": 51.6610221862793, "fcm_dpo/q_t": 0.4044990837574005, "grad_norm": 8.825883865356445, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.237393856048584, "logits/rejected": -1.9218673706054688, "logps/chosen": -153.82382202148438, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -224.54885864257812, "loss": 1.1003, "margin_dpo/margin_mean": 51.66102600097656, "margin_dpo/margin_std": 73.48239135742188, "step": 275 }, { "KL/chosen_KL_mean": -88.5787353515625, "KL/mean": -116.13322448730469, "KL/rejected_KL_mean": -143.68771362304688, "KL/std": 61.90583038330078, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.007816493511199951, "fcm_dpo/delta": -0.07866185158491135, "fcm_dpo/margin": 55.10897445678711, "fcm_dpo/q_t": 0.40066543221473694, "grad_norm": 9.1841459274292, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.9927542209625244, "logits/rejected": -1.4061203002929688, "logps/chosen": -175.11843872070312, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -221.54165649414062, "loss": 1.0744, "margin_dpo/margin_mean": 55.10897445678711, "margin_dpo/margin_std": 70.5630874633789, "step": 276 }, { "KL/chosen_KL_mean": -95.37164306640625, "KL/mean": -115.33956909179688, "KL/rejected_KL_mean": -135.3074951171875, "KL/std": 60.023258209228516, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.007816493511199951, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.93585968017578, "fcm_dpo/q_t": 0.4267790913581848, "grad_norm": 8.98894214630127, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.0896387100219727, "logits/rejected": -1.6759669780731201, "logps/chosen": -179.61575317382812, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -234.1417236328125, "loss": 1.1536, "margin_dpo/margin_mean": 39.93585968017578, "margin_dpo/margin_std": 61.03748321533203, "step": 277 }, { "KL/chosen_KL_mean": -91.13648986816406, "KL/mean": -116.9901123046875, "KL/rejected_KL_mean": -142.84375, "KL/std": 64.69404602050781, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.007787350565195084, "fcm_dpo/delta": -0.018676651641726494, "fcm_dpo/margin": 51.70725631713867, "fcm_dpo/q_t": 0.40535128116607666, "grad_norm": 9.952676773071289, "learning_rate": 3.705283756425872e-07, "logits/chosen": -1.8482965230941772, "logits/rejected": -1.6244611740112305, "logps/chosen": -173.56837463378906, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -240.70065307617188, "loss": 1.0864, "margin_dpo/margin_mean": 51.70725631713867, "margin_dpo/margin_std": 65.13477325439453, "step": 278 }, { "KL/chosen_KL_mean": -93.9708480834961, "KL/mean": -126.0738525390625, "KL/rejected_KL_mean": -158.17686462402344, "KL/std": 68.52223205566406, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.007652524393051863, "fcm_dpo/delta": -0.09655530750751495, "fcm_dpo/margin": 64.20600891113281, "fcm_dpo/q_t": 0.3859434723854065, "grad_norm": 9.140878677368164, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.7755544185638428, "logits/rejected": -1.561834454536438, "logps/chosen": -179.13885498046875, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -252.30349731445312, "loss": 1.0175, "margin_dpo/margin_mean": 64.20601654052734, "margin_dpo/margin_std": 68.24128723144531, "step": 279 }, { "KL/chosen_KL_mean": -98.3156509399414, "KL/mean": -129.6642608642578, "KL/rejected_KL_mean": -161.0128936767578, "KL/std": 62.356170654296875, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.007565831299871206, "fcm_dpo/delta": -0.07798557728528976, "fcm_dpo/margin": 62.69722366333008, "fcm_dpo/q_t": 0.3892950415611267, "grad_norm": 8.623045921325684, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.812194585800171, "logits/rejected": -1.6148416996002197, "logps/chosen": -184.17205810546875, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -265.1314697265625, "loss": 1.0251, "margin_dpo/margin_mean": 62.69721984863281, "margin_dpo/margin_std": 65.28692626953125, "step": 280 }, { "KL/chosen_KL_mean": -103.45527648925781, "KL/mean": -134.40423583984375, "KL/rejected_KL_mean": -165.35317993164062, "KL/std": 72.66020965576172, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.007442726753652096, "fcm_dpo/delta": -0.05177629366517067, "fcm_dpo/margin": 61.89791488647461, "fcm_dpo/q_t": 0.3939868211746216, "grad_norm": 7.1076555252075195, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.0981998443603516, "logits/rejected": -1.3581761121749878, "logps/chosen": -190.006103515625, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -253.9818572998047, "loss": 1.0465, "margin_dpo/margin_mean": 61.89791488647461, "margin_dpo/margin_std": 73.05177307128906, "step": 281 }, { "KL/chosen_KL_mean": -101.26097869873047, "KL/mean": -140.095947265625, "KL/rejected_KL_mean": -178.93087768554688, "KL/std": 70.34326171875, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.00724798534065485, "fcm_dpo/delta": -0.1728368103504181, "fcm_dpo/margin": 77.669921875, "fcm_dpo/q_t": 0.3687569499015808, "grad_norm": 8.713729858398438, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.8850352764129639, "logits/rejected": -1.509209394454956, "logps/chosen": -206.3660888671875, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -281.78424072265625, "loss": 0.9594, "margin_dpo/margin_mean": 77.669921875, "margin_dpo/margin_std": 70.297119140625, "step": 282 }, { "KL/chosen_KL_mean": -116.04240417480469, "KL/mean": -143.6357421875, "KL/rejected_KL_mean": -171.22909545898438, "KL/std": 69.8297119140625, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0071898470632731915, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.18667221069336, "fcm_dpo/q_t": 0.40851056575775146, "grad_norm": 9.791443824768066, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.4609336853027344, "logits/rejected": -2.1403560638427734, "logps/chosen": -211.09500122070312, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -274.77362060546875, "loss": 1.1015, "margin_dpo/margin_mean": 55.186676025390625, "margin_dpo/margin_std": 77.51045227050781, "step": 283 }, { "KL/chosen_KL_mean": -106.004150390625, "KL/mean": -140.8265380859375, "KL/rejected_KL_mean": -175.64894104003906, "KL/std": 68.5107650756836, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.007066941354423761, "fcm_dpo/delta": -0.08621073514223099, "fcm_dpo/margin": 69.64479064941406, "fcm_dpo/q_t": 0.3836461901664734, "grad_norm": 10.095714569091797, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.7811498641967773, "logits/rejected": -1.4442086219787598, "logps/chosen": -193.67062377929688, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -274.39996337890625, "loss": 0.9978, "margin_dpo/margin_mean": 69.64479064941406, "margin_dpo/margin_std": 61.21271514892578, "step": 284 }, { "KL/chosen_KL_mean": -117.52798461914062, "KL/mean": -142.04000854492188, "KL/rejected_KL_mean": -166.5520782470703, "KL/std": 63.406005859375, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.007066941354423761, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.024085998535156, "fcm_dpo/q_t": 0.417436420917511, "grad_norm": 9.456453323364258, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.276732921600342, "logits/rejected": -1.992375373840332, "logps/chosen": -211.96102905273438, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -270.6240234375, "loss": 1.1133, "margin_dpo/margin_mean": 49.024085998535156, "margin_dpo/margin_std": 60.002197265625, "step": 285 }, { "KL/chosen_KL_mean": -120.58680725097656, "KL/mean": -153.96461486816406, "KL/rejected_KL_mean": -187.34243774414062, "KL/std": 68.05718994140625, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.006983332335948944, "fcm_dpo/delta": -0.0695696771144867, "fcm_dpo/margin": 66.755615234375, "fcm_dpo/q_t": 0.3908499479293823, "grad_norm": 8.039011001586914, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.1657848358154297, "logits/rejected": -1.718252182006836, "logps/chosen": -224.3072052001953, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -283.6001892089844, "loss": 1.0323, "margin_dpo/margin_mean": 66.755615234375, "margin_dpo/margin_std": 70.82421112060547, "step": 286 }, { "KL/chosen_KL_mean": -138.42245483398438, "KL/mean": -157.6659698486328, "KL/rejected_KL_mean": -176.90948486328125, "KL/std": 61.71984100341797, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.006969293113797903, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.487030029296875, "fcm_dpo/q_t": 0.4381248354911804, "grad_norm": 8.112489700317383, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.1574950218200684, "logits/rejected": -1.552812933921814, "logps/chosen": -232.31234741210938, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -260.2431335449219, "loss": 1.1992, "margin_dpo/margin_mean": 38.48702621459961, "margin_dpo/margin_std": 72.0923843383789, "step": 287 }, { "KL/chosen_KL_mean": -130.00035095214844, "KL/mean": -164.72418212890625, "KL/rejected_KL_mean": -199.44798278808594, "KL/std": 70.35610961914062, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.006872690748423338, "fcm_dpo/delta": -0.08138720691204071, "fcm_dpo/margin": 69.44764709472656, "fcm_dpo/q_t": 0.39061349630355835, "grad_norm": 7.6049017906188965, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.8731772899627686, "logits/rejected": -1.4608891010284424, "logps/chosen": -218.1563720703125, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -292.72991943359375, "loss": 1.0322, "margin_dpo/margin_mean": 69.44763946533203, "margin_dpo/margin_std": 78.84242248535156, "step": 288 }, { "KL/chosen_KL_mean": -126.39546203613281, "KL/mean": -161.4593505859375, "KL/rejected_KL_mean": -196.5232391357422, "KL/std": 73.23516082763672, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.006815279833972454, "fcm_dpo/delta": -0.06087765097618103, "fcm_dpo/margin": 70.12779235839844, "fcm_dpo/q_t": 0.39209920167922974, "grad_norm": 7.420281887054443, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.9458308219909668, "logits/rejected": -1.7415971755981445, "logps/chosen": -201.78839111328125, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -289.6775207519531, "loss": 1.0396, "margin_dpo/margin_mean": 70.12779235839844, "margin_dpo/margin_std": 84.16267395019531, "step": 289 }, { "KL/chosen_KL_mean": -135.32473754882812, "KL/mean": -171.98463439941406, "KL/rejected_KL_mean": -208.64454650878906, "KL/std": 78.12646484375, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.00670152623206377, "fcm_dpo/delta": -0.10783704370260239, "fcm_dpo/margin": 73.31983947753906, "fcm_dpo/q_t": 0.3878808617591858, "grad_norm": 7.78359842300415, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.3866932392120361, "logits/rejected": -1.4321730136871338, "logps/chosen": -223.36669921875, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -331.856689453125, "loss": 1.0252, "margin_dpo/margin_mean": 73.31983947753906, "margin_dpo/margin_std": 81.9217300415039, "step": 290 }, { "KL/chosen_KL_mean": -127.79792785644531, "KL/mean": -159.52516174316406, "KL/rejected_KL_mean": -191.25241088867188, "KL/std": 74.26415252685547, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.006613646633923054, "fcm_dpo/delta": -0.023610683158040047, "fcm_dpo/margin": 63.4544677734375, "fcm_dpo/q_t": 0.4014871120452881, "grad_norm": 9.205639839172363, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.452571392059326, "logits/rejected": -1.9949864149093628, "logps/chosen": -221.89317321777344, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -287.302490234375, "loss": 1.0664, "margin_dpo/margin_mean": 63.4544677734375, "margin_dpo/margin_std": 73.697021484375, "step": 291 }, { "KL/chosen_KL_mean": -132.69924926757812, "KL/mean": -164.7520751953125, "KL/rejected_KL_mean": -196.80490112304688, "KL/std": 71.55592346191406, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.006598031148314476, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.10564422607422, "fcm_dpo/q_t": 0.400913268327713, "grad_norm": 8.09289836883545, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -1.8334577083587646, "logits/rejected": -1.3833339214324951, "logps/chosen": -220.9496612548828, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -293.2225341796875, "loss": 1.0533, "margin_dpo/margin_mean": 64.10564422607422, "margin_dpo/margin_std": 67.21332550048828, "step": 292 }, { "KL/chosen_KL_mean": -132.30722045898438, "KL/mean": -158.15435791015625, "KL/rejected_KL_mean": -184.00149536132812, "KL/std": 76.02566528320312, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.006598031148314476, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.69426727294922, "fcm_dpo/q_t": 0.4218252897262573, "grad_norm": 7.706785678863525, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.3672924041748047, "logits/rejected": -1.7987432479858398, "logps/chosen": -219.68377685546875, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -269.7572937011719, "loss": 1.139, "margin_dpo/margin_mean": 51.69426727294922, "margin_dpo/margin_std": 79.22975158691406, "step": 293 }, { "KL/chosen_KL_mean": -135.4763946533203, "KL/mean": -156.85235595703125, "KL/rejected_KL_mean": -178.22833251953125, "KL/std": 71.4581298828125, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.006598031148314476, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 42.751930236816406, "fcm_dpo/q_t": 0.4348605275154114, "grad_norm": 8.15061092376709, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.7131493091583252, "logits/rejected": -1.548429012298584, "logps/chosen": -208.98434448242188, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -266.3171081542969, "loss": 1.1983, "margin_dpo/margin_mean": 42.751930236816406, "margin_dpo/margin_std": 83.15204620361328, "step": 294 }, { "KL/chosen_KL_mean": -129.01327514648438, "KL/mean": -161.70785522460938, "KL/rejected_KL_mean": -194.40240478515625, "KL/std": 73.25961303710938, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.006530324462801218, "fcm_dpo/delta": -0.051573496311903, "fcm_dpo/margin": 65.38912963867188, "fcm_dpo/q_t": 0.40099573135375977, "grad_norm": 8.142817497253418, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.4977588653564453, "logits/rejected": -1.337082862854004, "logps/chosen": -206.7787628173828, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -288.649658203125, "loss": 1.0665, "margin_dpo/margin_mean": 65.38912963867188, "margin_dpo/margin_std": 78.81843566894531, "step": 295 }, { "KL/chosen_KL_mean": -138.83216857910156, "KL/mean": -173.9437255859375, "KL/rejected_KL_mean": -209.0552978515625, "KL/std": 73.18639373779297, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.006452551577240229, "fcm_dpo/delta": -0.05990508943796158, "fcm_dpo/margin": 70.2231216430664, "fcm_dpo/q_t": 0.39351141452789307, "grad_norm": 11.68242073059082, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.4861342906951904, "logits/rejected": -1.9963510036468506, "logps/chosen": -236.68858337402344, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -309.8716125488281, "loss": 1.0379, "margin_dpo/margin_mean": 70.2231216430664, "margin_dpo/margin_std": 73.76071166992188, "step": 296 }, { "KL/chosen_KL_mean": -140.3211669921875, "KL/mean": -169.66246032714844, "KL/rejected_KL_mean": -199.00375366210938, "KL/std": 73.3882827758789, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.006452551577240229, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.682579040527344, "fcm_dpo/q_t": 0.41284310817718506, "grad_norm": 10.617995262145996, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2554931640625, "logits/rejected": -0.948306679725647, "logps/chosen": -228.88699340820312, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -300.560302734375, "loss": 1.1035, "margin_dpo/margin_mean": 58.682579040527344, "margin_dpo/margin_std": 78.89262390136719, "step": 297 }, { "KL/chosen_KL_mean": -118.25181579589844, "KL/mean": -155.93020629882812, "KL/rejected_KL_mean": -193.60861206054688, "KL/std": 79.63209533691406, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.006385612301528454, "fcm_dpo/delta": -0.08521604537963867, "fcm_dpo/margin": 75.35680389404297, "fcm_dpo/q_t": 0.38948845863342285, "grad_norm": 11.476375579833984, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.075559139251709, "logits/rejected": -2.0530855655670166, "logps/chosen": -213.13226318359375, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -315.91961669921875, "loss": 1.0228, "margin_dpo/margin_mean": 75.35679626464844, "margin_dpo/margin_std": 80.68865203857422, "step": 298 }, { "KL/chosen_KL_mean": -122.69903564453125, "KL/mean": -160.02227783203125, "KL/rejected_KL_mean": -197.34552001953125, "KL/std": 77.28433227539062, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.006274399813264608, "fcm_dpo/delta": -0.07174316048622131, "fcm_dpo/margin": 74.64649200439453, "fcm_dpo/q_t": 0.3910841941833496, "grad_norm": 10.321080207824707, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.335298776626587, "logits/rejected": -1.1189202070236206, "logps/chosen": -203.10739135742188, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -286.8826904296875, "loss": 1.0307, "margin_dpo/margin_mean": 74.646484375, "margin_dpo/margin_std": 80.10662078857422, "step": 299 }, { "KL/chosen_KL_mean": -129.75216674804688, "KL/mean": -160.71630859375, "KL/rejected_KL_mean": -191.68045043945312, "KL/std": 80.23526763916016, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.006214071996510029, "fcm_dpo/delta": -0.031336650252342224, "fcm_dpo/margin": 61.92830276489258, "fcm_dpo/q_t": 0.4104677736759186, "grad_norm": 8.152363777160645, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.3780734539031982, "logits/rejected": -1.1608507633209229, "logps/chosen": -217.9110565185547, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -292.61962890625, "loss": 1.1068, "margin_dpo/margin_mean": 61.92830276489258, "margin_dpo/margin_std": 86.06282043457031, "step": 300 }, { "KL/chosen_KL_mean": -118.9486083984375, "KL/mean": -157.39703369140625, "KL/rejected_KL_mean": -195.845458984375, "KL/std": 78.47935485839844, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.006110908929258585, "fcm_dpo/delta": -0.08370425552129745, "fcm_dpo/margin": 76.89684295654297, "fcm_dpo/q_t": 0.3913896679878235, "grad_norm": 9.252001762390137, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7411823272705078, "logits/rejected": -1.309745192527771, "logps/chosen": -204.45352172851562, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -294.7622985839844, "loss": 1.0399, "margin_dpo/margin_mean": 76.89684295654297, "margin_dpo/margin_std": 88.70509338378906, "step": 301 }, { "KL/chosen_KL_mean": -114.6790771484375, "KL/mean": -147.81092834472656, "KL/rejected_KL_mean": -180.9427947998047, "KL/std": 80.71609497070312, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0060869609005749226, "fcm_dpo/delta": -0.019632818177342415, "fcm_dpo/margin": 66.26371765136719, "fcm_dpo/q_t": 0.40520817041397095, "grad_norm": 8.145109176635742, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3130149841308594, "logits/rejected": -1.1382200717926025, "logps/chosen": -194.12948608398438, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -284.00189208984375, "loss": 1.0784, "margin_dpo/margin_mean": 66.26371765136719, "margin_dpo/margin_std": 79.24909973144531, "step": 302 }, { "KL/chosen_KL_mean": -107.30046081542969, "KL/mean": -143.54725646972656, "KL/rejected_KL_mean": -179.7940673828125, "KL/std": 75.2556381225586, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.00606036651879549, "fcm_dpo/delta": -0.04388236999511719, "fcm_dpo/margin": 72.49358367919922, "fcm_dpo/q_t": 0.3969067335128784, "grad_norm": 8.158637046813965, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.8562278747558594, "logits/rejected": -1.3965389728546143, "logps/chosen": -202.8987579345703, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -271.858642578125, "loss": 1.0514, "margin_dpo/margin_mean": 72.49358367919922, "margin_dpo/margin_std": 80.17366027832031, "step": 303 }, { "KL/chosen_KL_mean": -106.96308898925781, "KL/mean": -144.4047088623047, "KL/rejected_KL_mean": -181.8463134765625, "KL/std": 77.97357177734375, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0059895385056734085, "fcm_dpo/delta": -0.05078504607081413, "fcm_dpo/margin": 74.88323974609375, "fcm_dpo/q_t": 0.3948588967323303, "grad_norm": 8.869270324707031, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.069180488586426, "logits/rejected": -1.644373893737793, "logps/chosen": -206.41622924804688, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -279.9285888671875, "loss": 1.0424, "margin_dpo/margin_mean": 74.88323974609375, "margin_dpo/margin_std": 80.93460845947266, "step": 304 }, { "KL/chosen_KL_mean": -117.22103881835938, "KL/mean": -152.83163452148438, "KL/rejected_KL_mean": -188.44219970703125, "KL/std": 81.01174926757812, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.005897901952266693, "fcm_dpo/delta": -0.06309368461370468, "fcm_dpo/margin": 71.22117614746094, "fcm_dpo/q_t": 0.40141817927360535, "grad_norm": 7.585349082946777, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.9081419706344604, "logits/rejected": -1.4037458896636963, "logps/chosen": -220.48443603515625, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -286.27117919921875, "loss": 1.0766, "margin_dpo/margin_mean": 71.22117614746094, "margin_dpo/margin_std": 87.74917602539062, "step": 305 }, { "KL/chosen_KL_mean": -105.78399658203125, "KL/mean": -147.35618591308594, "KL/rejected_KL_mean": -188.92837524414062, "KL/std": 77.20852661132812, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.005807062610983849, "fcm_dpo/delta": -0.08737780153751373, "fcm_dpo/margin": 83.14437866210938, "fcm_dpo/q_t": 0.385091096162796, "grad_norm": 9.452533721923828, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.079766035079956, "logits/rejected": -1.5082734823226929, "logps/chosen": -206.08782958984375, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -281.15924072265625, "loss": 1.0024, "margin_dpo/margin_mean": 83.14437866210938, "margin_dpo/margin_std": 71.55195617675781, "step": 306 }, { "KL/chosen_KL_mean": -110.2905044555664, "KL/mean": -141.8279266357422, "KL/rejected_KL_mean": -173.36532592773438, "KL/std": 77.66659545898438, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.00577638391405344, "fcm_dpo/delta": -0.0167161263525486, "fcm_dpo/margin": 63.07482147216797, "fcm_dpo/q_t": 0.41548019647598267, "grad_norm": 8.978883743286133, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.9153618812561035, "logits/rejected": -1.564443588256836, "logps/chosen": -206.94027709960938, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -266.71392822265625, "loss": 1.1224, "margin_dpo/margin_mean": 63.07482147216797, "margin_dpo/margin_std": 91.97120666503906, "step": 307 }, { "KL/chosen_KL_mean": -115.33697509765625, "KL/mean": -146.91268920898438, "KL/rejected_KL_mean": -178.48843383789062, "KL/std": 79.47337341308594, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.00573023222386837, "fcm_dpo/delta": -0.04010900855064392, "fcm_dpo/margin": 63.15143966674805, "fcm_dpo/q_t": 0.41625654697418213, "grad_norm": 9.804304122924805, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.6221284866333008, "logits/rejected": -1.3343465328216553, "logps/chosen": -201.0616455078125, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -264.91162109375, "loss": 1.1276, "margin_dpo/margin_mean": 63.15143585205078, "margin_dpo/margin_std": 92.7095947265625, "step": 308 }, { "KL/chosen_KL_mean": -130.15362548828125, "KL/mean": -158.55874633789062, "KL/rejected_KL_mean": -186.9638671875, "KL/std": 85.02793884277344, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.00573023222386837, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.81024169921875, "fcm_dpo/q_t": 0.4236733615398407, "grad_norm": 11.614842414855957, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.967792272567749, "logits/rejected": -1.552687168121338, "logps/chosen": -213.30508422851562, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -273.54986572265625, "loss": 1.1465, "margin_dpo/margin_mean": 56.810245513916016, "margin_dpo/margin_std": 85.15476989746094, "step": 309 }, { "KL/chosen_KL_mean": -118.37864685058594, "KL/mean": -149.83416748046875, "KL/rejected_KL_mean": -181.28970336914062, "KL/std": 77.02806091308594, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.00573023222386837, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 62.911048889160156, "fcm_dpo/q_t": 0.41524451971054077, "grad_norm": 8.607857704162598, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2730772495269775, "logits/rejected": -1.8608001470565796, "logps/chosen": -208.11663818359375, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -268.36053466796875, "loss": 1.1095, "margin_dpo/margin_mean": 62.91104507446289, "margin_dpo/margin_std": 81.10086822509766, "step": 310 }, { "KL/chosen_KL_mean": -138.62362670898438, "KL/mean": -178.92236328125, "KL/rejected_KL_mean": -219.22108459472656, "KL/std": 77.94852447509766, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.005680982023477554, "fcm_dpo/delta": -0.06062931939959526, "fcm_dpo/margin": 80.59745025634766, "fcm_dpo/q_t": 0.3928367495536804, "grad_norm": 8.963432312011719, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.497267246246338, "logits/rejected": -0.7486953139305115, "logps/chosen": -227.15008544921875, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -301.4871826171875, "loss": 1.0273, "margin_dpo/margin_mean": 80.59744262695312, "margin_dpo/margin_std": 79.70989227294922, "step": 311 }, { "KL/chosen_KL_mean": -133.10203552246094, "KL/mean": -166.98587036132812, "KL/rejected_KL_mean": -200.86968994140625, "KL/std": 85.89141845703125, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.005649399943649769, "fcm_dpo/delta": -0.020829809829592705, "fcm_dpo/margin": 67.76765441894531, "fcm_dpo/q_t": 0.4098922908306122, "grad_norm": 8.74950122833252, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.7832987308502197, "logits/rejected": -1.273546814918518, "logps/chosen": -213.5027313232422, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -283.31683349609375, "loss": 1.0917, "margin_dpo/margin_mean": 67.76765441894531, "margin_dpo/margin_std": 82.27255249023438, "step": 312 }, { "KL/chosen_KL_mean": -128.47833251953125, "KL/mean": -161.27532958984375, "KL/rejected_KL_mean": -194.07232666015625, "KL/std": 84.19730377197266, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.005629425868391991, "fcm_dpo/delta": -0.014578178524971008, "fcm_dpo/margin": 65.59400939941406, "fcm_dpo/q_t": 0.41379910707473755, "grad_norm": 9.291561126708984, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.304236888885498, "logits/rejected": -1.893003225326538, "logps/chosen": -212.6202392578125, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -287.4991760253906, "loss": 1.1141, "margin_dpo/margin_mean": 65.59400939941406, "margin_dpo/margin_std": 91.33814239501953, "step": 313 }, { "KL/chosen_KL_mean": -130.6454315185547, "KL/mean": -162.91830444335938, "KL/rejected_KL_mean": -195.191162109375, "KL/std": 84.91688537597656, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.005621219519525766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.54573822021484, "fcm_dpo/q_t": 0.41579824686050415, "grad_norm": 10.218788146972656, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8227198123931885, "logits/rejected": -1.602492094039917, "logps/chosen": -219.4551544189453, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -299.35009765625, "loss": 1.1114, "margin_dpo/margin_mean": 64.54573822021484, "margin_dpo/margin_std": 85.7214126586914, "step": 314 }, { "KL/chosen_KL_mean": -125.82084655761719, "KL/mean": -158.85763549804688, "KL/rejected_KL_mean": -191.89443969726562, "KL/std": 88.89170837402344, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.005621219519525766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.07357788085938, "fcm_dpo/q_t": 0.41466933488845825, "grad_norm": 8.958334922790527, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.4013843536376953, "logits/rejected": -1.2262012958526611, "logps/chosen": -206.6145477294922, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -284.03155517578125, "loss": 1.1279, "margin_dpo/margin_mean": 66.07357788085938, "margin_dpo/margin_std": 103.02519226074219, "step": 315 }, { "KL/chosen_KL_mean": -128.62818908691406, "KL/mean": -164.5709228515625, "KL/rejected_KL_mean": -200.51364135742188, "KL/std": 87.34502410888672, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.005621219519525766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.88544464111328, "fcm_dpo/q_t": 0.4069307744503021, "grad_norm": 8.934972763061523, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.6891182661056519, "logits/rejected": -1.6898720264434814, "logps/chosen": -212.81243896484375, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -306.2044677734375, "loss": 1.0851, "margin_dpo/margin_mean": 71.88543701171875, "margin_dpo/margin_std": 92.42520904541016, "step": 316 }, { "KL/chosen_KL_mean": -133.34890747070312, "KL/mean": -177.57528686523438, "KL/rejected_KL_mean": -221.80165100097656, "KL/std": 96.08078002929688, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.005575684830546379, "fcm_dpo/delta": -0.08166833221912384, "fcm_dpo/margin": 88.45275115966797, "fcm_dpo/q_t": 0.3872162997722626, "grad_norm": 10.137746810913086, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.904482126235962, "logits/rejected": -1.4261128902435303, "logps/chosen": -221.22239685058594, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -330.06683349609375, "loss": 1.0309, "margin_dpo/margin_mean": 88.45275115966797, "margin_dpo/margin_std": 103.23140716552734, "step": 317 }, { "KL/chosen_KL_mean": -129.76409912109375, "KL/mean": -168.71932983398438, "KL/rejected_KL_mean": -207.67453002929688, "KL/std": 80.61587524414062, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.005503932014107704, "fcm_dpo/delta": -0.023761218413710594, "fcm_dpo/margin": 77.91041564941406, "fcm_dpo/q_t": 0.3999139666557312, "grad_norm": 10.525120735168457, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.322593927383423, "logits/rejected": -1.6211872100830078, "logps/chosen": -227.96963500976562, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -295.8108215332031, "loss": 1.0595, "margin_dpo/margin_mean": 77.91041564941406, "margin_dpo/margin_std": 89.2889175415039, "step": 318 }, { "KL/chosen_KL_mean": -112.40916442871094, "KL/mean": -155.85494995117188, "KL/rejected_KL_mean": -199.30072021484375, "KL/std": 84.84902954101562, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.005433366168290377, "fcm_dpo/delta": -0.07586348056793213, "fcm_dpo/margin": 86.89157104492188, "fcm_dpo/q_t": 0.390333354473114, "grad_norm": 9.57177448272705, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.7172632217407227, "logits/rejected": -1.650120735168457, "logps/chosen": -194.91253662109375, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -317.951416015625, "loss": 1.0326, "margin_dpo/margin_mean": 86.8915786743164, "margin_dpo/margin_std": 96.36482238769531, "step": 319 }, { "KL/chosen_KL_mean": -114.59977722167969, "KL/mean": -149.68569946289062, "KL/rejected_KL_mean": -184.77166748046875, "KL/std": 85.78457641601562, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005421052686870098, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.17189025878906, "fcm_dpo/q_t": 0.4108913540840149, "grad_norm": 8.423419952392578, "learning_rate": 3.208807785813777e-07, "logits/chosen": -1.9459959268569946, "logits/rejected": -1.755147933959961, "logps/chosen": -201.73699951171875, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -287.99578857421875, "loss": 1.0896, "margin_dpo/margin_mean": 70.17189025878906, "margin_dpo/margin_std": 83.14974975585938, "step": 320 }, { "KL/chosen_KL_mean": -127.79622650146484, "KL/mean": -168.6947021484375, "KL/rejected_KL_mean": -209.59320068359375, "KL/std": 88.10226440429688, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0053630406036973, "fcm_dpo/delta": -0.053794536739587784, "fcm_dpo/margin": 81.79696655273438, "fcm_dpo/q_t": 0.3962160348892212, "grad_norm": 8.563617706298828, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.9017610549926758, "logits/rejected": -1.50909423828125, "logps/chosen": -220.90089416503906, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -307.06134033203125, "loss": 1.0489, "margin_dpo/margin_mean": 81.79696655273438, "margin_dpo/margin_std": 88.33961486816406, "step": 321 }, { "KL/chosen_KL_mean": -119.35481262207031, "KL/mean": -163.74227905273438, "KL/rejected_KL_mean": -208.1297607421875, "KL/std": 87.52552795410156, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0052577476017177105, "fcm_dpo/delta": -0.099141925573349, "fcm_dpo/margin": 88.77493286132812, "fcm_dpo/q_t": 0.39066368341445923, "grad_norm": 13.189000129699707, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.6973204612731934, "logits/rejected": -1.33841872215271, "logps/chosen": -211.5477294921875, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -309.0229797363281, "loss": 1.0344, "margin_dpo/margin_mean": 88.7749252319336, "margin_dpo/margin_std": 94.58563232421875, "step": 322 }, { "KL/chosen_KL_mean": -137.91192626953125, "KL/mean": -174.52194213867188, "KL/rejected_KL_mean": -211.13197326660156, "KL/std": 87.97076416015625, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.0052577476017177105, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.22003173828125, "fcm_dpo/q_t": 0.41082701086997986, "grad_norm": 10.541221618652344, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.8751602172851562, "logits/rejected": -1.4866108894348145, "logps/chosen": -218.1840362548828, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -291.435302734375, "loss": 1.0936, "margin_dpo/margin_mean": 73.22003173828125, "margin_dpo/margin_std": 91.98731994628906, "step": 323 }, { "KL/chosen_KL_mean": -136.89447021484375, "KL/mean": -175.3061065673828, "KL/rejected_KL_mean": -213.71774291992188, "KL/std": 88.07098388671875, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.005248694680631161, "fcm_dpo/delta": -0.017248284071683884, "fcm_dpo/margin": 76.82327270507812, "fcm_dpo/q_t": 0.4046363830566406, "grad_norm": 8.557024955749512, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.7485270500183105, "logits/rejected": -1.1038527488708496, "logps/chosen": -232.6238250732422, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -307.886962890625, "loss": 1.0699, "margin_dpo/margin_mean": 76.82327270507812, "margin_dpo/margin_std": 84.22348022460938, "step": 324 }, { "KL/chosen_KL_mean": -125.45704650878906, "KL/mean": -167.20828247070312, "KL/rejected_KL_mean": -208.95947265625, "KL/std": 83.9979019165039, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.005224970169365406, "fcm_dpo/delta": -0.02807956002652645, "fcm_dpo/margin": 83.50245666503906, "fcm_dpo/q_t": 0.3973432183265686, "grad_norm": 7.728775501251221, "learning_rate": 3.147047612756302e-07, "logits/chosen": -2.085505485534668, "logits/rejected": -1.3885273933410645, "logps/chosen": -224.16390991210938, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -285.3675842285156, "loss": 1.0477, "margin_dpo/margin_mean": 83.50244903564453, "margin_dpo/margin_std": 88.02544403076172, "step": 325 }, { "KL/chosen_KL_mean": -139.88232421875, "KL/mean": -177.19561767578125, "KL/rejected_KL_mean": -214.5089111328125, "KL/std": 83.44471740722656, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.005210298579186201, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.6265869140625, "fcm_dpo/q_t": 0.40835127234458923, "grad_norm": 7.308746337890625, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.349360466003418, "logits/rejected": -1.6654181480407715, "logps/chosen": -244.54364013671875, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -301.33428955078125, "loss": 1.0882, "margin_dpo/margin_mean": 74.6265869140625, "margin_dpo/margin_std": 90.59278869628906, "step": 326 }, { "KL/chosen_KL_mean": -134.6219482421875, "KL/mean": -170.7459716796875, "KL/rejected_KL_mean": -206.86996459960938, "KL/std": 76.69439697265625, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005210298579186201, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.24800109863281, "fcm_dpo/q_t": 0.41220515966415405, "grad_norm": 8.439931869506836, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.8652900457382202, "logits/rejected": -1.2869786024093628, "logps/chosen": -242.06219482421875, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -323.9231872558594, "loss": 1.0936, "margin_dpo/margin_mean": 72.24800109863281, "margin_dpo/margin_std": 87.2104263305664, "step": 327 }, { "KL/chosen_KL_mean": -130.83026123046875, "KL/mean": -178.26942443847656, "KL/rejected_KL_mean": -225.70858764648438, "KL/std": 91.13330078125, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0051423897966742516, "fcm_dpo/delta": -0.09236164391040802, "fcm_dpo/margin": 94.87832641601562, "fcm_dpo/q_t": 0.38510289788246155, "grad_norm": 9.626375198364258, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6872029304504395, "logits/rejected": -1.3892412185668945, "logps/chosen": -217.4409637451172, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -323.1104736328125, "loss": 1.0048, "margin_dpo/margin_mean": 94.87832641601562, "margin_dpo/margin_std": 87.95875549316406, "step": 328 }, { "KL/chosen_KL_mean": -136.053466796875, "KL/mean": -173.33670043945312, "KL/rejected_KL_mean": -210.61996459960938, "KL/std": 87.42394256591797, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.005086897406727076, "fcm_dpo/delta": -0.027483679354190826, "fcm_dpo/margin": 74.56649780273438, "fcm_dpo/q_t": 0.4114059805870056, "grad_norm": 9.7688627243042, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.4989808797836304, "logits/rejected": -0.9965137243270874, "logps/chosen": -236.66653442382812, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -315.57366943359375, "loss": 1.0974, "margin_dpo/margin_mean": 74.5665054321289, "margin_dpo/margin_std": 93.90438842773438, "step": 329 }, { "KL/chosen_KL_mean": -115.92872619628906, "KL/mean": -155.57330322265625, "KL/rejected_KL_mean": -195.21788024902344, "KL/std": 85.39158630371094, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005075965542346239, "fcm_dpo/delta": -0.010756894946098328, "fcm_dpo/margin": 79.28915405273438, "fcm_dpo/q_t": 0.4036453068256378, "grad_norm": 8.278905868530273, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.9060704708099365, "logits/rejected": -1.828526496887207, "logps/chosen": -192.8871612548828, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -288.0328369140625, "loss": 1.0611, "margin_dpo/margin_mean": 79.28915405273438, "margin_dpo/margin_std": 76.25235748291016, "step": 330 }, { "KL/chosen_KL_mean": -125.29647827148438, "KL/mean": -169.889892578125, "KL/rejected_KL_mean": -214.48329162597656, "KL/std": 83.70223999023438, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0050462898798286915, "fcm_dpo/delta": -0.058807749301195145, "fcm_dpo/margin": 89.18682861328125, "fcm_dpo/q_t": 0.393661767244339, "grad_norm": 9.451396942138672, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.7455837726593018, "logits/rejected": -1.435776710510254, "logps/chosen": -208.35589599609375, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -308.61444091796875, "loss": 1.0301, "margin_dpo/margin_mean": 89.18682861328125, "margin_dpo/margin_std": 84.6458740234375, "step": 331 }, { "KL/chosen_KL_mean": -133.3221893310547, "KL/mean": -163.64898681640625, "KL/rejected_KL_mean": -193.97579956054688, "KL/std": 91.76033020019531, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005016614217311144, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.65361022949219, "fcm_dpo/q_t": 0.4278526306152344, "grad_norm": 7.801400184631348, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.2583491802215576, "logits/rejected": -1.8388924598693848, "logps/chosen": -215.25308227539062, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -276.82171630859375, "loss": 1.1536, "margin_dpo/margin_mean": 60.65361022949219, "margin_dpo/margin_std": 89.62928771972656, "step": 332 }, { "KL/chosen_KL_mean": -153.32791137695312, "KL/mean": -196.48287963867188, "KL/rejected_KL_mean": -239.6378631591797, "KL/std": 96.53070068359375, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.004949228838086128, "fcm_dpo/delta": -0.06761734187602997, "fcm_dpo/margin": 86.30999755859375, "fcm_dpo/q_t": 0.4005696177482605, "grad_norm": 7.7869696617126465, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.6776665449142456, "logits/rejected": -1.3431476354599, "logps/chosen": -241.8604736328125, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -342.1038818359375, "loss": 1.0683, "margin_dpo/margin_mean": 86.30998992919922, "margin_dpo/margin_std": 103.57662963867188, "step": 333 }, { "KL/chosen_KL_mean": -139.57281494140625, "KL/mean": -177.90782165527344, "KL/rejected_KL_mean": -216.2428436279297, "KL/std": 91.14015197753906, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.004907602909952402, "fcm_dpo/delta": -0.042230866849422455, "fcm_dpo/margin": 76.67002868652344, "fcm_dpo/q_t": 0.410877525806427, "grad_norm": 8.666426658630371, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.2169189453125, "logits/rejected": -0.9099801778793335, "logps/chosen": -237.33059692382812, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -316.2486572265625, "loss": 1.1069, "margin_dpo/margin_mean": 76.67002868652344, "margin_dpo/margin_std": 101.49089050292969, "step": 334 }, { "KL/chosen_KL_mean": -139.00852966308594, "KL/mean": -171.97457885742188, "KL/rejected_KL_mean": -204.94061279296875, "KL/std": 90.3468017578125, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.004907602909952402, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.93209075927734, "fcm_dpo/q_t": 0.42391860485076904, "grad_norm": 11.67782211303711, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.271895408630371, "logits/rejected": -1.8442790508270264, "logps/chosen": -235.3845672607422, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -299.8646240234375, "loss": 1.1456, "margin_dpo/margin_mean": 65.93209075927734, "margin_dpo/margin_std": 99.186767578125, "step": 335 }, { "KL/chosen_KL_mean": -141.4232177734375, "KL/mean": -183.9738006591797, "KL/rejected_KL_mean": -226.52438354492188, "KL/std": 92.91388702392578, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.004892796277999878, "fcm_dpo/delta": -0.030261574313044548, "fcm_dpo/margin": 85.10115051269531, "fcm_dpo/q_t": 0.40259307622909546, "grad_norm": 10.014776229858398, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.5831501483917236, "logits/rejected": -1.432072639465332, "logps/chosen": -229.95274353027344, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -336.0801696777344, "loss": 1.0659, "margin_dpo/margin_mean": 85.10115814208984, "margin_dpo/margin_std": 98.00466918945312, "step": 336 }, { "KL/chosen_KL_mean": -140.3487091064453, "KL/mean": -180.80728149414062, "KL/rejected_KL_mean": -221.2658233642578, "KL/std": 93.71890258789062, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.004877990111708641, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.91712951660156, "fcm_dpo/q_t": 0.40876948833465576, "grad_norm": 8.34636116027832, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.473423480987549, "logits/rejected": -2.1949963569641113, "logps/chosen": -237.66110229492188, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -331.39715576171875, "loss": 1.0987, "margin_dpo/margin_mean": 80.91712951660156, "margin_dpo/margin_std": 110.64788818359375, "step": 337 }, { "KL/chosen_KL_mean": -146.07769775390625, "KL/mean": -182.96328735351562, "KL/rejected_KL_mean": -219.84886169433594, "KL/std": 94.81451416015625, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.004877990111708641, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.77116394042969, "fcm_dpo/q_t": 0.4146643877029419, "grad_norm": 8.451740264892578, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.6556761264801025, "logits/rejected": -1.5003132820129395, "logps/chosen": -229.7078094482422, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -313.54119873046875, "loss": 1.1082, "margin_dpo/margin_mean": 73.77116394042969, "margin_dpo/margin_std": 93.25320434570312, "step": 338 }, { "KL/chosen_KL_mean": -154.720703125, "KL/mean": -194.30882263183594, "KL/rejected_KL_mean": -233.89691162109375, "KL/std": 93.79939270019531, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.004851840436458588, "fcm_dpo/delta": -0.05389716848731041, "fcm_dpo/margin": 79.17621612548828, "fcm_dpo/q_t": 0.40974730253219604, "grad_norm": 7.380728721618652, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.841108798980713, "logits/rejected": -2.4075353145599365, "logps/chosen": -254.57254028320312, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -342.9712829589844, "loss": 1.1024, "margin_dpo/margin_mean": 79.17621612548828, "margin_dpo/margin_std": 103.74622344970703, "step": 339 }, { "KL/chosen_KL_mean": -166.8508758544922, "KL/mean": -201.06024169921875, "KL/rejected_KL_mean": -235.26959228515625, "KL/std": 86.42567443847656, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.004825690295547247, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.418701171875, "fcm_dpo/q_t": 0.4217659831047058, "grad_norm": 9.35424518585205, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.4108572006225586, "logits/rejected": -0.9256479144096375, "logps/chosen": -251.82484436035156, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -314.0699462890625, "loss": 1.1328, "margin_dpo/margin_mean": 68.418701171875, "margin_dpo/margin_std": 95.08515930175781, "step": 340 }, { "KL/chosen_KL_mean": -174.92437744140625, "KL/mean": -208.1255340576172, "KL/rejected_KL_mean": -241.32667541503906, "KL/std": 90.787109375, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.004825690295547247, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.40231323242188, "fcm_dpo/q_t": 0.4252350330352783, "grad_norm": 9.018841743469238, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.4261701107025146, "logits/rejected": -1.1767590045928955, "logps/chosen": -260.1883544921875, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -347.4126281738281, "loss": 1.154, "margin_dpo/margin_mean": 66.40231323242188, "margin_dpo/margin_std": 104.87745666503906, "step": 341 }, { "KL/chosen_KL_mean": -140.4943084716797, "KL/mean": -189.23406982421875, "KL/rejected_KL_mean": -237.97386169433594, "KL/std": 91.13187408447266, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.004746757913380861, "fcm_dpo/delta": -0.08245976269245148, "fcm_dpo/margin": 97.47955322265625, "fcm_dpo/q_t": 0.389830619096756, "grad_norm": 7.195984363555908, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.3359246253967285, "logits/rejected": -0.5663126111030579, "logps/chosen": -241.53289794921875, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -336.87237548828125, "loss": 1.0312, "margin_dpo/margin_mean": 97.47955322265625, "margin_dpo/margin_std": 97.33995056152344, "step": 342 }, { "KL/chosen_KL_mean": -151.91021728515625, "KL/mean": -192.64913940429688, "KL/rejected_KL_mean": -233.38804626464844, "KL/std": 94.88468933105469, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.004723217338323593, "fcm_dpo/delta": -0.04984119534492493, "fcm_dpo/margin": 81.47785949707031, "fcm_dpo/q_t": 0.40945255756378174, "grad_norm": 9.344711303710938, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.122952699661255, "logits/rejected": -2.014542579650879, "logps/chosen": -235.65756225585938, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -349.9671325683594, "loss": 1.0942, "margin_dpo/margin_mean": 81.47785949707031, "margin_dpo/margin_std": 99.79579162597656, "step": 343 }, { "KL/chosen_KL_mean": -151.64663696289062, "KL/mean": -195.04501342773438, "KL/rejected_KL_mean": -238.44345092773438, "KL/std": 100.59310150146484, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.004675786476582289, "fcm_dpo/delta": -0.05109323561191559, "fcm_dpo/margin": 86.79680633544922, "fcm_dpo/q_t": 0.4053107798099518, "grad_norm": 8.901230812072754, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.5972583293914795, "logits/rejected": -1.3481910228729248, "logps/chosen": -229.90060424804688, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -333.87213134765625, "loss": 1.0821, "margin_dpo/margin_mean": 86.79681396484375, "margin_dpo/margin_std": 106.55726623535156, "step": 344 }, { "KL/chosen_KL_mean": -156.7344970703125, "KL/mean": -202.5009765625, "KL/rejected_KL_mean": -248.26744079589844, "KL/std": 100.87413024902344, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.004628432914614677, "fcm_dpo/delta": -0.0252832118421793, "fcm_dpo/margin": 91.53292846679688, "fcm_dpo/q_t": 0.4011690616607666, "grad_norm": 8.872772216796875, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.5966269969940186, "logits/rejected": -1.3043098449707031, "logps/chosen": -254.81399536132812, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -352.3022155761719, "loss": 1.0619, "margin_dpo/margin_mean": 91.53292846679688, "margin_dpo/margin_std": 104.58908081054688, "step": 345 }, { "KL/chosen_KL_mean": -174.12857055664062, "KL/mean": -209.63845825195312, "KL/rejected_KL_mean": -245.1483154296875, "KL/std": 93.87400817871094, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.004628432914614677, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.01972961425781, "fcm_dpo/q_t": 0.42325425148010254, "grad_norm": 7.354798316955566, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.770930528640747, "logits/rejected": -1.4555552005767822, "logps/chosen": -254.81207275390625, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -331.0409240722656, "loss": 1.1434, "margin_dpo/margin_mean": 71.01972198486328, "margin_dpo/margin_std": 107.71234130859375, "step": 346 }, { "KL/chosen_KL_mean": -163.99874877929688, "KL/mean": -197.8900604248047, "KL/rejected_KL_mean": -231.7813720703125, "KL/std": 93.06059265136719, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.004628432914614677, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.78260803222656, "fcm_dpo/q_t": 0.4264860153198242, "grad_norm": 7.8731465339660645, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.7907865047454834, "logits/rejected": -1.465621829032898, "logps/chosen": -257.6781311035156, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -329.3238830566406, "loss": 1.15, "margin_dpo/margin_mean": 67.78260803222656, "margin_dpo/margin_std": 101.0419692993164, "step": 347 }, { "KL/chosen_KL_mean": -142.4119415283203, "KL/mean": -180.94671630859375, "KL/rejected_KL_mean": -219.48147583007812, "KL/std": 87.54843139648438, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.004578843247145414, "fcm_dpo/delta": -0.05385979264974594, "fcm_dpo/margin": 77.06953430175781, "fcm_dpo/q_t": 0.415646493434906, "grad_norm": 7.466777324676514, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.124338150024414, "logits/rejected": -1.8605356216430664, "logps/chosen": -232.6641845703125, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -315.25177001953125, "loss": 1.1096, "margin_dpo/margin_mean": 77.06953430175781, "margin_dpo/margin_std": 89.25281524658203, "step": 348 }, { "KL/chosen_KL_mean": -153.6639862060547, "KL/mean": -189.66073608398438, "KL/rejected_KL_mean": -225.6575164794922, "KL/std": 96.47988891601562, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.004578843247145414, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.99352264404297, "fcm_dpo/q_t": 0.42267706990242004, "grad_norm": 8.671862602233887, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.7556614875793457, "logits/rejected": -1.5225857496261597, "logps/chosen": -245.72207641601562, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -329.622802734375, "loss": 1.1349, "margin_dpo/margin_mean": 71.99352264404297, "margin_dpo/margin_std": 99.89590454101562, "step": 349 }, { "KL/chosen_KL_mean": -162.42642211914062, "KL/mean": -192.94882202148438, "KL/rejected_KL_mean": -223.47122192382812, "KL/std": 88.38798522949219, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.004578843247145414, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.04481506347656, "fcm_dpo/q_t": 0.43368837237358093, "grad_norm": 9.475072860717773, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8736212253570557, "logits/rejected": -1.6489782333374023, "logps/chosen": -249.48062133789062, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -319.1931457519531, "loss": 1.1735, "margin_dpo/margin_mean": 61.04481506347656, "margin_dpo/margin_std": 96.19078826904297, "step": 350 }, { "KL/chosen_KL_mean": -162.72390747070312, "KL/mean": -206.0121307373047, "KL/rejected_KL_mean": -249.3003692626953, "KL/std": 98.76765441894531, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.004572265315800905, "fcm_dpo/delta": -0.014386854134500027, "fcm_dpo/margin": 86.5764389038086, "fcm_dpo/q_t": 0.40686696767807007, "grad_norm": 7.638264179229736, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.4635801315307617, "logits/rejected": -1.1528069972991943, "logps/chosen": -251.8929443359375, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -357.598388671875, "loss": 1.0846, "margin_dpo/margin_mean": 86.5764389038086, "margin_dpo/margin_std": 105.33868408203125, "step": 351 }, { "KL/chosen_KL_mean": -149.60768127441406, "KL/mean": -190.431884765625, "KL/rejected_KL_mean": -231.25607299804688, "KL/std": 97.47367858886719, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.004565687384456396, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.64839172363281, "fcm_dpo/q_t": 0.4136275053024292, "grad_norm": 7.5316009521484375, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.7973028421401978, "logits/rejected": -1.2397058010101318, "logps/chosen": -246.41915893554688, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -315.660888671875, "loss": 1.1041, "margin_dpo/margin_mean": 81.64839935302734, "margin_dpo/margin_std": 105.70852661132812, "step": 352 }, { "KL/chosen_KL_mean": -170.81956481933594, "KL/mean": -196.7728729248047, "KL/rejected_KL_mean": -222.72616577148438, "KL/std": 86.24139404296875, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.004565687384456396, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.90660858154297, "fcm_dpo/q_t": 0.44339868426322937, "grad_norm": 8.548822402954102, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.1557681560516357, "logits/rejected": -1.7706350088119507, "logps/chosen": -256.1820373535156, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -310.093505859375, "loss": 1.2131, "margin_dpo/margin_mean": 51.90660858154297, "margin_dpo/margin_std": 97.65215301513672, "step": 353 }, { "KL/chosen_KL_mean": -149.86448669433594, "KL/mean": -191.12498474121094, "KL/rejected_KL_mean": -232.38548278808594, "KL/std": 87.54669189453125, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.004565687384456396, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.52099609375, "fcm_dpo/q_t": 0.4103440046310425, "grad_norm": 7.202279090881348, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8092983961105347, "logits/rejected": -1.3350532054901123, "logps/chosen": -245.6656494140625, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -327.2516174316406, "loss": 1.0778, "margin_dpo/margin_mean": 82.52099609375, "margin_dpo/margin_std": 81.61939239501953, "step": 354 }, { "KL/chosen_KL_mean": -147.8013153076172, "KL/mean": -185.87112426757812, "KL/rejected_KL_mean": -223.94091796875, "KL/std": 89.1695556640625, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.004565687384456396, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.13961791992188, "fcm_dpo/q_t": 0.4174767732620239, "grad_norm": 8.800679206848145, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.495856761932373, "logits/rejected": -2.016538619995117, "logps/chosen": -233.98687744140625, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -312.4024963378906, "loss": 1.1155, "margin_dpo/margin_mean": 76.13961791992188, "margin_dpo/margin_std": 94.29069519042969, "step": 355 }, { "KL/chosen_KL_mean": -173.7869110107422, "KL/mean": -203.5086669921875, "KL/rejected_KL_mean": -233.2304229736328, "KL/std": 87.36798095703125, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.004565687384456396, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.44353485107422, "fcm_dpo/q_t": 0.4356841742992401, "grad_norm": 7.608415126800537, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8900471925735474, "logits/rejected": -1.4259315729141235, "logps/chosen": -264.6050109863281, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -315.35772705078125, "loss": 1.1841, "margin_dpo/margin_mean": 59.44353485107422, "margin_dpo/margin_std": 100.11868286132812, "step": 356 }, { "KL/chosen_KL_mean": -146.61019897460938, "KL/mean": -193.37269592285156, "KL/rejected_KL_mean": -240.13522338867188, "KL/std": 95.88402557373047, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": -0.06622452288866043, "fcm_dpo/margin": 93.5250244140625, "fcm_dpo/q_t": 0.39965254068374634, "grad_norm": 7.551096439361572, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.926755428314209, "logits/rejected": -1.5557773113250732, "logps/chosen": -234.1149444580078, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -332.38458251953125, "loss": 1.0499, "margin_dpo/margin_mean": 93.52501678466797, "margin_dpo/margin_std": 89.57090759277344, "step": 357 }, { "KL/chosen_KL_mean": -157.2572479248047, "KL/mean": -192.32559204101562, "KL/rejected_KL_mean": -227.39395141601562, "KL/std": 89.31974029541016, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.1366958618164, "fcm_dpo/q_t": 0.4259636104106903, "grad_norm": 7.547357082366943, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.5632538795471191, "logits/rejected": -1.0892590284347534, "logps/chosen": -252.31271362304688, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -323.87005615234375, "loss": 1.139, "margin_dpo/margin_mean": 70.1366958618164, "margin_dpo/margin_std": 95.69628143310547, "step": 358 }, { "KL/chosen_KL_mean": -175.76025390625, "KL/mean": -208.0497589111328, "KL/rejected_KL_mean": -240.33929443359375, "KL/std": 84.06350708007812, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.57902526855469, "fcm_dpo/q_t": 0.4314485192298889, "grad_norm": 8.64183521270752, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4683704376220703, "logits/rejected": -0.8896896839141846, "logps/chosen": -263.1335754394531, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -329.11114501953125, "loss": 1.1647, "margin_dpo/margin_mean": 64.57902526855469, "margin_dpo/margin_std": 97.5063705444336, "step": 359 }, { "KL/chosen_KL_mean": -160.458251953125, "KL/mean": -202.1102294921875, "KL/rejected_KL_mean": -243.76220703125, "KL/std": 84.7669677734375, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.30393981933594, "fcm_dpo/q_t": 0.4104546010494232, "grad_norm": 11.531301498413086, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.695784330368042, "logits/rejected": -1.3564321994781494, "logps/chosen": -235.68170166015625, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -319.856201171875, "loss": 1.085, "margin_dpo/margin_mean": 83.30393981933594, "margin_dpo/margin_std": 89.72037506103516, "step": 360 }, { "KL/chosen_KL_mean": -164.1429443359375, "KL/mean": -200.51097106933594, "KL/rejected_KL_mean": -236.8789825439453, "KL/std": 96.51451110839844, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.73603820800781, "fcm_dpo/q_t": 0.4233100414276123, "grad_norm": 8.774190902709961, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.7393983602523804, "logits/rejected": -1.4965567588806152, "logps/chosen": -246.09628295898438, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -333.55816650390625, "loss": 1.1366, "margin_dpo/margin_mean": 72.73603820800781, "margin_dpo/margin_std": 103.08126831054688, "step": 361 }, { "KL/chosen_KL_mean": -169.34576416015625, "KL/mean": -207.85906982421875, "KL/rejected_KL_mean": -246.3723907470703, "KL/std": 91.59767150878906, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.004505613818764687, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.0266342163086, "fcm_dpo/q_t": 0.41770225763320923, "grad_norm": 8.202536582946777, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.7628631591796875, "logits/rejected": -1.558828353881836, "logps/chosen": -258.68548583984375, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -356.8743591308594, "loss": 1.117, "margin_dpo/margin_mean": 77.0266342163086, "margin_dpo/margin_std": 97.63632202148438, "step": 362 }, { "KL/chosen_KL_mean": -157.54837036132812, "KL/mean": -207.40463256835938, "KL/rejected_KL_mean": -257.2608947753906, "KL/std": 96.06962585449219, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.004477773793041706, "fcm_dpo/delta": -0.062174614518880844, "fcm_dpo/margin": 99.71250915527344, "fcm_dpo/q_t": 0.395487904548645, "grad_norm": 9.43197250366211, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.8057832717895508, "logits/rejected": -1.372434377670288, "logps/chosen": -238.3233642578125, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -336.6935729980469, "loss": 1.041, "margin_dpo/margin_mean": 99.71250915527344, "margin_dpo/margin_std": 104.28041076660156, "step": 363 }, { "KL/chosen_KL_mean": -156.28396606445312, "KL/mean": -196.60545349121094, "KL/rejected_KL_mean": -236.92694091796875, "KL/std": 89.32424926757812, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.004435393493622541, "fcm_dpo/delta": -0.0327824242413044, "fcm_dpo/margin": 80.64295959472656, "fcm_dpo/q_t": 0.41430360078811646, "grad_norm": 10.794096946716309, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.082165241241455, "logits/rejected": -1.9413553476333618, "logps/chosen": -232.81356811523438, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -334.4886779785156, "loss": 1.0977, "margin_dpo/margin_mean": 80.64295959472656, "margin_dpo/margin_std": 85.35018920898438, "step": 364 }, { "KL/chosen_KL_mean": -152.05703735351562, "KL/mean": -195.8902587890625, "KL/rejected_KL_mean": -239.72348022460938, "KL/std": 95.72281646728516, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.004408982116729021, "fcm_dpo/delta": -0.026924798265099525, "fcm_dpo/margin": 87.66644287109375, "fcm_dpo/q_t": 0.4100651144981384, "grad_norm": 8.116775512695312, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7498509883880615, "logits/rejected": -1.6262346506118774, "logps/chosen": -232.6019287109375, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -339.046630859375, "loss": 1.0923, "margin_dpo/margin_mean": 87.66644287109375, "margin_dpo/margin_std": 109.09190368652344, "step": 365 }, { "KL/chosen_KL_mean": -153.9936065673828, "KL/mean": -190.23486328125, "KL/rejected_KL_mean": -226.47610473632812, "KL/std": 99.45747375488281, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.004397111013531685, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.48251342773438, "fcm_dpo/q_t": 0.4258352220058441, "grad_norm": 7.571304798126221, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.7325334548950195, "logits/rejected": -1.7681221961975098, "logps/chosen": -226.23167419433594, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -318.14794921875, "loss": 1.1357, "margin_dpo/margin_mean": 72.48251342773438, "margin_dpo/margin_std": 97.60982513427734, "step": 366 }, { "KL/chosen_KL_mean": -139.41978454589844, "KL/mean": -183.57632446289062, "KL/rejected_KL_mean": -227.73284912109375, "KL/std": 87.45701599121094, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.004397111013531685, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.31307983398438, "fcm_dpo/q_t": 0.4078672528266907, "grad_norm": 7.347011566162109, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.6667547225952148, "logits/rejected": -1.311103105545044, "logps/chosen": -224.841796875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -311.2506408691406, "loss": 1.0727, "margin_dpo/margin_mean": 88.31307983398438, "margin_dpo/margin_std": 90.43326568603516, "step": 367 }, { "KL/chosen_KL_mean": -129.2643280029297, "KL/mean": -177.939208984375, "KL/rejected_KL_mean": -226.6140594482422, "KL/std": 96.72942352294922, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0043753888458013535, "fcm_dpo/delta": -0.049646470695734024, "fcm_dpo/margin": 97.34974670410156, "fcm_dpo/q_t": 0.39874643087387085, "grad_norm": 8.47080135345459, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.5841729640960693, "logits/rejected": -1.5513195991516113, "logps/chosen": -203.10545349121094, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -331.309326171875, "loss": 1.0512, "margin_dpo/margin_mean": 97.34974670410156, "margin_dpo/margin_std": 98.21133422851562, "step": 368 }, { "KL/chosen_KL_mean": -143.57693481445312, "KL/mean": -189.10586547851562, "KL/rejected_KL_mean": -234.6348114013672, "KL/std": 89.6650619506836, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0043464526534080505, "fcm_dpo/delta": -0.01659727469086647, "fcm_dpo/margin": 91.05787658691406, "fcm_dpo/q_t": 0.40614593029022217, "grad_norm": 8.055490493774414, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.5803322792053223, "logits/rejected": -1.2069971561431885, "logps/chosen": -242.03890991210938, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -333.7915344238281, "loss": 1.0665, "margin_dpo/margin_mean": 91.0578842163086, "margin_dpo/margin_std": 90.8848648071289, "step": 369 }, { "KL/chosen_KL_mean": -160.365966796875, "KL/mean": -190.66909790039062, "KL/rejected_KL_mean": -220.97222900390625, "KL/std": 85.95237731933594, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.004339238628745079, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.60629653930664, "fcm_dpo/q_t": 0.4369906187057495, "grad_norm": 7.728323459625244, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.9726451635360718, "logits/rejected": -1.775976300239563, "logps/chosen": -257.1416931152344, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -323.7163391113281, "loss": 1.1868, "margin_dpo/margin_mean": 60.606292724609375, "margin_dpo/margin_std": 100.43878936767578, "step": 370 }, { "KL/chosen_KL_mean": -152.195068359375, "KL/mean": -186.28826904296875, "KL/rejected_KL_mean": -220.3814697265625, "KL/std": 86.58132934570312, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.004339238628745079, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.18638610839844, "fcm_dpo/q_t": 0.42918169498443604, "grad_norm": 9.97611141204834, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -2.005422353744507, "logits/rejected": -1.6448743343353271, "logps/chosen": -255.18527221679688, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -325.7320861816406, "loss": 1.1531, "margin_dpo/margin_mean": 68.1863784790039, "margin_dpo/margin_std": 93.98680114746094, "step": 371 }, { "KL/chosen_KL_mean": -130.27630615234375, "KL/mean": -180.32229614257812, "KL/rejected_KL_mean": -230.3682861328125, "KL/std": 100.70570373535156, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.004327091854065657, "fcm_dpo/delta": -0.028071513399481773, "fcm_dpo/margin": 100.09196472167969, "fcm_dpo/q_t": 0.39851370453834534, "grad_norm": 6.8407769203186035, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.96476149559021, "logits/rejected": -1.3370912075042725, "logps/chosen": -221.30255126953125, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -315.978515625, "loss": 1.0474, "margin_dpo/margin_mean": 100.09196472167969, "margin_dpo/margin_std": 103.58212280273438, "step": 372 }, { "KL/chosen_KL_mean": -151.31735229492188, "KL/mean": -188.22654724121094, "KL/rejected_KL_mean": -225.13572692871094, "KL/std": 96.5085220336914, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.004314945079386234, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.8183822631836, "fcm_dpo/q_t": 0.42457956075668335, "grad_norm": 9.399214744567871, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.115131139755249, "logits/rejected": -1.5906925201416016, "logps/chosen": -251.39044189453125, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -316.2520751953125, "loss": 1.1408, "margin_dpo/margin_mean": 73.81837463378906, "margin_dpo/margin_std": 103.36750793457031, "step": 373 }, { "KL/chosen_KL_mean": -131.15357971191406, "KL/mean": -172.12078857421875, "KL/rejected_KL_mean": -213.08799743652344, "KL/std": 91.51872253417969, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.004314945079386234, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.9344253540039, "fcm_dpo/q_t": 0.41606882214546204, "grad_norm": 7.232913494110107, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.1891205310821533, "logits/rejected": -1.7963275909423828, "logps/chosen": -229.16592407226562, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -313.8402404785156, "loss": 1.0988, "margin_dpo/margin_mean": 81.93441772460938, "margin_dpo/margin_std": 88.47526550292969, "step": 374 }, { "KL/chosen_KL_mean": -131.91470336914062, "KL/mean": -178.7541961669922, "KL/rejected_KL_mean": -225.59368896484375, "KL/std": 88.1768569946289, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.004269293509423733, "fcm_dpo/delta": -0.05318109318614006, "fcm_dpo/margin": 93.6789779663086, "fcm_dpo/q_t": 0.4043002128601074, "grad_norm": 8.471725463867188, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.480994462966919, "logits/rejected": -1.2425600290298462, "logps/chosen": -216.3109893798828, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -321.3125305175781, "loss": 1.0668, "margin_dpo/margin_mean": 93.67897033691406, "margin_dpo/margin_std": 94.08068084716797, "step": 375 }, { "KL/chosen_KL_mean": -128.636962890625, "KL/mean": -180.00961303710938, "KL/rejected_KL_mean": -231.38226318359375, "KL/std": 101.59724426269531, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004246876109391451, "fcm_dpo/delta": -0.02632340043783188, "fcm_dpo/margin": 102.74528503417969, "fcm_dpo/q_t": 0.39757248759269714, "grad_norm": 8.523371696472168, "learning_rate": 2.5e-07, "logits/chosen": -1.912109375, "logits/rejected": -1.6073026657104492, "logps/chosen": -224.37466430664062, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -343.3555603027344, "loss": 1.0476, "margin_dpo/margin_mean": 102.74528503417969, "margin_dpo/margin_std": 109.21365356445312, "step": 376 }, { "KL/chosen_KL_mean": -135.9942626953125, "KL/mean": -189.4968719482422, "KL/rejected_KL_mean": -242.99948120117188, "KL/std": 101.66473388671875, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004198828712105751, "fcm_dpo/delta": -0.0568900927901268, "fcm_dpo/margin": 107.00521850585938, "fcm_dpo/q_t": 0.3935031592845917, "grad_norm": 7.302873611450195, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5164813995361328, "logits/rejected": -1.3998558521270752, "logps/chosen": -214.92120361328125, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -348.8984069824219, "loss": 1.0304, "margin_dpo/margin_mean": 107.00521850585938, "margin_dpo/margin_std": 101.62017822265625, "step": 377 }, { "KL/chosen_KL_mean": -132.3656005859375, "KL/mean": -174.53353881835938, "KL/rejected_KL_mean": -216.70144653320312, "KL/std": 96.52075958251953, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004193047992885113, "fcm_dpo/delta": -0.013785770162940025, "fcm_dpo/margin": 84.33583068847656, "fcm_dpo/q_t": 0.41534560918807983, "grad_norm": 8.869207382202148, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.8962169885635376, "logits/rejected": -1.614516019821167, "logps/chosen": -207.73800659179688, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -310.4055480957031, "loss": 1.1078, "margin_dpo/margin_mean": 84.33584594726562, "margin_dpo/margin_std": 100.5412368774414, "step": 378 }, { "KL/chosen_KL_mean": -156.30067443847656, "KL/mean": -193.59304809570312, "KL/rejected_KL_mean": -230.88540649414062, "KL/std": 99.84820556640625, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004187267739325762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.584716796875, "fcm_dpo/q_t": 0.427146852016449, "grad_norm": 8.179179191589355, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.7718091011047363, "logits/rejected": -1.700627088546753, "logps/chosen": -245.15635681152344, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -352.26654052734375, "loss": 1.1541, "margin_dpo/margin_mean": 74.584716796875, "margin_dpo/margin_std": 115.35541534423828, "step": 379 }, { "KL/chosen_KL_mean": -170.94046020507812, "KL/mean": -216.87330627441406, "KL/rejected_KL_mean": -262.80615234375, "KL/std": 103.1238021850586, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004187267739325762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.86571502685547, "fcm_dpo/q_t": 0.410881906747818, "grad_norm": 11.36270523071289, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.2138094902038574, "logits/rejected": -1.1893771886825562, "logps/chosen": -250.77366638183594, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -373.5509033203125, "loss": 1.094, "margin_dpo/margin_mean": 91.86570739746094, "margin_dpo/margin_std": 117.33644104003906, "step": 380 }, { "KL/chosen_KL_mean": -150.72341918945312, "KL/mean": -190.0155029296875, "KL/rejected_KL_mean": -229.30758666992188, "KL/std": 98.36593627929688, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004187267739325762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.58416748046875, "fcm_dpo/q_t": 0.42159706354141235, "grad_norm": 8.7929105758667, "learning_rate": 2.435840528363426e-07, "logits/chosen": -2.0123066902160645, "logits/rejected": -1.4830715656280518, "logps/chosen": -237.29025268554688, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -313.7243347167969, "loss": 1.1287, "margin_dpo/margin_mean": 78.58416748046875, "margin_dpo/margin_std": 102.7260513305664, "step": 381 }, { "KL/chosen_KL_mean": -147.2804412841797, "KL/mean": -193.22747802734375, "KL/rejected_KL_mean": -239.1745147705078, "KL/std": 97.0614013671875, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.004152303095906973, "fcm_dpo/delta": -0.04192652553319931, "fcm_dpo/margin": 91.89407348632812, "fcm_dpo/q_t": 0.40928754210472107, "grad_norm": 8.231247901916504, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.9114573001861572, "logits/rejected": -1.64837646484375, "logps/chosen": -251.74224853515625, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -353.7767333984375, "loss": 1.0834, "margin_dpo/margin_mean": 91.8940658569336, "margin_dpo/margin_std": 99.2021484375, "step": 382 }, { "KL/chosen_KL_mean": -162.24615478515625, "KL/mean": -201.09519958496094, "KL/rejected_KL_mean": -239.94424438476562, "KL/std": 100.27937316894531, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.004152303095906973, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.6980972290039, "fcm_dpo/q_t": 0.4232904314994812, "grad_norm": 7.962710857391357, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.461895227432251, "logits/rejected": -1.3361754417419434, "logps/chosen": -244.36221313476562, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -338.7214050292969, "loss": 1.1314, "margin_dpo/margin_mean": 77.69810485839844, "margin_dpo/margin_std": 98.93550109863281, "step": 383 }, { "KL/chosen_KL_mean": -172.86517333984375, "KL/mean": -210.96612548828125, "KL/rejected_KL_mean": -249.06704711914062, "KL/std": 103.7215805053711, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.004147011321038008, "fcm_dpo/delta": -0.012760485522449017, "fcm_dpo/margin": 76.20188903808594, "fcm_dpo/q_t": 0.4251400828361511, "grad_norm": 10.232134819030762, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.5315279960632324, "logits/rejected": -1.300133228302002, "logps/chosen": -253.45042419433594, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -340.51495361328125, "loss": 1.1552, "margin_dpo/margin_mean": 76.20188903808594, "margin_dpo/margin_std": 115.95773315429688, "step": 384 }, { "KL/chosen_KL_mean": -143.530029296875, "KL/mean": -192.982421875, "KL/rejected_KL_mean": -242.434814453125, "KL/std": 93.25091552734375, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": -0.04600986838340759, "fcm_dpo/margin": 98.90480041503906, "fcm_dpo/q_t": 0.40425851941108704, "grad_norm": 9.202186584472656, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.1868796348571777, "logits/rejected": -1.574935793876648, "logps/chosen": -236.29019165039062, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -326.894775390625, "loss": 1.0689, "margin_dpo/margin_mean": 98.90480041503906, "margin_dpo/margin_std": 107.68914031982422, "step": 385 }, { "KL/chosen_KL_mean": -151.9226531982422, "KL/mean": -199.53268432617188, "KL/rejected_KL_mean": -247.14271545410156, "KL/std": 95.37660217285156, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.22006225585938, "fcm_dpo/q_t": 0.4066125154495239, "grad_norm": 7.020884990692139, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.806860089302063, "logits/rejected": -1.5267260074615479, "logps/chosen": -237.2816162109375, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -344.13336181640625, "loss": 1.0772, "margin_dpo/margin_mean": 95.22006225585938, "margin_dpo/margin_std": 103.84027099609375, "step": 386 }, { "KL/chosen_KL_mean": -177.31272888183594, "KL/mean": -219.57440185546875, "KL/rejected_KL_mean": -261.8360290527344, "KL/std": 92.86285400390625, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.52330017089844, "fcm_dpo/q_t": 0.4172847867012024, "grad_norm": 9.553810119628906, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.4506683349609375, "logits/rejected": -1.7682149410247803, "logps/chosen": -280.0008544921875, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -361.2646179199219, "loss": 1.1058, "margin_dpo/margin_mean": 84.52330017089844, "margin_dpo/margin_std": 94.86949157714844, "step": 387 }, { "KL/chosen_KL_mean": -178.86007690429688, "KL/mean": -225.8180694580078, "KL/rejected_KL_mean": -272.7760009765625, "KL/std": 103.1664810180664, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.91596221923828, "fcm_dpo/q_t": 0.40872350335121155, "grad_norm": 8.181188583374023, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.7165148258209229, "logits/rejected": -1.6004592180252075, "logps/chosen": -262.4453125, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -382.87640380859375, "loss": 1.0832, "margin_dpo/margin_mean": 93.91596221923828, "margin_dpo/margin_std": 106.29582977294922, "step": 388 }, { "KL/chosen_KL_mean": -184.2982940673828, "KL/mean": -224.17138671875, "KL/rejected_KL_mean": -264.04449462890625, "KL/std": 104.75208282470703, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.74620056152344, "fcm_dpo/q_t": 0.42282283306121826, "grad_norm": 10.780573844909668, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.4227919578552246, "logits/rejected": -1.9777250289916992, "logps/chosen": -274.12054443359375, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -348.89825439453125, "loss": 1.1367, "margin_dpo/margin_mean": 79.74620056152344, "margin_dpo/margin_std": 112.81922912597656, "step": 389 }, { "KL/chosen_KL_mean": -184.95278930664062, "KL/mean": -223.4080810546875, "KL/rejected_KL_mean": -261.8634033203125, "KL/std": 102.48016357421875, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.91058349609375, "fcm_dpo/q_t": 0.4270437955856323, "grad_norm": 7.706514835357666, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.6198937892913818, "logits/rejected": -1.415130376815796, "logps/chosen": -262.4852294921875, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -354.4472961425781, "loss": 1.1521, "margin_dpo/margin_mean": 76.91059112548828, "margin_dpo/margin_std": 118.01615905761719, "step": 390 }, { "KL/chosen_KL_mean": -167.2500762939453, "KL/mean": -209.08824157714844, "KL/rejected_KL_mean": -250.92642211914062, "KL/std": 98.26612854003906, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004103782586753368, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.67634582519531, "fcm_dpo/q_t": 0.41859135031700134, "grad_norm": 7.779163837432861, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.9235318899154663, "logits/rejected": -1.4732369184494019, "logps/chosen": -255.48300170898438, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -343.9339599609375, "loss": 1.1165, "margin_dpo/margin_mean": 83.67634582519531, "margin_dpo/margin_std": 103.55424499511719, "step": 391 }, { "KL/chosen_KL_mean": -169.296630859375, "KL/mean": -221.00906372070312, "KL/rejected_KL_mean": -272.72149658203125, "KL/std": 110.81494140625, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004086670465767384, "fcm_dpo/delta": -0.041872791945934296, "fcm_dpo/margin": 103.42484283447266, "fcm_dpo/q_t": 0.4005693793296814, "grad_norm": 8.512868881225586, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.5337090492248535, "logits/rejected": -1.4164018630981445, "logps/chosen": -257.04107666015625, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -381.2922058105469, "loss": 1.0644, "margin_dpo/margin_mean": 103.42484283447266, "margin_dpo/margin_std": 118.93104553222656, "step": 392 }, { "KL/chosen_KL_mean": -150.52947998046875, "KL/mean": -196.25650024414062, "KL/rejected_KL_mean": -241.9835205078125, "KL/std": 103.97962951660156, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004069558810442686, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.45407104492188, "fcm_dpo/q_t": 0.41181743144989014, "grad_norm": 7.529955863952637, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.7760937213897705, "logits/rejected": -1.4581148624420166, "logps/chosen": -227.09329223632812, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -321.709716796875, "loss": 1.0894, "margin_dpo/margin_mean": 91.45407104492188, "margin_dpo/margin_std": 101.80986022949219, "step": 393 }, { "KL/chosen_KL_mean": -159.1811065673828, "KL/mean": -209.55174255371094, "KL/rejected_KL_mean": -259.92236328125, "KL/std": 99.087158203125, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004069558810442686, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.74126434326172, "fcm_dpo/q_t": 0.40247642993927, "grad_norm": 7.792952060699463, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.4691686630249023, "logits/rejected": -1.2349374294281006, "logps/chosen": -243.8797607421875, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -357.380126953125, "loss": 1.0556, "margin_dpo/margin_mean": 100.74127197265625, "margin_dpo/margin_std": 97.46701049804688, "step": 394 }, { "KL/chosen_KL_mean": -165.8938751220703, "KL/mean": -204.03717041015625, "KL/rejected_KL_mean": -242.18048095703125, "KL/std": 103.39730834960938, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004069558810442686, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.2865982055664, "fcm_dpo/q_t": 0.4288087487220764, "grad_norm": 8.30531120300293, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.6891672611236572, "logits/rejected": -2.29927921295166, "logps/chosen": -254.78152465820312, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -333.22601318359375, "loss": 1.1774, "margin_dpo/margin_mean": 76.28660583496094, "margin_dpo/margin_std": 139.5265655517578, "step": 395 }, { "KL/chosen_KL_mean": -174.5836181640625, "KL/mean": -209.01934814453125, "KL/rejected_KL_mean": -243.45510864257812, "KL/std": 91.92335510253906, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004069558810442686, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.87150573730469, "fcm_dpo/q_t": 0.4334295392036438, "grad_norm": 9.724390029907227, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -2.0346484184265137, "logits/rejected": -1.5534579753875732, "logps/chosen": -277.054931640625, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -336.68988037109375, "loss": 1.1727, "margin_dpo/margin_mean": 68.87150573730469, "margin_dpo/margin_std": 108.53622436523438, "step": 396 }, { "KL/chosen_KL_mean": -157.44802856445312, "KL/mean": -218.79403686523438, "KL/rejected_KL_mean": -280.1400451660156, "KL/std": 106.63572692871094, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.004006239585578442, "fcm_dpo/delta": -0.09643207490444183, "fcm_dpo/margin": 122.69200897216797, "fcm_dpo/q_t": 0.38481515645980835, "grad_norm": 7.735910415649414, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.347353935241699, "logits/rejected": -1.7566663026809692, "logps/chosen": -258.4924621582031, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -388.9912109375, "loss": 1.0155, "margin_dpo/margin_mean": 122.6920166015625, "margin_dpo/margin_std": 127.52362060546875, "step": 397 }, { "KL/chosen_KL_mean": -166.75302124023438, "KL/mean": -210.75833129882812, "KL/rejected_KL_mean": -254.7636260986328, "KL/std": 104.84674835205078, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.003991823643445969, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.01060485839844, "fcm_dpo/q_t": 0.41778630018234253, "grad_norm": 7.380377769470215, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -1.9043289422988892, "logits/rejected": -1.6166844367980957, "logps/chosen": -261.80224609375, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -368.7692565917969, "loss": 1.1237, "margin_dpo/margin_mean": 88.01060485839844, "margin_dpo/margin_std": 124.08979797363281, "step": 398 }, { "KL/chosen_KL_mean": -144.8331756591797, "KL/mean": -201.09735107421875, "KL/rejected_KL_mean": -257.3615417480469, "KL/std": 94.79011535644531, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.003975649364292622, "fcm_dpo/delta": -0.040683407336473465, "fcm_dpo/margin": 112.5283432006836, "fcm_dpo/q_t": 0.39391738176345825, "grad_norm": 7.605029106140137, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -1.9416100978851318, "logits/rejected": -1.115723729133606, "logps/chosen": -241.4717254638672, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -340.8979797363281, "loss": 1.0278, "margin_dpo/margin_mean": 112.5283432006836, "margin_dpo/margin_std": 102.76640319824219, "step": 399 }, { "KL/chosen_KL_mean": -151.684814453125, "KL/mean": -208.95559692382812, "KL/rejected_KL_mean": -266.22637939453125, "KL/std": 104.03498840332031, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.00392950139939785, "fcm_dpo/delta": -0.05244225263595581, "fcm_dpo/margin": 114.54156494140625, "fcm_dpo/q_t": 0.39431455731391907, "grad_norm": 11.551488876342773, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.2866878509521484, "logits/rejected": -1.8114168643951416, "logps/chosen": -256.98797607421875, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -374.88189697265625, "loss": 1.0402, "margin_dpo/margin_mean": 114.54156494140625, "margin_dpo/margin_std": 120.20166778564453, "step": 400 }, { "KL/chosen_KL_mean": -167.62222290039062, "KL/mean": -203.14031982421875, "KL/rejected_KL_mean": -238.65841674804688, "KL/std": 106.3094711303711, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.003918163478374481, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.03620147705078, "fcm_dpo/q_t": 0.43625694513320923, "grad_norm": 10.034729957580566, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.4383968114852905, "logits/rejected": -1.1918203830718994, "logps/chosen": -251.911376953125, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -329.6015319824219, "loss": 1.1889, "margin_dpo/margin_mean": 71.03620147705078, "margin_dpo/margin_std": 130.46397399902344, "step": 401 }, { "KL/chosen_KL_mean": -148.6624755859375, "KL/mean": -215.69447326660156, "KL/rejected_KL_mean": -282.7264709472656, "KL/std": 113.14290618896484, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0038541255053132772, "fcm_dpo/delta": -0.12290497869253159, "fcm_dpo/margin": 134.06399536132812, "fcm_dpo/q_t": 0.37716907262802124, "grad_norm": 7.389410495758057, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.8536090850830078, "logits/rejected": -1.3815048933029175, "logps/chosen": -232.255615234375, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -370.5367431640625, "loss": 0.9793, "margin_dpo/margin_mean": 134.06399536132812, "margin_dpo/margin_std": 111.29826354980469, "step": 402 }, { "KL/chosen_KL_mean": -151.36526489257812, "KL/mean": -201.20083618164062, "KL/rejected_KL_mean": -251.03643798828125, "KL/std": 108.67737579345703, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.671142578125, "fcm_dpo/q_t": 0.41004717350006104, "grad_norm": 8.522459983825684, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.3331522941589355, "logits/rejected": -1.9397037029266357, "logps/chosen": -225.11834716796875, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -334.95654296875, "loss": 1.0891, "margin_dpo/margin_mean": 99.671142578125, "margin_dpo/margin_std": 116.73790740966797, "step": 403 }, { "KL/chosen_KL_mean": -180.01834106445312, "KL/mean": -222.85113525390625, "KL/rejected_KL_mean": -265.6839294433594, "KL/std": 104.5828857421875, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.66559600830078, "fcm_dpo/q_t": 0.42314809560775757, "grad_norm": 11.481825828552246, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.0147602558135986, "logits/rejected": -1.6781134605407715, "logps/chosen": -259.69451904296875, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -349.9644470214844, "loss": 1.1428, "margin_dpo/margin_mean": 85.66558837890625, "margin_dpo/margin_std": 128.23257446289062, "step": 404 }, { "KL/chosen_KL_mean": -183.221435546875, "KL/mean": -226.2115478515625, "KL/rejected_KL_mean": -269.20166015625, "KL/std": 98.56543731689453, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.980224609375, "fcm_dpo/q_t": 0.42215287685394287, "grad_norm": 7.532588481903076, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.6171070337295532, "logits/rejected": -1.1449040174484253, "logps/chosen": -279.6833801269531, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -362.0723571777344, "loss": 1.1239, "margin_dpo/margin_mean": 85.980224609375, "margin_dpo/margin_std": 104.52601623535156, "step": 405 }, { "KL/chosen_KL_mean": -175.3079833984375, "KL/mean": -214.14288330078125, "KL/rejected_KL_mean": -252.97779846191406, "KL/std": 105.262451171875, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.66981506347656, "fcm_dpo/q_t": 0.4298367202281952, "grad_norm": 12.540308952331543, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.8778245449066162, "logits/rejected": -1.7103102207183838, "logps/chosen": -253.4419403076172, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -351.26141357421875, "loss": 1.1643, "margin_dpo/margin_mean": 77.66981506347656, "margin_dpo/margin_std": 122.85980224609375, "step": 406 }, { "KL/chosen_KL_mean": -181.552001953125, "KL/mean": -215.82144165039062, "KL/rejected_KL_mean": -250.09088134765625, "KL/std": 103.81057739257812, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.53887939453125, "fcm_dpo/q_t": 0.43747150897979736, "grad_norm": 8.457676887512207, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.0205883979797363, "logits/rejected": -1.4874842166900635, "logps/chosen": -265.5648193359375, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -332.8719177246094, "loss": 1.1843, "margin_dpo/margin_mean": 68.53887939453125, "margin_dpo/margin_std": 110.65715026855469, "step": 407 }, { "KL/chosen_KL_mean": -173.978759765625, "KL/mean": -208.39767456054688, "KL/rejected_KL_mean": -242.81661987304688, "KL/std": 99.234130859375, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.8378677368164, "fcm_dpo/q_t": 0.43800172209739685, "grad_norm": 8.018157005310059, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.1560182571411133, "logits/rejected": -1.5235161781311035, "logps/chosen": -278.44049072265625, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -339.1888122558594, "loss": 1.1937, "margin_dpo/margin_mean": 68.83786010742188, "margin_dpo/margin_std": 123.15728759765625, "step": 408 }, { "KL/chosen_KL_mean": -185.18179321289062, "KL/mean": -221.50390625, "KL/rejected_KL_mean": -257.8260192871094, "KL/std": 101.33197021484375, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.64421081542969, "fcm_dpo/q_t": 0.4329625964164734, "grad_norm": 9.050348281860352, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.7223212718963623, "logits/rejected": -1.2977075576782227, "logps/chosen": -282.85009765625, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -347.8718566894531, "loss": 1.1736, "margin_dpo/margin_mean": 72.64421081542969, "margin_dpo/margin_std": 111.51698303222656, "step": 409 }, { "KL/chosen_KL_mean": -179.2720947265625, "KL/mean": -223.65188598632812, "KL/rejected_KL_mean": -268.03167724609375, "KL/std": 107.48175048828125, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.75958251953125, "fcm_dpo/q_t": 0.4186663031578064, "grad_norm": 7.258873462677002, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.8609213829040527, "logits/rejected": -1.7334578037261963, "logps/chosen": -255.7413330078125, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -356.67230224609375, "loss": 1.1168, "margin_dpo/margin_mean": 88.75958251953125, "margin_dpo/margin_std": 108.35555267333984, "step": 410 }, { "KL/chosen_KL_mean": -192.88262939453125, "KL/mean": -233.3557586669922, "KL/rejected_KL_mean": -273.82891845703125, "KL/std": 100.20765686035156, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.94630432128906, "fcm_dpo/q_t": 0.4271332919597626, "grad_norm": 7.488071918487549, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.2656688690185547, "logits/rejected": -1.8556618690490723, "logps/chosen": -280.0489196777344, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -360.9249267578125, "loss": 1.1434, "margin_dpo/margin_mean": 80.94630432128906, "margin_dpo/margin_std": 109.0125503540039, "step": 411 }, { "KL/chosen_KL_mean": -181.86825561523438, "KL/mean": -222.45567321777344, "KL/rejected_KL_mean": -263.0430908203125, "KL/std": 100.51443481445312, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.1748275756836, "fcm_dpo/q_t": 0.4275646507740021, "grad_norm": 8.922504425048828, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.3391451835632324, "logits/rejected": -2.1232643127441406, "logps/chosen": -260.81561279296875, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -369.1486511230469, "loss": 1.1586, "margin_dpo/margin_mean": 81.17483520507812, "margin_dpo/margin_std": 129.36679077148438, "step": 412 }, { "KL/chosen_KL_mean": -195.15963745117188, "KL/mean": -239.5413055419922, "KL/rejected_KL_mean": -283.9229736328125, "KL/std": 106.93467712402344, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.7633056640625, "fcm_dpo/q_t": 0.4193413257598877, "grad_norm": 8.611116409301758, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.4096271991729736, "logits/rejected": -1.1595728397369385, "logps/chosen": -290.8543701171875, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -391.8837890625, "loss": 1.1327, "margin_dpo/margin_mean": 88.7633056640625, "margin_dpo/margin_std": 127.85453796386719, "step": 413 }, { "KL/chosen_KL_mean": -179.55294799804688, "KL/mean": -221.66036987304688, "KL/rejected_KL_mean": -263.76776123046875, "KL/std": 102.98111724853516, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.21482849121094, "fcm_dpo/q_t": 0.4245346486568451, "grad_norm": 10.91741943359375, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.632889986038208, "logits/rejected": -1.2788541316986084, "logps/chosen": -267.8296203613281, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -356.6378173828125, "loss": 1.1506, "margin_dpo/margin_mean": 84.21482849121094, "margin_dpo/margin_std": 131.74050903320312, "step": 414 }, { "KL/chosen_KL_mean": -194.98989868164062, "KL/mean": -230.41497802734375, "KL/rejected_KL_mean": -265.8400573730469, "KL/std": 94.70242309570312, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0038230251520872116, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.85015869140625, "fcm_dpo/q_t": 0.4357284605503082, "grad_norm": 9.19848346710205, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.6082158088684082, "logits/rejected": -1.2292755842208862, "logps/chosen": -303.9057922363281, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -373.3114013671875, "loss": 1.1819, "margin_dpo/margin_mean": 70.85015869140625, "margin_dpo/margin_std": 116.15193176269531, "step": 415 }, { "KL/chosen_KL_mean": -195.8232421875, "KL/mean": -244.4453887939453, "KL/rejected_KL_mean": -293.0675354003906, "KL/std": 108.51058959960938, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0038061244413256645, "fcm_dpo/delta": -0.04440442472696304, "fcm_dpo/margin": 97.2442626953125, "fcm_dpo/q_t": 0.41247332096099854, "grad_norm": 7.942221164703369, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.7322821617126465, "logits/rejected": -1.2213243246078491, "logps/chosen": -289.2221374511719, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -390.7405090332031, "loss": 1.0991, "margin_dpo/margin_mean": 97.24427032470703, "margin_dpo/margin_std": 114.10728454589844, "step": 416 }, { "KL/chosen_KL_mean": -182.06253051757812, "KL/mean": -227.11422729492188, "KL/rejected_KL_mean": -272.1658935546875, "KL/std": 101.36210632324219, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0037892237305641174, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.1033935546875, "fcm_dpo/q_t": 0.4185965657234192, "grad_norm": 7.829151153564453, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.2460672855377197, "logits/rejected": -1.3374770879745483, "logps/chosen": -265.59783935546875, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -346.6077575683594, "loss": 1.1111, "margin_dpo/margin_mean": 90.1033935546875, "margin_dpo/margin_std": 103.69532775878906, "step": 417 }, { "KL/chosen_KL_mean": -173.66685485839844, "KL/mean": -231.1494140625, "KL/rejected_KL_mean": -288.6319885253906, "KL/std": 108.66363525390625, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0037536313757300377, "fcm_dpo/delta": -0.09482387453317642, "fcm_dpo/margin": 114.96514129638672, "fcm_dpo/q_t": 0.39769232273101807, "grad_norm": 9.933135986328125, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.103422164916992, "logits/rejected": -1.437751054763794, "logps/chosen": -281.88836669921875, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -400.49664306640625, "loss": 1.0451, "margin_dpo/margin_mean": 114.96514129638672, "margin_dpo/margin_std": 107.40345001220703, "step": 418 }, { "KL/chosen_KL_mean": -181.10943603515625, "KL/mean": -230.719970703125, "KL/rejected_KL_mean": -280.33050537109375, "KL/std": 96.3857421875, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.003709537675604224, "fcm_dpo/delta": -0.02291746251285076, "fcm_dpo/margin": 99.2210693359375, "fcm_dpo/q_t": 0.4123213291168213, "grad_norm": 9.94516372680664, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.1981310844421387, "logits/rejected": -0.7331212759017944, "logps/chosen": -278.292724609375, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -378.51580810546875, "loss": 1.0902, "margin_dpo/margin_mean": 99.22106170654297, "margin_dpo/margin_std": 105.91574096679688, "step": 419 }, { "KL/chosen_KL_mean": -184.74082946777344, "KL/mean": -229.30364990234375, "KL/rejected_KL_mean": -273.866455078125, "KL/std": 102.14118957519531, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.12562561035156, "fcm_dpo/q_t": 0.4220064580440521, "grad_norm": 12.765666007995605, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.179891586303711, "logits/rejected": -1.0718528032302856, "logps/chosen": -299.04931640625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -349.5500183105469, "loss": 1.13, "margin_dpo/margin_mean": 89.12562561035156, "margin_dpo/margin_std": 118.65499877929688, "step": 420 }, { "KL/chosen_KL_mean": -175.8483123779297, "KL/mean": -225.43234252929688, "KL/rejected_KL_mean": -275.016357421875, "KL/std": 105.26570129394531, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.16805267333984, "fcm_dpo/q_t": 0.41277986764907837, "grad_norm": 6.710361480712891, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8465800285339355, "logits/rejected": -1.4745086431503296, "logps/chosen": -261.7281494140625, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -371.35284423828125, "loss": 1.0973, "margin_dpo/margin_mean": 99.16806030273438, "margin_dpo/margin_std": 115.97150421142578, "step": 421 }, { "KL/chosen_KL_mean": -173.37054443359375, "KL/mean": -220.83154296875, "KL/rejected_KL_mean": -268.29254150390625, "KL/std": 97.44050598144531, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.92204284667969, "fcm_dpo/q_t": 0.41659384965896606, "grad_norm": 10.592460632324219, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.329010486602783, "logits/rejected": -1.960078239440918, "logps/chosen": -273.85113525390625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -362.7008056640625, "loss": 1.1041, "margin_dpo/margin_mean": 94.92204284667969, "margin_dpo/margin_std": 108.57640075683594, "step": 422 }, { "KL/chosen_KL_mean": -160.5980682373047, "KL/mean": -208.9354248046875, "KL/rejected_KL_mean": -257.27276611328125, "KL/std": 116.63713073730469, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.67471313476562, "fcm_dpo/q_t": 0.4167243242263794, "grad_norm": 8.631891250610352, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9911987781524658, "logits/rejected": -1.8750890493392944, "logps/chosen": -239.04800415039062, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -349.3193054199219, "loss": 1.1184, "margin_dpo/margin_mean": 96.67471313476562, "margin_dpo/margin_std": 132.72886657714844, "step": 423 }, { "KL/chosen_KL_mean": -181.83438110351562, "KL/mean": -228.1916961669922, "KL/rejected_KL_mean": -274.54901123046875, "KL/std": 103.95587158203125, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.71463775634766, "fcm_dpo/q_t": 0.4179834723472595, "grad_norm": 7.753489017486572, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.8338696956634521, "logits/rejected": -1.2484863996505737, "logps/chosen": -269.47674560546875, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -346.9146728515625, "loss": 1.1146, "margin_dpo/margin_mean": 92.71464538574219, "margin_dpo/margin_std": 112.88055419921875, "step": 424 }, { "KL/chosen_KL_mean": -186.89630126953125, "KL/mean": -219.10934448242188, "KL/rejected_KL_mean": -251.3223876953125, "KL/std": 107.93467712402344, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.42608642578125, "fcm_dpo/q_t": 0.44494175910949707, "grad_norm": 9.17188835144043, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.8773984909057617, "logits/rejected": -1.8152968883514404, "logps/chosen": -265.41607666015625, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -354.071044921875, "loss": 1.2207, "margin_dpo/margin_mean": 64.42608642578125, "margin_dpo/margin_std": 133.9623260498047, "step": 425 }, { "KL/chosen_KL_mean": -161.483642578125, "KL/mean": -208.41476440429688, "KL/rejected_KL_mean": -255.34588623046875, "KL/std": 98.83863830566406, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.86227416992188, "fcm_dpo/q_t": 0.41743916273117065, "grad_norm": 8.107372283935547, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.7375352382659912, "logits/rejected": -0.9994393587112427, "logps/chosen": -269.98944091796875, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -343.64593505859375, "loss": 1.1135, "margin_dpo/margin_mean": 93.86227416992188, "margin_dpo/margin_std": 115.73319244384766, "step": 426 }, { "KL/chosen_KL_mean": -131.8596954345703, "KL/mean": -174.5131072998047, "KL/rejected_KL_mean": -217.16653442382812, "KL/std": 93.35734558105469, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.30684661865234, "fcm_dpo/q_t": 0.4243261516094208, "grad_norm": 7.925274848937988, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.502164363861084, "logits/rejected": -1.8794245719909668, "logps/chosen": -230.98016357421875, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -302.8911437988281, "loss": 1.1291, "margin_dpo/margin_mean": 85.30685424804688, "margin_dpo/margin_std": 101.00962829589844, "step": 427 }, { "KL/chosen_KL_mean": -189.48512268066406, "KL/mean": -226.6806640625, "KL/rejected_KL_mean": -263.876220703125, "KL/std": 105.41412353515625, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0037010363303124905, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.39109802246094, "fcm_dpo/q_t": 0.434742271900177, "grad_norm": 8.829818725585938, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.7616534233093262, "logits/rejected": -1.5044443607330322, "logps/chosen": -295.4543762207031, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -372.9783935546875, "loss": 1.1763, "margin_dpo/margin_mean": 74.39109802246094, "margin_dpo/margin_std": 117.28004455566406, "step": 428 }, { "KL/chosen_KL_mean": -174.12057495117188, "KL/mean": -221.45201110839844, "KL/rejected_KL_mean": -268.783447265625, "KL/std": 110.66049194335938, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0036894180811941624, "fcm_dpo/delta": -0.031490474939346313, "fcm_dpo/margin": 94.66284942626953, "fcm_dpo/q_t": 0.4179378151893616, "grad_norm": 7.013363838195801, "learning_rate": 1.828194884925749e-07, "logits/chosen": -2.0200490951538086, "logits/rejected": -1.3129656314849854, "logps/chosen": -287.66546630859375, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -367.02545166015625, "loss": 1.1215, "margin_dpo/margin_mean": 94.662841796875, "margin_dpo/margin_std": 125.67195129394531, "step": 429 }, { "KL/chosen_KL_mean": -164.96548461914062, "KL/mean": -201.32444763183594, "KL/rejected_KL_mean": -237.68341064453125, "KL/std": 102.43136596679688, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.003677800064906478, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.71791076660156, "fcm_dpo/q_t": 0.4367010295391083, "grad_norm": 7.792520046234131, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.3999238014221191, "logits/rejected": -1.36525297164917, "logps/chosen": -256.28485107421875, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -347.7930603027344, "loss": 1.1835, "margin_dpo/margin_mean": 72.71791076660156, "margin_dpo/margin_std": 119.28764343261719, "step": 430 }, { "KL/chosen_KL_mean": -166.02694702148438, "KL/mean": -209.38671875, "KL/rejected_KL_mean": -252.74647521972656, "KL/std": 105.70272064208984, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.003677800064906478, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.71953582763672, "fcm_dpo/q_t": 0.42416912317276, "grad_norm": 8.687884330749512, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -2.0580568313598633, "logits/rejected": -1.682270884513855, "logps/chosen": -259.2081604003906, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -350.87872314453125, "loss": 1.1395, "margin_dpo/margin_mean": 86.71954345703125, "margin_dpo/margin_std": 119.84416198730469, "step": 431 }, { "KL/chosen_KL_mean": -145.68161010742188, "KL/mean": -198.1104736328125, "KL/rejected_KL_mean": -250.53936767578125, "KL/std": 108.79702758789062, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0036704866215586662, "fcm_dpo/delta": -0.019925188273191452, "fcm_dpo/margin": 104.85779571533203, "fcm_dpo/q_t": 0.40814444422721863, "grad_norm": 7.376406192779541, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.4861321449279785, "logits/rejected": -0.9309872388839722, "logps/chosen": -250.1163787841797, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -355.6289367675781, "loss": 1.0762, "margin_dpo/margin_mean": 104.85779571533203, "margin_dpo/margin_std": 104.94450378417969, "step": 432 }, { "KL/chosen_KL_mean": -174.6761932373047, "KL/mean": -215.60940551757812, "KL/rejected_KL_mean": -256.5426025390625, "KL/std": 102.92887878417969, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.86639404296875, "fcm_dpo/q_t": 0.428830087184906, "grad_norm": 7.770791053771973, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.5186316967010498, "logits/rejected": -1.1939599514007568, "logps/chosen": -264.5194091796875, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -358.27606201171875, "loss": 1.1542, "margin_dpo/margin_mean": 81.86639404296875, "margin_dpo/margin_std": 117.09519958496094, "step": 433 }, { "KL/chosen_KL_mean": -150.4554443359375, "KL/mean": -196.809326171875, "KL/rejected_KL_mean": -243.1632080078125, "KL/std": 115.04313659667969, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.707763671875, "fcm_dpo/q_t": 0.4218069612979889, "grad_norm": 10.806130409240723, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.4759421348571777, "logits/rejected": -1.8088021278381348, "logps/chosen": -248.14797973632812, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -327.2545166015625, "loss": 1.1316, "margin_dpo/margin_mean": 92.707763671875, "margin_dpo/margin_std": 134.9464569091797, "step": 434 }, { "KL/chosen_KL_mean": -148.0664520263672, "KL/mean": -195.94573974609375, "KL/rejected_KL_mean": -243.82504272460938, "KL/std": 106.5086669921875, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.75857543945312, "fcm_dpo/q_t": 0.41754594445228577, "grad_norm": 8.670187950134277, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.497197151184082, "logits/rejected": -2.059156656265259, "logps/chosen": -234.23837280273438, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -337.57672119140625, "loss": 1.1115, "margin_dpo/margin_mean": 95.75858306884766, "margin_dpo/margin_std": 119.56307983398438, "step": 435 }, { "KL/chosen_KL_mean": -123.63499450683594, "KL/mean": -173.1519775390625, "KL/rejected_KL_mean": -222.6689453125, "KL/std": 89.74921417236328, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.03394317626953, "fcm_dpo/q_t": 0.4128342866897583, "grad_norm": 9.753068923950195, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.8909978866577148, "logits/rejected": -1.0461177825927734, "logps/chosen": -229.42208862304688, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -311.29364013671875, "loss": 1.0841, "margin_dpo/margin_mean": 99.033935546875, "margin_dpo/margin_std": 92.84120178222656, "step": 436 }, { "KL/chosen_KL_mean": -142.92214965820312, "KL/mean": -183.7037811279297, "KL/rejected_KL_mean": -224.48544311523438, "KL/std": 96.91071319580078, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.56330108642578, "fcm_dpo/q_t": 0.4275186061859131, "grad_norm": 9.075855255126953, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9783051013946533, "logits/rejected": -1.323150634765625, "logps/chosen": -238.98419189453125, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -313.4976501464844, "loss": 1.1398, "margin_dpo/margin_mean": 81.56329345703125, "margin_dpo/margin_std": 95.77186584472656, "step": 437 }, { "KL/chosen_KL_mean": -148.93246459960938, "KL/mean": -190.12509155273438, "KL/rejected_KL_mean": -231.31773376464844, "KL/std": 97.18467712402344, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.38526916503906, "fcm_dpo/q_t": 0.4276707172393799, "grad_norm": 8.271927833557129, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1209750175476074, "logits/rejected": -1.309360146522522, "logps/chosen": -250.787841796875, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -320.765380859375, "loss": 1.1444, "margin_dpo/margin_mean": 82.38526916503906, "margin_dpo/margin_std": 105.441650390625, "step": 438 }, { "KL/chosen_KL_mean": -158.548583984375, "KL/mean": -200.0796661376953, "KL/rejected_KL_mean": -241.61073303222656, "KL/std": 105.94914245605469, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.003663172945380211, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.06214904785156, "fcm_dpo/q_t": 0.42750439047813416, "grad_norm": 8.279786109924316, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.6542387008666992, "logits/rejected": -1.4205396175384521, "logps/chosen": -240.30422973632812, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -332.19708251953125, "loss": 1.1415, "margin_dpo/margin_mean": 83.06214904785156, "margin_dpo/margin_std": 102.57997131347656, "step": 439 }, { "KL/chosen_KL_mean": -142.78366088867188, "KL/mean": -194.11111450195312, "KL/rejected_KL_mean": -245.43856811523438, "KL/std": 107.18521118164062, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": -0.02960079163312912, "fcm_dpo/margin": 102.65493774414062, "fcm_dpo/q_t": 0.41181766986846924, "grad_norm": 6.652213096618652, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.425079822540283, "logits/rejected": -1.9281439781188965, "logps/chosen": -248.4247283935547, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -348.839599609375, "loss": 1.0937, "margin_dpo/margin_mean": 102.6549301147461, "margin_dpo/margin_std": 119.68273162841797, "step": 440 }, { "KL/chosen_KL_mean": -185.1339111328125, "KL/mean": -228.09225463867188, "KL/rejected_KL_mean": -271.05059814453125, "KL/std": 101.60466003417969, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.91670227050781, "fcm_dpo/q_t": 0.42634695768356323, "grad_norm": 7.8740715980529785, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.5001919269561768, "logits/rejected": -1.1317778825759888, "logps/chosen": -276.6634521484375, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -374.66998291015625, "loss": 1.1371, "margin_dpo/margin_mean": 85.91670227050781, "margin_dpo/margin_std": 112.26949310302734, "step": 441 }, { "KL/chosen_KL_mean": -180.07342529296875, "KL/mean": -235.09642028808594, "KL/rejected_KL_mean": -290.11944580078125, "KL/std": 107.53369140625, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.04603576660156, "fcm_dpo/q_t": 0.40477871894836426, "grad_norm": 9.925519943237305, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.6638357639312744, "logits/rejected": -1.193490982055664, "logps/chosen": -264.8509826660156, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -373.943603515625, "loss": 1.0637, "margin_dpo/margin_mean": 110.0460205078125, "margin_dpo/margin_std": 110.4421157836914, "step": 442 }, { "KL/chosen_KL_mean": -176.22767639160156, "KL/mean": -210.25033569335938, "KL/rejected_KL_mean": -244.27297973632812, "KL/std": 105.0416259765625, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.04527282714844, "fcm_dpo/q_t": 0.44058459997177124, "grad_norm": 8.969043731689453, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.825087547302246, "logits/rejected": -1.2656283378601074, "logps/chosen": -278.876953125, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -337.31103515625, "loss": 1.1883, "margin_dpo/margin_mean": 68.04527282714844, "margin_dpo/margin_std": 101.85441589355469, "step": 443 }, { "KL/chosen_KL_mean": -180.88507080078125, "KL/mean": -221.47659301757812, "KL/rejected_KL_mean": -262.068115234375, "KL/std": 102.50066375732422, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.18304443359375, "fcm_dpo/q_t": 0.4298546016216278, "grad_norm": 9.320587158203125, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6225669384002686, "logits/rejected": -1.4563040733337402, "logps/chosen": -268.8047790527344, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -365.39154052734375, "loss": 1.1573, "margin_dpo/margin_mean": 81.18304443359375, "margin_dpo/margin_std": 118.6432113647461, "step": 444 }, { "KL/chosen_KL_mean": -159.22897338867188, "KL/mean": -206.47286987304688, "KL/rejected_KL_mean": -253.7167510986328, "KL/std": 99.51150512695312, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.48779296875, "fcm_dpo/q_t": 0.4178006649017334, "grad_norm": 7.3936262130737305, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.8734161853790283, "logits/rejected": -1.4532876014709473, "logps/chosen": -260.62982177734375, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -352.754638671875, "loss": 1.1073, "margin_dpo/margin_mean": 94.48779296875, "margin_dpo/margin_std": 105.08795166015625, "step": 445 }, { "KL/chosen_KL_mean": -153.9788055419922, "KL/mean": -195.48651123046875, "KL/rejected_KL_mean": -236.9941864013672, "KL/std": 104.64459228515625, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.0153579711914, "fcm_dpo/q_t": 0.42762523889541626, "grad_norm": 11.044872283935547, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9425890445709229, "logits/rejected": -1.8059041500091553, "logps/chosen": -241.40115356445312, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -343.6949462890625, "loss": 1.14, "margin_dpo/margin_mean": 83.01536560058594, "margin_dpo/margin_std": 101.4017333984375, "step": 446 }, { "KL/chosen_KL_mean": -151.159912109375, "KL/mean": -206.8175811767578, "KL/rejected_KL_mean": -262.4752502441406, "KL/std": 110.01190185546875, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0036415504291653633, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.31533813476562, "fcm_dpo/q_t": 0.4046975374221802, "grad_norm": 7.47876501083374, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.14174747467041, "logits/rejected": -2.09769868850708, "logps/chosen": -234.77516174316406, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -367.38763427734375, "loss": 1.0728, "margin_dpo/margin_mean": 111.31533813476562, "margin_dpo/margin_std": 126.67897033691406, "step": 447 }, { "KL/chosen_KL_mean": -164.76406860351562, "KL/mean": -218.3070831298828, "KL/rejected_KL_mean": -271.85009765625, "KL/std": 114.6876449584961, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0036288611590862274, "fcm_dpo/delta": -0.03496779501438141, "fcm_dpo/margin": 107.08602905273438, "fcm_dpo/q_t": 0.40878742933273315, "grad_norm": 8.506490707397461, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.6992747783660889, "logits/rejected": -1.4293479919433594, "logps/chosen": -257.33984375, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -379.53985595703125, "loss": 1.0937, "margin_dpo/margin_mean": 107.08602905273438, "margin_dpo/margin_std": 134.44287109375, "step": 448 }, { "KL/chosen_KL_mean": -161.64865112304688, "KL/mean": -216.47604370117188, "KL/rejected_KL_mean": -271.303466796875, "KL/std": 107.31214904785156, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0036161718890070915, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.65481567382812, "fcm_dpo/q_t": 0.4062817692756653, "grad_norm": 7.932675838470459, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4656729698181152, "logits/rejected": -1.0218647718429565, "logps/chosen": -264.0475769042969, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -366.45233154296875, "loss": 1.0759, "margin_dpo/margin_mean": 109.65481567382812, "margin_dpo/margin_std": 122.98450469970703, "step": 449 }, { "KL/chosen_KL_mean": -145.1838836669922, "KL/mean": -196.794189453125, "KL/rejected_KL_mean": -248.40451049804688, "KL/std": 108.87329864501953, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0036161718890070915, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.22064971923828, "fcm_dpo/q_t": 0.4112847149372101, "grad_norm": 8.157800674438477, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.468385696411133, "logits/rejected": -2.2814369201660156, "logps/chosen": -232.17672729492188, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -356.9365539550781, "loss": 1.0915, "margin_dpo/margin_mean": 103.22064971923828, "margin_dpo/margin_std": 118.33140563964844, "step": 450 }, { "KL/chosen_KL_mean": -163.8168487548828, "KL/mean": -205.17733764648438, "KL/rejected_KL_mean": -246.537841796875, "KL/std": 107.68229675292969, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0036161718890070915, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.72098541259766, "fcm_dpo/q_t": 0.4297275245189667, "grad_norm": 11.571629524230957, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.3115339279174805, "logits/rejected": -1.7988758087158203, "logps/chosen": -250.6281280517578, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -326.3934326171875, "loss": 1.1658, "margin_dpo/margin_mean": 82.72097778320312, "margin_dpo/margin_std": 133.84030151367188, "step": 451 }, { "KL/chosen_KL_mean": -144.13482666015625, "KL/mean": -190.4474639892578, "KL/rejected_KL_mean": -236.76011657714844, "KL/std": 102.71875, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0035974713973701, "fcm_dpo/delta": -0.02592368610203266, "fcm_dpo/margin": 92.62530517578125, "fcm_dpo/q_t": 0.4200305640697479, "grad_norm": 7.468684196472168, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7296477556228638, "logits/rejected": -1.674553632736206, "logps/chosen": -223.51446533203125, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -340.47552490234375, "loss": 1.1137, "margin_dpo/margin_mean": 92.62529754638672, "margin_dpo/margin_std": 96.71379852294922, "step": 452 }, { "KL/chosen_KL_mean": -173.96148681640625, "KL/mean": -230.06982421875, "KL/rejected_KL_mean": -286.17816162109375, "KL/std": 104.87478637695312, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0035939691588282585, "fcm_dpo/delta": -0.009744642302393913, "fcm_dpo/margin": 112.2166748046875, "fcm_dpo/q_t": 0.40478187799453735, "grad_norm": 10.316544532775879, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -2.071340322494507, "logits/rejected": -1.6110639572143555, "logps/chosen": -261.65667724609375, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -376.23638916015625, "loss": 1.0631, "margin_dpo/margin_mean": 112.2166748046875, "margin_dpo/margin_std": 113.53723907470703, "step": 453 }, { "KL/chosen_KL_mean": -182.67547607421875, "KL/mean": -227.7379913330078, "KL/rejected_KL_mean": -272.8005065917969, "KL/std": 107.76519775390625, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.003590466920286417, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.12501525878906, "fcm_dpo/q_t": 0.4243161380290985, "grad_norm": 8.455181121826172, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.6731433868408203, "logits/rejected": -1.19912588596344, "logps/chosen": -272.24169921875, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -365.7215576171875, "loss": 1.1512, "margin_dpo/margin_mean": 90.12501525878906, "margin_dpo/margin_std": 140.74319458007812, "step": 454 }, { "KL/chosen_KL_mean": -171.49151611328125, "KL/mean": -215.20111083984375, "KL/rejected_KL_mean": -258.91070556640625, "KL/std": 106.60896301269531, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.003590466920286417, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.41919708251953, "fcm_dpo/q_t": 0.4255741834640503, "grad_norm": 9.087653160095215, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8771929740905762, "logits/rejected": -1.8401820659637451, "logps/chosen": -249.12149047851562, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -377.888671875, "loss": 1.1439, "margin_dpo/margin_mean": 87.41919708251953, "margin_dpo/margin_std": 121.63143920898438, "step": 455 }, { "KL/chosen_KL_mean": -174.88723754882812, "KL/mean": -218.83175659179688, "KL/rejected_KL_mean": -262.7762756347656, "KL/std": 108.046630859375, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.003590466920286417, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.8890380859375, "fcm_dpo/q_t": 0.42472055554389954, "grad_norm": 10.03993034362793, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.0770020484924316, "logits/rejected": -1.9483678340911865, "logps/chosen": -264.5041198730469, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -372.33599853515625, "loss": 1.1323, "margin_dpo/margin_mean": 87.8890380859375, "margin_dpo/margin_std": 108.95448303222656, "step": 456 }, { "KL/chosen_KL_mean": -142.23580932617188, "KL/mean": -199.90113830566406, "KL/rejected_KL_mean": -257.56646728515625, "KL/std": 108.0162353515625, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.003543007653206587, "fcm_dpo/delta": -0.06653130054473877, "fcm_dpo/margin": 115.33065795898438, "fcm_dpo/q_t": 0.40156179666519165, "grad_norm": 8.461836814880371, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.1931958198547363, "logits/rejected": -1.881136178970337, "logps/chosen": -229.5574951171875, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -348.33306884765625, "loss": 1.0504, "margin_dpo/margin_mean": 115.3306655883789, "margin_dpo/margin_std": 97.08042907714844, "step": 457 }, { "KL/chosen_KL_mean": -165.9761199951172, "KL/mean": -206.82937622070312, "KL/rejected_KL_mean": -247.6826171875, "KL/std": 106.56148529052734, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.003543007653206587, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.70649719238281, "fcm_dpo/q_t": 0.43211352825164795, "grad_norm": 8.534250259399414, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.832714557647705, "logits/rejected": -1.6081299781799316, "logps/chosen": -259.49658203125, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -351.0516052246094, "loss": 1.165, "margin_dpo/margin_mean": 81.70649719238281, "margin_dpo/margin_std": 125.64759826660156, "step": 458 }, { "KL/chosen_KL_mean": -156.81216430664062, "KL/mean": -208.63536071777344, "KL/rejected_KL_mean": -260.4585876464844, "KL/std": 111.30593872070312, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.003529222682118416, "fcm_dpo/delta": -0.01949174702167511, "fcm_dpo/margin": 103.64640808105469, "fcm_dpo/q_t": 0.4136005640029907, "grad_norm": 6.5523834228515625, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6535613536834717, "logits/rejected": -1.405625581741333, "logps/chosen": -232.5003662109375, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -352.6290588378906, "loss": 1.0971, "margin_dpo/margin_mean": 103.64640808105469, "margin_dpo/margin_std": 119.43470764160156, "step": 459 }, { "KL/chosen_KL_mean": -161.82501220703125, "KL/mean": -209.39825439453125, "KL/rejected_KL_mean": -256.97149658203125, "KL/std": 108.63713836669922, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.003529222682118416, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.146484375, "fcm_dpo/q_t": 0.42054271697998047, "grad_norm": 9.576332092285156, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -1.9110339879989624, "logits/rejected": -1.9415394067764282, "logps/chosen": -242.94290161132812, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -367.28387451171875, "loss": 1.1259, "margin_dpo/margin_mean": 95.146484375, "margin_dpo/margin_std": 124.64227294921875, "step": 460 }, { "KL/chosen_KL_mean": -198.65940856933594, "KL/mean": -238.14801025390625, "KL/rejected_KL_mean": -277.6365661621094, "KL/std": 95.63964080810547, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.003529222682118416, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.97716522216797, "fcm_dpo/q_t": 0.4337084889411926, "grad_norm": 7.279162406921387, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.4350428581237793, "logits/rejected": -1.2394301891326904, "logps/chosen": -280.242919921875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -371.513671875, "loss": 1.1675, "margin_dpo/margin_mean": 78.9771728515625, "margin_dpo/margin_std": 115.29185485839844, "step": 461 }, { "KL/chosen_KL_mean": -151.26431274414062, "KL/mean": -200.8085479736328, "KL/rejected_KL_mean": -250.35275268554688, "KL/std": 102.4466552734375, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.003529222682118416, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.08845520019531, "fcm_dpo/q_t": 0.41680508852005005, "grad_norm": 7.676913738250732, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.2305116653442383, "logits/rejected": -1.7276296615600586, "logps/chosen": -245.46286010742188, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -335.984375, "loss": 1.1035, "margin_dpo/margin_mean": 99.08844757080078, "margin_dpo/margin_std": 110.71627807617188, "step": 462 }, { "KL/chosen_KL_mean": -146.0782470703125, "KL/mean": -206.6436309814453, "KL/rejected_KL_mean": -267.208984375, "KL/std": 111.3388671875, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.00350542226806283, "fcm_dpo/delta": -0.06789717078208923, "fcm_dpo/margin": 121.13074493408203, "fcm_dpo/q_t": 0.398717999458313, "grad_norm": 11.124042510986328, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.8921117782592773, "logits/rejected": -1.4690706729888916, "logps/chosen": -232.00299072265625, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -364.11083984375, "loss": 1.0524, "margin_dpo/margin_mean": 121.13075256347656, "margin_dpo/margin_std": 118.82306671142578, "step": 463 }, { "KL/chosen_KL_mean": -180.74749755859375, "KL/mean": -230.1597137451172, "KL/rejected_KL_mean": -279.5719299316406, "KL/std": 109.47645568847656, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.003481621854007244, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.82440948486328, "fcm_dpo/q_t": 0.4185960590839386, "grad_norm": 11.563304901123047, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.7588783502578735, "logits/rejected": -1.8399561643600464, "logps/chosen": -260.43670654296875, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -386.8642578125, "loss": 1.1175, "margin_dpo/margin_mean": 98.82440948486328, "margin_dpo/margin_std": 125.87593078613281, "step": 464 }, { "KL/chosen_KL_mean": -171.5797576904297, "KL/mean": -228.90499877929688, "KL/rejected_KL_mean": -286.230224609375, "KL/std": 118.9791259765625, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.003481621854007244, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.65049743652344, "fcm_dpo/q_t": 0.40499651432037354, "grad_norm": 7.617553234100342, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.654434323310852, "logits/rejected": -1.353251576423645, "logps/chosen": -263.4399719238281, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -404.94024658203125, "loss": 1.0635, "margin_dpo/margin_mean": 114.65049743652344, "margin_dpo/margin_std": 112.99559020996094, "step": 465 }, { "KL/chosen_KL_mean": -152.46583557128906, "KL/mean": -210.42578125, "KL/rejected_KL_mean": -268.3857421875, "KL/std": 113.14037322998047, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.003481621854007244, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.91989135742188, "fcm_dpo/q_t": 0.40440747141838074, "grad_norm": 6.3236846923828125, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.4483768939971924, "logits/rejected": -1.1259346008300781, "logps/chosen": -237.167236328125, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -364.43658447265625, "loss": 1.0638, "margin_dpo/margin_mean": 115.91989135742188, "margin_dpo/margin_std": 119.01202392578125, "step": 466 }, { "KL/chosen_KL_mean": -197.99533081054688, "KL/mean": -246.110107421875, "KL/rejected_KL_mean": -294.224853515625, "KL/std": 118.2468490600586, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.003481621854007244, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.22953033447266, "fcm_dpo/q_t": 0.4207920432090759, "grad_norm": 11.334175109863281, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.3910608291625977, "logits/rejected": -1.8220816850662231, "logps/chosen": -307.29364013671875, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -403.0685119628906, "loss": 1.1334, "margin_dpo/margin_mean": 96.22953033447266, "margin_dpo/margin_std": 134.78573608398438, "step": 467 }, { "KL/chosen_KL_mean": -153.37420654296875, "KL/mean": -198.64462280273438, "KL/rejected_KL_mean": -243.9150390625, "KL/std": 117.04736328125, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.003481621854007244, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.54083251953125, "fcm_dpo/q_t": 0.42628103494644165, "grad_norm": 8.669154167175293, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.7600032091140747, "logits/rejected": -1.5498385429382324, "logps/chosen": -251.69586181640625, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -350.59552001953125, "loss": 1.148, "margin_dpo/margin_mean": 90.54083251953125, "margin_dpo/margin_std": 134.69952392578125, "step": 468 }, { "KL/chosen_KL_mean": -187.65769958496094, "KL/mean": -243.13168334960938, "KL/rejected_KL_mean": -298.60565185546875, "KL/std": 116.16435241699219, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0034574605524539948, "fcm_dpo/delta": -0.03481918200850487, "fcm_dpo/margin": 110.94795989990234, "fcm_dpo/q_t": 0.4091476798057556, "grad_norm": 8.989618301391602, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.3008790016174316, "logits/rejected": -1.8869959115982056, "logps/chosen": -284.4219055175781, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -408.2006530761719, "loss": 1.0849, "margin_dpo/margin_mean": 110.94795989990234, "margin_dpo/margin_std": 124.79432678222656, "step": 469 }, { "KL/chosen_KL_mean": -172.62655639648438, "KL/mean": -223.51864624023438, "KL/rejected_KL_mean": -274.4107360839844, "KL/std": 107.62390899658203, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0034574605524539948, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.78416442871094, "fcm_dpo/q_t": 0.4158906936645508, "grad_norm": 8.794624328613281, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.7221657037734985, "logits/rejected": -1.3079010248184204, "logps/chosen": -256.12322998046875, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -362.8965148925781, "loss": 1.0993, "margin_dpo/margin_mean": 101.78416442871094, "margin_dpo/margin_std": 108.78021240234375, "step": 470 }, { "KL/chosen_KL_mean": -197.77061462402344, "KL/mean": -245.08401489257812, "KL/rejected_KL_mean": -292.39739990234375, "KL/std": 106.6776123046875, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0034574605524539948, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.62680053710938, "fcm_dpo/q_t": 0.42178910970687866, "grad_norm": 7.90871524810791, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.52286696434021, "logits/rejected": -1.098524570465088, "logps/chosen": -286.24493408203125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -382.8791198730469, "loss": 1.1193, "margin_dpo/margin_mean": 94.62680053710938, "margin_dpo/margin_std": 108.30757141113281, "step": 471 }, { "KL/chosen_KL_mean": -185.84048461914062, "KL/mean": -242.78103637695312, "KL/rejected_KL_mean": -299.7215576171875, "KL/std": 108.53646850585938, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": -0.01742708310484886, "fcm_dpo/margin": 113.88108825683594, "fcm_dpo/q_t": 0.40723273158073425, "grad_norm": 7.415139675140381, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.101285457611084, "logits/rejected": -1.395476222038269, "logps/chosen": -311.0741882324219, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -420.77508544921875, "loss": 1.0711, "margin_dpo/margin_mean": 113.88108825683594, "margin_dpo/margin_std": 116.49694061279297, "step": 472 }, { "KL/chosen_KL_mean": -211.150390625, "KL/mean": -245.96975708007812, "KL/rejected_KL_mean": -280.78912353515625, "KL/std": 108.80541229248047, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.63873291015625, "fcm_dpo/q_t": 0.4436192512512207, "grad_norm": 8.89926528930664, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.7546712160110474, "logits/rejected": -1.2594163417816162, "logps/chosen": -306.76177978515625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -368.9402770996094, "loss": 1.2062, "margin_dpo/margin_mean": 69.63873291015625, "margin_dpo/margin_std": 126.59794616699219, "step": 473 }, { "KL/chosen_KL_mean": -196.3642120361328, "KL/mean": -232.613037109375, "KL/rejected_KL_mean": -268.8618469238281, "KL/std": 103.38455200195312, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.49763488769531, "fcm_dpo/q_t": 0.44085901975631714, "grad_norm": 14.313152313232422, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.3852801322937012, "logits/rejected": -1.193461298942566, "logps/chosen": -277.8439636230469, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -365.3274841308594, "loss": 1.2033, "margin_dpo/margin_mean": 72.49763488769531, "margin_dpo/margin_std": 127.64671325683594, "step": 474 }, { "KL/chosen_KL_mean": -148.33688354492188, "KL/mean": -204.86834716796875, "KL/rejected_KL_mean": -261.3997802734375, "KL/std": 110.5709457397461, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.06293487548828, "fcm_dpo/q_t": 0.4083915948867798, "grad_norm": 7.375909805297852, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.1760787963867188, "logits/rejected": -2.124093532562256, "logps/chosen": -222.53286743164062, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -356.09222412109375, "loss": 1.0803, "margin_dpo/margin_mean": 113.06292724609375, "margin_dpo/margin_std": 127.98579406738281, "step": 475 }, { "KL/chosen_KL_mean": -172.64646911621094, "KL/mean": -222.93531799316406, "KL/rejected_KL_mean": -273.22418212890625, "KL/std": 112.55107116699219, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.57769012451172, "fcm_dpo/q_t": 0.4183441400527954, "grad_norm": 8.676711082458496, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.7305399179458618, "logits/rejected": -1.3420495986938477, "logps/chosen": -244.61756896972656, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -353.48638916015625, "loss": 1.1117, "margin_dpo/margin_mean": 100.57769012451172, "margin_dpo/margin_std": 121.6563491821289, "step": 476 }, { "KL/chosen_KL_mean": -172.94113159179688, "KL/mean": -222.84429931640625, "KL/rejected_KL_mean": -272.74749755859375, "KL/std": 98.91001892089844, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.80635070800781, "fcm_dpo/q_t": 0.41792958974838257, "grad_norm": 7.784486293792725, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.118290424346924, "logits/rejected": -1.1916790008544922, "logps/chosen": -277.9466857910156, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -354.62591552734375, "loss": 1.1146, "margin_dpo/margin_mean": 99.80635070800781, "margin_dpo/margin_std": 122.40753173828125, "step": 477 }, { "KL/chosen_KL_mean": -168.89895629882812, "KL/mean": -220.816162109375, "KL/rejected_KL_mean": -272.7333679199219, "KL/std": 107.77391052246094, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.0034454308915883303, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.83441162109375, "fcm_dpo/q_t": 0.4149329960346222, "grad_norm": 8.032904624938965, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.3838605880737305, "logits/rejected": -1.2020361423492432, "logps/chosen": -245.68716430664062, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -363.1733093261719, "loss": 1.1008, "margin_dpo/margin_mean": 103.83441162109375, "margin_dpo/margin_std": 119.13009643554688, "step": 478 }, { "KL/chosen_KL_mean": -170.88421630859375, "KL/mean": -223.80311584472656, "KL/rejected_KL_mean": -276.7220458984375, "KL/std": 110.08104705810547, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.003440340980887413, "fcm_dpo/delta": -0.014794536866247654, "fcm_dpo/margin": 105.83782958984375, "fcm_dpo/q_t": 0.41301044821739197, "grad_norm": 10.730645179748535, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.5187063217163086, "logits/rejected": -1.3761881589889526, "logps/chosen": -252.24652099609375, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -377.81317138671875, "loss": 1.0937, "margin_dpo/margin_mean": 105.83782958984375, "margin_dpo/margin_std": 114.75797271728516, "step": 479 }, { "KL/chosen_KL_mean": -198.71426391601562, "KL/mean": -239.4034881591797, "KL/rejected_KL_mean": -280.09271240234375, "KL/std": 103.86823272705078, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0034352513030171394, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.37846374511719, "fcm_dpo/q_t": 0.4330998957157135, "grad_norm": 8.441927909851074, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7332559823989868, "logits/rejected": -1.0534849166870117, "logps/chosen": -295.4881591796875, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -366.4974365234375, "loss": 1.163, "margin_dpo/margin_mean": 81.37846374511719, "margin_dpo/margin_std": 113.44856262207031, "step": 480 }, { "KL/chosen_KL_mean": -193.66848754882812, "KL/mean": -241.52822875976562, "KL/rejected_KL_mean": -289.38800048828125, "KL/std": 108.02978515625, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0034352513030171394, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.71952819824219, "fcm_dpo/q_t": 0.421600341796875, "grad_norm": 10.686586380004883, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.9811701774597168, "logits/rejected": -1.5507304668426514, "logps/chosen": -285.3386535644531, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -388.0828857421875, "loss": 1.1247, "margin_dpo/margin_mean": 95.71952056884766, "margin_dpo/margin_std": 119.31742858886719, "step": 481 }, { "KL/chosen_KL_mean": -187.73928833007812, "KL/mean": -233.50332641601562, "KL/rejected_KL_mean": -279.2673645019531, "KL/std": 107.09455871582031, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0034352513030171394, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.528076171875, "fcm_dpo/q_t": 0.4246319532394409, "grad_norm": 10.31877613067627, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.626564383506775, "logits/rejected": -1.273120641708374, "logps/chosen": -286.2593994140625, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -374.0968017578125, "loss": 1.1383, "margin_dpo/margin_mean": 91.52808380126953, "margin_dpo/margin_std": 120.31767272949219, "step": 482 }, { "KL/chosen_KL_mean": -193.0203399658203, "KL/mean": -242.24212646484375, "KL/rejected_KL_mean": -291.46392822265625, "KL/std": 112.40119171142578, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.003429742995649576, "fcm_dpo/delta": -0.016060275956988335, "fcm_dpo/margin": 98.44355773925781, "fcm_dpo/q_t": 0.419140487909317, "grad_norm": 6.975028038024902, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.060020923614502, "logits/rejected": -1.4221384525299072, "logps/chosen": -300.1389465332031, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -392.57891845703125, "loss": 1.1149, "margin_dpo/margin_mean": 98.44355773925781, "margin_dpo/margin_std": 112.80430603027344, "step": 483 }, { "KL/chosen_KL_mean": -175.74607849121094, "KL/mean": -231.17672729492188, "KL/rejected_KL_mean": -286.60736083984375, "KL/std": 114.05818176269531, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.003419180167838931, "fcm_dpo/delta": -0.01478292141109705, "fcm_dpo/margin": 110.86129760742188, "fcm_dpo/q_t": 0.4097769260406494, "grad_norm": 7.691289901733398, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.3838446140289307, "logits/rejected": -1.9066808223724365, "logps/chosen": -262.72601318359375, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -377.3310241699219, "loss": 1.0871, "margin_dpo/margin_mean": 110.86128997802734, "margin_dpo/margin_std": 124.89863586425781, "step": 484 }, { "KL/chosen_KL_mean": -167.87294006347656, "KL/mean": -227.23599243164062, "KL/rejected_KL_mean": -286.5990295410156, "KL/std": 119.5997314453125, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.72611999511719, "fcm_dpo/q_t": 0.40414565801620483, "grad_norm": 7.532182216644287, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.8168668746948242, "logits/rejected": -1.4055249691009521, "logps/chosen": -261.93170166015625, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -395.1620178222656, "loss": 1.0623, "margin_dpo/margin_mean": 118.72611236572266, "margin_dpo/margin_std": 121.46720123291016, "step": 485 }, { "KL/chosen_KL_mean": -192.95993041992188, "KL/mean": -245.27255249023438, "KL/rejected_KL_mean": -297.5851745605469, "KL/std": 114.98240661621094, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.62525177001953, "fcm_dpo/q_t": 0.4149983525276184, "grad_norm": 9.295222282409668, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.1343398094177246, "logits/rejected": -1.4460769891738892, "logps/chosen": -286.705810546875, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -395.65582275390625, "loss": 1.1036, "margin_dpo/margin_mean": 104.62525939941406, "margin_dpo/margin_std": 122.81935119628906, "step": 486 }, { "KL/chosen_KL_mean": -185.85263061523438, "KL/mean": -233.24609375, "KL/rejected_KL_mean": -280.63958740234375, "KL/std": 108.77975463867188, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.78694152832031, "fcm_dpo/q_t": 0.4227539598941803, "grad_norm": 8.84157657623291, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.273259162902832, "logits/rejected": -1.9701170921325684, "logps/chosen": -273.8861999511719, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -379.1116943359375, "loss": 1.1269, "margin_dpo/margin_mean": 94.78694152832031, "margin_dpo/margin_std": 116.5904312133789, "step": 487 }, { "KL/chosen_KL_mean": -166.26364135742188, "KL/mean": -219.23251342773438, "KL/rejected_KL_mean": -272.20135498046875, "KL/std": 115.95780944824219, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.93771362304688, "fcm_dpo/q_t": 0.4132499694824219, "grad_norm": 7.965515613555908, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.6637756824493408, "logits/rejected": -1.4030412435531616, "logps/chosen": -251.0532989501953, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -362.474853515625, "loss": 1.089, "margin_dpo/margin_mean": 105.93771362304688, "margin_dpo/margin_std": 104.26072692871094, "step": 488 }, { "KL/chosen_KL_mean": -188.92745971679688, "KL/mean": -236.78684997558594, "KL/rejected_KL_mean": -284.646240234375, "KL/std": 105.75407409667969, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.71878814697266, "fcm_dpo/q_t": 0.42226240038871765, "grad_norm": 8.427530288696289, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.564948558807373, "logits/rejected": -1.0701770782470703, "logps/chosen": -279.3967590332031, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -371.04388427734375, "loss": 1.1249, "margin_dpo/margin_mean": 95.71879577636719, "margin_dpo/margin_std": 118.09747314453125, "step": 489 }, { "KL/chosen_KL_mean": -188.20443725585938, "KL/mean": -233.75973510742188, "KL/rejected_KL_mean": -279.3150634765625, "KL/std": 114.91810607910156, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.11060333251953, "fcm_dpo/q_t": 0.4276156723499298, "grad_norm": 7.694245338439941, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.0541791915893555, "logits/rejected": -2.138061285018921, "logps/chosen": -273.5245666503906, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -395.30889892578125, "loss": 1.1666, "margin_dpo/margin_mean": 91.110595703125, "margin_dpo/margin_std": 152.2172393798828, "step": 490 }, { "KL/chosen_KL_mean": -191.28285217285156, "KL/mean": -233.448486328125, "KL/rejected_KL_mean": -275.6141052246094, "KL/std": 117.21177673339844, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.33124542236328, "fcm_dpo/q_t": 0.43105894327163696, "grad_norm": 9.01755142211914, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.9717955589294434, "logits/rejected": -1.4925322532653809, "logps/chosen": -277.4263916015625, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -356.2935791015625, "loss": 1.1586, "margin_dpo/margin_mean": 84.33125305175781, "margin_dpo/margin_std": 118.42976379394531, "step": 491 }, { "KL/chosen_KL_mean": -181.25424194335938, "KL/mean": -214.68194580078125, "KL/rejected_KL_mean": -248.10963439941406, "KL/std": 115.5567398071289, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.85538482666016, "fcm_dpo/q_t": 0.44610458612442017, "grad_norm": 12.294608116149902, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.889963150024414, "logits/rejected": -1.4967410564422607, "logps/chosen": -262.36181640625, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -328.86163330078125, "loss": 1.2293, "margin_dpo/margin_mean": 66.85537719726562, "margin_dpo/margin_std": 143.98446655273438, "step": 492 }, { "KL/chosen_KL_mean": -178.69589233398438, "KL/mean": -223.39395141601562, "KL/rejected_KL_mean": -268.092041015625, "KL/std": 103.88409423828125, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0034141256473958492, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.39613342285156, "fcm_dpo/q_t": 0.4267809987068176, "grad_norm": 15.048552513122559, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.477911949157715, "logits/rejected": -1.7579889297485352, "logps/chosen": -290.90325927734375, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -361.69921875, "loss": 1.1409, "margin_dpo/margin_mean": 89.39613342285156, "margin_dpo/margin_std": 111.75929260253906, "step": 493 }, { "KL/chosen_KL_mean": -173.08877563476562, "KL/mean": -231.89781188964844, "KL/rejected_KL_mean": -290.70684814453125, "KL/std": 114.9976806640625, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": -0.04505976289510727, "fcm_dpo/margin": 117.61809539794922, "fcm_dpo/q_t": 0.4053962528705597, "grad_norm": 8.723076820373535, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.5490723848342896, "logits/rejected": -1.4756224155426025, "logps/chosen": -246.20367431640625, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -382.869873046875, "loss": 1.0727, "margin_dpo/margin_mean": 117.61808776855469, "margin_dpo/margin_std": 126.00625610351562, "step": 494 }, { "KL/chosen_KL_mean": -178.42495727539062, "KL/mean": -228.89169311523438, "KL/rejected_KL_mean": -279.35845947265625, "KL/std": 108.76109313964844, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.93351745605469, "fcm_dpo/q_t": 0.41925811767578125, "grad_norm": 9.70799446105957, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.093061923980713, "logits/rejected": -1.6170687675476074, "logps/chosen": -277.94525146484375, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -377.28936767578125, "loss": 1.1209, "margin_dpo/margin_mean": 100.93352508544922, "margin_dpo/margin_std": 131.6840057373047, "step": 495 }, { "KL/chosen_KL_mean": -198.2783660888672, "KL/mean": -241.52362060546875, "KL/rejected_KL_mean": -284.76885986328125, "KL/std": 109.83924865722656, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.49050903320312, "fcm_dpo/q_t": 0.4295841455459595, "grad_norm": 7.970189094543457, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.089627742767334, "logits/rejected": -1.4353868961334229, "logps/chosen": -306.1351318359375, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -377.5394287109375, "loss": 1.1506, "margin_dpo/margin_mean": 86.4905014038086, "margin_dpo/margin_std": 111.47935485839844, "step": 496 }, { "KL/chosen_KL_mean": -211.078369140625, "KL/mean": -238.99798583984375, "KL/rejected_KL_mean": -266.9176025390625, "KL/std": 110.53617858886719, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.83922576904297, "fcm_dpo/q_t": 0.45489537715911865, "grad_norm": 7.411959648132324, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.9977952241897583, "logits/rejected": -1.5707545280456543, "logps/chosen": -303.16162109375, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -348.712646484375, "loss": 1.2462, "margin_dpo/margin_mean": 55.83922576904297, "margin_dpo/margin_std": 118.08739471435547, "step": 497 }, { "KL/chosen_KL_mean": -180.35585021972656, "KL/mean": -227.35069274902344, "KL/rejected_KL_mean": -274.34552001953125, "KL/std": 109.93843078613281, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.98966217041016, "fcm_dpo/q_t": 0.4238489866256714, "grad_norm": 10.499435424804688, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.9336884021759033, "logits/rejected": -1.4464361667633057, "logps/chosen": -278.55023193359375, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -365.032958984375, "loss": 1.1348, "margin_dpo/margin_mean": 93.98965454101562, "margin_dpo/margin_std": 122.49534606933594, "step": 498 }, { "KL/chosen_KL_mean": -187.91543579101562, "KL/mean": -237.508544921875, "KL/rejected_KL_mean": -287.1016845703125, "KL/std": 118.04207611083984, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.18623352050781, "fcm_dpo/q_t": 0.4203417897224426, "grad_norm": 7.954686641693115, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.0367279052734375, "logits/rejected": -1.986741542816162, "logps/chosen": -268.955322265625, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -394.02337646484375, "loss": 1.1219, "margin_dpo/margin_mean": 99.18623352050781, "margin_dpo/margin_std": 125.63957977294922, "step": 499 }, { "KL/chosen_KL_mean": -208.7354736328125, "KL/mean": -251.08828735351562, "KL/rejected_KL_mean": -293.44110107421875, "KL/std": 113.52961730957031, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.003383495844900608, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.70561218261719, "fcm_dpo/q_t": 0.4324054718017578, "grad_norm": 9.598217010498047, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.4862442016601562, "logits/rejected": -1.3106663227081299, "logps/chosen": -297.9835205078125, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -393.851318359375, "loss": 1.1654, "margin_dpo/margin_mean": 84.70561218261719, "margin_dpo/margin_std": 128.2379608154297, "step": 500 }, { "KL/chosen_KL_mean": -182.74697875976562, "KL/mean": -229.3802032470703, "KL/rejected_KL_mean": -276.013427734375, "KL/std": 122.40470886230469, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.0033783160615712404, "fcm_dpo/delta": -0.015332462266087532, "fcm_dpo/margin": 93.26643371582031, "fcm_dpo/q_t": 0.4257209897041321, "grad_norm": 7.753704071044922, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.3967821598052979, "logits/rejected": -0.9468486905097961, "logps/chosen": -273.34393310546875, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -361.3106994628906, "loss": 1.1506, "margin_dpo/margin_mean": 93.26643371582031, "margin_dpo/margin_std": 137.30825805664062, "step": 501 }, { "KL/chosen_KL_mean": -184.1671142578125, "KL/mean": -225.82968139648438, "KL/rejected_KL_mean": -267.49224853515625, "KL/std": 102.22221374511719, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.003373136278241873, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.32511901855469, "fcm_dpo/q_t": 0.43269187211990356, "grad_norm": 8.385882377624512, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.849120020866394, "logits/rejected": -1.108466625213623, "logps/chosen": -287.6378479003906, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -356.536376953125, "loss": 1.1699, "margin_dpo/margin_mean": 83.32511901855469, "margin_dpo/margin_std": 126.28573608398438, "step": 502 }, { "KL/chosen_KL_mean": -183.68106079101562, "KL/mean": -231.09710693359375, "KL/rejected_KL_mean": -278.5131530761719, "KL/std": 109.38270568847656, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.003373136278241873, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.83209991455078, "fcm_dpo/q_t": 0.42374318838119507, "grad_norm": 7.935153961181641, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7599844932556152, "logits/rejected": -1.263484239578247, "logps/chosen": -277.4110412597656, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -366.12213134765625, "loss": 1.1329, "margin_dpo/margin_mean": 94.83209228515625, "margin_dpo/margin_std": 120.8095932006836, "step": 503 }, { "KL/chosen_KL_mean": -187.03981018066406, "KL/mean": -248.20298767089844, "KL/rejected_KL_mean": -309.36614990234375, "KL/std": 114.46544647216797, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.003367688972502947, "fcm_dpo/delta": -0.016175489872694016, "fcm_dpo/margin": 122.32638549804688, "fcm_dpo/q_t": 0.4016532003879547, "grad_norm": 8.427231788635254, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.098501205444336, "logits/rejected": -1.7682828903198242, "logps/chosen": -279.3316650390625, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -418.40380859375, "loss": 1.0528, "margin_dpo/margin_mean": 122.32637786865234, "margin_dpo/margin_std": 115.720458984375, "step": 504 }, { "KL/chosen_KL_mean": -205.08184814453125, "KL/mean": -248.5288543701172, "KL/rejected_KL_mean": -291.9758605957031, "KL/std": 113.08055114746094, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.89398956298828, "fcm_dpo/q_t": 0.42985278367996216, "grad_norm": 8.073877334594727, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.260624885559082, "logits/rejected": -1.692556381225586, "logps/chosen": -311.3587951660156, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -386.1819763183594, "loss": 1.1486, "margin_dpo/margin_mean": 86.89399719238281, "margin_dpo/margin_std": 109.494384765625, "step": 505 }, { "KL/chosen_KL_mean": -197.16265869140625, "KL/mean": -246.70291137695312, "KL/rejected_KL_mean": -296.2431640625, "KL/std": 107.29255676269531, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.08053588867188, "fcm_dpo/q_t": 0.41973912715911865, "grad_norm": 8.027770042419434, "learning_rate": 9.527690882192635e-08, "logits/chosen": -2.1629104614257812, "logits/rejected": -1.8157403469085693, "logps/chosen": -278.603515625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -381.9075927734375, "loss": 1.1164, "margin_dpo/margin_mean": 99.08053588867188, "margin_dpo/margin_std": 112.92709350585938, "step": 506 }, { "KL/chosen_KL_mean": -207.05508422851562, "KL/mean": -256.11151123046875, "KL/rejected_KL_mean": -305.16796875, "KL/std": 111.85960388183594, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.11292266845703, "fcm_dpo/q_t": 0.422024130821228, "grad_norm": 8.144790649414062, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.395522117614746, "logits/rejected": -1.9754009246826172, "logps/chosen": -306.4820251464844, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -407.42095947265625, "loss": 1.132, "margin_dpo/margin_mean": 98.1129150390625, "margin_dpo/margin_std": 131.945068359375, "step": 507 }, { "KL/chosen_KL_mean": -198.92111206054688, "KL/mean": -241.08432006835938, "KL/rejected_KL_mean": -283.24755859375, "KL/std": 119.756591796875, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.32643127441406, "fcm_dpo/q_t": 0.4332975149154663, "grad_norm": 13.951045036315918, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.2239325046539307, "logits/rejected": -1.7573646306991577, "logps/chosen": -296.8367614746094, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -387.04534912109375, "loss": 1.1778, "margin_dpo/margin_mean": 84.32643127441406, "margin_dpo/margin_std": 139.7979278564453, "step": 508 }, { "KL/chosen_KL_mean": -185.1092529296875, "KL/mean": -223.8022003173828, "KL/rejected_KL_mean": -262.49517822265625, "KL/std": 94.70466613769531, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.38593292236328, "fcm_dpo/q_t": 0.43707790970802307, "grad_norm": 13.090678215026855, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9617998600006104, "logits/rejected": -1.2931082248687744, "logps/chosen": -285.839111328125, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -349.43853759765625, "loss": 1.1871, "margin_dpo/margin_mean": 77.38593292236328, "margin_dpo/margin_std": 125.59359741210938, "step": 509 }, { "KL/chosen_KL_mean": -218.98501586914062, "KL/mean": -265.94781494140625, "KL/rejected_KL_mean": -312.91064453125, "KL/std": 113.07213592529297, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.92558288574219, "fcm_dpo/q_t": 0.42438262701034546, "grad_norm": 9.498225212097168, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.701727032661438, "logits/rejected": -1.522562026977539, "logps/chosen": -308.1544189453125, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -420.83001708984375, "loss": 1.1317, "margin_dpo/margin_mean": 93.92558288574219, "margin_dpo/margin_std": 113.46527099609375, "step": 510 }, { "KL/chosen_KL_mean": -200.13671875, "KL/mean": -250.87844848632812, "KL/rejected_KL_mean": -301.62017822265625, "KL/std": 111.72164916992188, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.48345947265625, "fcm_dpo/q_t": 0.4192960262298584, "grad_norm": 9.615492820739746, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.075472116470337, "logits/rejected": -1.5030133724212646, "logps/chosen": -289.8865051269531, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -399.3514404296875, "loss": 1.1169, "margin_dpo/margin_mean": 101.48345947265625, "margin_dpo/margin_std": 125.36493682861328, "step": 511 }, { "KL/chosen_KL_mean": -190.61883544921875, "KL/mean": -239.13717651367188, "KL/rejected_KL_mean": -287.6555480957031, "KL/std": 110.0096664428711, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.03670501708984, "fcm_dpo/q_t": 0.42226627469062805, "grad_norm": 11.531759262084961, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.9124705791473389, "logits/rejected": -1.4805744886398315, "logps/chosen": -276.3539733886719, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -375.2804260253906, "loss": 1.1339, "margin_dpo/margin_mean": 97.03671264648438, "margin_dpo/margin_std": 129.79408264160156, "step": 512 }, { "KL/chosen_KL_mean": -200.87391662597656, "KL/mean": -242.10150146484375, "KL/rejected_KL_mean": -283.3290710449219, "KL/std": 105.09806823730469, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.45515441894531, "fcm_dpo/q_t": 0.43306249380111694, "grad_norm": 9.258931159973145, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.252840042114258, "logits/rejected": -1.549091100692749, "logps/chosen": -297.0644226074219, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -366.246826171875, "loss": 1.1693, "margin_dpo/margin_mean": 82.45515441894531, "margin_dpo/margin_std": 120.02608489990234, "step": 513 }, { "KL/chosen_KL_mean": -196.7435302734375, "KL/mean": -241.22250366210938, "KL/rejected_KL_mean": -285.70147705078125, "KL/std": 105.96126556396484, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.95793151855469, "fcm_dpo/q_t": 0.42910587787628174, "grad_norm": 9.1299409866333, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.5200533866882324, "logits/rejected": -0.930212676525116, "logps/chosen": -294.5671691894531, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -371.0712890625, "loss": 1.1522, "margin_dpo/margin_mean": 88.95793151855469, "margin_dpo/margin_std": 125.56547546386719, "step": 514 }, { "KL/chosen_KL_mean": -186.52767944335938, "KL/mean": -230.82656860351562, "KL/rejected_KL_mean": -275.1254577636719, "KL/std": 110.0330810546875, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.59779357910156, "fcm_dpo/q_t": 0.4288720488548279, "grad_norm": 9.294062614440918, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.7147738933563232, "logits/rejected": -1.4813618659973145, "logps/chosen": -268.46124267578125, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -368.7020263671875, "loss": 1.1521, "margin_dpo/margin_mean": 88.5978012084961, "margin_dpo/margin_std": 121.98088073730469, "step": 515 }, { "KL/chosen_KL_mean": -193.44900512695312, "KL/mean": -235.36648559570312, "KL/rejected_KL_mean": -277.2839660644531, "KL/std": 126.25431823730469, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0033622414339333773, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.83495330810547, "fcm_dpo/q_t": 0.43412619829177856, "grad_norm": 10.396279335021973, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.199636697769165, "logits/rejected": -2.027867078781128, "logps/chosen": -290.5406799316406, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -395.84063720703125, "loss": 1.1849, "margin_dpo/margin_mean": 83.83495330810547, "margin_dpo/margin_std": 149.7241668701172, "step": 516 }, { "KL/chosen_KL_mean": -205.06704711914062, "KL/mean": -257.4508056640625, "KL/rejected_KL_mean": -309.8345642089844, "KL/std": 112.68952941894531, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": -0.028169605880975723, "fcm_dpo/margin": 104.76752471923828, "fcm_dpo/q_t": 0.4168575704097748, "grad_norm": 7.444843292236328, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.8439829349517822, "logits/rejected": -1.4165236949920654, "logps/chosen": -298.25360107421875, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -405.4273681640625, "loss": 1.1137, "margin_dpo/margin_mean": 104.76752471923828, "margin_dpo/margin_std": 128.36602783203125, "step": 517 }, { "KL/chosen_KL_mean": -214.19830322265625, "KL/mean": -253.08970642089844, "KL/rejected_KL_mean": -291.9811096191406, "KL/std": 106.21487426757812, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.7828140258789, "fcm_dpo/q_t": 0.4384167194366455, "grad_norm": 9.706255912780762, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.9851510524749756, "logits/rejected": -1.662151575088501, "logps/chosen": -301.21966552734375, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -383.2256774902344, "loss": 1.1904, "margin_dpo/margin_mean": 77.78282165527344, "margin_dpo/margin_std": 132.5013427734375, "step": 518 }, { "KL/chosen_KL_mean": -199.25042724609375, "KL/mean": -241.2652587890625, "KL/rejected_KL_mean": -283.28009033203125, "KL/std": 99.11959838867188, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.02967071533203, "fcm_dpo/q_t": 0.43266552686691284, "grad_norm": 9.032164573669434, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.8016257286071777, "logits/rejected": -1.4205193519592285, "logps/chosen": -299.93798828125, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -392.31048583984375, "loss": 1.155, "margin_dpo/margin_mean": 84.02967071533203, "margin_dpo/margin_std": 105.06468200683594, "step": 519 }, { "KL/chosen_KL_mean": -184.95950317382812, "KL/mean": -235.533203125, "KL/rejected_KL_mean": -286.10693359375, "KL/std": 112.03166198730469, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.1474609375, "fcm_dpo/q_t": 0.4188072085380554, "grad_norm": 7.930410861968994, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.4233133792877197, "logits/rejected": -0.9773313403129578, "logps/chosen": -283.73712158203125, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -393.0526428222656, "loss": 1.1094, "margin_dpo/margin_mean": 101.1474609375, "margin_dpo/margin_std": 109.78894805908203, "step": 520 }, { "KL/chosen_KL_mean": -185.49609375, "KL/mean": -237.0944061279297, "KL/rejected_KL_mean": -288.69268798828125, "KL/std": 104.49290466308594, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.19660949707031, "fcm_dpo/q_t": 0.4174082577228546, "grad_norm": 8.747714042663574, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.8506009578704834, "logits/rejected": -1.5068330764770508, "logps/chosen": -271.84716796875, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -379.08587646484375, "loss": 1.1028, "margin_dpo/margin_mean": 103.19660949707031, "margin_dpo/margin_std": 107.39787292480469, "step": 521 }, { "KL/chosen_KL_mean": -194.733154296875, "KL/mean": -240.319091796875, "KL/rejected_KL_mean": -285.905029296875, "KL/std": 114.99195098876953, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.1718978881836, "fcm_dpo/q_t": 0.42712900042533875, "grad_norm": 8.720601081848145, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.7345507144927979, "logits/rejected": -1.3704659938812256, "logps/chosen": -270.7412109375, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -360.3463134765625, "loss": 1.1437, "margin_dpo/margin_mean": 91.1718978881836, "margin_dpo/margin_std": 120.23829650878906, "step": 522 }, { "KL/chosen_KL_mean": -215.80918884277344, "KL/mean": -249.06907653808594, "KL/rejected_KL_mean": -282.3289489746094, "KL/std": 113.04805755615234, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.51976013183594, "fcm_dpo/q_t": 0.44730162620544434, "grad_norm": 9.624588012695312, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.839151382446289, "logits/rejected": -1.1237852573394775, "logps/chosen": -312.30548095703125, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -373.34423828125, "loss": 1.2241, "margin_dpo/margin_mean": 66.51976013183594, "margin_dpo/margin_std": 133.1144256591797, "step": 523 }, { "KL/chosen_KL_mean": -196.87620544433594, "KL/mean": -244.75839233398438, "KL/rejected_KL_mean": -292.64056396484375, "KL/std": 112.40684509277344, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0033433521166443825, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.76437377929688, "fcm_dpo/q_t": 0.4239140450954437, "grad_norm": 10.483524322509766, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.1903719902038574, "logits/rejected": -1.4967714548110962, "logps/chosen": -300.2817077636719, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -380.55072021484375, "loss": 1.1301, "margin_dpo/margin_mean": 95.76437377929688, "margin_dpo/margin_std": 119.75094604492188, "step": 524 }, { "KL/chosen_KL_mean": -183.9818115234375, "KL/mean": -239.25071716308594, "KL/rejected_KL_mean": -294.5196228027344, "KL/std": 113.20848083496094, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": -0.054066602140665054, "fcm_dpo/margin": 110.53781127929688, "fcm_dpo/q_t": 0.41406142711639404, "grad_norm": 9.360244750976562, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.453290343284607, "logits/rejected": -1.365220308303833, "logps/chosen": -268.18829345703125, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -400.78118896484375, "loss": 1.1037, "margin_dpo/margin_mean": 110.53781127929688, "margin_dpo/margin_std": 132.47119140625, "step": 525 }, { "KL/chosen_KL_mean": -190.40582275390625, "KL/mean": -239.6215362548828, "KL/rejected_KL_mean": -288.83721923828125, "KL/std": 106.98213195800781, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.43142700195312, "fcm_dpo/q_t": 0.4224291443824768, "grad_norm": 6.954518795013428, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.747727870941162, "logits/rejected": -1.0965616703033447, "logps/chosen": -293.25994873046875, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -382.48870849609375, "loss": 1.1293, "margin_dpo/margin_mean": 98.43142700195312, "margin_dpo/margin_std": 126.39039611816406, "step": 526 }, { "KL/chosen_KL_mean": -193.59677124023438, "KL/mean": -243.36756896972656, "KL/rejected_KL_mean": -293.13836669921875, "KL/std": 111.62156677246094, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.5416030883789, "fcm_dpo/q_t": 0.4204142689704895, "grad_norm": 9.105607032775879, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.0025923252105713, "logits/rejected": -2.1721744537353516, "logps/chosen": -290.3621826171875, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -378.5335693359375, "loss": 1.1092, "margin_dpo/margin_mean": 99.54159545898438, "margin_dpo/margin_std": 97.62309265136719, "step": 527 }, { "KL/chosen_KL_mean": -180.984130859375, "KL/mean": -229.83531188964844, "KL/rejected_KL_mean": -278.6864929199219, "KL/std": 113.92654418945312, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.70236206054688, "fcm_dpo/q_t": 0.4231950640678406, "grad_norm": 7.7196831703186035, "learning_rate": 7.413308141366254e-08, "logits/chosen": -1.983785629272461, "logits/rejected": -1.4581798315048218, "logps/chosen": -275.1490478515625, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -379.0588073730469, "loss": 1.1336, "margin_dpo/margin_mean": 97.7023696899414, "margin_dpo/margin_std": 130.06954956054688, "step": 528 }, { "KL/chosen_KL_mean": -208.9997100830078, "KL/mean": -254.0175323486328, "KL/rejected_KL_mean": -299.03533935546875, "KL/std": 112.21867370605469, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.03567504882812, "fcm_dpo/q_t": 0.4295962452888489, "grad_norm": 9.77739143371582, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8885607719421387, "logits/rejected": -1.7152621746063232, "logps/chosen": -289.7764892578125, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -396.22650146484375, "loss": 1.1573, "margin_dpo/margin_mean": 90.03567504882812, "margin_dpo/margin_std": 132.88954162597656, "step": 529 }, { "KL/chosen_KL_mean": -192.32513427734375, "KL/mean": -237.15798950195312, "KL/rejected_KL_mean": -281.9908447265625, "KL/std": 109.30624389648438, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.66569519042969, "fcm_dpo/q_t": 0.429033100605011, "grad_norm": 8.377902030944824, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.678199291229248, "logits/rejected": -1.364229679107666, "logps/chosen": -272.10089111328125, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -374.04473876953125, "loss": 1.1414, "margin_dpo/margin_mean": 89.66569519042969, "margin_dpo/margin_std": 105.81976318359375, "step": 530 }, { "KL/chosen_KL_mean": -184.55325317382812, "KL/mean": -240.87889099121094, "KL/rejected_KL_mean": -297.20452880859375, "KL/std": 115.5262222290039, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.003307394217699766, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.65127563476562, "fcm_dpo/q_t": 0.4106708765029907, "grad_norm": 9.312383651733398, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.9771723747253418, "logits/rejected": -1.3904871940612793, "logps/chosen": -263.8724365234375, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -375.3128662109375, "loss": 1.0826, "margin_dpo/margin_mean": 112.65128326416016, "margin_dpo/margin_std": 114.58784484863281, "step": 531 }, { "KL/chosen_KL_mean": -200.41238403320312, "KL/mean": -260.78790283203125, "KL/rejected_KL_mean": -321.16339111328125, "KL/std": 109.99523162841797, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0033011375926434994, "fcm_dpo/delta": -0.018952464684844017, "fcm_dpo/margin": 120.75102996826172, "fcm_dpo/q_t": 0.40518367290496826, "grad_norm": 9.592004776000977, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.7380117177963257, "logits/rejected": -1.1200871467590332, "logps/chosen": -302.0492858886719, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -414.5087890625, "loss": 1.0672, "margin_dpo/margin_mean": 120.75102233886719, "margin_dpo/margin_std": 122.54512023925781, "step": 532 }, { "KL/chosen_KL_mean": -193.37159729003906, "KL/mean": -228.61279296875, "KL/rejected_KL_mean": -263.85400390625, "KL/std": 116.39776611328125, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0032948812004178762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.48243713378906, "fcm_dpo/q_t": 0.4451850652694702, "grad_norm": 10.166023254394531, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.4347105026245117, "logits/rejected": -1.834648847579956, "logps/chosen": -284.11822509765625, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -346.74249267578125, "loss": 1.208, "margin_dpo/margin_mean": 70.48243713378906, "margin_dpo/margin_std": 124.21114349365234, "step": 533 }, { "KL/chosen_KL_mean": -172.22439575195312, "KL/mean": -224.40533447265625, "KL/rejected_KL_mean": -276.58624267578125, "KL/std": 109.15692138671875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0032948812004178762, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.36187744140625, "fcm_dpo/q_t": 0.41735345125198364, "grad_norm": 7.845878601074219, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.8333697319030762, "logits/rejected": -1.427631139755249, "logps/chosen": -258.855224609375, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -361.4582214355469, "loss": 1.1023, "margin_dpo/margin_mean": 104.36188507080078, "margin_dpo/margin_std": 107.57058715820312, "step": 534 }, { "KL/chosen_KL_mean": -174.12899780273438, "KL/mean": -229.60406494140625, "KL/rejected_KL_mean": -285.0791015625, "KL/std": 110.1668472290039, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0032890948932617903, "fcm_dpo/delta": -0.017592424526810646, "fcm_dpo/margin": 110.95011901855469, "fcm_dpo/q_t": 0.4134537875652313, "grad_norm": 6.714627742767334, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.7282235622406006, "logits/rejected": -1.5426005125045776, "logps/chosen": -264.474365234375, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -385.85711669921875, "loss": 1.0966, "margin_dpo/margin_mean": 110.95011901855469, "margin_dpo/margin_std": 126.34043884277344, "step": 535 }, { "KL/chosen_KL_mean": -172.468017578125, "KL/mean": -227.81350708007812, "KL/rejected_KL_mean": -283.15899658203125, "KL/std": 110.89315795898438, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0032833085861057043, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.69098663330078, "fcm_dpo/q_t": 0.41305407881736755, "grad_norm": 7.855533599853516, "learning_rate": 6.698662514899638e-08, "logits/chosen": -2.001776695251465, "logits/rejected": -1.7550241947174072, "logps/chosen": -251.42759704589844, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -376.2467956542969, "loss": 1.0912, "margin_dpo/margin_mean": 110.69099426269531, "margin_dpo/margin_std": 116.6318588256836, "step": 536 }, { "KL/chosen_KL_mean": -151.35423278808594, "KL/mean": -208.62570190429688, "KL/rejected_KL_mean": -265.89715576171875, "KL/std": 113.93359375, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0032718051224946976, "fcm_dpo/delta": -0.035159412771463394, "fcm_dpo/margin": 114.54290771484375, "fcm_dpo/q_t": 0.4101157486438751, "grad_norm": 8.625238418579102, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.5014822483062744, "logits/rejected": -0.8075401186943054, "logps/chosen": -238.06362915039062, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -344.8048400878906, "loss": 1.0808, "margin_dpo/margin_mean": 114.54290771484375, "margin_dpo/margin_std": 111.77937316894531, "step": 537 }, { "KL/chosen_KL_mean": -208.54110717773438, "KL/mean": -254.97496032714844, "KL/rejected_KL_mean": -301.4088439941406, "KL/std": 102.7526626586914, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.003260301658883691, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.86772155761719, "fcm_dpo/q_t": 0.4273139238357544, "grad_norm": 7.674952030181885, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.8302216529846191, "logits/rejected": -1.2919201850891113, "logps/chosen": -306.9832763671875, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -392.50347900390625, "loss": 1.138, "margin_dpo/margin_mean": 92.86772155761719, "margin_dpo/margin_std": 111.14151763916016, "step": 538 }, { "KL/chosen_KL_mean": -179.94091796875, "KL/mean": -229.10693359375, "KL/rejected_KL_mean": -278.2729797363281, "KL/std": 111.52729034423828, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.003260301658883691, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.33206176757812, "fcm_dpo/q_t": 0.42340824007987976, "grad_norm": 7.348552703857422, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.3774914741516113, "logits/rejected": -1.893991231918335, "logps/chosen": -270.873046875, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -375.71844482421875, "loss": 1.1326, "margin_dpo/margin_mean": 98.33206176757812, "margin_dpo/margin_std": 127.13123321533203, "step": 539 }, { "KL/chosen_KL_mean": -193.78387451171875, "KL/mean": -254.03903198242188, "KL/rejected_KL_mean": -314.294189453125, "KL/std": 123.70571899414062, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.003249871777370572, "fcm_dpo/delta": -0.03209342062473297, "fcm_dpo/margin": 120.51031494140625, "fcm_dpo/q_t": 0.40855878591537476, "grad_norm": 9.347468376159668, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.168853282928467, "logits/rejected": -1.9400277137756348, "logps/chosen": -301.6537780761719, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -435.68768310546875, "loss": 1.0864, "margin_dpo/margin_mean": 120.51031494140625, "margin_dpo/margin_std": 145.7858428955078, "step": 540 }, { "KL/chosen_KL_mean": -212.3988494873047, "KL/mean": -249.87445068359375, "KL/rejected_KL_mean": -287.35003662109375, "KL/std": 100.91443634033203, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.95118713378906, "fcm_dpo/q_t": 0.4419167637825012, "grad_norm": 8.432121276855469, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.5567681789398193, "logits/rejected": -2.1123948097229004, "logps/chosen": -318.9663391113281, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -396.4163818359375, "loss": 1.1924, "margin_dpo/margin_mean": 74.95118713378906, "margin_dpo/margin_std": 109.7323226928711, "step": 541 }, { "KL/chosen_KL_mean": -181.07977294921875, "KL/mean": -238.52732849121094, "KL/rejected_KL_mean": -295.97491455078125, "KL/std": 114.15882873535156, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.89512634277344, "fcm_dpo/q_t": 0.4110707640647888, "grad_norm": 8.677638053894043, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.1104118824005127, "logits/rejected": -1.9838924407958984, "logps/chosen": -268.4490661621094, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -398.79522705078125, "loss": 1.0872, "margin_dpo/margin_mean": 114.89512634277344, "margin_dpo/margin_std": 123.78125, "step": 542 }, { "KL/chosen_KL_mean": -207.6693115234375, "KL/mean": -238.26022338867188, "KL/rejected_KL_mean": -268.85113525390625, "KL/std": 121.67361450195312, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.181800842285156, "fcm_dpo/q_t": 0.45325812697410583, "grad_norm": 10.164315223693848, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.9744274616241455, "logits/rejected": -2.0226099491119385, "logps/chosen": -294.8218994140625, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -380.3828430175781, "loss": 1.2549, "margin_dpo/margin_mean": 61.18180465698242, "margin_dpo/margin_std": 149.09906005859375, "step": 543 }, { "KL/chosen_KL_mean": -218.19400024414062, "KL/mean": -256.64874267578125, "KL/rejected_KL_mean": -295.10345458984375, "KL/std": 113.5095443725586, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.909423828125, "fcm_dpo/q_t": 0.4406411647796631, "grad_norm": 7.824803829193115, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.7367229461669922, "logits/rejected": -1.644344687461853, "logps/chosen": -304.3409118652344, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -401.49737548828125, "loss": 1.1893, "margin_dpo/margin_mean": 76.909423828125, "margin_dpo/margin_std": 120.879638671875, "step": 544 }, { "KL/chosen_KL_mean": -192.41775512695312, "KL/mean": -240.5507049560547, "KL/rejected_KL_mean": -288.68365478515625, "KL/std": 113.20428466796875, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.26589965820312, "fcm_dpo/q_t": 0.4258841872215271, "grad_norm": 7.805768966674805, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4300789833068848, "logits/rejected": -0.9983561635017395, "logps/chosen": -288.84197998046875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -389.417724609375, "loss": 1.1483, "margin_dpo/margin_mean": 96.26589965820312, "margin_dpo/margin_std": 139.8582763671875, "step": 545 }, { "KL/chosen_KL_mean": -197.97601318359375, "KL/mean": -240.30874633789062, "KL/rejected_KL_mean": -282.6414794921875, "KL/std": 114.08150482177734, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.66549682617188, "fcm_dpo/q_t": 0.4339200258255005, "grad_norm": 7.863534927368164, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.5547171831130981, "logits/rejected": -1.2028568983078003, "logps/chosen": -283.99713134765625, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -370.0141296386719, "loss": 1.1714, "margin_dpo/margin_mean": 84.66549682617188, "margin_dpo/margin_std": 126.0037841796875, "step": 546 }, { "KL/chosen_KL_mean": -190.51580810546875, "KL/mean": -242.37034606933594, "KL/rejected_KL_mean": -294.2248840332031, "KL/std": 111.89376831054688, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.70909118652344, "fcm_dpo/q_t": 0.41898149251937866, "grad_norm": 7.909639358520508, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.6168100833892822, "logits/rejected": -1.1383620500564575, "logps/chosen": -291.16680908203125, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -392.1366271972656, "loss": 1.1046, "margin_dpo/margin_mean": 103.70909118652344, "margin_dpo/margin_std": 100.5467300415039, "step": 547 }, { "KL/chosen_KL_mean": -186.82266235351562, "KL/mean": -229.37928771972656, "KL/rejected_KL_mean": -271.9359130859375, "KL/std": 113.71398162841797, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.11323547363281, "fcm_dpo/q_t": 0.43445175886154175, "grad_norm": 9.023825645446777, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.4547295570373535, "logits/rejected": -2.1096935272216797, "logps/chosen": -282.852783203125, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -370.34912109375, "loss": 1.1822, "margin_dpo/margin_mean": 85.11322784423828, "margin_dpo/margin_std": 143.5918731689453, "step": 548 }, { "KL/chosen_KL_mean": -178.01046752929688, "KL/mean": -220.97662353515625, "KL/rejected_KL_mean": -263.94281005859375, "KL/std": 108.5689697265625, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.93233489990234, "fcm_dpo/q_t": 0.43330612778663635, "grad_norm": 8.601058006286621, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.9332427978515625, "logits/rejected": -1.5241358280181885, "logps/chosen": -261.46142578125, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -345.3230285644531, "loss": 1.175, "margin_dpo/margin_mean": 85.93233489990234, "margin_dpo/margin_std": 133.61184692382812, "step": 549 }, { "KL/chosen_KL_mean": -197.52651977539062, "KL/mean": -239.56918334960938, "KL/rejected_KL_mean": -281.6118469238281, "KL/std": 102.40521240234375, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.08533477783203, "fcm_dpo/q_t": 0.4347371459007263, "grad_norm": 7.686150074005127, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.535252332687378, "logits/rejected": -1.3214629888534546, "logps/chosen": -287.120849609375, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -382.9273681640625, "loss": 1.1658, "margin_dpo/margin_mean": 84.0853271484375, "margin_dpo/margin_std": 112.4507827758789, "step": 550 }, { "KL/chosen_KL_mean": -175.69412231445312, "KL/mean": -228.12338256835938, "KL/rejected_KL_mean": -280.55267333984375, "KL/std": 117.85772705078125, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.0032394418958574533, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.8585433959961, "fcm_dpo/q_t": 0.41963905096054077, "grad_norm": 8.086027145385742, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.680680274963379, "logits/rejected": -1.542121171951294, "logps/chosen": -270.7762756347656, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -398.27191162109375, "loss": 1.1123, "margin_dpo/margin_mean": 104.85853576660156, "margin_dpo/margin_std": 120.5511474609375, "step": 551 }, { "KL/chosen_KL_mean": -167.742919921875, "KL/mean": -242.27987670898438, "KL/rejected_KL_mean": -316.81683349609375, "KL/std": 126.91950988769531, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": -0.09537231177091599, "fcm_dpo/margin": 149.0739288330078, "fcm_dpo/q_t": 0.38699859380722046, "grad_norm": 11.166348457336426, "learning_rate": 5.363104864490034e-08, "logits/chosen": -1.8683826923370361, "logits/rejected": -1.4978245496749878, "logps/chosen": -269.152099609375, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -427.6552734375, "loss": 1.0124, "margin_dpo/margin_mean": 149.07391357421875, "margin_dpo/margin_std": 132.4610595703125, "step": 552 }, { "KL/chosen_KL_mean": -181.687255859375, "KL/mean": -226.242919921875, "KL/rejected_KL_mean": -270.798583984375, "KL/std": 117.8589859008789, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.111328125, "fcm_dpo/q_t": 0.43188416957855225, "grad_norm": 7.968804359436035, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.6206395626068115, "logits/rejected": -1.2946882247924805, "logps/chosen": -279.651123046875, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -374.37579345703125, "loss": 1.1629, "margin_dpo/margin_mean": 89.11132049560547, "margin_dpo/margin_std": 126.66194152832031, "step": 553 }, { "KL/chosen_KL_mean": -189.9703369140625, "KL/mean": -239.2952880859375, "KL/rejected_KL_mean": -288.6202392578125, "KL/std": 118.50912475585938, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.64991760253906, "fcm_dpo/q_t": 0.4257536828517914, "grad_norm": 6.979705333709717, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8502933979034424, "logits/rejected": -1.0691074132919312, "logps/chosen": -293.6571350097656, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -381.61236572265625, "loss": 1.1541, "margin_dpo/margin_mean": 98.64991760253906, "margin_dpo/margin_std": 152.28224182128906, "step": 554 }, { "KL/chosen_KL_mean": -197.7806854248047, "KL/mean": -242.71263122558594, "KL/rejected_KL_mean": -287.64459228515625, "KL/std": 122.04646301269531, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.86390686035156, "fcm_dpo/q_t": 0.43246766924858093, "grad_norm": 7.68993616104126, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.8757071495056152, "logits/rejected": -1.7856922149658203, "logps/chosen": -283.98431396484375, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -402.1917724609375, "loss": 1.1691, "margin_dpo/margin_mean": 89.86390686035156, "margin_dpo/margin_std": 140.88052368164062, "step": 555 }, { "KL/chosen_KL_mean": -178.98655700683594, "KL/mean": -227.77467346191406, "KL/rejected_KL_mean": -276.5627746582031, "KL/std": 107.54499816894531, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.57621765136719, "fcm_dpo/q_t": 0.4257696568965912, "grad_norm": 8.835697174072266, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.698270320892334, "logits/rejected": -1.4044609069824219, "logps/chosen": -267.3900451660156, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -383.201904296875, "loss": 1.1372, "margin_dpo/margin_mean": 97.57621765136719, "margin_dpo/margin_std": 122.75873565673828, "step": 556 }, { "KL/chosen_KL_mean": -192.45277404785156, "KL/mean": -248.14927673339844, "KL/rejected_KL_mean": -303.84576416015625, "KL/std": 125.76103210449219, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.003178236773237586, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.39299774169922, "fcm_dpo/q_t": 0.41653555631637573, "grad_norm": 7.036273002624512, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -2.1374125480651855, "logits/rejected": -1.9680849313735962, "logps/chosen": -291.28802490234375, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -419.4946594238281, "loss": 1.11, "margin_dpo/margin_mean": 111.39299774169922, "margin_dpo/margin_std": 137.6937255859375, "step": 557 }, { "KL/chosen_KL_mean": -187.46713256835938, "KL/mean": -247.79244995117188, "KL/rejected_KL_mean": -308.1177673339844, "KL/std": 108.94729614257812, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": -0.008940746076405048, "fcm_dpo/margin": 120.65062713623047, "fcm_dpo/q_t": 0.40843868255615234, "grad_norm": 9.33481216430664, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3130805492401123, "logits/rejected": -1.7002408504486084, "logps/chosen": -293.4789733886719, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -416.50909423828125, "loss": 1.0712, "margin_dpo/margin_mean": 120.65061950683594, "margin_dpo/margin_std": 112.40098571777344, "step": 558 }, { "KL/chosen_KL_mean": -205.79547119140625, "KL/mean": -252.44276428222656, "KL/rejected_KL_mean": -299.09002685546875, "KL/std": 111.34978485107422, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.29456329345703, "fcm_dpo/q_t": 0.42875921726226807, "grad_norm": 8.744844436645508, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.243088722229004, "logits/rejected": -1.759009599685669, "logps/chosen": -306.22021484375, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -399.50274658203125, "loss": 1.1519, "margin_dpo/margin_mean": 93.29456329345703, "margin_dpo/margin_std": 127.34200286865234, "step": 559 }, { "KL/chosen_KL_mean": -193.0711669921875, "KL/mean": -240.04632568359375, "KL/rejected_KL_mean": -287.021484375, "KL/std": 108.99869537353516, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.95028686523438, "fcm_dpo/q_t": 0.4286956787109375, "grad_norm": 10.012557983398438, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.3834807872772217, "logits/rejected": -0.9399782419204712, "logps/chosen": -280.7066955566406, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -374.4013366699219, "loss": 1.1491, "margin_dpo/margin_mean": 93.95028686523438, "margin_dpo/margin_std": 126.19662475585938, "step": 560 }, { "KL/chosen_KL_mean": -203.0263671875, "KL/mean": -246.73471069335938, "KL/rejected_KL_mean": -290.4430847167969, "KL/std": 106.65776062011719, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.41671752929688, "fcm_dpo/q_t": 0.4330689013004303, "grad_norm": 7.072727680206299, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.849059820175171, "logits/rejected": -1.8214304447174072, "logps/chosen": -277.90777587890625, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -396.49102783203125, "loss": 1.1541, "margin_dpo/margin_mean": 87.41671752929688, "margin_dpo/margin_std": 103.17327880859375, "step": 561 }, { "KL/chosen_KL_mean": -184.5010986328125, "KL/mean": -228.7815399169922, "KL/rejected_KL_mean": -273.06195068359375, "KL/std": 108.70648193359375, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.56088256835938, "fcm_dpo/q_t": 0.4331892430782318, "grad_norm": 9.964574813842773, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.0186550617218018, "logits/rejected": -1.5577726364135742, "logps/chosen": -285.164306640625, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -375.3028564453125, "loss": 1.169, "margin_dpo/margin_mean": 88.56088256835938, "margin_dpo/margin_std": 134.45828247070312, "step": 562 }, { "KL/chosen_KL_mean": -193.9988555908203, "KL/mean": -230.42367553710938, "KL/rejected_KL_mean": -266.84844970703125, "KL/std": 109.38494873046875, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.84962463378906, "fcm_dpo/q_t": 0.4443323314189911, "grad_norm": 8.740386962890625, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.6381338834762573, "logits/rejected": -1.29889714717865, "logps/chosen": -290.258056640625, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -368.05352783203125, "loss": 1.2022, "margin_dpo/margin_mean": 72.84962463378906, "margin_dpo/margin_std": 116.33465576171875, "step": 563 }, { "KL/chosen_KL_mean": -189.8822784423828, "KL/mean": -231.30453491210938, "KL/rejected_KL_mean": -272.726806640625, "KL/std": 109.18863677978516, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.84454345703125, "fcm_dpo/q_t": 0.43623530864715576, "grad_norm": 9.6243314743042, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.317279100418091, "logits/rejected": -1.7432258129119873, "logps/chosen": -302.8426818847656, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -372.25482177734375, "loss": 1.166, "margin_dpo/margin_mean": 82.84452819824219, "margin_dpo/margin_std": 100.39226531982422, "step": 564 }, { "KL/chosen_KL_mean": -187.97640991210938, "KL/mean": -238.2678680419922, "KL/rejected_KL_mean": -288.559326171875, "KL/std": 118.5359115600586, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.58293151855469, "fcm_dpo/q_t": 0.42403581738471985, "grad_norm": 7.664135456085205, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.6582660675048828, "logits/rejected": -1.1515098810195923, "logps/chosen": -283.34197998046875, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -380.7767028808594, "loss": 1.1339, "margin_dpo/margin_mean": 100.58293151855469, "margin_dpo/margin_std": 130.72488403320312, "step": 565 }, { "KL/chosen_KL_mean": -196.1460723876953, "KL/mean": -244.1182403564453, "KL/rejected_KL_mean": -292.09039306640625, "KL/std": 121.36354064941406, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.94434356689453, "fcm_dpo/q_t": 0.4267733097076416, "grad_norm": 6.947885036468506, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9184284210205078, "logits/rejected": -1.4897897243499756, "logps/chosen": -295.1192626953125, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -402.91131591796875, "loss": 1.1405, "margin_dpo/margin_mean": 95.94434356689453, "margin_dpo/margin_std": 119.72806549072266, "step": 566 }, { "KL/chosen_KL_mean": -189.90228271484375, "KL/mean": -232.5454864501953, "KL/rejected_KL_mean": -275.188720703125, "KL/std": 109.42069244384766, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.2863998413086, "fcm_dpo/q_t": 0.43454045057296753, "grad_norm": 8.139785766601562, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.1963772773742676, "logits/rejected": -1.514394998550415, "logps/chosen": -282.60589599609375, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -366.41302490234375, "loss": 1.1688, "margin_dpo/margin_mean": 85.2863998413086, "margin_dpo/margin_std": 116.04129028320312, "step": 567 }, { "KL/chosen_KL_mean": -179.33209228515625, "KL/mean": -236.41708374023438, "KL/rejected_KL_mean": -293.5020751953125, "KL/std": 115.00941467285156, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.16999816894531, "fcm_dpo/q_t": 0.4135008156299591, "grad_norm": 7.385034561157227, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.2787621021270752, "logits/rejected": -1.1494121551513672, "logps/chosen": -252.46435546875, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -388.4547119140625, "loss": 1.0902, "margin_dpo/margin_mean": 114.16999816894531, "margin_dpo/margin_std": 117.23577117919922, "step": 568 }, { "KL/chosen_KL_mean": -189.62258911132812, "KL/mean": -236.51947021484375, "KL/rejected_KL_mean": -283.4163818359375, "KL/std": 117.49772644042969, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.79376220703125, "fcm_dpo/q_t": 0.42880523204803467, "grad_norm": 10.143410682678223, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.848665475845337, "logits/rejected": -1.4635474681854248, "logps/chosen": -278.83966064453125, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -369.8274230957031, "loss": 1.1603, "margin_dpo/margin_mean": 93.79376983642578, "margin_dpo/margin_std": 141.90924072265625, "step": 569 }, { "KL/chosen_KL_mean": -170.39723205566406, "KL/mean": -230.64675903320312, "KL/rejected_KL_mean": -290.8962707519531, "KL/std": 111.27423095703125, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.4990234375, "fcm_dpo/q_t": 0.40827298164367676, "grad_norm": 7.6482720375061035, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.4274916648864746, "logits/rejected": -1.760793685913086, "logps/chosen": -265.1600036621094, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -379.7431945800781, "loss": 1.0733, "margin_dpo/margin_mean": 120.49903106689453, "margin_dpo/margin_std": 116.72915649414062, "step": 570 }, { "KL/chosen_KL_mean": -178.57810974121094, "KL/mean": -231.5008544921875, "KL/rejected_KL_mean": -284.423583984375, "KL/std": 108.5894546508789, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.845458984375, "fcm_dpo/q_t": 0.42009437084198, "grad_norm": 10.952875137329102, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7936301231384277, "logits/rejected": -1.6394460201263428, "logps/chosen": -259.3994140625, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -381.4388732910156, "loss": 1.121, "margin_dpo/margin_mean": 105.845458984375, "margin_dpo/margin_std": 131.96640014648438, "step": 571 }, { "KL/chosen_KL_mean": -182.99339294433594, "KL/mean": -236.2998046875, "KL/rejected_KL_mean": -289.606201171875, "KL/std": 116.04554748535156, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.61280822753906, "fcm_dpo/q_t": 0.4198009669780731, "grad_norm": 7.543960094451904, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.6445589065551758, "logits/rejected": -1.4730207920074463, "logps/chosen": -260.3733215332031, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -380.17218017578125, "loss": 1.1139, "margin_dpo/margin_mean": 106.61280822753906, "margin_dpo/margin_std": 124.4512939453125, "step": 572 }, { "KL/chosen_KL_mean": -177.89004516601562, "KL/mean": -227.35751342773438, "KL/rejected_KL_mean": -276.8249816894531, "KL/std": 115.03153228759766, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.93495178222656, "fcm_dpo/q_t": 0.4246794283390045, "grad_norm": 7.593951225280762, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.081737995147705, "logits/rejected": -1.8492331504821777, "logps/chosen": -261.59613037109375, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -367.5224609375, "loss": 1.1355, "margin_dpo/margin_mean": 98.93495178222656, "margin_dpo/margin_std": 122.96702575683594, "step": 573 }, { "KL/chosen_KL_mean": -187.81163024902344, "KL/mean": -241.83914184570312, "KL/rejected_KL_mean": -295.86663818359375, "KL/std": 114.37353515625, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.05500793457031, "fcm_dpo/q_t": 0.41819465160369873, "grad_norm": 7.878485202789307, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.320751667022705, "logits/rejected": -2.0034642219543457, "logps/chosen": -272.94110107421875, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -394.9206848144531, "loss": 1.1134, "margin_dpo/margin_mean": 108.05500793457031, "margin_dpo/margin_std": 128.8806610107422, "step": 574 }, { "KL/chosen_KL_mean": -194.15322875976562, "KL/mean": -243.363525390625, "KL/rejected_KL_mean": -292.57379150390625, "KL/std": 116.79057312011719, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.42057037353516, "fcm_dpo/q_t": 0.42574119567871094, "grad_norm": 8.874051094055176, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.126708745956421, "logits/rejected": -0.6403757333755493, "logps/chosen": -299.924072265625, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -393.52325439453125, "loss": 1.1433, "margin_dpo/margin_mean": 98.42057037353516, "margin_dpo/margin_std": 136.57981872558594, "step": 575 }, { "KL/chosen_KL_mean": -192.27597045898438, "KL/mean": -240.41152954101562, "KL/rejected_KL_mean": -288.547119140625, "KL/std": 108.98869323730469, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.003172558732330799, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.27114868164062, "fcm_dpo/q_t": 0.4268459379673004, "grad_norm": 9.353522300720215, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.9957120418548584, "logits/rejected": -1.9369118213653564, "logps/chosen": -273.4804992675781, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -393.9565124511719, "loss": 1.1457, "margin_dpo/margin_mean": 96.27114868164062, "margin_dpo/margin_std": 130.0897216796875, "step": 576 }, { "KL/chosen_KL_mean": -172.32423400878906, "KL/mean": -231.8932342529297, "KL/rejected_KL_mean": -291.46221923828125, "KL/std": 117.7230224609375, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": -0.015163488686084747, "fcm_dpo/margin": 119.13798522949219, "fcm_dpo/q_t": 0.4103591740131378, "grad_norm": 6.850360870361328, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.9609417915344238, "logits/rejected": -1.259368896484375, "logps/chosen": -273.7884521484375, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -380.7886047363281, "loss": 1.0888, "margin_dpo/margin_mean": 119.13798522949219, "margin_dpo/margin_std": 134.53831481933594, "step": 577 }, { "KL/chosen_KL_mean": -205.72935485839844, "KL/mean": -249.46255493164062, "KL/rejected_KL_mean": -293.19573974609375, "KL/std": 112.45337677001953, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.46638488769531, "fcm_dpo/q_t": 0.4337363839149475, "grad_norm": 11.565193176269531, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.350067138671875, "logits/rejected": -1.1102031469345093, "logps/chosen": -283.02178955078125, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -381.4427490234375, "loss": 1.1688, "margin_dpo/margin_mean": 87.46638488769531, "margin_dpo/margin_std": 128.52316284179688, "step": 578 }, { "KL/chosen_KL_mean": -184.61907958984375, "KL/mean": -228.60357666015625, "KL/rejected_KL_mean": -272.588134765625, "KL/std": 111.45071411132812, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.9690170288086, "fcm_dpo/q_t": 0.4333972930908203, "grad_norm": 6.914161205291748, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.6848514080047607, "logits/rejected": -1.392615795135498, "logps/chosen": -271.9621276855469, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -374.7091064453125, "loss": 1.1614, "margin_dpo/margin_mean": 87.9690170288086, "margin_dpo/margin_std": 113.71694946289062, "step": 579 }, { "KL/chosen_KL_mean": -176.16622924804688, "KL/mean": -230.3125, "KL/rejected_KL_mean": -284.45880126953125, "KL/std": 112.80050659179688, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.29254150390625, "fcm_dpo/q_t": 0.4192730784416199, "grad_norm": 8.782660484313965, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.7384636402130127, "logits/rejected": -1.435481309890747, "logps/chosen": -265.09503173828125, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -373.1553955078125, "loss": 1.1184, "margin_dpo/margin_mean": 108.29252624511719, "margin_dpo/margin_std": 137.513671875, "step": 580 }, { "KL/chosen_KL_mean": -186.46261596679688, "KL/mean": -235.7542724609375, "KL/rejected_KL_mean": -285.04595947265625, "KL/std": 112.422607421875, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.5833511352539, "fcm_dpo/q_t": 0.4256318211555481, "grad_norm": 8.792887687683105, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.9580172300338745, "logits/rejected": -1.6088459491729736, "logps/chosen": -278.68585205078125, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -384.4353942871094, "loss": 1.1467, "margin_dpo/margin_mean": 98.58335876464844, "margin_dpo/margin_std": 140.75860595703125, "step": 581 }, { "KL/chosen_KL_mean": -165.3528594970703, "KL/mean": -209.95791625976562, "KL/rejected_KL_mean": -254.5629425048828, "KL/std": 117.87429809570312, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.21009063720703, "fcm_dpo/q_t": 0.43364256620407104, "grad_norm": 8.67624282836914, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.185642719268799, "logits/rejected": -2.1779932975769043, "logps/chosen": -252.6884765625, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -369.71734619140625, "loss": 1.1683, "margin_dpo/margin_mean": 89.21009063720703, "margin_dpo/margin_std": 130.05125427246094, "step": 582 }, { "KL/chosen_KL_mean": -198.5111083984375, "KL/mean": -248.30860900878906, "KL/rejected_KL_mean": -298.10614013671875, "KL/std": 111.59788513183594, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.003162951907142997, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.59500122070312, "fcm_dpo/q_t": 0.4253222346305847, "grad_norm": 10.099176406860352, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4626103639602661, "logits/rejected": -1.2271606922149658, "logps/chosen": -282.2281799316406, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -392.6907958984375, "loss": 1.1389, "margin_dpo/margin_mean": 99.59501647949219, "margin_dpo/margin_std": 132.80418395996094, "step": 583 }, { "KL/chosen_KL_mean": -176.8873291015625, "KL/mean": -229.1917724609375, "KL/rejected_KL_mean": -281.4962463378906, "KL/std": 115.08943939208984, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.003156842663884163, "fcm_dpo/delta": -0.019352423027157784, "fcm_dpo/margin": 104.6089096069336, "fcm_dpo/q_t": 0.42092764377593994, "grad_norm": 6.547236442565918, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -2.0490238666534424, "logits/rejected": -1.6032390594482422, "logps/chosen": -271.1468811035156, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -385.97894287109375, "loss": 1.127, "margin_dpo/margin_mean": 104.60890197753906, "margin_dpo/margin_std": 130.51597595214844, "step": 584 }, { "KL/chosen_KL_mean": -188.05422973632812, "KL/mean": -246.909912109375, "KL/rejected_KL_mean": -305.765625, "KL/std": 131.92446899414062, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.71139526367188, "fcm_dpo/q_t": 0.4131510853767395, "grad_norm": 7.4224019050598145, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.8644791841506958, "logits/rejected": -1.587982177734375, "logps/chosen": -273.69097900390625, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -396.05108642578125, "loss": 1.1075, "margin_dpo/margin_mean": 117.71138000488281, "margin_dpo/margin_std": 156.39085388183594, "step": 585 }, { "KL/chosen_KL_mean": -209.29556274414062, "KL/mean": -248.21836853027344, "KL/rejected_KL_mean": -287.1412048339844, "KL/std": 114.60346984863281, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.84562683105469, "fcm_dpo/q_t": 0.4407142102718353, "grad_norm": 8.694780349731445, "learning_rate": 2.974695142855388e-08, "logits/chosen": -2.053757429122925, "logits/rejected": -1.8007800579071045, "logps/chosen": -292.9766845703125, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -385.5003662109375, "loss": 1.1861, "margin_dpo/margin_mean": 77.84562683105469, "margin_dpo/margin_std": 110.4765625, "step": 586 }, { "KL/chosen_KL_mean": -168.72769165039062, "KL/mean": -211.37164306640625, "KL/rejected_KL_mean": -254.01559448242188, "KL/std": 111.85173034667969, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.28793334960938, "fcm_dpo/q_t": 0.4354180693626404, "grad_norm": 9.052767753601074, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.327927589416504, "logits/rejected": -2.0455079078674316, "logps/chosen": -246.30519104003906, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -343.3359375, "loss": 1.1681, "margin_dpo/margin_mean": 85.28793334960938, "margin_dpo/margin_std": 116.35154724121094, "step": 587 }, { "KL/chosen_KL_mean": -185.30630493164062, "KL/mean": -225.53485107421875, "KL/rejected_KL_mean": -265.76336669921875, "KL/std": 113.30691528320312, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.45707702636719, "fcm_dpo/q_t": 0.43997564911842346, "grad_norm": 11.912144660949707, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.706125259399414, "logits/rejected": -1.6709671020507812, "logps/chosen": -269.1105651855469, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -378.8828125, "loss": 1.1939, "margin_dpo/margin_mean": 80.45707702636719, "margin_dpo/margin_std": 136.847900390625, "step": 588 }, { "KL/chosen_KL_mean": -198.62750244140625, "KL/mean": -251.43942260742188, "KL/rejected_KL_mean": -304.2513427734375, "KL/std": 111.55789184570312, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.62387084960938, "fcm_dpo/q_t": 0.41980454325675964, "grad_norm": 7.36534309387207, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.5611798763275146, "logits/rejected": -1.1669960021972656, "logps/chosen": -295.092529296875, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -420.2885437011719, "loss": 1.1087, "margin_dpo/margin_mean": 105.62387084960938, "margin_dpo/margin_std": 107.24911499023438, "step": 589 }, { "KL/chosen_KL_mean": -181.0082244873047, "KL/mean": -233.885009765625, "KL/rejected_KL_mean": -286.7618103027344, "KL/std": 103.61112976074219, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.75358581542969, "fcm_dpo/q_t": 0.4201553463935852, "grad_norm": 9.985930442810059, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.8213601112365723, "logits/rejected": -1.678665041923523, "logps/chosen": -269.43609619140625, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -396.9605712890625, "loss": 1.1097, "margin_dpo/margin_mean": 105.75358581542969, "margin_dpo/margin_std": 109.1496810913086, "step": 590 }, { "KL/chosen_KL_mean": -173.15542602539062, "KL/mean": -229.03675842285156, "KL/rejected_KL_mean": -284.9180908203125, "KL/std": 110.99236297607422, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.76268768310547, "fcm_dpo/q_t": 0.4154403805732727, "grad_norm": 6.189868927001953, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.8986079692840576, "logits/rejected": -1.2839877605438232, "logps/chosen": -261.005126953125, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -374.497314453125, "loss": 1.0931, "margin_dpo/margin_mean": 111.7626953125, "margin_dpo/margin_std": 107.58427429199219, "step": 591 }, { "KL/chosen_KL_mean": -205.88653564453125, "KL/mean": -248.31439208984375, "KL/rejected_KL_mean": -290.74224853515625, "KL/std": 109.34471893310547, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.855712890625, "fcm_dpo/q_t": 0.43581974506378174, "grad_norm": 9.133187294006348, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.910834550857544, "logits/rejected": -1.58935546875, "logps/chosen": -280.8622741699219, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -372.29486083984375, "loss": 1.1828, "margin_dpo/margin_mean": 84.855712890625, "margin_dpo/margin_std": 138.40345764160156, "step": 592 }, { "KL/chosen_KL_mean": -186.73794555664062, "KL/mean": -224.00009155273438, "KL/rejected_KL_mean": -261.26220703125, "KL/std": 113.83757019042969, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.52426147460938, "fcm_dpo/q_t": 0.44387704133987427, "grad_norm": 11.905708312988281, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.6447526812553406, "logits/rejected": -0.3333684802055359, "logps/chosen": -258.97698974609375, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -331.3587646484375, "loss": 1.2039, "margin_dpo/margin_mean": 74.52426147460938, "margin_dpo/margin_std": 125.87034606933594, "step": 593 }, { "KL/chosen_KL_mean": -167.7000732421875, "KL/mean": -224.64271545410156, "KL/rejected_KL_mean": -281.5853576660156, "KL/std": 110.71220397949219, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.88529968261719, "fcm_dpo/q_t": 0.41392990946769714, "grad_norm": 7.832910537719727, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9329581260681152, "logits/rejected": -1.4097390174865723, "logps/chosen": -258.1377868652344, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -373.8529357910156, "loss": 1.0881, "margin_dpo/margin_mean": 113.88530731201172, "margin_dpo/margin_std": 108.28773498535156, "step": 594 }, { "KL/chosen_KL_mean": -180.20303344726562, "KL/mean": -226.70504760742188, "KL/rejected_KL_mean": -273.20709228515625, "KL/std": 111.3542251586914, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.00405883789062, "fcm_dpo/q_t": 0.43030524253845215, "grad_norm": 7.672323226928711, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.9246888160705566, "logits/rejected": -1.5283567905426025, "logps/chosen": -261.71783447265625, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -359.7275390625, "loss": 1.1541, "margin_dpo/margin_mean": 93.00405883789062, "margin_dpo/margin_std": 127.79142761230469, "step": 595 }, { "KL/chosen_KL_mean": -188.50784301757812, "KL/mean": -229.85594177246094, "KL/rejected_KL_mean": -271.20404052734375, "KL/std": 109.97270965576172, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.6961669921875, "fcm_dpo/q_t": 0.4380682110786438, "grad_norm": 9.98623275756836, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.7072044610977173, "logits/rejected": -1.1358757019042969, "logps/chosen": -276.71588134765625, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -357.29779052734375, "loss": 1.1899, "margin_dpo/margin_mean": 82.6961669921875, "margin_dpo/margin_std": 139.55116271972656, "step": 596 }, { "KL/chosen_KL_mean": -183.04916381835938, "KL/mean": -238.55377197265625, "KL/rejected_KL_mean": -294.05841064453125, "KL/std": 117.93161010742188, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.00923919677734, "fcm_dpo/q_t": 0.41686010360717773, "grad_norm": 7.822099685668945, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.4878970384597778, "logits/rejected": -1.3745884895324707, "logps/chosen": -266.6391296386719, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -401.0028076171875, "loss": 1.1055, "margin_dpo/margin_mean": 111.00923919677734, "margin_dpo/margin_std": 127.21815490722656, "step": 597 }, { "KL/chosen_KL_mean": -174.09844970703125, "KL/mean": -221.5681610107422, "KL/rejected_KL_mean": -269.037841796875, "KL/std": 110.25518798828125, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.93939208984375, "fcm_dpo/q_t": 0.4279315769672394, "grad_norm": 8.203577995300293, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.9749321937561035, "logits/rejected": -1.6642273664474487, "logps/chosen": -264.28106689453125, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -363.83642578125, "loss": 1.148, "margin_dpo/margin_mean": 94.93939971923828, "margin_dpo/margin_std": 124.31063079833984, "step": 598 }, { "KL/chosen_KL_mean": -175.36721801757812, "KL/mean": -217.30955505371094, "KL/rejected_KL_mean": -259.25189208984375, "KL/std": 115.16212463378906, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.88467407226562, "fcm_dpo/q_t": 0.4364261031150818, "grad_norm": 7.345400333404541, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.7156729698181152, "logits/rejected": -1.5954644680023193, "logps/chosen": -253.20948791503906, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -348.44952392578125, "loss": 1.1803, "margin_dpo/margin_mean": 83.88468170166016, "margin_dpo/margin_std": 127.73529052734375, "step": 599 }, { "KL/chosen_KL_mean": -197.98379516601562, "KL/mean": -245.10723876953125, "KL/rejected_KL_mean": -292.2307434082031, "KL/std": 108.67285919189453, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.2469482421875, "fcm_dpo/q_t": 0.4289044439792633, "grad_norm": 8.116325378417969, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.341055393218994, "logits/rejected": -1.9934622049331665, "logps/chosen": -290.1716003417969, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -385.1850280761719, "loss": 1.1454, "margin_dpo/margin_mean": 94.2469482421875, "margin_dpo/margin_std": 117.85382080078125, "step": 600 }, { "KL/chosen_KL_mean": -197.83251953125, "KL/mean": -248.5717010498047, "KL/rejected_KL_mean": -299.3109130859375, "KL/std": 110.60934448242188, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.4783706665039, "fcm_dpo/q_t": 0.4230826795101166, "grad_norm": 7.698277950286865, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7124583721160889, "logits/rejected": -1.385046124458313, "logps/chosen": -287.04864501953125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -399.4814453125, "loss": 1.1194, "margin_dpo/margin_mean": 101.4783706665039, "margin_dpo/margin_std": 105.77505493164062, "step": 601 }, { "KL/chosen_KL_mean": -158.03439331054688, "KL/mean": -222.68907165527344, "KL/rejected_KL_mean": -287.34375, "KL/std": 121.12591552734375, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.30935668945312, "fcm_dpo/q_t": 0.4039660096168518, "grad_norm": 8.272041320800781, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.1480319499969482, "logits/rejected": -1.88950514793396, "logps/chosen": -238.0919952392578, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -380.541259765625, "loss": 1.0652, "margin_dpo/margin_mean": 129.30935668945312, "margin_dpo/margin_std": 138.99658203125, "step": 602 }, { "KL/chosen_KL_mean": -168.51333618164062, "KL/mean": -222.58685302734375, "KL/rejected_KL_mean": -276.660400390625, "KL/std": 109.20457458496094, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.14704132080078, "fcm_dpo/q_t": 0.4187895655632019, "grad_norm": 6.718961715698242, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.219219207763672, "logits/rejected": -1.649443507194519, "logps/chosen": -270.82293701171875, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -384.729248046875, "loss": 1.1117, "margin_dpo/margin_mean": 108.14703369140625, "margin_dpo/margin_std": 125.78421020507812, "step": 603 }, { "KL/chosen_KL_mean": -177.0880126953125, "KL/mean": -232.6240234375, "KL/rejected_KL_mean": -288.1600341796875, "KL/std": 114.2822036743164, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.07205200195312, "fcm_dpo/q_t": 0.4156600832939148, "grad_norm": 7.920666694641113, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.7532963752746582, "logits/rejected": -1.6019339561462402, "logps/chosen": -255.2620849609375, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -387.6561584472656, "loss": 1.1006, "margin_dpo/margin_mean": 111.07205200195312, "margin_dpo/margin_std": 117.76295471191406, "step": 604 }, { "KL/chosen_KL_mean": -193.49639892578125, "KL/mean": -239.09225463867188, "KL/rejected_KL_mean": -284.6881103515625, "KL/std": 102.01020812988281, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.19171905517578, "fcm_dpo/q_t": 0.43108677864074707, "grad_norm": 8.000343322753906, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8180413246154785, "logits/rejected": -1.3234832286834717, "logps/chosen": -288.26971435546875, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -391.9930114746094, "loss": 1.1515, "margin_dpo/margin_mean": 91.19171905517578, "margin_dpo/margin_std": 115.30633544921875, "step": 605 }, { "KL/chosen_KL_mean": -186.07504272460938, "KL/mean": -233.80377197265625, "KL/rejected_KL_mean": -281.532470703125, "KL/std": 112.26156616210938, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.45744323730469, "fcm_dpo/q_t": 0.42759907245635986, "grad_norm": 6.96083927154541, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.9330947399139404, "logits/rejected": -1.555093765258789, "logps/chosen": -273.82843017578125, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -385.7746887207031, "loss": 1.1442, "margin_dpo/margin_mean": 95.45743560791016, "margin_dpo/margin_std": 120.3873062133789, "step": 606 }, { "KL/chosen_KL_mean": -180.44595336914062, "KL/mean": -233.86260986328125, "KL/rejected_KL_mean": -287.279296875, "KL/std": 119.70184326171875, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.003150733420625329, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.83331298828125, "fcm_dpo/q_t": 0.42035096883773804, "grad_norm": 9.626907348632812, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7588562965393066, "logits/rejected": -1.2327852249145508, "logps/chosen": -268.7749938964844, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -374.04736328125, "loss": 1.1298, "margin_dpo/margin_mean": 106.83332061767578, "margin_dpo/margin_std": 147.4442596435547, "step": 607 }, { "KL/chosen_KL_mean": -164.15951538085938, "KL/mean": -219.70062255859375, "KL/rejected_KL_mean": -275.2417297363281, "KL/std": 117.38926696777344, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0031354136299341917, "fcm_dpo/delta": -0.024370815604925156, "fcm_dpo/margin": 111.08221435546875, "fcm_dpo/q_t": 0.41636747121810913, "grad_norm": 7.095462322235107, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.9512650966644287, "logits/rejected": -1.6596546173095703, "logps/chosen": -249.92889404296875, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -375.47454833984375, "loss": 1.1103, "margin_dpo/margin_mean": 111.08221435546875, "margin_dpo/margin_std": 128.59034729003906, "step": 608 }, { "KL/chosen_KL_mean": -184.5853729248047, "KL/mean": -235.88861083984375, "KL/rejected_KL_mean": -287.19183349609375, "KL/std": 118.77293395996094, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0031354136299341917, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.6064453125, "fcm_dpo/q_t": 0.4235478937625885, "grad_norm": 8.931703567504883, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.0193607807159424, "logits/rejected": -1.5825717449188232, "logps/chosen": -277.55194091796875, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -383.1099853515625, "loss": 1.1316, "margin_dpo/margin_mean": 102.6064453125, "margin_dpo/margin_std": 126.21072387695312, "step": 609 }, { "KL/chosen_KL_mean": -168.45947265625, "KL/mean": -224.1175994873047, "KL/rejected_KL_mean": -279.7757263183594, "KL/std": 112.54822540283203, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0031354136299341917, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.31623840332031, "fcm_dpo/q_t": 0.4160533547401428, "grad_norm": 7.413478851318359, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.5659962892532349, "logits/rejected": -1.0065752267837524, "logps/chosen": -263.1597595214844, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -369.463134765625, "loss": 1.1017, "margin_dpo/margin_mean": 111.31624603271484, "margin_dpo/margin_std": 119.59022521972656, "step": 610 }, { "KL/chosen_KL_mean": -177.46363830566406, "KL/mean": -232.42068481445312, "KL/rejected_KL_mean": -287.37774658203125, "KL/std": 116.86863708496094, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0031354136299341917, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.91413879394531, "fcm_dpo/q_t": 0.417890727519989, "grad_norm": 8.851881980895996, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.1245343685150146, "logits/rejected": -1.7850650548934937, "logps/chosen": -264.2469787597656, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -377.2208251953125, "loss": 1.1058, "margin_dpo/margin_mean": 109.91413879394531, "margin_dpo/margin_std": 119.0347900390625, "step": 611 }, { "KL/chosen_KL_mean": -196.66156005859375, "KL/mean": -252.7989501953125, "KL/rejected_KL_mean": -308.936279296875, "KL/std": 111.10398864746094, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0031354136299341917, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.27471160888672, "fcm_dpo/q_t": 0.41577237844467163, "grad_norm": 7.9901885986328125, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.2296996116638184, "logits/rejected": -1.4129842519760132, "logps/chosen": -297.68170166015625, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -402.7193298339844, "loss": 1.0977, "margin_dpo/margin_mean": 112.27471160888672, "margin_dpo/margin_std": 117.75897216796875, "step": 612 }, { "KL/chosen_KL_mean": -156.64892578125, "KL/mean": -220.47991943359375, "KL/rejected_KL_mean": -284.3109130859375, "KL/std": 113.50990295410156, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": -0.011844899505376816, "fcm_dpo/margin": 127.6620101928711, "fcm_dpo/q_t": 0.40424418449401855, "grad_norm": 7.193422794342041, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.8755114078521729, "logits/rejected": -1.6777077913284302, "logps/chosen": -245.63754272460938, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -394.30645751953125, "loss": 1.0557, "margin_dpo/margin_mean": 127.6620101928711, "margin_dpo/margin_std": 111.80007934570312, "step": 613 }, { "KL/chosen_KL_mean": -195.73831176757812, "KL/mean": -247.72488403320312, "KL/rejected_KL_mean": -299.71142578125, "KL/std": 107.72708129882812, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.9731216430664, "fcm_dpo/q_t": 0.422094464302063, "grad_norm": 9.200819969177246, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.331017017364502, "logits/rejected": -1.3248867988586426, "logps/chosen": -275.9383544921875, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -409.5738220214844, "loss": 1.1298, "margin_dpo/margin_mean": 103.97311401367188, "margin_dpo/margin_std": 133.04771423339844, "step": 614 }, { "KL/chosen_KL_mean": -183.98760986328125, "KL/mean": -225.282470703125, "KL/rejected_KL_mean": -266.57733154296875, "KL/std": 108.51956176757812, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.58973693847656, "fcm_dpo/q_t": 0.43742650747299194, "grad_norm": 7.979496002197266, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.6506582498550415, "logits/rejected": -1.2492780685424805, "logps/chosen": -284.4228515625, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -367.75738525390625, "loss": 1.1728, "margin_dpo/margin_mean": 82.58973693847656, "margin_dpo/margin_std": 104.12202453613281, "step": 615 }, { "KL/chosen_KL_mean": -186.85012817382812, "KL/mean": -244.12313842773438, "KL/rejected_KL_mean": -301.3961181640625, "KL/std": 108.01141357421875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.54598999023438, "fcm_dpo/q_t": 0.4145500063896179, "grad_norm": 7.713818550109863, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.8352738618850708, "logits/rejected": -1.3425803184509277, "logps/chosen": -279.34307861328125, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -383.456787109375, "loss": 1.0984, "margin_dpo/margin_mean": 114.54598999023438, "margin_dpo/margin_std": 127.227294921875, "step": 616 }, { "KL/chosen_KL_mean": -193.22308349609375, "KL/mean": -238.76116943359375, "KL/rejected_KL_mean": -284.2992858886719, "KL/std": 105.90592956542969, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.07620239257812, "fcm_dpo/q_t": 0.43133533000946045, "grad_norm": 7.76587438583374, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6316235065460205, "logits/rejected": -0.9982467889785767, "logps/chosen": -284.320068359375, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -354.7093505859375, "loss": 1.1563, "margin_dpo/margin_mean": 91.07620239257812, "margin_dpo/margin_std": 119.64820861816406, "step": 617 }, { "KL/chosen_KL_mean": -195.73208618164062, "KL/mean": -247.57852172851562, "KL/rejected_KL_mean": -299.42498779296875, "KL/std": 117.928466796875, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.69287109375, "fcm_dpo/q_t": 0.4219128489494324, "grad_norm": 7.370098114013672, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.8884258270263672, "logits/rejected": -1.5189456939697266, "logps/chosen": -297.752685546875, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -399.2261657714844, "loss": 1.1288, "margin_dpo/margin_mean": 103.69287109375, "margin_dpo/margin_std": 130.20013427734375, "step": 618 }, { "KL/chosen_KL_mean": -189.84146118164062, "KL/mean": -235.31118774414062, "KL/rejected_KL_mean": -280.7808837890625, "KL/std": 115.89726257324219, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.003127994714304805, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.93948364257812, "fcm_dpo/q_t": 0.4329695701599121, "grad_norm": 7.3020429611206055, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.3978182077407837, "logits/rejected": -1.0291481018066406, "logps/chosen": -279.58282470703125, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -380.6822814941406, "loss": 1.1673, "margin_dpo/margin_mean": 90.9394760131836, "margin_dpo/margin_std": 139.17025756835938, "step": 619 }, { "KL/chosen_KL_mean": -181.78330993652344, "KL/mean": -231.0501708984375, "KL/rejected_KL_mean": -280.3170166015625, "KL/std": 121.60171508789062, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.003112836740911007, "fcm_dpo/delta": -0.02428833208978176, "fcm_dpo/margin": 98.5337142944336, "fcm_dpo/q_t": 0.4267125129699707, "grad_norm": 8.21114444732666, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.34804368019104, "logits/rejected": -1.8376202583312988, "logps/chosen": -266.9076232910156, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -365.72955322265625, "loss": 1.1482, "margin_dpo/margin_mean": 98.5337142944336, "margin_dpo/margin_std": 131.5443115234375, "step": 620 }, { "KL/chosen_KL_mean": -184.6741943359375, "KL/mean": -233.54531860351562, "KL/rejected_KL_mean": -282.4164733886719, "KL/std": 117.61843872070312, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.003106296295300126, "fcm_dpo/delta": -0.021055439487099648, "fcm_dpo/margin": 97.74227142333984, "fcm_dpo/q_t": 0.42762893438339233, "grad_norm": 7.997491836547852, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -2.0748517513275146, "logits/rejected": -1.8179851770401, "logps/chosen": -273.922607421875, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -377.881103515625, "loss": 1.1545, "margin_dpo/margin_mean": 97.74227142333984, "margin_dpo/margin_std": 137.4414520263672, "step": 621 }, { "KL/chosen_KL_mean": -183.0408172607422, "KL/mean": -235.33740234375, "KL/rejected_KL_mean": -287.6339416503906, "KL/std": 105.35299682617188, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.5931167602539, "fcm_dpo/q_t": 0.42204397916793823, "grad_norm": 8.387816429138184, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.3841347694396973, "logits/rejected": -0.7205435633659363, "logps/chosen": -282.8349609375, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -378.462158203125, "loss": 1.1196, "margin_dpo/margin_mean": 104.59312438964844, "margin_dpo/margin_std": 115.57538604736328, "step": 622 }, { "KL/chosen_KL_mean": -189.61868286132812, "KL/mean": -226.5292205810547, "KL/rejected_KL_mean": -263.4397888183594, "KL/std": 113.50151824951172, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.82110595703125, "fcm_dpo/q_t": 0.44558805227279663, "grad_norm": 7.883606910705566, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.2601571083068848, "logits/rejected": -1.8253154754638672, "logps/chosen": -280.07421875, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -354.7625427246094, "loss": 1.2162, "margin_dpo/margin_mean": 73.82110595703125, "margin_dpo/margin_std": 139.2568817138672, "step": 623 }, { "KL/chosen_KL_mean": -198.02015686035156, "KL/mean": -256.9413146972656, "KL/rejected_KL_mean": -315.8624572753906, "KL/std": 112.91099548339844, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.84229278564453, "fcm_dpo/q_t": 0.41305655241012573, "grad_norm": 8.496374130249023, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.585076332092285, "logits/rejected": -1.6841087341308594, "logps/chosen": -307.8953857421875, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -420.63568115234375, "loss": 1.0934, "margin_dpo/margin_mean": 117.84229278564453, "margin_dpo/margin_std": 130.50270080566406, "step": 624 }, { "KL/chosen_KL_mean": -204.0675506591797, "KL/mean": -246.09152221679688, "KL/rejected_KL_mean": -288.115478515625, "KL/std": 110.99606323242188, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.04794311523438, "fcm_dpo/q_t": 0.43748342990875244, "grad_norm": 7.542810916900635, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.8343758583068848, "logits/rejected": -1.470095157623291, "logps/chosen": -291.2357177734375, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -379.97698974609375, "loss": 1.176, "margin_dpo/margin_mean": 84.04793548583984, "margin_dpo/margin_std": 119.44215393066406, "step": 625 }, { "KL/chosen_KL_mean": -192.91876220703125, "KL/mean": -235.40042114257812, "KL/rejected_KL_mean": -277.882080078125, "KL/std": 123.1500473022461, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.96333312988281, "fcm_dpo/q_t": 0.4373732805252075, "grad_norm": 9.474620819091797, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.4029515981674194, "logits/rejected": -0.9693394899368286, "logps/chosen": -297.1429748535156, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -382.0595703125, "loss": 1.1825, "margin_dpo/margin_mean": 84.96333312988281, "margin_dpo/margin_std": 134.7731475830078, "step": 626 }, { "KL/chosen_KL_mean": -170.04278564453125, "KL/mean": -224.51651000976562, "KL/rejected_KL_mean": -278.9902648925781, "KL/std": 126.52397155761719, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.94744873046875, "fcm_dpo/q_t": 0.4203978180885315, "grad_norm": 6.113049030303955, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.7298521995544434, "logits/rejected": -1.8734867572784424, "logps/chosen": -244.97740173339844, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -391.5631408691406, "loss": 1.1215, "margin_dpo/margin_mean": 108.94746398925781, "margin_dpo/margin_std": 140.08351135253906, "step": 627 }, { "KL/chosen_KL_mean": -181.14303588867188, "KL/mean": -236.32846069335938, "KL/rejected_KL_mean": -291.51385498046875, "KL/std": 114.7253189086914, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.37083435058594, "fcm_dpo/q_t": 0.4184967279434204, "grad_norm": 8.910543441772461, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.9561101198196411, "logits/rejected": -1.6950714588165283, "logps/chosen": -274.8341064453125, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -398.85784912109375, "loss": 1.1128, "margin_dpo/margin_mean": 110.37083435058594, "margin_dpo/margin_std": 130.40377807617188, "step": 628 }, { "KL/chosen_KL_mean": -194.1273956298828, "KL/mean": -244.88919067382812, "KL/rejected_KL_mean": -295.6510009765625, "KL/std": 114.90979766845703, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.52360534667969, "fcm_dpo/q_t": 0.42418527603149414, "grad_norm": 7.930535793304443, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.017317295074463, "logits/rejected": -1.5781127214431763, "logps/chosen": -297.3577880859375, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -392.81939697265625, "loss": 1.1317, "margin_dpo/margin_mean": 101.52360534667969, "margin_dpo/margin_std": 122.4766845703125, "step": 629 }, { "KL/chosen_KL_mean": -177.43154907226562, "KL/mean": -209.03939819335938, "KL/rejected_KL_mean": -240.64723205566406, "KL/std": 106.48727416992188, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.215675354003906, "fcm_dpo/q_t": 0.4526708424091339, "grad_norm": 9.483389854431152, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.0577945709228516, "logits/rejected": -1.540987253189087, "logps/chosen": -271.3291015625, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -330.0216064453125, "loss": 1.2351, "margin_dpo/margin_mean": 63.215675354003906, "margin_dpo/margin_std": 121.06292724609375, "step": 630 }, { "KL/chosen_KL_mean": -173.7066650390625, "KL/mean": -222.6975555419922, "KL/rejected_KL_mean": -271.6884460449219, "KL/std": 106.37603759765625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.98178100585938, "fcm_dpo/q_t": 0.4268760085105896, "grad_norm": 7.507592678070068, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.4163804054260254, "logits/rejected": -1.9853250980377197, "logps/chosen": -261.47747802734375, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -360.370849609375, "loss": 1.1401, "margin_dpo/margin_mean": 97.98178100585938, "margin_dpo/margin_std": 121.59942626953125, "step": 631 }, { "KL/chosen_KL_mean": -171.16160583496094, "KL/mean": -224.3045654296875, "KL/rejected_KL_mean": -277.447509765625, "KL/std": 111.40956115722656, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.28591918945312, "fcm_dpo/q_t": 0.4217221736907959, "grad_norm": 9.331432342529297, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.838683843612671, "logits/rejected": -1.3980896472930908, "logps/chosen": -259.78814697265625, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -368.8984375, "loss": 1.1291, "margin_dpo/margin_mean": 106.28591918945312, "margin_dpo/margin_std": 138.56890869140625, "step": 632 }, { "KL/chosen_KL_mean": -213.7779998779297, "KL/mean": -244.25025939941406, "KL/rejected_KL_mean": -274.7225341796875, "KL/std": 113.05734252929688, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.94453430175781, "fcm_dpo/q_t": 0.4557117819786072, "grad_norm": 8.471674919128418, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.266923189163208, "logits/rejected": -2.0806422233581543, "logps/chosen": -295.15240478515625, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -373.3482666015625, "loss": 1.2618, "margin_dpo/margin_mean": 60.94453430175781, "margin_dpo/margin_std": 153.16500854492188, "step": 633 }, { "KL/chosen_KL_mean": -194.97300720214844, "KL/mean": -235.79995727539062, "KL/rejected_KL_mean": -276.626953125, "KL/std": 102.48681640625, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.65393829345703, "fcm_dpo/q_t": 0.4389786720275879, "grad_norm": 8.425758361816406, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.3584342002868652, "logits/rejected": -1.1858956813812256, "logps/chosen": -291.0958557128906, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -389.4747314453125, "loss": 1.1777, "margin_dpo/margin_mean": 81.65393829345703, "margin_dpo/margin_std": 104.93417358398438, "step": 634 }, { "KL/chosen_KL_mean": -207.04168701171875, "KL/mean": -237.12960815429688, "KL/rejected_KL_mean": -267.2175598144531, "KL/std": 112.93395233154297, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.17587661743164, "fcm_dpo/q_t": 0.4556182026863098, "grad_norm": 9.880289077758789, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.5453529357910156, "logits/rejected": -0.9858849048614502, "logps/chosen": -305.7257995605469, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -357.11676025390625, "loss": 1.2531, "margin_dpo/margin_mean": 60.175872802734375, "margin_dpo/margin_std": 136.85101318359375, "step": 635 }, { "KL/chosen_KL_mean": -182.02252197265625, "KL/mean": -236.13734436035156, "KL/rejected_KL_mean": -290.252197265625, "KL/std": 114.74525451660156, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.22966003417969, "fcm_dpo/q_t": 0.4195432960987091, "grad_norm": 7.490612030029297, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.494437336921692, "logits/rejected": -1.013523817062378, "logps/chosen": -272.44049072265625, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -377.95904541015625, "loss": 1.1092, "margin_dpo/margin_mean": 108.22966003417969, "margin_dpo/margin_std": 114.38992309570312, "step": 636 }, { "KL/chosen_KL_mean": -182.83071899414062, "KL/mean": -240.45718383789062, "KL/rejected_KL_mean": -298.08367919921875, "KL/std": 127.26216888427734, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0030997558496892452, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.2529296875, "fcm_dpo/q_t": 0.4144030213356018, "grad_norm": 7.149726867675781, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.8954975605010986, "logits/rejected": -1.7920608520507812, "logps/chosen": -265.28045654296875, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -402.1122741699219, "loss": 1.0949, "margin_dpo/margin_mean": 115.2529296875, "margin_dpo/margin_std": 120.5191650390625, "step": 637 }, { "KL/chosen_KL_mean": -172.8487548828125, "KL/mean": -228.28024291992188, "KL/rejected_KL_mean": -283.7117004394531, "KL/std": 113.40983581542969, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0030966082122176886, "fcm_dpo/delta": -0.010164685547351837, "fcm_dpo/margin": 110.86293029785156, "fcm_dpo/q_t": 0.41732466220855713, "grad_norm": 7.078164577484131, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.5666935443878174, "logits/rejected": -0.888373851776123, "logps/chosen": -264.77374267578125, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -369.99871826171875, "loss": 1.101, "margin_dpo/margin_mean": 110.86293029785156, "margin_dpo/margin_std": 107.95150756835938, "step": 638 }, { "KL/chosen_KL_mean": -229.256103515625, "KL/mean": -254.897705078125, "KL/rejected_KL_mean": -280.539306640625, "KL/std": 104.0133285522461, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.003093460574746132, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.28325271606445, "fcm_dpo/q_t": 0.46155595779418945, "grad_norm": 7.711297035217285, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.6318411827087402, "logits/rejected": -0.9030921459197998, "logps/chosen": -333.78363037109375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -362.0196533203125, "loss": 1.274, "margin_dpo/margin_mean": 51.28325653076172, "margin_dpo/margin_std": 129.9130401611328, "step": 639 }, { "KL/chosen_KL_mean": -193.6444549560547, "KL/mean": -243.78179931640625, "KL/rejected_KL_mean": -293.91912841796875, "KL/std": 114.75259399414062, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0030869338661432266, "fcm_dpo/delta": -0.021142521873116493, "fcm_dpo/margin": 100.27467346191406, "fcm_dpo/q_t": 0.4255128502845764, "grad_norm": 10.239623069763184, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8664889335632324, "logits/rejected": -1.354414701461792, "logps/chosen": -291.52972412109375, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -406.6241455078125, "loss": 1.136, "margin_dpo/margin_mean": 100.27467346191406, "margin_dpo/margin_std": 115.51165771484375, "step": 640 }, { "KL/chosen_KL_mean": -177.5532684326172, "KL/mean": -229.89230346679688, "KL/rejected_KL_mean": -282.2313232421875, "KL/std": 132.56735229492188, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.6780776977539, "fcm_dpo/q_t": 0.4250600337982178, "grad_norm": 6.849096775054932, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.197295904159546, "logits/rejected": -1.6481389999389648, "logps/chosen": -273.99896240234375, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -377.3670349121094, "loss": 1.1383, "margin_dpo/margin_mean": 104.67808532714844, "margin_dpo/margin_std": 148.38204956054688, "step": 641 }, { "KL/chosen_KL_mean": -197.46902465820312, "KL/mean": -246.984375, "KL/rejected_KL_mean": -296.49969482421875, "KL/std": 113.94924926757812, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.03068542480469, "fcm_dpo/q_t": 0.4283752143383026, "grad_norm": 9.827589988708496, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.5794352293014526, "logits/rejected": -1.2492644786834717, "logps/chosen": -298.2288818359375, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -411.20733642578125, "loss": 1.1502, "margin_dpo/margin_mean": 99.03067016601562, "margin_dpo/margin_std": 142.74305725097656, "step": 642 }, { "KL/chosen_KL_mean": -185.9024658203125, "KL/mean": -232.4935302734375, "KL/rejected_KL_mean": -279.0845947265625, "KL/std": 105.85090637207031, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.18214416503906, "fcm_dpo/q_t": 0.4308868944644928, "grad_norm": 10.097166061401367, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.259896755218506, "logits/rejected": -1.8961149454116821, "logps/chosen": -270.6461181640625, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -373.40301513671875, "loss": 1.1569, "margin_dpo/margin_mean": 93.18214416503906, "margin_dpo/margin_std": 122.92306518554688, "step": 643 }, { "KL/chosen_KL_mean": -191.47409057617188, "KL/mean": -229.99607849121094, "KL/rejected_KL_mean": -268.51806640625, "KL/std": 107.43290710449219, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.04399871826172, "fcm_dpo/q_t": 0.4427317976951599, "grad_norm": 8.681756019592285, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.9766466617584229, "logits/rejected": -1.7162691354751587, "logps/chosen": -276.79681396484375, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -363.1266784667969, "loss": 1.1948, "margin_dpo/margin_mean": 77.04399871826172, "margin_dpo/margin_std": 117.29782104492188, "step": 644 }, { "KL/chosen_KL_mean": -204.91650390625, "KL/mean": -229.4888916015625, "KL/rejected_KL_mean": -254.06126403808594, "KL/std": 116.08135986328125, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.144752502441406, "fcm_dpo/q_t": 0.4636668562889099, "grad_norm": 8.173365592956543, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.3817024230957031, "logits/rejected": -1.2160388231277466, "logps/chosen": -287.50677490234375, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -357.06500244140625, "loss": 1.2822, "margin_dpo/margin_mean": 49.144744873046875, "margin_dpo/margin_std": 132.84146118164062, "step": 645 }, { "KL/chosen_KL_mean": -180.12185668945312, "KL/mean": -229.5357666015625, "KL/rejected_KL_mean": -278.94970703125, "KL/std": 101.58773803710938, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.82783508300781, "fcm_dpo/q_t": 0.426535427570343, "grad_norm": 7.3757829666137695, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.7562671899795532, "logits/rejected": -1.1511667966842651, "logps/chosen": -271.31390380859375, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -362.72802734375, "loss": 1.1341, "margin_dpo/margin_mean": 98.82783508300781, "margin_dpo/margin_std": 110.9209976196289, "step": 646 }, { "KL/chosen_KL_mean": -193.26722717285156, "KL/mean": -237.708251953125, "KL/rejected_KL_mean": -282.1492919921875, "KL/std": 121.65162658691406, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.88206481933594, "fcm_dpo/q_t": 0.4348573684692383, "grad_norm": 8.32433032989502, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.5131592750549316, "logits/rejected": -1.3775649070739746, "logps/chosen": -282.1116943359375, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -381.6476135253906, "loss": 1.1733, "margin_dpo/margin_mean": 88.88206481933594, "margin_dpo/margin_std": 135.55612182617188, "step": 647 }, { "KL/chosen_KL_mean": -181.40371704101562, "KL/mean": -241.581298828125, "KL/rejected_KL_mean": -301.7589111328125, "KL/std": 125.13018798828125, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.35519409179688, "fcm_dpo/q_t": 0.41166093945503235, "grad_norm": 8.120327949523926, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.893972396850586, "logits/rejected": -1.8228175640106201, "logps/chosen": -269.2518310546875, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -406.428955078125, "loss": 1.09, "margin_dpo/margin_mean": 120.35519409179688, "margin_dpo/margin_std": 132.73785400390625, "step": 648 }, { "KL/chosen_KL_mean": -167.15603637695312, "KL/mean": -217.71864318847656, "KL/rejected_KL_mean": -268.2812805175781, "KL/std": 108.96568298339844, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.1252212524414, "fcm_dpo/q_t": 0.42491960525512695, "grad_norm": 9.19161605834961, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.4598820209503174, "logits/rejected": -2.061753034591675, "logps/chosen": -256.8485412597656, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -356.98785400390625, "loss": 1.1265, "margin_dpo/margin_mean": 101.1252212524414, "margin_dpo/margin_std": 106.7547378540039, "step": 649 }, { "KL/chosen_KL_mean": -198.1519775390625, "KL/mean": -245.6478729248047, "KL/rejected_KL_mean": -293.143798828125, "KL/std": 118.9176025390625, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.99179077148438, "fcm_dpo/q_t": 0.4302680194377899, "grad_norm": 7.666706562042236, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.1139910221099854, "logits/rejected": -0.9099173545837402, "logps/chosen": -288.08258056640625, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -395.756591796875, "loss": 1.1614, "margin_dpo/margin_mean": 94.99179077148438, "margin_dpo/margin_std": 143.11358642578125, "step": 650 }, { "KL/chosen_KL_mean": -162.3658905029297, "KL/mean": -221.10821533203125, "KL/rejected_KL_mean": -279.85052490234375, "KL/std": 113.80860137939453, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.48464965820312, "fcm_dpo/q_t": 0.41280728578567505, "grad_norm": 7.491771221160889, "learning_rate": 3.158738163478475e-09, "logits/chosen": -1.9696449041366577, "logits/rejected": -1.7925975322723389, "logps/chosen": -241.55322265625, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -385.78387451171875, "loss": 1.0857, "margin_dpo/margin_mean": 117.48464965820312, "margin_dpo/margin_std": 111.6327896118164, "step": 651 }, { "KL/chosen_KL_mean": -171.71823120117188, "KL/mean": -220.78616333007812, "KL/rejected_KL_mean": -269.8541259765625, "KL/std": 105.96775817871094, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.13587188720703, "fcm_dpo/q_t": 0.4269661605358124, "grad_norm": 8.48566722869873, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.5244576930999756, "logits/rejected": -2.2332282066345215, "logps/chosen": -273.5084533691406, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -386.1787109375, "loss": 1.1307, "margin_dpo/margin_mean": 98.13587951660156, "margin_dpo/margin_std": 101.92900085449219, "step": 652 }, { "KL/chosen_KL_mean": -196.31361389160156, "KL/mean": -244.05804443359375, "KL/rejected_KL_mean": -291.802490234375, "KL/std": 116.5035400390625, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.48887634277344, "fcm_dpo/q_t": 0.42855823040008545, "grad_norm": 7.866513252258301, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.1842567920684814, "logits/rejected": -1.9320077896118164, "logps/chosen": -289.6672058105469, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -400.92572021484375, "loss": 1.1444, "margin_dpo/margin_mean": 95.48887634277344, "margin_dpo/margin_std": 114.59435272216797, "step": 653 }, { "KL/chosen_KL_mean": -196.72314453125, "KL/mean": -237.71563720703125, "KL/rejected_KL_mean": -278.7081298828125, "KL/std": 111.320068359375, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.98500061035156, "fcm_dpo/q_t": 0.4404606819152832, "grad_norm": 9.2181978225708, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.2647712230682373, "logits/rejected": -2.0847699642181396, "logps/chosen": -285.83868408203125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -383.6280822753906, "loss": 1.1946, "margin_dpo/margin_mean": 81.98500061035156, "margin_dpo/margin_std": 140.48434448242188, "step": 654 }, { "KL/chosen_KL_mean": -180.02325439453125, "KL/mean": -229.09085083007812, "KL/rejected_KL_mean": -278.158447265625, "KL/std": 121.22976684570312, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.13520812988281, "fcm_dpo/q_t": 0.42741870880126953, "grad_norm": 7.476641654968262, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.9303321838378906, "logits/rejected": -1.9337120056152344, "logps/chosen": -261.0593566894531, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -380.9607849121094, "loss": 1.1464, "margin_dpo/margin_mean": 98.13520812988281, "margin_dpo/margin_std": 130.70579528808594, "step": 655 }, { "KL/chosen_KL_mean": -181.28262329101562, "KL/mean": -244.01901245117188, "KL/rejected_KL_mean": -306.75543212890625, "KL/std": 121.66258239746094, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.4728012084961, "fcm_dpo/q_t": 0.408223956823349, "grad_norm": 8.25948715209961, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.5391499996185303, "logits/rejected": -1.6076674461364746, "logps/chosen": -266.60797119140625, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -425.0940856933594, "loss": 1.0727, "margin_dpo/margin_mean": 125.4728012084961, "margin_dpo/margin_std": 126.49400329589844, "step": 656 }, { "KL/chosen_KL_mean": -197.75714111328125, "KL/mean": -244.07852172851562, "KL/rejected_KL_mean": -290.39984130859375, "KL/std": 112.60087585449219, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.64273071289062, "fcm_dpo/q_t": 0.4323294162750244, "grad_norm": 8.64854621887207, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.88322114944458, "logits/rejected": -1.7166547775268555, "logps/chosen": -277.9548645019531, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -392.9811706542969, "loss": 1.1649, "margin_dpo/margin_mean": 92.64273071289062, "margin_dpo/margin_std": 136.12265014648438, "step": 657 }, { "KL/chosen_KL_mean": -192.57154846191406, "KL/mean": -235.60763549804688, "KL/rejected_KL_mean": -278.6436767578125, "KL/std": 105.48158264160156, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.0721435546875, "fcm_dpo/q_t": 0.43603914976119995, "grad_norm": 7.03842306137085, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.206242322921753, "logits/rejected": -1.7537527084350586, "logps/chosen": -287.2060546875, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -380.27532958984375, "loss": 1.1691, "margin_dpo/margin_mean": 86.0721435546875, "margin_dpo/margin_std": 113.34773254394531, "step": 658 }, { "KL/chosen_KL_mean": -176.58456420898438, "KL/mean": -228.2755126953125, "KL/rejected_KL_mean": -279.9664611816406, "KL/std": 113.19835662841797, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.38191223144531, "fcm_dpo/q_t": 0.42455700039863586, "grad_norm": 7.075484752655029, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.1691057682037354, "logits/rejected": -1.2132914066314697, "logps/chosen": -289.14044189453125, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -373.8880615234375, "loss": 1.1442, "margin_dpo/margin_mean": 103.38191223144531, "margin_dpo/margin_std": 146.8539276123047, "step": 659 }, { "KL/chosen_KL_mean": -185.58200073242188, "KL/mean": -238.95811462402344, "KL/rejected_KL_mean": -292.334228515625, "KL/std": 109.85297393798828, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.75224304199219, "fcm_dpo/q_t": 0.42124852538108826, "grad_norm": 9.148067474365234, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8065153360366821, "logits/rejected": -1.179041862487793, "logps/chosen": -278.95941162109375, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -373.72906494140625, "loss": 1.1174, "margin_dpo/margin_mean": 106.75224304199219, "margin_dpo/margin_std": 120.66143798828125, "step": 660 }, { "KL/chosen_KL_mean": -182.1873779296875, "KL/mean": -242.16925048828125, "KL/rejected_KL_mean": -302.151123046875, "KL/std": 124.26422882080078, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.9637222290039, "fcm_dpo/q_t": 0.41259461641311646, "grad_norm": 8.688028335571289, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -2.0124053955078125, "logits/rejected": -1.518552303314209, "logps/chosen": -270.04254150390625, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -394.5544128417969, "loss": 1.0951, "margin_dpo/margin_mean": 119.96372985839844, "margin_dpo/margin_std": 138.39036560058594, "step": 661 }, { "KL/chosen_KL_mean": -178.04721069335938, "KL/mean": -233.50401306152344, "KL/rejected_KL_mean": -288.9608154296875, "KL/std": 107.94063568115234, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.91361999511719, "fcm_dpo/q_t": 0.4180074632167816, "grad_norm": 9.100963592529297, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.7857964038848877, "logits/rejected": -1.2203419208526611, "logps/chosen": -280.8269958496094, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -384.1861572265625, "loss": 1.1045, "margin_dpo/margin_mean": 110.91362762451172, "margin_dpo/margin_std": 115.03153228759766, "step": 662 }, { "KL/chosen_KL_mean": -180.49668884277344, "KL/mean": -230.11288452148438, "KL/rejected_KL_mean": -279.7291259765625, "KL/std": 112.23318481445312, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.23240661621094, "fcm_dpo/q_t": 0.4269559979438782, "grad_norm": 10.073052406311035, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.3457047939300537, "logits/rejected": -2.135727882385254, "logps/chosen": -276.84326171875, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -400.25555419921875, "loss": 1.143, "margin_dpo/margin_mean": 99.23240661621094, "margin_dpo/margin_std": 130.19790649414062, "step": 663 }, { "KL/chosen_KL_mean": -201.02127075195312, "KL/mean": -245.7024688720703, "KL/rejected_KL_mean": -290.3836669921875, "KL/std": 115.56576538085938, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.3624267578125, "fcm_dpo/q_t": 0.43373623490333557, "grad_norm": 7.003365993499756, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.7450544834136963, "logits/rejected": -1.57627272605896, "logps/chosen": -292.8636779785156, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -402.2203674316406, "loss": 1.1612, "margin_dpo/margin_mean": 89.3624267578125, "margin_dpo/margin_std": 114.2310562133789, "step": 664 }, { "KL/chosen_KL_mean": -172.12567138671875, "KL/mean": -221.0740966796875, "KL/rejected_KL_mean": -270.02252197265625, "KL/std": 110.64122009277344, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.89686584472656, "fcm_dpo/q_t": 0.4271303117275238, "grad_norm": 6.7707624435424805, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.4094455242156982, "logits/rejected": -1.9834065437316895, "logps/chosen": -260.31182861328125, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -361.9345397949219, "loss": 1.1378, "margin_dpo/margin_mean": 97.89686584472656, "margin_dpo/margin_std": 114.4814682006836, "step": 665 }, { "KL/chosen_KL_mean": -187.51409912109375, "KL/mean": -238.20079040527344, "KL/rejected_KL_mean": -288.8874816894531, "KL/std": 108.68548583984375, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.3733901977539, "fcm_dpo/q_t": 0.42562001943588257, "grad_norm": 9.229649543762207, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.7616984844207764, "logits/rejected": -1.39094877243042, "logps/chosen": -288.48870849609375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -390.13739013671875, "loss": 1.1377, "margin_dpo/margin_mean": 101.3733901977539, "margin_dpo/margin_std": 129.4086456298828, "step": 666 }, { "KL/chosen_KL_mean": -206.09291076660156, "KL/mean": -257.1612548828125, "KL/rejected_KL_mean": -308.2295837402344, "KL/std": 113.10394287109375, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.13664245605469, "fcm_dpo/q_t": 0.42403918504714966, "grad_norm": 7.475996494293213, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.4767992496490479, "logits/rejected": -0.8219588994979858, "logps/chosen": -303.6640625, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -401.8143310546875, "loss": 1.1244, "margin_dpo/margin_mean": 102.13664245605469, "margin_dpo/margin_std": 110.43060302734375, "step": 667 }, { "KL/chosen_KL_mean": -182.59725952148438, "KL/mean": -242.39706420898438, "KL/rejected_KL_mean": -302.1968994140625, "KL/std": 120.19706726074219, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.59963989257812, "fcm_dpo/q_t": 0.4128602147102356, "grad_norm": 8.5903902053833, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.250476121902466, "logits/rejected": -1.729312539100647, "logps/chosen": -285.17230224609375, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -411.01458740234375, "loss": 1.0907, "margin_dpo/margin_mean": 119.59963989257812, "margin_dpo/margin_std": 131.95204162597656, "step": 668 }, { "KL/chosen_KL_mean": -165.75021362304688, "KL/mean": -223.2188720703125, "KL/rejected_KL_mean": -280.68756103515625, "KL/std": 117.87957763671875, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.003080407390370965, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.9373550415039, "fcm_dpo/q_t": 0.41559362411499023, "grad_norm": 8.172355651855469, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.7118463516235352, "logits/rejected": -1.3195197582244873, "logps/chosen": -255.2449951171875, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -363.2070617675781, "loss": 1.1005, "margin_dpo/margin_mean": 114.9373550415039, "margin_dpo/margin_std": 127.96144104003906, "step": 669 }, { "KL/chosen_KL_mean": -167.01919555664062, "KL/mean": -230.22787475585938, "KL/rejected_KL_mean": -293.4365539550781, "KL/std": 119.76484680175781, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.003077741712331772, "fcm_dpo/delta": -0.008660750463604927, "fcm_dpo/margin": 126.4173583984375, "fcm_dpo/q_t": 0.4065937399864197, "grad_norm": 6.473464488983154, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7067947387695312, "logits/rejected": -1.1683510541915894, "logps/chosen": -262.47381591796875, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -394.969482421875, "loss": 1.0709, "margin_dpo/margin_mean": 126.4173583984375, "margin_dpo/margin_std": 125.62957763671875, "step": 670 }, { "KL/chosen_KL_mean": -203.7989044189453, "KL/mean": -246.853759765625, "KL/rejected_KL_mean": -289.90863037109375, "KL/std": 111.45346069335938, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.10972595214844, "fcm_dpo/q_t": 0.43680694699287415, "grad_norm": 8.133575439453125, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6234817504882812, "logits/rejected": -1.3986479043960571, "logps/chosen": -285.9220275878906, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -380.1283264160156, "loss": 1.1764, "margin_dpo/margin_mean": 86.10972595214844, "margin_dpo/margin_std": 125.15023040771484, "step": 671 }, { "KL/chosen_KL_mean": -203.19253540039062, "KL/mean": -240.09298706054688, "KL/rejected_KL_mean": -276.993408203125, "KL/std": 106.5054702758789, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.80091857910156, "fcm_dpo/q_t": 0.4453005790710449, "grad_norm": 11.505027770996094, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.9999785423278809, "logits/rejected": -1.6733595132827759, "logps/chosen": -293.25445556640625, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -377.4466552734375, "loss": 1.2039, "margin_dpo/margin_mean": 73.80091094970703, "margin_dpo/margin_std": 113.52955627441406, "step": 672 }, { "KL/chosen_KL_mean": -182.769775390625, "KL/mean": -224.15093994140625, "KL/rejected_KL_mean": -265.5321350097656, "KL/std": 110.40188598632812, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.76239013671875, "fcm_dpo/q_t": 0.43871402740478516, "grad_norm": 9.090703010559082, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.9016464948654175, "logits/rejected": -1.536212682723999, "logps/chosen": -262.03289794921875, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -356.88140869140625, "loss": 1.1777, "margin_dpo/margin_mean": 82.76239013671875, "margin_dpo/margin_std": 113.34103393554688, "step": 673 }, { "KL/chosen_KL_mean": -169.93386840820312, "KL/mean": -230.50704956054688, "KL/rejected_KL_mean": -291.0802307128906, "KL/std": 110.26815795898438, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.14639282226562, "fcm_dpo/q_t": 0.41137152910232544, "grad_norm": 6.478679180145264, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.6095741987228394, "logits/rejected": -1.208085298538208, "logps/chosen": -245.39218139648438, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -367.28387451171875, "loss": 1.0813, "margin_dpo/margin_mean": 121.14639282226562, "margin_dpo/margin_std": 121.66947937011719, "step": 674 }, { "KL/chosen_KL_mean": -186.06626892089844, "KL/mean": -239.4055938720703, "KL/rejected_KL_mean": -292.74493408203125, "KL/std": 119.14442443847656, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.67864990234375, "fcm_dpo/q_t": 0.42190229892730713, "grad_norm": 8.697331428527832, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.4818004369735718, "logits/rejected": -1.1414153575897217, "logps/chosen": -265.9757995605469, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -373.9631652832031, "loss": 1.1358, "margin_dpo/margin_mean": 106.67864990234375, "margin_dpo/margin_std": 146.65902709960938, "step": 675 }, { "KL/chosen_KL_mean": -200.35025024414062, "KL/mean": -252.31124877929688, "KL/rejected_KL_mean": -304.272216796875, "KL/std": 114.20941162109375, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.92196655273438, "fcm_dpo/q_t": 0.42333754897117615, "grad_norm": 7.216146945953369, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.8966562747955322, "logits/rejected": -1.3054472208023071, "logps/chosen": -298.5213623046875, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -399.296875, "loss": 1.1313, "margin_dpo/margin_mean": 103.92196655273438, "margin_dpo/margin_std": 129.1129150390625, "step": 676 }, { "KL/chosen_KL_mean": -180.68450927734375, "KL/mean": -232.55213928222656, "KL/rejected_KL_mean": -284.4197998046875, "KL/std": 113.11341857910156, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0030750762671232224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.73529052734375, "fcm_dpo/q_t": 0.4232900142669678, "grad_norm": 7.116570472717285, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.672013759613037, "logits/rejected": -0.9983895421028137, "logps/chosen": -272.06378173828125, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -367.29754638671875, "loss": 1.1225, "margin_dpo/margin_mean": 103.73529052734375, "margin_dpo/margin_std": 112.15374755859375, "step": 677 }, { "KL/chosen_KL_mean": -168.87075805664062, "KL/mean": -231.21067810058594, "KL/rejected_KL_mean": -293.5506286621094, "KL/std": 120.92158508300781, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.003067562822252512, "fcm_dpo/delta": -0.024493634700775146, "fcm_dpo/margin": 124.67985534667969, "fcm_dpo/q_t": 0.40851497650146484, "grad_norm": 8.479134559631348, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.6514018774032593, "logits/rejected": -1.325568675994873, "logps/chosen": -244.51661682128906, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -380.5167236328125, "loss": 1.0759, "margin_dpo/margin_mean": 124.67985534667969, "margin_dpo/margin_std": 124.2884750366211, "step": 678 }, { "KL/chosen_KL_mean": -196.74951171875, "KL/mean": -247.8406982421875, "KL/rejected_KL_mean": -298.931884765625, "KL/std": 101.35913848876953, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.003060049144551158, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.18235778808594, "fcm_dpo/q_t": 0.4249420762062073, "grad_norm": 7.2457380294799805, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.177043914794922, "logits/rejected": -1.659234642982483, "logps/chosen": -277.52294921875, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -381.81036376953125, "loss": 1.1245, "margin_dpo/margin_mean": 102.18235778808594, "margin_dpo/margin_std": 108.94142150878906, "step": 679 }, { "KL/chosen_KL_mean": -197.22393798828125, "KL/mean": -253.7880096435547, "KL/rejected_KL_mean": -310.35211181640625, "KL/std": 119.72410583496094, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.003053477965295315, "fcm_dpo/delta": -0.02152036502957344, "fcm_dpo/margin": 113.12812805175781, "fcm_dpo/q_t": 0.4178704023361206, "grad_norm": 5.985490322113037, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.8823460340499878, "logits/rejected": -1.5481247901916504, "logps/chosen": -304.9068603515625, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -426.44696044921875, "loss": 1.111, "margin_dpo/margin_mean": 113.12812805175781, "margin_dpo/margin_std": 130.92922973632812, "step": 680 }, { "KL/chosen_KL_mean": -179.46432495117188, "KL/mean": -228.14822387695312, "KL/rejected_KL_mean": -276.8321228027344, "KL/std": 113.00016784667969, "epoch": 1.0, "fcm_dpo/beta": 0.0030469067860394716, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.36778259277344, "fcm_dpo/q_t": 0.42905163764953613, "grad_norm": 7.68331241607666, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.269704818725586, "logits/rejected": -1.9578585624694824, "logps/chosen": -272.47540283203125, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -371.654296875, "loss": 1.1469, "margin_dpo/margin_mean": 97.36778259277344, "margin_dpo/margin_std": 124.69633483886719, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1272611473625451, "train_runtime": 1979.2244, "train_samples_per_second": 22.028, "train_steps_per_second": 0.344 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }