{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 681,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "KL/chosen_KL_mean": 0.05419158935546875,
      "KL/mean": 0.023573115468025208,
      "KL/rejected_KL_mean": -0.007045745849609375,
      "KL/std": 0.33097243309020996,
      "epoch": 0.0014684287812041115,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.06123492121696472,
      "fcm_dpo/q_t": 0.4984707534313202,
      "grad_norm": 38.819252014160156,
      "learning_rate": 0.0,
      "logits/chosen": 0.2677389979362488,
      "logits/rejected": 0.6191319227218628,
      "logps/chosen": -85.37664031982422,
      "logps/ref_chosen": -85.43083190917969,
      "logps/ref_rejected": -79.90458679199219,
      "logps/rejected": -79.91163635253906,
      "loss": 1.3807,
      "margin_dpo/margin_mean": 0.061235010623931885,
      "margin_dpo/margin_std": 0.4424324035644531,
      "step": 1
    },
    {
      "KL/chosen_KL_mean": -0.044902801513671875,
      "KL/mean": 0.017303764820098877,
      "KL/rejected_KL_mean": 0.07951736450195312,
      "KL/std": 0.2933448553085327,
      "epoch": 0.002936857562408223,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.12442219257354736,
      "fcm_dpo/q_t": 0.5031089186668396,
      "grad_norm": 29.75910758972168,
      "learning_rate": 7.246376811594203e-09,
      "logits/chosen": 0.47771453857421875,
      "logits/rejected": 0.5908545851707458,
      "logps/chosen": -82.11383056640625,
      "logps/ref_chosen": -82.06893157958984,
      "logps/ref_rejected": -81.65457916259766,
      "logps/rejected": -81.57505798339844,
      "loss": 1.3992,
      "margin_dpo/margin_mean": -0.12442246079444885,
      "margin_dpo/margin_std": 0.41358208656311035,
      "step": 2
    },
    {
      "KL/chosen_KL_mean": 0.045459747314453125,
      "KL/mean": 0.017605803906917572,
      "KL/rejected_KL_mean": -0.01024627685546875,
      "KL/std": 0.30478668212890625,
      "epoch": 0.004405286343612335,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05570589005947113,
      "fcm_dpo/q_t": 0.498607873916626,
      "grad_norm": 25.993059158325195,
      "learning_rate": 1.4492753623188406e-08,
      "logits/chosen": 0.42671653628349304,
      "logits/rejected": 0.6499756574630737,
      "logps/chosen": -93.7655258178711,
      "logps/ref_chosen": -93.81098937988281,
      "logps/ref_rejected": -74.22950744628906,
      "logps/rejected": -74.23974609375,
      "loss": 1.3813,
      "margin_dpo/margin_mean": 0.05570597946643829,
      "margin_dpo/margin_std": 0.4541243314743042,
      "step": 3
    },
    {
      "KL/chosen_KL_mean": -0.04769134521484375,
      "KL/mean": -0.008433207869529724,
      "KL/rejected_KL_mean": 0.03082275390625,
      "KL/std": 0.3678542971611023,
      "epoch": 0.005873715124816446,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0785132348537445,
      "fcm_dpo/q_t": 0.5019584894180298,
      "grad_norm": 32.774818420410156,
      "learning_rate": 2.1739130434782606e-08,
      "logits/chosen": 0.6469099521636963,
      "logits/rejected": 0.7786818742752075,
      "logps/chosen": -87.34016418457031,
      "logps/ref_chosen": -87.29246520996094,
      "logps/ref_rejected": -93.82425689697266,
      "logps/rejected": -93.7934341430664,
      "loss": 1.3948,
      "margin_dpo/margin_mean": -0.07851245999336243,
      "margin_dpo/margin_std": 0.508175253868103,
      "step": 4
    },
    {
      "KL/chosen_KL_mean": -0.039257049560546875,
      "KL/mean": -0.029438771307468414,
      "KL/rejected_KL_mean": -0.019618988037109375,
      "KL/std": 0.3082965314388275,
      "epoch": 0.007342143906020558,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.019636347889900208,
      "fcm_dpo/q_t": 0.5004903078079224,
      "grad_norm": 35.35218811035156,
      "learning_rate": 2.898550724637681e-08,
      "logits/chosen": 0.44584977626800537,
      "logits/rejected": 0.6573811769485474,
      "logps/chosen": -89.37601470947266,
      "logps/ref_chosen": -89.33676147460938,
      "logps/ref_rejected": -88.74783325195312,
      "logps/rejected": -88.7674560546875,
      "loss": 1.3888,
      "margin_dpo/margin_mean": -0.0196361243724823,
      "margin_dpo/margin_std": 0.44399672746658325,
      "step": 5
    },
    {
      "KL/chosen_KL_mean": -0.000194549560546875,
      "KL/mean": -0.008608967065811157,
      "KL/rejected_KL_mean": -0.017017364501953125,
      "KL/std": 0.2754075527191162,
      "epoch": 0.00881057268722467,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.016812950372695923,
      "fcm_dpo/q_t": 0.49957966804504395,
      "grad_norm": 36.00873565673828,
      "learning_rate": 3.6231884057971014e-08,
      "logits/chosen": 0.730034351348877,
      "logits/rejected": 1.0671439170837402,
      "logps/chosen": -97.32167053222656,
      "logps/ref_chosen": -97.32147216796875,
      "logps/ref_rejected": -97.88345336914062,
      "logps/rejected": -97.90046691894531,
      "loss": 1.3851,
      "margin_dpo/margin_mean": 0.016811877489089966,
      "margin_dpo/margin_std": 0.4103967249393463,
      "step": 6
    },
    {
      "KL/chosen_KL_mean": 0.01053619384765625,
      "KL/mean": -0.026270486414432526,
      "KL/rejected_KL_mean": -0.06308364868164062,
      "KL/std": 0.3485536575317383,
      "epoch": 0.010279001468428781,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.0736158937215805,
      "fcm_dpo/q_t": 0.49816328287124634,
      "grad_norm": 37.77275085449219,
      "learning_rate": 4.347826086956521e-08,
      "logits/chosen": 0.4895734190940857,
      "logits/rejected": 0.5370491743087769,
      "logps/chosen": -86.63800048828125,
      "logps/ref_chosen": -86.64852905273438,
      "logps/ref_rejected": -109.61618041992188,
      "logps/rejected": -109.67926025390625,
      "loss": 1.3795,
      "margin_dpo/margin_mean": 0.07361528277397156,
      "margin_dpo/margin_std": 0.4408665895462036,
      "step": 7
    },
    {
      "KL/chosen_KL_mean": -0.033481597900390625,
      "KL/mean": 0.001060090959072113,
      "KL/rejected_KL_mean": 0.035610198974609375,
      "KL/std": 0.3318078815937042,
      "epoch": 0.011747430249632892,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.06908880174160004,
      "fcm_dpo/q_t": 0.5017245411872864,
      "grad_norm": 32.84318542480469,
      "learning_rate": 5.0724637681159424e-08,
      "logits/chosen": 0.4353947639465332,
      "logits/rejected": 0.7500125765800476,
      "logps/chosen": -89.95716857910156,
      "logps/ref_chosen": -89.9236831665039,
      "logps/ref_rejected": -86.22804260253906,
      "logps/rejected": -86.19242858886719,
      "loss": 1.3939,
      "margin_dpo/margin_mean": -0.06908857822418213,
      "margin_dpo/margin_std": 0.5175743699073792,
      "step": 8
    },
    {
      "KL/chosen_KL_mean": 0.0496063232421875,
      "KL/mean": 0.012138597667217255,
      "KL/rejected_KL_mean": -0.025325775146484375,
      "KL/std": 0.3826137185096741,
      "epoch": 0.013215859030837005,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07492710649967194,
      "fcm_dpo/q_t": 0.4981287121772766,
      "grad_norm": 35.888336181640625,
      "learning_rate": 5.797101449275362e-08,
      "logits/chosen": 0.740329384803772,
      "logits/rejected": 0.9239120483398438,
      "logps/chosen": -103.80752563476562,
      "logps/ref_chosen": -103.85713195800781,
      "logps/ref_rejected": -104.31932067871094,
      "logps/rejected": -104.34465026855469,
      "loss": 1.3796,
      "margin_dpo/margin_mean": 0.07492664456367493,
      "margin_dpo/margin_std": 0.5700302124023438,
      "step": 9
    },
    {
      "KL/chosen_KL_mean": 0.0328369140625,
      "KL/mean": 0.06327671557664871,
      "KL/rejected_KL_mean": 0.09371185302734375,
      "KL/std": 0.28018298745155334,
      "epoch": 0.014684287812041116,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.0608818382024765,
      "fcm_dpo/q_t": 0.501521110534668,
      "grad_norm": 33.679630279541016,
      "learning_rate": 6.521739130434782e-08,
      "logits/chosen": 0.5119473934173584,
      "logits/rejected": 0.7244791984558105,
      "logps/chosen": -76.17152404785156,
      "logps/ref_chosen": -76.20436096191406,
      "logps/ref_rejected": -87.152099609375,
      "logps/rejected": -87.05839538574219,
      "loss": 1.3928,
      "margin_dpo/margin_mean": -0.060881733894348145,
      "margin_dpo/margin_std": 0.3876721262931824,
      "step": 10
    },
    {
      "KL/chosen_KL_mean": 0.020168304443359375,
      "KL/mean": -0.019120410084724426,
      "KL/rejected_KL_mean": -0.058406829833984375,
      "KL/std": 0.32755106687545776,
      "epoch": 0.016152716593245228,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.07857182621955872,
      "fcm_dpo/q_t": 0.49803751707077026,
      "grad_norm": 36.930084228515625,
      "learning_rate": 7.246376811594203e-08,
      "logits/chosen": 0.22594323754310608,
      "logits/rejected": 0.43925705552101135,
      "logps/chosen": -82.34632873535156,
      "logps/ref_chosen": -82.36650085449219,
      "logps/ref_rejected": -94.26461791992188,
      "logps/rejected": -94.32302856445312,
      "loss": 1.379,
      "margin_dpo/margin_mean": 0.07857242226600647,
      "margin_dpo/margin_std": 0.45420587062835693,
      "step": 11
    },
    {
      "KL/chosen_KL_mean": 0.046306610107421875,
      "KL/mean": 0.026197895407676697,
      "KL/rejected_KL_mean": 0.0060882568359375,
      "KL/std": 0.347372829914093,
      "epoch": 0.01762114537444934,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04021328687667847,
      "fcm_dpo/q_t": 0.49899566173553467,
      "grad_norm": 39.970237731933594,
      "learning_rate": 7.971014492753623e-08,
      "logits/chosen": 0.4583059549331665,
      "logits/rejected": 0.5883156061172485,
      "logps/chosen": -99.05918884277344,
      "logps/ref_chosen": -99.10549926757812,
      "logps/ref_rejected": -110.27140808105469,
      "logps/rejected": -110.26531982421875,
      "loss": 1.3829,
      "margin_dpo/margin_mean": 0.04021361470222473,
      "margin_dpo/margin_std": 0.5016989707946777,
      "step": 12
    },
    {
      "KL/chosen_KL_mean": 0.028961181640625,
      "KL/mean": 0.021265730261802673,
      "KL/rejected_KL_mean": 0.01357269287109375,
      "KL/std": 0.31696659326553345,
      "epoch": 0.01908957415565345,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.015392988920211792,
      "fcm_dpo/q_t": 0.4996158480644226,
      "grad_norm": 42.037940979003906,
      "learning_rate": 8.695652173913042e-08,
      "logits/chosen": 0.41135668754577637,
      "logits/rejected": 0.6608414053916931,
      "logps/chosen": -90.53076171875,
      "logps/ref_chosen": -90.55972290039062,
      "logps/ref_rejected": -93.69110107421875,
      "logps/rejected": -93.67752075195312,
      "loss": 1.3853,
      "margin_dpo/margin_mean": 0.015392720699310303,
      "margin_dpo/margin_std": 0.4781372547149658,
      "step": 13
    },
    {
      "KL/chosen_KL_mean": -0.0421295166015625,
      "KL/mean": 0.0007936879992485046,
      "KL/rejected_KL_mean": 0.0437164306640625,
      "KL/std": 0.30456358194351196,
      "epoch": 0.020558002936857563,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.08585520088672638,
      "fcm_dpo/q_t": 0.5021449327468872,
      "grad_norm": 35.98721694946289,
      "learning_rate": 9.420289855072464e-08,
      "logits/chosen": 0.9760057926177979,
      "logits/rejected": 1.0674599409103394,
      "logps/chosen": -99.86930847167969,
      "logps/ref_chosen": -99.82717895507812,
      "logps/ref_rejected": -108.94200134277344,
      "logps/rejected": -108.89828491210938,
      "loss": 1.3954,
      "margin_dpo/margin_mean": -0.08585470914840698,
      "margin_dpo/margin_std": 0.4400973916053772,
      "step": 14
    },
    {
      "KL/chosen_KL_mean": -0.001239776611328125,
      "KL/mean": 0.01269543170928955,
      "KL/rejected_KL_mean": 0.026630401611328125,
      "KL/std": 0.3145955801010132,
      "epoch": 0.022026431718061675,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.027870237827301025,
      "fcm_dpo/q_t": 0.5006976127624512,
      "grad_norm": 31.74387550354004,
      "learning_rate": 1.0144927536231885e-07,
      "logits/chosen": 0.3185517489910126,
      "logits/rejected": 0.48128974437713623,
      "logps/chosen": -78.91120910644531,
      "logps/ref_chosen": -78.90997314453125,
      "logps/ref_rejected": -90.06234741210938,
      "logps/rejected": -90.03571319580078,
      "loss": 1.3897,
      "margin_dpo/margin_mean": -0.027870893478393555,
      "margin_dpo/margin_std": 0.4726863503456116,
      "step": 15
    },
    {
      "KL/chosen_KL_mean": 0.006221771240234375,
      "KL/mean": -0.0011203661561012268,
      "KL/rejected_KL_mean": -0.0084686279296875,
      "KL/std": 0.3218470811843872,
      "epoch": 0.023494860499265784,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.014686957001686096,
      "fcm_dpo/q_t": 0.4996318817138672,
      "grad_norm": 34.74339294433594,
      "learning_rate": 1.0869565217391303e-07,
      "logits/chosen": 0.5471932888031006,
      "logits/rejected": 0.8027733564376831,
      "logps/chosen": -97.41705322265625,
      "logps/ref_chosen": -97.42327880859375,
      "logps/ref_rejected": -90.59945678710938,
      "logps/rejected": -90.60792541503906,
      "loss": 1.3854,
      "margin_dpo/margin_mean": 0.014686673879623413,
      "margin_dpo/margin_std": 0.47504281997680664,
      "step": 16
    },
    {
      "KL/chosen_KL_mean": 0.051303863525390625,
      "KL/mean": 0.005533188581466675,
      "KL/rejected_KL_mean": -0.040241241455078125,
      "KL/std": 0.3313257694244385,
      "epoch": 0.024963289280469897,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09154921770095825,
      "fcm_dpo/q_t": 0.49771296977996826,
      "grad_norm": 35.61276626586914,
      "learning_rate": 1.1594202898550725e-07,
      "logits/chosen": 0.49732881784439087,
      "logits/rejected": 0.7290574312210083,
      "logps/chosen": -104.31300354003906,
      "logps/ref_chosen": -104.36430358886719,
      "logps/ref_rejected": -90.46772766113281,
      "logps/rejected": -90.50796508789062,
      "loss": 1.3778,
      "margin_dpo/margin_mean": 0.09154891967773438,
      "margin_dpo/margin_std": 0.49299106001853943,
      "step": 17
    },
    {
      "KL/chosen_KL_mean": -0.026485443115234375,
      "KL/mean": -0.038269490003585815,
      "KL/rejected_KL_mean": -0.050048828125,
      "KL/std": 0.323337197303772,
      "epoch": 0.02643171806167401,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.023557662963867188,
      "fcm_dpo/q_t": 0.49941104650497437,
      "grad_norm": 41.933494567871094,
      "learning_rate": 1.2318840579710146e-07,
      "logits/chosen": 0.16588415205478668,
      "logits/rejected": 0.4849891662597656,
      "logps/chosen": -87.11843872070312,
      "logps/ref_chosen": -87.09195709228516,
      "logps/ref_rejected": -81.85072326660156,
      "logps/rejected": -81.90077209472656,
      "loss": 1.3846,
      "margin_dpo/margin_mean": 0.023558229207992554,
      "margin_dpo/margin_std": 0.519328236579895,
      "step": 18
    },
    {
      "KL/chosen_KL_mean": -0.01941680908203125,
      "KL/mean": -0.03904549777507782,
      "KL/rejected_KL_mean": -0.05867767333984375,
      "KL/std": 0.3405440151691437,
      "epoch": 0.027900146842878122,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.039252668619155884,
      "fcm_dpo/q_t": 0.4990195333957672,
      "grad_norm": 31.184173583984375,
      "learning_rate": 1.3043478260869563e-07,
      "logits/chosen": 0.7113861441612244,
      "logits/rejected": 0.9976291656494141,
      "logps/chosen": -105.89295959472656,
      "logps/ref_chosen": -105.87354278564453,
      "logps/ref_rejected": -96.93023681640625,
      "logps/rejected": -96.98890686035156,
      "loss": 1.3829,
      "margin_dpo/margin_mean": 0.03925308585166931,
      "margin_dpo/margin_std": 0.4445246160030365,
      "step": 19
    },
    {
      "KL/chosen_KL_mean": 0.034393310546875,
      "KL/mean": 0.005580544471740723,
      "KL/rejected_KL_mean": -0.023235321044921875,
      "KL/std": 0.2989480197429657,
      "epoch": 0.02936857562408223,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.057629168033599854,
      "fcm_dpo/q_t": 0.49855953454971313,
      "grad_norm": 32.38508224487305,
      "learning_rate": 1.3768115942028986e-07,
      "logits/chosen": 0.2745107412338257,
      "logits/rejected": 0.5551785826683044,
      "logps/chosen": -90.72372436523438,
      "logps/ref_chosen": -90.75811767578125,
      "logps/ref_rejected": -85.91232299804688,
      "logps/rejected": -85.93555450439453,
      "loss": 1.3809,
      "margin_dpo/margin_mean": 0.057629287242889404,
      "margin_dpo/margin_std": 0.39042335748672485,
      "step": 20
    },
    {
      "KL/chosen_KL_mean": 0.043949127197265625,
      "KL/mean": -0.009302347898483276,
      "KL/rejected_KL_mean": -0.06255722045898438,
      "KL/std": 0.2668537497520447,
      "epoch": 0.030837004405286344,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.10650789737701416,
      "fcm_dpo/q_t": 0.4973394572734833,
      "grad_norm": 31.334672927856445,
      "learning_rate": 1.4492753623188405e-07,
      "logits/chosen": 0.6649617552757263,
      "logits/rejected": 0.8533320426940918,
      "logps/chosen": -80.28950500488281,
      "logps/ref_chosen": -80.33345794677734,
      "logps/ref_rejected": -83.93373107910156,
      "logps/rejected": -83.99629211425781,
      "loss": 1.3761,
      "margin_dpo/margin_mean": 0.10650783777236938,
      "margin_dpo/margin_std": 0.3960753381252289,
      "step": 21
    },
    {
      "KL/chosen_KL_mean": 0.0425872802734375,
      "KL/mean": 0.016435056924819946,
      "KL/rejected_KL_mean": -0.009716033935546875,
      "KL/std": 0.3606400489807129,
      "epoch": 0.032305433186490456,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05228877067565918,
      "fcm_dpo/q_t": 0.4986942410469055,
      "grad_norm": 42.832733154296875,
      "learning_rate": 1.5217391304347825e-07,
      "logits/chosen": 0.5063530206680298,
      "logits/rejected": 0.6201586723327637,
      "logps/chosen": -95.35271453857422,
      "logps/ref_chosen": -95.39530181884766,
      "logps/ref_rejected": -103.47351837158203,
      "logps/rejected": -103.48323059082031,
      "loss": 1.3818,
      "margin_dpo/margin_mean": 0.05228865146636963,
      "margin_dpo/margin_std": 0.5111033916473389,
      "step": 22
    },
    {
      "KL/chosen_KL_mean": 0.028690338134765625,
      "KL/mean": 0.008205808699131012,
      "KL/rejected_KL_mean": -0.012271881103515625,
      "KL/std": 0.2899686098098755,
      "epoch": 0.033773861967694566,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.04096432030200958,
      "fcm_dpo/q_t": 0.4989771246910095,
      "grad_norm": 31.360807418823242,
      "learning_rate": 1.5942028985507245e-07,
      "logits/chosen": 0.7632906436920166,
      "logits/rejected": 0.9437848329544067,
      "logps/chosen": -90.60882568359375,
      "logps/ref_chosen": -90.63751220703125,
      "logps/ref_rejected": -86.59425354003906,
      "logps/rejected": -86.60652923583984,
      "loss": 1.3826,
      "margin_dpo/margin_mean": 0.0409637987613678,
      "margin_dpo/margin_std": 0.39144009351730347,
      "step": 23
    },
    {
      "KL/chosen_KL_mean": 0.0003566741943359375,
      "KL/mean": 0.0464811846613884,
      "KL/rejected_KL_mean": 0.0926055908203125,
      "KL/std": 0.3461281657218933,
      "epoch": 0.03524229074889868,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": -0.09224571287631989,
      "fcm_dpo/q_t": 0.5023068189620972,
      "grad_norm": 44.4437370300293,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 0.6687687635421753,
      "logits/rejected": 0.6997181177139282,
      "logps/chosen": -69.91693115234375,
      "logps/ref_chosen": -69.91728973388672,
      "logps/ref_rejected": -106.60990142822266,
      "logps/rejected": -106.51729583740234,
      "loss": 1.3962,
      "margin_dpo/margin_mean": -0.0922449380159378,
      "margin_dpo/margin_std": 0.5254719257354736,
      "step": 24
    },
    {
      "KL/chosen_KL_mean": 0.001270294189453125,
      "KL/mean": -0.02792833000421524,
      "KL/rejected_KL_mean": -0.05712890625,
      "KL/std": 0.3060055375099182,
      "epoch": 0.03671071953010279,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.05838797986507416,
      "fcm_dpo/q_t": 0.4985414147377014,
      "grad_norm": 37.039588928222656,
      "learning_rate": 1.7391304347826085e-07,
      "logits/chosen": 0.5007416009902954,
      "logits/rejected": 0.6674846410751343,
      "logps/chosen": -80.82421875,
      "logps/ref_chosen": -80.82548522949219,
      "logps/ref_rejected": -95.95710754394531,
      "logps/rejected": -96.01423645019531,
      "loss": 1.3809,
      "margin_dpo/margin_mean": 0.05838817358016968,
      "margin_dpo/margin_std": 0.41962748765945435,
      "step": 25
    },
    {
      "KL/chosen_KL_mean": -0.010829925537109375,
      "KL/mean": -0.0158480703830719,
      "KL/rejected_KL_mean": -0.020862579345703125,
      "KL/std": 0.2907237410545349,
      "epoch": 0.0381791483113069,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.010033130645751953,
      "fcm_dpo/q_t": 0.4997495114803314,
      "grad_norm": 41.3673095703125,
      "learning_rate": 1.8115942028985507e-07,
      "logits/chosen": 0.3731698989868164,
      "logits/rejected": 0.5834112167358398,
      "logps/chosen": -88.91200256347656,
      "logps/ref_chosen": -88.90116882324219,
      "logps/ref_rejected": -109.82819366455078,
      "logps/rejected": -109.84906005859375,
      "loss": 1.3857,
      "margin_dpo/margin_mean": 0.010033339262008667,
      "margin_dpo/margin_std": 0.4099235534667969,
      "step": 26
    },
    {
      "KL/chosen_KL_mean": 0.04615020751953125,
      "KL/mean": -0.032234713435173035,
      "KL/rejected_KL_mean": -0.11062240600585938,
      "KL/std": 0.2985093593597412,
      "epoch": 0.039647577092511016,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1567753255367279,
      "fcm_dpo/q_t": 0.49608227610588074,
      "grad_norm": 44.02979278564453,
      "learning_rate": 1.8840579710144927e-07,
      "logits/chosen": 0.15856103599071503,
      "logits/rejected": 0.3470751643180847,
      "logps/chosen": -77.54986572265625,
      "logps/ref_chosen": -77.59600830078125,
      "logps/ref_rejected": -103.93850708007812,
      "logps/rejected": -104.04913330078125,
      "loss": 1.3711,
      "margin_dpo/margin_mean": 0.15677517652511597,
      "margin_dpo/margin_std": 0.40555810928344727,
      "step": 27
    },
    {
      "KL/chosen_KL_mean": 0.047145843505859375,
      "KL/mean": -0.030098095536231995,
      "KL/rejected_KL_mean": -0.10734176635742188,
      "KL/std": 0.35706520080566406,
      "epoch": 0.041116005873715125,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.15448924899101257,
      "fcm_dpo/q_t": 0.496139258146286,
      "grad_norm": 36.62578582763672,
      "learning_rate": 1.9565217391304347e-07,
      "logits/chosen": 0.4918988049030304,
      "logits/rejected": 0.7027618885040283,
      "logps/chosen": -102.18142700195312,
      "logps/ref_chosen": -102.22856903076172,
      "logps/ref_rejected": -96.9594955444336,
      "logps/rejected": -97.06683349609375,
      "loss": 1.3716,
      "margin_dpo/margin_mean": 0.15448984503746033,
      "margin_dpo/margin_std": 0.5215355753898621,
      "step": 28
    },
    {
      "KL/chosen_KL_mean": -0.018512725830078125,
      "KL/mean": -0.0793382003903389,
      "KL/rejected_KL_mean": -0.14015960693359375,
      "KL/std": 0.37649092078208923,
      "epoch": 0.042584434654919234,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1216522604227066,
      "fcm_dpo/q_t": 0.4969598650932312,
      "grad_norm": 42.38597869873047,
      "learning_rate": 2.028985507246377e-07,
      "logits/chosen": 0.2806279957294464,
      "logits/rejected": 0.5361133813858032,
      "logps/chosen": -88.66556549072266,
      "logps/ref_chosen": -88.64704895019531,
      "logps/ref_rejected": -102.96011352539062,
      "logps/rejected": -103.10027313232422,
      "loss": 1.3747,
      "margin_dpo/margin_mean": 0.12165278196334839,
      "margin_dpo/margin_std": 0.4404868483543396,
      "step": 29
    },
    {
      "KL/chosen_KL_mean": 0.005184173583984375,
      "KL/mean": -0.044647976756095886,
      "KL/rejected_KL_mean": -0.09448623657226562,
      "KL/std": 0.3062649667263031,
      "epoch": 0.04405286343612335,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.09966346621513367,
      "fcm_dpo/q_t": 0.49751079082489014,
      "grad_norm": 39.168643951416016,
      "learning_rate": 2.1014492753623187e-07,
      "logits/chosen": 0.5004847049713135,
      "logits/rejected": 0.6124160289764404,
      "logps/chosen": -88.3832015991211,
      "logps/ref_chosen": -88.38838195800781,
      "logps/ref_rejected": -102.31889343261719,
      "logps/rejected": -102.41337585449219,
      "loss": 1.3768,
      "margin_dpo/margin_mean": 0.09966397285461426,
      "margin_dpo/margin_std": 0.4101472795009613,
      "step": 30
    },
    {
      "KL/chosen_KL_mean": -0.023273468017578125,
      "KL/mean": -0.0849444791674614,
      "KL/rejected_KL_mean": -0.14661026000976562,
      "KL/std": 0.34307873249053955,
      "epoch": 0.04552129221732746,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.12333105504512787,
      "fcm_dpo/q_t": 0.4969198703765869,
      "grad_norm": 31.17573356628418,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": 0.9666788578033447,
      "logits/rejected": 1.2424330711364746,
      "logps/chosen": -101.14893341064453,
      "logps/ref_chosen": -101.12565612792969,
      "logps/ref_rejected": -79.8584213256836,
      "logps/rejected": -80.00503540039062,
      "loss": 1.3746,
      "margin_dpo/margin_mean": 0.12333130836486816,
      "margin_dpo/margin_std": 0.4796608090400696,
      "step": 31
    },
    {
      "KL/chosen_KL_mean": 0.001277923583984375,
      "KL/mean": -0.06870098412036896,
      "KL/rejected_KL_mean": -0.13867568969726562,
      "KL/std": 0.3285205364227295,
      "epoch": 0.04698972099853157,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1399599015712738,
      "fcm_dpo/q_t": 0.4965028762817383,
      "grad_norm": 35.13859558105469,
      "learning_rate": 2.2463768115942027e-07,
      "logits/chosen": 0.3857934772968292,
      "logits/rejected": 0.5240524411201477,
      "logps/chosen": -96.61575317382812,
      "logps/ref_chosen": -96.61702728271484,
      "logps/ref_rejected": -96.224365234375,
      "logps/rejected": -96.363037109375,
      "loss": 1.3729,
      "margin_dpo/margin_mean": 0.13996022939682007,
      "margin_dpo/margin_std": 0.46658003330230713,
      "step": 32
    },
    {
      "KL/chosen_KL_mean": 0.061641693115234375,
      "KL/mean": -0.02603095769882202,
      "KL/rejected_KL_mean": -0.11370468139648438,
      "KL/std": 0.32698971033096313,
      "epoch": 0.048458149779735685,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.17535078525543213,
      "fcm_dpo/q_t": 0.4956197738647461,
      "grad_norm": 33.6834716796875,
      "learning_rate": 2.318840579710145e-07,
      "logits/chosen": 0.5319309830665588,
      "logits/rejected": 0.7106450796127319,
      "logps/chosen": -81.4593734741211,
      "logps/ref_chosen": -81.52101135253906,
      "logps/ref_rejected": -93.80595397949219,
      "logps/rejected": -93.91966247558594,
      "loss": 1.3694,
      "margin_dpo/margin_mean": 0.175351083278656,
      "margin_dpo/margin_std": 0.4586974084377289,
      "step": 33
    },
    {
      "KL/chosen_KL_mean": 0.0251007080078125,
      "KL/mean": -0.04835952818393707,
      "KL/rejected_KL_mean": -0.12181472778320312,
      "KL/std": 0.32071372866630554,
      "epoch": 0.049926578560939794,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.14692452549934387,
      "fcm_dpo/q_t": 0.496332049369812,
      "grad_norm": 40.93767166137695,
      "learning_rate": 2.391304347826087e-07,
      "logits/chosen": 0.46491187810897827,
      "logits/rejected": 0.6379407644271851,
      "logps/chosen": -77.17694091796875,
      "logps/ref_chosen": -77.20204162597656,
      "logps/ref_rejected": -106.71876525878906,
      "logps/rejected": -106.840576171875,
      "loss": 1.3722,
      "margin_dpo/margin_mean": 0.14692476391792297,
      "margin_dpo/margin_std": 0.47234487533569336,
      "step": 34
    },
    {
      "KL/chosen_KL_mean": -0.060741424560546875,
      "KL/mean": -0.14784874022006989,
      "KL/rejected_KL_mean": -0.23495101928710938,
      "KL/std": 0.3914700746536255,
      "epoch": 0.0513950073421439,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.1742132604122162,
      "fcm_dpo/q_t": 0.495650053024292,
      "grad_norm": 42.669586181640625,
      "learning_rate": 2.463768115942029e-07,
      "logits/chosen": 0.22820869088172913,
      "logits/rejected": 0.3843857944011688,
      "logps/chosen": -77.63108825683594,
      "logps/ref_chosen": -77.57035064697266,
      "logps/ref_rejected": -112.18855285644531,
      "logps/rejected": -112.42350769042969,
      "loss": 1.3697,
      "margin_dpo/margin_mean": 0.17421391606330872,
      "margin_dpo/margin_std": 0.5403324365615845,
      "step": 35
    },
    {
      "KL/chosen_KL_mean": 0.0303192138671875,
      "KL/mean": -0.06879466027021408,
      "KL/rejected_KL_mean": -0.16790771484375,
      "KL/std": 0.4058462381362915,
      "epoch": 0.05286343612334802,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.19822145998477936,
      "fcm_dpo/q_t": 0.4950520396232605,
      "grad_norm": 32.55595779418945,
      "learning_rate": 2.536231884057971e-07,
      "logits/chosen": 0.4027172327041626,
      "logits/rejected": 0.5912371873855591,
      "logps/chosen": -83.25791931152344,
      "logps/ref_chosen": -83.28823852539062,
      "logps/ref_rejected": -91.75741577148438,
      "logps/rejected": -91.92532348632812,
      "loss": 1.3673,
      "margin_dpo/margin_mean": 0.19822166860103607,
      "margin_dpo/margin_std": 0.5464945435523987,
      "step": 36
    },
    {
      "KL/chosen_KL_mean": -0.058078765869140625,
      "KL/mean": -0.17308874428272247,
      "KL/rejected_KL_mean": -0.2881050109863281,
      "KL/std": 0.4333382248878479,
      "epoch": 0.05433186490455213,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.23002305626869202,
      "fcm_dpo/q_t": 0.4942634105682373,
      "grad_norm": 35.80500030517578,
      "learning_rate": 2.6086956521739126e-07,
      "logits/chosen": 0.699413537979126,
      "logits/rejected": 0.8630663156509399,
      "logps/chosen": -94.82917022705078,
      "logps/ref_chosen": -94.77108764648438,
      "logps/ref_rejected": -85.71726989746094,
      "logps/rejected": -86.00537109375,
      "loss": 1.3644,
      "margin_dpo/margin_mean": 0.23002314567565918,
      "margin_dpo/margin_std": 0.6174068450927734,
      "step": 37
    },
    {
      "KL/chosen_KL_mean": 0.0362091064453125,
      "KL/mean": -0.09902308881282806,
      "KL/rejected_KL_mean": -0.23425674438476562,
      "KL/std": 0.4296520948410034,
      "epoch": 0.055800293685756244,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2704668343067169,
      "fcm_dpo/q_t": 0.4932498335838318,
      "grad_norm": 36.792579650878906,
      "learning_rate": 2.681159420289855e-07,
      "logits/chosen": 0.20736345648765564,
      "logits/rejected": 0.34377244114875793,
      "logps/chosen": -75.94229125976562,
      "logps/ref_chosen": -75.97850036621094,
      "logps/ref_rejected": -104.10401916503906,
      "logps/rejected": -104.33827209472656,
      "loss": 1.3603,
      "margin_dpo/margin_mean": 0.27046751976013184,
      "margin_dpo/margin_std": 0.5921492576599121,
      "step": 38
    },
    {
      "KL/chosen_KL_mean": -0.0030517578125,
      "KL/mean": -0.16817954182624817,
      "KL/rejected_KL_mean": -0.3333091735839844,
      "KL/std": 0.3673657774925232,
      "epoch": 0.05726872246696035,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.3302544951438904,
      "fcm_dpo/q_t": 0.491748571395874,
      "grad_norm": 36.370079040527344,
      "learning_rate": 2.753623188405797e-07,
      "logits/chosen": 0.27447906136512756,
      "logits/rejected": 0.4493808150291443,
      "logps/chosen": -81.18882751464844,
      "logps/ref_chosen": -81.18577575683594,
      "logps/ref_rejected": -84.1959228515625,
      "logps/rejected": -84.52922821044922,
      "loss": 1.3541,
      "margin_dpo/margin_mean": 0.3302551507949829,
      "margin_dpo/margin_std": 0.44706541299819946,
      "step": 39
    },
    {
      "KL/chosen_KL_mean": -0.030109405517578125,
      "KL/mean": -0.1016501784324646,
      "KL/rejected_KL_mean": -0.17319107055664062,
      "KL/std": 0.35342079401016235,
      "epoch": 0.05873715124816446,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.14308512210845947,
      "fcm_dpo/q_t": 0.4964262545108795,
      "grad_norm": 29.365358352661133,
      "learning_rate": 2.8260869565217386e-07,
      "logits/chosen": 0.7558138370513916,
      "logits/rejected": 0.9623916149139404,
      "logps/chosen": -83.36268615722656,
      "logps/ref_chosen": -83.33257293701172,
      "logps/ref_rejected": -80.25591278076172,
      "logps/rejected": -80.42910766601562,
      "loss": 1.3726,
      "margin_dpo/margin_mean": 0.1430847942829132,
      "margin_dpo/margin_std": 0.4806174039840698,
      "step": 40
    },
    {
      "KL/chosen_KL_mean": -0.07754135131835938,
      "KL/mean": -0.22058053314685822,
      "KL/rejected_KL_mean": -0.3636207580566406,
      "KL/std": 0.4311068058013916,
      "epoch": 0.06020558002936858,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.2860817015171051,
      "fcm_dpo/q_t": 0.4928615391254425,
      "grad_norm": 33.35248947143555,
      "learning_rate": 2.898550724637681e-07,
      "logits/chosen": 0.6289557218551636,
      "logits/rejected": 0.8152822256088257,
      "logps/chosen": -93.22620391845703,
      "logps/ref_chosen": -93.14866638183594,
      "logps/ref_rejected": -102.07920837402344,
      "logps/rejected": -102.44282531738281,
      "loss": 1.359,
      "margin_dpo/margin_mean": 0.286081999540329,
      "margin_dpo/margin_std": 0.6327430605888367,
      "step": 41
    },
    {
      "KL/chosen_KL_mean": 0.04155731201171875,
      "KL/mean": -0.16173964738845825,
      "KL/rejected_KL_mean": -0.3650321960449219,
      "KL/std": 0.5146182775497437,
      "epoch": 0.06167400881057269,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4065883755683899,
      "fcm_dpo/q_t": 0.4898473620414734,
      "grad_norm": 41.106712341308594,
      "learning_rate": 2.971014492753623e-07,
      "logits/chosen": 0.6057265400886536,
      "logits/rejected": 0.7496967315673828,
      "logps/chosen": -90.66006469726562,
      "logps/ref_chosen": -90.70162200927734,
      "logps/ref_rejected": -113.8646469116211,
      "logps/rejected": -114.22967529296875,
      "loss": 1.347,
      "margin_dpo/margin_mean": 0.40658843517303467,
      "margin_dpo/margin_std": 0.6019660830497742,
      "step": 42
    },
    {
      "KL/chosen_KL_mean": 0.0411376953125,
      "KL/mean": -0.20209653675556183,
      "KL/rejected_KL_mean": -0.4453315734863281,
      "KL/std": 0.5224407911300659,
      "epoch": 0.0631424375917768,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4864697754383087,
      "fcm_dpo/q_t": 0.48786601424217224,
      "grad_norm": 37.75397872924805,
      "learning_rate": 3.043478260869565e-07,
      "logits/chosen": 0.5100123882293701,
      "logits/rejected": 0.6964707970619202,
      "logps/chosen": -89.60289001464844,
      "logps/ref_chosen": -89.64402770996094,
      "logps/ref_rejected": -103.95185852050781,
      "logps/rejected": -104.39718627929688,
      "loss": 1.3396,
      "margin_dpo/margin_mean": 0.4864700734615326,
      "margin_dpo/margin_std": 0.7371401786804199,
      "step": 43
    },
    {
      "KL/chosen_KL_mean": -0.0260162353515625,
      "KL/mean": -0.24083814024925232,
      "KL/rejected_KL_mean": -0.45566558837890625,
      "KL/std": 0.5008527040481567,
      "epoch": 0.06461086637298091,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4296528697013855,
      "fcm_dpo/q_t": 0.48926910758018494,
      "grad_norm": 34.530487060546875,
      "learning_rate": 3.115942028985507e-07,
      "logits/chosen": 0.6433513760566711,
      "logits/rejected": 0.7694642543792725,
      "logps/chosen": -81.90433502197266,
      "logps/ref_chosen": -81.8783187866211,
      "logps/ref_rejected": -113.421630859375,
      "logps/rejected": -113.8772964477539,
      "loss": 1.3445,
      "margin_dpo/margin_mean": 0.4296536147594452,
      "margin_dpo/margin_std": 0.5389829277992249,
      "step": 44
    },
    {
      "KL/chosen_KL_mean": -0.11905670166015625,
      "KL/mean": -0.21861934661865234,
      "KL/rejected_KL_mean": -0.3181800842285156,
      "KL/std": 0.4403315782546997,
      "epoch": 0.06607929515418502,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.199128195643425,
      "fcm_dpo/q_t": 0.49502819776535034,
      "grad_norm": 24.52897834777832,
      "learning_rate": 3.188405797101449e-07,
      "logits/chosen": 0.4717053771018982,
      "logits/rejected": 0.5798736810684204,
      "logps/chosen": -77.46365356445312,
      "logps/ref_chosen": -77.34459686279297,
      "logps/ref_rejected": -84.24774169921875,
      "logps/rejected": -84.56592559814453,
      "loss": 1.3673,
      "margin_dpo/margin_mean": 0.19912879168987274,
      "margin_dpo/margin_std": 0.5796105861663818,
      "step": 45
    },
    {
      "KL/chosen_KL_mean": -0.04913330078125,
      "KL/mean": -0.26750367879867554,
      "KL/rejected_KL_mean": -0.4858741760253906,
      "KL/std": 0.5423171520233154,
      "epoch": 0.06754772393538913,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.43673908710479736,
      "fcm_dpo/q_t": 0.48909813165664673,
      "grad_norm": 33.06425476074219,
      "learning_rate": 3.260869565217391e-07,
      "logits/chosen": 0.5884057283401489,
      "logits/rejected": 0.8358145952224731,
      "logps/chosen": -90.38995361328125,
      "logps/ref_chosen": -90.3408203125,
      "logps/ref_rejected": -100.676513671875,
      "logps/rejected": -101.16239166259766,
      "loss": 1.3441,
      "margin_dpo/margin_mean": 0.43673956394195557,
      "margin_dpo/margin_std": 0.6209474802017212,
      "step": 46
    },
    {
      "KL/chosen_KL_mean": -0.028057098388671875,
      "KL/mean": -0.24152395129203796,
      "KL/rejected_KL_mean": -0.454986572265625,
      "KL/std": 0.5000725984573364,
      "epoch": 0.06901615271659324,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.4269258379936218,
      "fcm_dpo/q_t": 0.4893423020839691,
      "grad_norm": 34.044002532958984,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": 0.47639983892440796,
      "logits/rejected": 0.6607139110565186,
      "logps/chosen": -104.43936157226562,
      "logps/ref_chosen": -104.41130065917969,
      "logps/ref_rejected": -101.25489807128906,
      "logps/rejected": -101.70988464355469,
      "loss": 1.3451,
      "margin_dpo/margin_mean": 0.42692551016807556,
      "margin_dpo/margin_std": 0.6505356431007385,
      "step": 47
    },
    {
      "KL/chosen_KL_mean": -0.032489776611328125,
      "KL/mean": -0.29501965641975403,
      "KL/rejected_KL_mean": -0.5575523376464844,
      "KL/std": 0.6199899315834045,
      "epoch": 0.07048458149779736,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.5250635743141174,
      "fcm_dpo/q_t": 0.48689717054367065,
      "grad_norm": 39.09486389160156,
      "learning_rate": 3.4057971014492755e-07,
      "logits/chosen": 0.10753734409809113,
      "logits/rejected": 0.2988663911819458,
      "logps/chosen": -91.13276672363281,
      "logps/ref_chosen": -91.10027313232422,
      "logps/ref_rejected": -95.08058166503906,
      "logps/rejected": -95.63813018798828,
      "loss": 1.3361,
      "margin_dpo/margin_mean": 0.5250641107559204,
      "margin_dpo/margin_std": 0.7990906238555908,
      "step": 48
    },
    {
      "KL/chosen_KL_mean": 0.04256439208984375,
      "KL/mean": -0.3208075165748596,
      "KL/rejected_KL_mean": -0.6841773986816406,
      "KL/std": 0.7505483627319336,
      "epoch": 0.07195301027900147,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.7267380952835083,
      "fcm_dpo/q_t": 0.48189887404441833,
      "grad_norm": 39.786983489990234,
      "learning_rate": 3.478260869565217e-07,
      "logits/chosen": 0.07788099348545074,
      "logits/rejected": 0.5461311340332031,
      "logps/chosen": -92.96112060546875,
      "logps/ref_chosen": -93.00367736816406,
      "logps/ref_rejected": -91.74899291992188,
      "logps/rejected": -92.43317413330078,
      "loss": 1.3172,
      "margin_dpo/margin_mean": 0.7267380952835083,
      "margin_dpo/margin_std": 0.950691819190979,
      "step": 49
    },
    {
      "KL/chosen_KL_mean": -0.09926223754882812,
      "KL/mean": -0.4128139913082123,
      "KL/rejected_KL_mean": -0.7263565063476562,
      "KL/std": 0.7748845219612122,
      "epoch": 0.07342143906020558,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6270943284034729,
      "fcm_dpo/q_t": 0.48438096046447754,
      "grad_norm": 33.1424446105957,
      "learning_rate": 3.5507246376811595e-07,
      "logits/chosen": 0.5201438069343567,
      "logits/rejected": 0.6898816227912903,
      "logps/chosen": -94.72607421875,
      "logps/ref_chosen": -94.62681579589844,
      "logps/ref_rejected": -103.57435607910156,
      "logps/rejected": -104.30072021484375,
      "loss": 1.3271,
      "margin_dpo/margin_mean": 0.6270943880081177,
      "margin_dpo/margin_std": 0.983003556728363,
      "step": 50
    },
    {
      "KL/chosen_KL_mean": -0.07915496826171875,
      "KL/mean": -0.41101139783859253,
      "KL/rejected_KL_mean": -0.7428703308105469,
      "KL/std": 0.8373640179634094,
      "epoch": 0.07488986784140969,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.6637116074562073,
      "fcm_dpo/q_t": 0.48349109292030334,
      "grad_norm": 30.053504943847656,
      "learning_rate": 3.6231884057971015e-07,
      "logits/chosen": 0.3313475251197815,
      "logits/rejected": 0.4574078321456909,
      "logps/chosen": -87.58643341064453,
      "logps/ref_chosen": -87.50727844238281,
      "logps/ref_rejected": -83.47235107421875,
      "logps/rejected": -84.21522521972656,
      "loss": 1.3238,
      "margin_dpo/margin_mean": 0.6637121438980103,
      "margin_dpo/margin_std": 1.0461779832839966,
      "step": 51
    },
    {
      "KL/chosen_KL_mean": -0.023807525634765625,
      "KL/mean": -0.43643075227737427,
      "KL/rejected_KL_mean": -0.84906005859375,
      "KL/std": 0.7662195563316345,
      "epoch": 0.0763582966226138,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 0.825249969959259,
      "fcm_dpo/q_t": 0.4794442355632782,
      "grad_norm": 32.13584899902344,
      "learning_rate": 3.695652173913043e-07,
      "logits/chosen": 0.3861672282218933,
      "logits/rejected": 0.5959784984588623,
      "logps/chosen": -90.65406799316406,
      "logps/ref_chosen": -90.63026428222656,
      "logps/ref_rejected": -87.0390625,
      "logps/rejected": -87.88812255859375,
      "loss": 1.3076,
      "margin_dpo/margin_mean": 0.825249433517456,
      "margin_dpo/margin_std": 0.9151663780212402,
      "step": 52
    },
    {
      "KL/chosen_KL_mean": 0.0486297607421875,
      "KL/mean": -0.5051177740097046,
      "KL/rejected_KL_mean": -1.0588607788085938,
      "KL/std": 1.097104787826538,
      "epoch": 0.07782672540381791,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.107494831085205,
      "fcm_dpo/q_t": 0.472596675157547,
      "grad_norm": 39.35917282104492,
      "learning_rate": 3.7681159420289855e-07,
      "logits/chosen": 0.22227796912193298,
      "logits/rejected": 0.33877307176589966,
      "logps/chosen": -81.5344467163086,
      "logps/ref_chosen": -81.58307647705078,
      "logps/ref_rejected": -95.66152954101562,
      "logps/rejected": -96.72039031982422,
      "loss": 1.2837,
      "margin_dpo/margin_mean": 1.1074947118759155,
      "margin_dpo/margin_std": 1.3432953357696533,
      "step": 53
    },
    {
      "KL/chosen_KL_mean": -0.015171051025390625,
      "KL/mean": -0.579878032207489,
      "KL/rejected_KL_mean": -1.1445808410644531,
      "KL/std": 1.302019715309143,
      "epoch": 0.07929515418502203,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.129401683807373,
      "fcm_dpo/q_t": 0.47198718786239624,
      "grad_norm": 39.807918548583984,
      "learning_rate": 3.8405797101449274e-07,
      "logits/chosen": 0.6011637449264526,
      "logits/rejected": 0.7894606590270996,
      "logps/chosen": -88.92533874511719,
      "logps/ref_chosen": -88.91017150878906,
      "logps/ref_rejected": -99.1175537109375,
      "logps/rejected": -100.26213073730469,
      "loss": 1.2808,
      "margin_dpo/margin_mean": 1.129401683807373,
      "margin_dpo/margin_std": 1.303290843963623,
      "step": 54
    },
    {
      "KL/chosen_KL_mean": 0.13454818725585938,
      "KL/mean": -0.4299110770225525,
      "KL/rejected_KL_mean": -0.9943656921386719,
      "KL/std": 1.1683313846588135,
      "epoch": 0.08076358296622614,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.128913402557373,
      "fcm_dpo/q_t": 0.4720560312271118,
      "grad_norm": 35.54600524902344,
      "learning_rate": 3.9130434782608694e-07,
      "logits/chosen": 0.12294469773769379,
      "logits/rejected": 0.42934727668762207,
      "logps/chosen": -92.32137298583984,
      "logps/ref_chosen": -92.45591735839844,
      "logps/ref_rejected": -92.97093200683594,
      "logps/rejected": -93.96530151367188,
      "loss": 1.2824,
      "margin_dpo/margin_mean": 1.128913164138794,
      "margin_dpo/margin_std": 1.5035631656646729,
      "step": 55
    },
    {
      "KL/chosen_KL_mean": -0.16281890869140625,
      "KL/mean": -0.6875529289245605,
      "KL/rejected_KL_mean": -1.2122840881347656,
      "KL/std": 1.2281386852264404,
      "epoch": 0.08223201174743025,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0494670867919922,
      "fcm_dpo/q_t": 0.4740346074104309,
      "grad_norm": 33.242610931396484,
      "learning_rate": 3.9855072463768114e-07,
      "logits/chosen": 0.20589306950569153,
      "logits/rejected": 0.5453356504440308,
      "logps/chosen": -87.39947509765625,
      "logps/ref_chosen": -87.23665618896484,
      "logps/ref_rejected": -100.64553833007812,
      "logps/rejected": -101.85782623291016,
      "loss": 1.291,
      "margin_dpo/margin_mean": 1.0494662523269653,
      "margin_dpo/margin_std": 1.6543631553649902,
      "step": 56
    },
    {
      "KL/chosen_KL_mean": -0.06944656372070312,
      "KL/mean": -0.6393994092941284,
      "KL/rejected_KL_mean": -1.2093544006347656,
      "KL/std": 1.2679367065429688,
      "epoch": 0.08370044052863436,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1399201154708862,
      "fcm_dpo/q_t": 0.4718002676963806,
      "grad_norm": 33.5506477355957,
      "learning_rate": 4.057971014492754e-07,
      "logits/chosen": 0.4567074775695801,
      "logits/rejected": 0.6803990602493286,
      "logps/chosen": -98.22018432617188,
      "logps/ref_chosen": -98.15074157714844,
      "logps/ref_rejected": -101.05284118652344,
      "logps/rejected": -102.26219177246094,
      "loss": 1.2822,
      "margin_dpo/margin_mean": 1.1399204730987549,
      "margin_dpo/margin_std": 1.6380152702331543,
      "step": 57
    },
    {
      "KL/chosen_KL_mean": 0.2353668212890625,
      "KL/mean": -0.5463656187057495,
      "KL/rejected_KL_mean": -1.328094482421875,
      "KL/std": 1.4204570055007935,
      "epoch": 0.08516886930983847,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.5634589195251465,
      "fcm_dpo/q_t": 0.4614941477775574,
      "grad_norm": 38.17324447631836,
      "learning_rate": 4.1304347826086954e-07,
      "logits/chosen": 0.043736301362514496,
      "logits/rejected": 0.3536825180053711,
      "logps/chosen": -99.3456039428711,
      "logps/ref_chosen": -99.58097076416016,
      "logps/ref_rejected": -91.22227478027344,
      "logps/rejected": -92.55036926269531,
      "loss": 1.2447,
      "margin_dpo/margin_mean": 1.5634586811065674,
      "margin_dpo/margin_std": 1.7996635437011719,
      "step": 58
    },
    {
      "KL/chosen_KL_mean": 0.02886199951171875,
      "KL/mean": -0.5994742512702942,
      "KL/rejected_KL_mean": -1.2278099060058594,
      "KL/std": 1.183937668800354,
      "epoch": 0.08663729809104258,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2566630840301514,
      "fcm_dpo/q_t": 0.46886834502220154,
      "grad_norm": 34.16793441772461,
      "learning_rate": 4.2028985507246374e-07,
      "logits/chosen": 0.0859917551279068,
      "logits/rejected": 0.2755841016769409,
      "logps/chosen": -89.77346801757812,
      "logps/ref_chosen": -89.80232238769531,
      "logps/ref_rejected": -94.27667236328125,
      "logps/rejected": -95.50448608398438,
      "loss": 1.2699,
      "margin_dpo/margin_mean": 1.2566626071929932,
      "margin_dpo/margin_std": 1.4483157396316528,
      "step": 59
    },
    {
      "KL/chosen_KL_mean": -0.04311370849609375,
      "KL/mean": -0.6052899360656738,
      "KL/rejected_KL_mean": -1.1674728393554688,
      "KL/std": 1.2212886810302734,
      "epoch": 0.0881057268722467,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.1243648529052734,
      "fcm_dpo/q_t": 0.47208476066589355,
      "grad_norm": 29.016010284423828,
      "learning_rate": 4.2753623188405794e-07,
      "logits/chosen": 0.36096423864364624,
      "logits/rejected": 0.6281805038452148,
      "logps/chosen": -95.19882202148438,
      "logps/ref_chosen": -95.15571594238281,
      "logps/ref_rejected": -91.3724365234375,
      "logps/rejected": -92.53990936279297,
      "loss": 1.2816,
      "margin_dpo/margin_mean": 1.1243648529052734,
      "margin_dpo/margin_std": 1.3506968021392822,
      "step": 60
    },
    {
      "KL/chosen_KL_mean": -0.20401382446289062,
      "KL/mean": -0.7684180736541748,
      "KL/rejected_KL_mean": -1.3328208923339844,
      "KL/std": 1.3854691982269287,
      "epoch": 0.08957415565345081,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.12880277633667,
      "fcm_dpo/q_t": 0.47229456901550293,
      "grad_norm": 28.453763961791992,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": 0.28292596340179443,
      "logits/rejected": 0.4254472255706787,
      "logps/chosen": -85.93632507324219,
      "logps/ref_chosen": -85.73230743408203,
      "logps/ref_rejected": -97.96575927734375,
      "logps/rejected": -99.298583984375,
      "loss": 1.285,
      "margin_dpo/margin_mean": 1.1288026571273804,
      "margin_dpo/margin_std": 1.7909438610076904,
      "step": 61
    },
    {
      "KL/chosen_KL_mean": -0.16046142578125,
      "KL/mean": -0.7982224822044373,
      "KL/rejected_KL_mean": -1.4359855651855469,
      "KL/std": 1.475393533706665,
      "epoch": 0.09104258443465492,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.275530219078064,
      "fcm_dpo/q_t": 0.46859806776046753,
      "grad_norm": 28.625490188598633,
      "learning_rate": 4.420289855072464e-07,
      "logits/chosen": 0.22347919642925262,
      "logits/rejected": 0.42385077476501465,
      "logps/chosen": -81.79584503173828,
      "logps/ref_chosen": -81.63538360595703,
      "logps/ref_rejected": -84.03831481933594,
      "logps/rejected": -85.47429656982422,
      "loss": 1.2734,
      "margin_dpo/margin_mean": 1.2755298614501953,
      "margin_dpo/margin_std": 1.9424405097961426,
      "step": 62
    },
    {
      "KL/chosen_KL_mean": -0.005611419677734375,
      "KL/mean": -0.800529956817627,
      "KL/rejected_KL_mean": -1.5954513549804688,
      "KL/std": 1.4559605121612549,
      "epoch": 0.09251101321585903,
      "fcm_dpo/beta": 0.10000000149011612,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.589846134185791,
      "fcm_dpo/q_t": 0.4606913626194,
      "grad_norm": 34.8215217590332,
      "learning_rate": 4.4927536231884053e-07,
      "logits/chosen": 0.5357827544212341,
      "logits/rejected": 0.760285496711731,
      "logps/chosen": -103.62966918945312,
      "logps/ref_chosen": -103.62405395507812,
      "logps/ref_rejected": -103.40303039550781,
      "logps/rejected": -104.99848175048828,
      "loss": 1.241,
      "margin_dpo/margin_mean": 1.5898463726043701,
      "margin_dpo/margin_std": 1.6962254047393799,
      "step": 63
    },
    {
      "KL/chosen_KL_mean": -0.1227874755859375,
      "KL/mean": -0.9918836355209351,
      "KL/rejected_KL_mean": -1.8609848022460938,
      "KL/std": 1.7937242984771729,
      "epoch": 0.09397944199706314,
      "fcm_dpo/beta": 0.10574857890605927,
      "fcm_dpo/delta": 0.09315701574087143,
      "fcm_dpo/margin": 1.7381972074508667,
      "fcm_dpo/q_t": 0.4564475119113922,
      "grad_norm": 36.42966842651367,
      "learning_rate": 4.5652173913043473e-07,
      "logits/chosen": 0.6088030934333801,
      "logits/rejected": 0.8315421342849731,
      "logps/chosen": -87.12437438964844,
      "logps/ref_chosen": -87.0015869140625,
      "logps/ref_rejected": -100.5854721069336,
      "logps/rejected": -102.44645690917969,
      "loss": 1.2233,
      "margin_dpo/margin_mean": 1.7381969690322876,
      "margin_dpo/margin_std": 2.0424246788024902,
      "step": 64
    },
    {
      "KL/chosen_KL_mean": -0.08455657958984375,
      "KL/mean": -0.9359855055809021,
      "KL/rejected_KL_mean": -1.7874183654785156,
      "KL/std": 1.6582458019256592,
      "epoch": 0.09544787077826726,
      "fcm_dpo/beta": 0.10574857890605927,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.7028577327728271,
      "fcm_dpo/q_t": 0.45592403411865234,
      "grad_norm": 40.25533676147461,
      "learning_rate": 4.63768115942029e-07,
      "logits/chosen": 0.05840514972805977,
      "logits/rejected": 0.20318356156349182,
      "logps/chosen": -91.30647277832031,
      "logps/ref_chosen": -91.22191619873047,
      "logps/ref_rejected": -115.33553314208984,
      "logps/rejected": -117.12295532226562,
      "loss": 1.2267,
      "margin_dpo/margin_mean": 1.702858567237854,
      "margin_dpo/margin_std": 2.1073904037475586,
      "step": 65
    },
    {
      "KL/chosen_KL_mean": -0.31862640380859375,
      "KL/mean": -0.9341774582862854,
      "KL/rejected_KL_mean": -1.5497283935546875,
      "KL/std": 1.6019315719604492,
      "epoch": 0.09691629955947137,
      "fcm_dpo/beta": 0.10574857890605927,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.2311046123504639,
      "fcm_dpo/q_t": 0.4680859446525574,
      "grad_norm": 26.769336700439453,
      "learning_rate": 4.7101449275362313e-07,
      "logits/chosen": 0.09048471599817276,
      "logits/rejected": 0.289577841758728,
      "logps/chosen": -84.10285186767578,
      "logps/ref_chosen": -83.78422546386719,
      "logps/ref_rejected": -82.7520980834961,
      "logps/rejected": -84.30182647705078,
      "loss": 1.2717,
      "margin_dpo/margin_mean": 1.2311049699783325,
      "margin_dpo/margin_std": 1.9570016860961914,
      "step": 66
    },
    {
      "KL/chosen_KL_mean": -0.8981285095214844,
      "KL/mean": -1.4339194297790527,
      "KL/rejected_KL_mean": -1.9697036743164062,
      "KL/std": 2.0047428607940674,
      "epoch": 0.09838472834067548,
      "fcm_dpo/beta": 0.10574857890605927,
      "fcm_dpo/delta": 0.0,
      "fcm_dpo/margin": 1.0715798139572144,
      "fcm_dpo/q_t": 0.47269880771636963,
      "grad_norm": 25.728153228759766,
      "learning_rate": 4.782608695652174e-07,
      "logits/chosen": 0.41361337900161743,
      "logits/rejected": 0.6960128545761108,
      "logps/chosen": -88.57109069824219,
      "logps/ref_chosen": -87.67295837402344,
      "logps/ref_rejected": -79.08674621582031,
      "logps/rejected": -81.05645751953125,
      "loss": 1.2962,
      "margin_dpo/margin_mean": 1.071579933166504,
      "margin_dpo/margin_std": 2.678385019302368,
      "step": 67
    },
    {
      "KL/chosen_KL_mean": -0.23705291748046875,
      "KL/mean": -1.306268334388733,
      "KL/rejected_KL_mean": -2.3754806518554688,
      "KL/std": 2.279259443283081,
      "epoch": 0.09985315712187959,
      "fcm_dpo/beta": 0.11429806053638458,
      "fcm_dpo/delta": 0.16729214787483215,
      "fcm_dpo/margin": 2.1384334564208984,
      "fcm_dpo/q_t": 0.4444364011287689,
      "grad_norm": 30.86060333251953,
      "learning_rate": 4.855072463768116e-07,
      "logits/chosen": 0.051854174584150314,
      "logits/rejected": 0.3580140471458435,
      "logps/chosen": -97.31590270996094,
      "logps/ref_chosen": -97.07884216308594,
      "logps/ref_rejected": -84.11872863769531,
      "logps/rejected": -86.49420928955078,
      "loss": 1.1826,
      "margin_dpo/margin_mean": 2.1384332180023193,
      "margin_dpo/margin_std": 2.863788604736328,
      "step": 68
    },
    {
      "KL/chosen_KL_mean": -0.26673126220703125,
      "KL/mean": -1.4098851680755615,
      "KL/rejected_KL_mean": -2.55303955078125,
      "KL/std": 2.43912410736084,
      "epoch": 0.1013215859030837,
      "fcm_dpo/beta": 0.1242436096072197,
      "fcm_dpo/delta": 0.12596552073955536,
      "fcm_dpo/margin": 2.2863035202026367,
      "fcm_dpo/q_t": 0.4351448714733124,
      "grad_norm": 35.16178894042969,
      "learning_rate": 4.927536231884058e-07,
      "logits/chosen": 0.32528191804885864,
      "logits/rejected": 0.4329597055912018,
      "logps/chosen": -85.98644256591797,
      "logps/ref_chosen": -85.71971130371094,
      "logps/ref_rejected": -109.48020935058594,
      "logps/rejected": -112.03324890136719,
      "loss": 1.1548,
      "margin_dpo/margin_mean": 2.286303997039795,
      "margin_dpo/margin_std": 2.9793944358825684,
      "step": 69
    },
    {
      "KL/chosen_KL_mean": -0.5641059875488281,
      "KL/mean": -1.8750416040420532,
      "KL/rejected_KL_mean": -3.1859779357910156,
      "KL/std": 2.762816905975342,
      "epoch": 0.1027900146842878,
      "fcm_dpo/beta": 0.13005325198173523,
      "fcm_dpo/delta": 0.06559156626462936,
      "fcm_dpo/margin": 2.6218714714050293,
      "fcm_dpo/q_t": 0.4227621555328369,
      "grad_norm": 34.204044342041016,
      "learning_rate": 5e-07,
      "logits/chosen": 0.22942177951335907,
      "logits/rejected": 0.40416640043258667,
      "logps/chosen": -95.57405090332031,
      "logps/ref_chosen": -95.00994873046875,
      "logps/ref_rejected": -96.21272277832031,
      "logps/rejected": -99.39869689941406,
      "loss": 1.1204,
      "margin_dpo/margin_mean": 2.6218717098236084,
      "margin_dpo/margin_std": 3.3703863620758057,
      "step": 70
    },
    {
      "KL/chosen_KL_mean": -0.10558319091796875,
      "KL/mean": -1.9037671089172363,
      "KL/rejected_KL_mean": -3.7019577026367188,
      "KL/std": 3.4358456134796143,
      "epoch": 0.10425844346549193,
      "fcm_dpo/beta": 0.12580490112304688,
      "fcm_dpo/delta": -0.06296645104885101,
      "fcm_dpo/margin": 3.5963757038116455,
      "fcm_dpo/q_t": 0.39504915475845337,
      "grad_norm": 39.50156784057617,
      "learning_rate": 4.999967061337492e-07,
      "logits/chosen": -0.11060446500778198,
      "logits/rejected": 0.2538440227508545,
      "logps/chosen": -91.08293151855469,
      "logps/ref_chosen": -90.97735595703125,
      "logps/ref_rejected": -102.59103393554688,
      "logps/rejected": -106.29298400878906,
      "loss": 1.0376,
      "margin_dpo/margin_mean": 3.5963759422302246,
      "margin_dpo/margin_std": 3.8943240642547607,
      "step": 71
    },
    {
      "KL/chosen_KL_mean": -0.6036224365234375,
      "KL/mean": -2.043973922729492,
      "KL/rejected_KL_mean": -3.484333038330078,
      "KL/std": 2.938852310180664,
      "epoch": 0.10572687224669604,
      "fcm_dpo/beta": 0.12773177027702332,
      "fcm_dpo/delta": 0.03603281453251839,
      "fcm_dpo/margin": 2.8807120323181152,
      "fcm_dpo/q_t": 0.41478389501571655,
      "grad_norm": 34.58380126953125,
      "learning_rate": 4.999868246217933e-07,
      "logits/chosen": 0.0036033689975738525,
      "logits/rejected": 0.3418530821800232,
      "logps/chosen": -98.49514770507812,
      "logps/ref_chosen": -97.89152526855469,
      "logps/ref_rejected": -100.19171142578125,
      "logps/rejected": -103.6760482788086,
      "loss": 1.0955,
      "margin_dpo/margin_mean": 2.8807125091552734,
      "margin_dpo/margin_std": 3.377729892730713,
      "step": 72
    },
    {
      "KL/chosen_KL_mean": -0.7455329895019531,
      "KL/mean": -2.28900146484375,
      "KL/rejected_KL_mean": -3.8324737548828125,
      "KL/std": 3.7662665843963623,
      "epoch": 0.10719530102790015,
      "fcm_dpo/beta": 0.12929880619049072,
      "fcm_dpo/delta": 0.0009626960381865501,
      "fcm_dpo/margin": 3.0869386196136475,
      "fcm_dpo/q_t": 0.41355645656585693,
      "grad_norm": 32.428592681884766,
      "learning_rate": 4.999703557245192e-07,
      "logits/chosen": -0.1532449871301651,
      "logits/rejected": 0.16656705737113953,
      "logps/chosen": -96.51455688476562,
      "logps/ref_chosen": -95.7690200805664,
      "logps/ref_rejected": -95.93243408203125,
      "logps/rejected": -99.76490783691406,
      "loss": 1.1121,
      "margin_dpo/margin_mean": 3.08693790435791,
      "margin_dpo/margin_std": 4.839898109436035,
      "step": 73
    },
    {
      "KL/chosen_KL_mean": -0.9901046752929688,
      "KL/mean": -2.72599720954895,
      "KL/rejected_KL_mean": -4.461891174316406,
      "KL/std": 3.752127170562744,
      "epoch": 0.10866372980910426,
      "fcm_dpo/beta": 0.1257437765598297,
      "fcm_dpo/delta": -0.04282357543706894,
      "fcm_dpo/margin": 3.471791982650757,
      "fcm_dpo/q_t": 0.40088897943496704,
      "grad_norm": 34.84873962402344,
      "learning_rate": 4.999472998758977e-07,
      "logits/chosen": 0.00663900189101696,
      "logits/rejected": 0.1502746343612671,
      "logps/chosen": -79.79850006103516,
      "logps/ref_chosen": -78.80839538574219,
      "logps/ref_rejected": -101.64677429199219,
      "logps/rejected": -106.10865783691406,
      "loss": 1.0673,
      "margin_dpo/margin_mean": 3.4717917442321777,
      "margin_dpo/margin_std": 4.448845863342285,
      "step": 74
    },
    {
      "KL/chosen_KL_mean": -1.0334205627441406,
      "KL/mean": -3.2896485328674316,
      "KL/rejected_KL_mean": -5.5458831787109375,
      "KL/std": 4.480316638946533,
      "epoch": 0.11013215859030837,
      "fcm_dpo/beta": 0.11868904531002045,
      "fcm_dpo/delta": -0.16064026951789856,
      "fcm_dpo/margin": 4.512465476989746,
      "fcm_dpo/q_t": 0.3789573311805725,
      "grad_norm": 34.83369064331055,
      "learning_rate": 4.999176576834721e-07,
      "logits/chosen": -0.32559818029403687,
      "logits/rejected": -0.1226997897028923,
      "logps/chosen": -79.31527709960938,
      "logps/ref_chosen": -78.28185272216797,
      "logps/ref_rejected": -115.40311431884766,
      "logps/rejected": -120.9489974975586,
      "loss": 1.0056,
      "margin_dpo/margin_mean": 4.512465476989746,
      "margin_dpo/margin_std": 5.260388374328613,
      "step": 75
    },
    {
      "KL/chosen_KL_mean": -1.4338340759277344,
      "KL/mean": -3.0040442943573,
      "KL/rejected_KL_mean": -4.5742645263671875,
      "KL/std": 3.8455944061279297,
      "epoch": 0.11160058737151249,
      "fcm_dpo/beta": 0.11624883115291595,
      "fcm_dpo/delta": 0.039236292243003845,
      "fcm_dpo/margin": 3.140429735183716,
      "fcm_dpo/q_t": 0.4185473620891571,
      "grad_norm": 25.690200805664062,
      "learning_rate": 4.998814299283415e-07,
      "logits/chosen": -0.6047390699386597,
      "logits/rejected": -0.22543105483055115,
      "logps/chosen": -89.31097412109375,
      "logps/ref_chosen": -87.87713623046875,
      "logps/ref_rejected": -85.71968078613281,
      "logps/rejected": -90.2939453125,
      "loss": 1.1207,
      "margin_dpo/margin_mean": 3.140429973602295,
      "margin_dpo/margin_std": 4.656819820404053,
      "step": 76
    },
    {
      "KL/chosen_KL_mean": -1.6432876586914062,
      "KL/mean": -3.319711685180664,
      "KL/rejected_KL_mean": -4.996135711669922,
      "KL/std": 4.458495616912842,
      "epoch": 0.1130690161527166,
      "fcm_dpo/beta": 0.11663267016410828,
      "fcm_dpo/delta": 0.009803339838981628,
      "fcm_dpo/margin": 3.3528432846069336,
      "fcm_dpo/q_t": 0.4120829105377197,
      "grad_norm": 30.479032516479492,
      "learning_rate": 4.998386175651409e-07,
      "logits/chosen": -0.017038114368915558,
      "logits/rejected": 0.3690095543861389,
      "logps/chosen": -101.34363555908203,
      "logps/ref_chosen": -99.70034790039062,
      "logps/ref_rejected": -98.20576477050781,
      "logps/rejected": -103.20189666748047,
      "loss": 1.1132,
      "margin_dpo/margin_mean": 3.3528430461883545,
      "margin_dpo/margin_std": 5.037170886993408,
      "step": 77
    },
    {
      "KL/chosen_KL_mean": -0.6886749267578125,
      "KL/mean": -2.518968105316162,
      "KL/rejected_KL_mean": -4.349266052246094,
      "KL/std": 3.7086033821105957,
      "epoch": 0.1145374449339207,
      "fcm_dpo/beta": 0.11592504382133484,
      "fcm_dpo/delta": -0.028045756742358208,
      "fcm_dpo/margin": 3.660592555999756,
      "fcm_dpo/q_t": 0.40231800079345703,
      "grad_norm": 27.284442901611328,
      "learning_rate": 4.997892217220159e-07,
      "logits/chosen": 0.3182200789451599,
      "logits/rejected": 0.5652003288269043,
      "logps/chosen": -90.98538208007812,
      "logps/ref_chosen": -90.29670715332031,
      "logps/ref_rejected": -91.13772583007812,
      "logps/rejected": -95.48699188232422,
      "loss": 1.0633,
      "margin_dpo/margin_mean": 3.6605923175811768,
      "margin_dpo/margin_std": 4.364482879638672,
      "step": 78
    },
    {
      "KL/chosen_KL_mean": -0.9002113342285156,
      "KL/mean": -3.2634940147399902,
      "KL/rejected_KL_mean": -5.6267852783203125,
      "KL/std": 5.357280731201172,
      "epoch": 0.11600587371512482,
      "fcm_dpo/beta": 0.10934148728847504,
      "fcm_dpo/delta": -0.13787810504436493,
      "fcm_dpo/margin": 4.726571083068848,
      "fcm_dpo/q_t": 0.3826444149017334,
      "grad_norm": 28.235061645507812,
      "learning_rate": 4.997332437005931e-07,
      "logits/chosen": -0.6626664400100708,
      "logits/rejected": -0.28403806686401367,
      "logps/chosen": -87.27853393554688,
      "logps/ref_chosen": -86.37832641601562,
      "logps/ref_rejected": -94.10777282714844,
      "logps/rejected": -99.73455810546875,
      "loss": 1.0235,
      "margin_dpo/margin_mean": 4.726571083068848,
      "margin_dpo/margin_std": 5.863595962524414,
      "step": 79
    },
    {
      "KL/chosen_KL_mean": -1.5418586730957031,
      "KL/mean": -3.458197593688965,
      "KL/rejected_KL_mean": -5.374538421630859,
      "KL/std": 4.751584529876709,
      "epoch": 0.11747430249632893,
      "fcm_dpo/beta": 0.10612836480140686,
      "fcm_dpo/delta": -0.007701331749558449,
      "fcm_dpo/margin": 3.8326797485351562,
      "fcm_dpo/q_t": 0.4087960422039032,
      "grad_norm": 26.289152145385742,
      "learning_rate": 4.996706849759452e-07,
      "logits/chosen": -0.7835044860839844,
      "logits/rejected": -0.4342138171195984,
      "logps/chosen": -95.51217651367188,
      "logps/ref_chosen": -93.97032165527344,
      "logps/ref_rejected": -92.57441711425781,
      "logps/rejected": -97.94895935058594,
      "loss": 1.0872,
      "margin_dpo/margin_mean": 3.8326797485351562,
      "margin_dpo/margin_std": 5.155909538269043,
      "step": 80
    },
    {
      "KL/chosen_KL_mean": -1.0715065002441406,
      "KL/mean": -3.7517361640930176,
      "KL/rejected_KL_mean": -6.431964874267578,
      "KL/std": 5.863026142120361,
      "epoch": 0.11894273127753303,
      "fcm_dpo/beta": 0.09724778681993484,
      "fcm_dpo/delta": -0.15014737844467163,
      "fcm_dpo/margin": 5.3604583740234375,
      "fcm_dpo/q_t": 0.3851175308227539,
      "grad_norm": 27.164127349853516,
      "learning_rate": 4.996015471965529e-07,
      "logits/chosen": -0.5423442125320435,
      "logits/rejected": -0.11592958122491837,
      "logps/chosen": -100.90162658691406,
      "logps/ref_chosen": -99.83012390136719,
      "logps/ref_rejected": -133.67247009277344,
      "logps/rejected": -140.10443115234375,
      "loss": 1.03,
      "margin_dpo/margin_mean": 5.360459327697754,
      "margin_dpo/margin_std": 7.009407997131348,
      "step": 81
    },
    {
      "KL/chosen_KL_mean": -1.9401092529296875,
      "KL/mean": -3.7880847454071045,
      "KL/rejected_KL_mean": -5.636058807373047,
      "KL/std": 4.525975704193115,
      "epoch": 0.12041116005873716,
      "fcm_dpo/beta": 0.10018362104892731,
      "fcm_dpo/delta": 0.03008412942290306,
      "fcm_dpo/margin": 3.695950508117676,
      "fcm_dpo/q_t": 0.41528964042663574,
      "grad_norm": 25.989595413208008,
      "learning_rate": 4.995258321842611e-07,
      "logits/chosen": -0.1208338513970375,
      "logits/rejected": 0.09825039654970169,
      "logps/chosen": -84.98609161376953,
      "logps/ref_chosen": -83.04598236083984,
      "logps/ref_rejected": -94.52595520019531,
      "logps/rejected": -100.1620101928711,
      "loss": 1.111,
      "margin_dpo/margin_mean": 3.695950746536255,
      "margin_dpo/margin_std": 4.948929309844971,
      "step": 82
    },
    {
      "KL/chosen_KL_mean": -2.2260665893554688,
      "KL/mean": -4.306647300720215,
      "KL/rejected_KL_mean": -6.387226104736328,
      "KL/std": 5.088846206665039,
      "epoch": 0.12187958883994127,
      "fcm_dpo/beta": 0.09825055301189423,
      "fcm_dpo/delta": -0.010072429664433002,
      "fcm_dpo/margin": 4.16117000579834,
      "fcm_dpo/q_t": 0.40939655900001526,
      "grad_norm": 26.697071075439453,
      "learning_rate": 4.994435419342304e-07,
      "logits/chosen": -0.6126989722251892,
      "logits/rejected": -0.19759254157543182,
      "logps/chosen": -94.40228271484375,
      "logps/ref_chosen": -92.17621612548828,
      "logps/ref_rejected": -107.74464416503906,
      "logps/rejected": -114.13186645507812,
      "loss": 1.1087,
      "margin_dpo/margin_mean": 4.161169052124023,
      "margin_dpo/margin_std": 6.4333062171936035,
      "step": 83
    },
    {
      "KL/chosen_KL_mean": -2.9193878173828125,
      "KL/mean": -4.193624496459961,
      "KL/rejected_KL_mean": -5.467861175537109,
      "KL/std": 4.584219932556152,
      "epoch": 0.12334801762114538,
      "fcm_dpo/beta": 0.09807070344686508,
      "fcm_dpo/delta": -0.0003195726894773543,
      "fcm_dpo/margin": 2.548470973968506,
      "fcm_dpo/q_t": 0.4433935582637787,
      "grad_norm": 27.83990478515625,
      "learning_rate": 4.993546786148857e-07,
      "logits/chosen": -0.4146873950958252,
      "logits/rejected": -0.028379738330841064,
      "logps/chosen": -104.44587707519531,
      "logps/ref_chosen": -101.5264892578125,
      "logps/ref_rejected": -92.42608642578125,
      "logps/rejected": -97.89395141601562,
      "loss": 1.2121,
      "margin_dpo/margin_mean": 2.548471689224243,
      "margin_dpo/margin_std": 4.922743797302246,
      "step": 84
    },
    {
      "KL/chosen_KL_mean": -2.5317916870117188,
      "KL/mean": -4.295086860656738,
      "KL/rejected_KL_mean": -6.058380126953125,
      "KL/std": 5.017945766448975,
      "epoch": 0.12481644640234948,
      "fcm_dpo/beta": 0.10136842727661133,
      "fcm_dpo/delta": 0.0458756648004055,
      "fcm_dpo/margin": 3.5265915393829346,
      "fcm_dpo/q_t": 0.42067793011665344,
      "grad_norm": 27.12324333190918,
      "learning_rate": 4.992592445678582e-07,
      "logits/chosen": -0.22527801990509033,
      "logits/rejected": 0.1162186861038208,
      "logps/chosen": -98.65916442871094,
      "logps/ref_chosen": -96.12738037109375,
      "logps/ref_rejected": -85.05519104003906,
      "logps/rejected": -91.11357116699219,
      "loss": 1.1502,
      "margin_dpo/margin_mean": 3.526592493057251,
      "margin_dpo/margin_std": 6.103909492492676,
      "step": 85
    },
    {
      "KL/chosen_KL_mean": -2.0357131958007812,
      "KL/mean": -4.248371124267578,
      "KL/rejected_KL_mean": -6.461029052734375,
      "KL/std": 5.855093002319336,
      "epoch": 0.1262848751835536,
      "fcm_dpo/beta": 0.09849658608436584,
      "fcm_dpo/delta": -0.0416511595249176,
      "fcm_dpo/margin": 4.425321578979492,
      "fcm_dpo/q_t": 0.4077085852622986,
      "grad_norm": 22.06851577758789,
      "learning_rate": 4.991572423079235e-07,
      "logits/chosen": -0.46623319387435913,
      "logits/rejected": -0.21914854645729065,
      "logps/chosen": -83.73997497558594,
      "logps/ref_chosen": -81.70426177978516,
      "logps/ref_rejected": -93.65550231933594,
      "logps/rejected": -100.11653137207031,
      "loss": 1.1041,
      "margin_dpo/margin_mean": 4.42532205581665,
      "margin_dpo/margin_std": 7.2861175537109375,
      "step": 86
    },
    {
      "KL/chosen_KL_mean": -2.3987808227539062,
      "KL/mean": -4.743775367736816,
      "KL/rejected_KL_mean": -7.088764190673828,
      "KL/std": 6.3629608154296875,
      "epoch": 0.1277533039647577,
      "fcm_dpo/beta": 0.09521473199129105,
      "fcm_dpo/delta": -0.05466712266206741,
      "fcm_dpo/margin": 4.689988136291504,
      "fcm_dpo/q_t": 0.39907458424568176,
      "grad_norm": 25.35971450805664,
      "learning_rate": 4.990486745229364e-07,
      "logits/chosen": -0.775127112865448,
      "logits/rejected": -0.4026734232902527,
      "logps/chosen": -95.08474731445312,
      "logps/ref_chosen": -92.68596649169922,
      "logps/ref_rejected": -102.91818237304688,
      "logps/rejected": -110.00694274902344,
      "loss": 1.1038,
      "margin_dpo/margin_mean": 4.689988136291504,
      "margin_dpo/margin_std": 7.550135612487793,
      "step": 87
    },
    {
      "KL/chosen_KL_mean": -2.8831100463867188,
      "KL/mean": -4.781992435455322,
      "KL/rejected_KL_mean": -6.680870056152344,
      "KL/std": 6.1795148849487305,
      "epoch": 0.12922173274596183,
      "fcm_dpo/beta": 0.09525230526924133,
      "fcm_dpo/delta": 0.0414726585149765,
      "fcm_dpo/margin": 3.797753095626831,
      "fcm_dpo/q_t": 0.42012062668800354,
      "grad_norm": 23.575328826904297,
      "learning_rate": 4.989335440737586e-07,
      "logits/chosen": -0.2280137538909912,
      "logits/rejected": -0.03912961483001709,
      "logps/chosen": -103.64608764648438,
      "logps/ref_chosen": -100.76298522949219,
      "logps/ref_rejected": -113.15037536621094,
      "logps/rejected": -119.83123779296875,
      "loss": 1.1452,
      "margin_dpo/margin_mean": 3.797752857208252,
      "margin_dpo/margin_std": 6.371947288513184,
      "step": 88
    },
    {
      "KL/chosen_KL_mean": -2.3461875915527344,
      "KL/mean": -4.412843704223633,
      "KL/rejected_KL_mean": -6.479499816894531,
      "KL/std": 5.301060676574707,
      "epoch": 0.13069016152716592,
      "fcm_dpo/beta": 0.09678910672664642,
      "fcm_dpo/delta": -0.0004812609404325485,
      "fcm_dpo/margin": 4.133310317993164,
      "fcm_dpo/q_t": 0.4150283932685852,
      "grad_norm": 23.296411514282227,
      "learning_rate": 4.988118539941847e-07,
      "logits/chosen": -0.5917224287986755,
      "logits/rejected": -0.2460830807685852,
      "logps/chosen": -92.03726959228516,
      "logps/ref_chosen": -89.69108581542969,
      "logps/ref_rejected": -88.56832885742188,
      "logps/rejected": -95.04783630371094,
      "loss": 1.1218,
      "margin_dpo/margin_mean": 4.133309841156006,
      "margin_dpo/margin_std": 7.060988903045654,
      "step": 89
    },
    {
      "KL/chosen_KL_mean": -2.0983924865722656,
      "KL/mean": -4.808633804321289,
      "KL/rejected_KL_mean": -7.5188751220703125,
      "KL/std": 6.206541538238525,
      "epoch": 0.13215859030837004,
      "fcm_dpo/beta": 0.09331987798213959,
      "fcm_dpo/delta": -0.1242503896355629,
      "fcm_dpo/margin": 5.4204816818237305,
      "fcm_dpo/q_t": 0.3881657123565674,
      "grad_norm": 25.370624542236328,
      "learning_rate": 4.986836074908615e-07,
      "logits/chosen": -0.17194883525371552,
      "logits/rejected": -0.05834429711103439,
      "logps/chosen": -83.48094940185547,
      "logps/ref_chosen": -81.38255310058594,
      "logps/ref_rejected": -117.77714538574219,
      "logps/rejected": -125.2960205078125,
      "loss": 1.0481,
      "margin_dpo/margin_mean": 5.4204816818237305,
      "margin_dpo/margin_std": 7.445097923278809,
      "step": 90
    },
    {
      "KL/chosen_KL_mean": -2.4222450256347656,
      "KL/mean": -4.687593460083008,
      "KL/rejected_KL_mean": -6.95294189453125,
      "KL/std": 6.084618091583252,
      "epoch": 0.13362701908957417,
      "fcm_dpo/beta": 0.09016484767198563,
      "fcm_dpo/delta": -0.009695576503872871,
      "fcm_dpo/margin": 4.530700206756592,
      "fcm_dpo/q_t": 0.41130465269088745,
      "grad_norm": 23.73488998413086,
      "learning_rate": 4.985488079432037e-07,
      "logits/chosen": -0.2937658727169037,
      "logits/rejected": 0.1296396106481552,
      "logps/chosen": -99.64412689208984,
      "logps/ref_chosen": -97.22187805175781,
      "logps/ref_rejected": -92.97674560546875,
      "logps/rejected": -99.9296875,
      "loss": 1.1074,
      "margin_dpo/margin_mean": 4.530699729919434,
      "margin_dpo/margin_std": 7.080096244812012,
      "step": 91
    },
    {
      "KL/chosen_KL_mean": -2.6651649475097656,
      "KL/mean": -4.686550140380859,
      "KL/rejected_KL_mean": -6.707935333251953,
      "KL/std": 6.086717128753662,
      "epoch": 0.13509544787077826,
      "fcm_dpo/beta": 0.09177425503730774,
      "fcm_dpo/delta": 0.03221934661269188,
      "fcm_dpo/margin": 4.0427680015563965,
      "fcm_dpo/q_t": 0.4183937609195709,
      "grad_norm": 22.06598472595215,
      "learning_rate": 4.984074589033043e-07,
      "logits/chosen": -0.8036127090454102,
      "logits/rejected": -0.4346522390842438,
      "logps/chosen": -87.19544219970703,
      "logps/ref_chosen": -84.5302734375,
      "logps/ref_rejected": -84.5013198852539,
      "logps/rejected": -91.20925903320312,
      "loss": 1.1314,
      "margin_dpo/margin_mean": 4.042767524719238,
      "margin_dpo/margin_std": 6.603899002075195,
      "step": 92
    },
    {
      "KL/chosen_KL_mean": -2.945281982421875,
      "KL/mean": -4.841535568237305,
      "KL/rejected_KL_mean": -6.73779296875,
      "KL/std": 5.725800514221191,
      "epoch": 0.13656387665198239,
      "fcm_dpo/beta": 0.09409474581480026,
      "fcm_dpo/delta": 0.047999195754528046,
      "fcm_dpo/margin": 3.7925119400024414,
      "fcm_dpo/q_t": 0.42175865173339844,
      "grad_norm": 25.131797790527344,
      "learning_rate": 4.982595640958425e-07,
      "logits/chosen": -0.807417631149292,
      "logits/rejected": -0.22067147493362427,
      "logps/chosen": -93.19571685791016,
      "logps/ref_chosen": -90.25043487548828,
      "logps/ref_rejected": -84.0942153930664,
      "logps/rejected": -90.8320083618164,
      "loss": 1.1568,
      "margin_dpo/margin_mean": 3.7925119400024414,
      "margin_dpo/margin_std": 6.841826438903809,
      "step": 93
    },
    {
      "KL/chosen_KL_mean": -3.0549240112304688,
      "KL/mean": -5.960987091064453,
      "KL/rejected_KL_mean": -8.867050170898438,
      "KL/std": 6.788209915161133,
      "epoch": 0.13803230543318648,
      "fcm_dpo/beta": 0.08679958432912827,
      "fcm_dpo/delta": -0.1317116767168045,
      "fcm_dpo/margin": 5.812126159667969,
      "fcm_dpo/q_t": 0.3840278387069702,
      "grad_norm": 24.332944869995117,
      "learning_rate": 4.98105127417984e-07,
      "logits/chosen": -0.8378304243087769,
      "logits/rejected": -0.5412890911102295,
      "logps/chosen": -95.50914001464844,
      "logps/ref_chosen": -92.4542236328125,
      "logps/ref_rejected": -105.2472915649414,
      "logps/rejected": -114.11434173583984,
      "loss": 1.0494,
      "margin_dpo/margin_mean": 5.812125205993652,
      "margin_dpo/margin_std": 7.783188819885254,
      "step": 94
    },
    {
      "KL/chosen_KL_mean": -2.5052261352539062,
      "KL/mean": -4.787609577178955,
      "KL/rejected_KL_mean": -7.069988250732422,
      "KL/std": 5.63349723815918,
      "epoch": 0.1395007342143906,
      "fcm_dpo/beta": 0.08820401132106781,
      "fcm_dpo/delta": -0.004071902483701706,
      "fcm_dpo/margin": 4.564762115478516,
      "fcm_dpo/q_t": 0.41168665885925293,
      "grad_norm": 20.839780807495117,
      "learning_rate": 4.979441529392784e-07,
      "logits/chosen": -1.0057801008224487,
      "logits/rejected": -0.6787027716636658,
      "logps/chosen": -81.37893676757812,
      "logps/ref_chosen": -78.87370300292969,
      "logps/ref_rejected": -83.59121704101562,
      "logps/rejected": -90.66120147705078,
      "loss": 1.109,
      "margin_dpo/margin_mean": 4.564761638641357,
      "margin_dpo/margin_std": 6.9142165184021,
      "step": 95
    },
    {
      "KL/chosen_KL_mean": -2.1085472106933594,
      "KL/mean": -4.975281715393066,
      "KL/rejected_KL_mean": -7.842014312744141,
      "KL/std": 6.743013381958008,
      "epoch": 0.14096916299559473,
      "fcm_dpo/beta": 0.08310525119304657,
      "fcm_dpo/delta": -0.09016522765159607,
      "fcm_dpo/margin": 5.7334771156311035,
      "fcm_dpo/q_t": 0.39262157678604126,
      "grad_norm": 21.038515090942383,
      "learning_rate": 4.977766449015534e-07,
      "logits/chosen": -0.7446866035461426,
      "logits/rejected": -0.2978627681732178,
      "logps/chosen": -108.70071411132812,
      "logps/ref_chosen": -106.5921630859375,
      "logps/ref_rejected": -101.76802825927734,
      "logps/rejected": -109.61004638671875,
      "loss": 1.053,
      "margin_dpo/margin_mean": 5.733477592468262,
      "margin_dpo/margin_std": 7.4695820808410645,
      "step": 96
    },
    {
      "KL/chosen_KL_mean": -3.1854591369628906,
      "KL/mean": -5.7789177894592285,
      "KL/rejected_KL_mean": -8.372371673583984,
      "KL/std": 5.956425189971924,
      "epoch": 0.14243759177679882,
      "fcm_dpo/beta": 0.08288799226284027,
      "fcm_dpo/delta": -0.03676958009600639,
      "fcm_dpo/margin": 5.186917304992676,
      "fcm_dpo/q_t": 0.40120813250541687,
      "grad_norm": 23.0380802154541,
      "learning_rate": 4.976026077188012e-07,
      "logits/chosen": -0.4803990125656128,
      "logits/rejected": -0.07440396398305893,
      "logps/chosen": -91.86534118652344,
      "logps/ref_chosen": -88.67988586425781,
      "logps/ref_rejected": -84.81229400634766,
      "logps/rejected": -93.18466186523438,
      "loss": 1.0708,
      "margin_dpo/margin_mean": 5.186916828155518,
      "margin_dpo/margin_std": 6.351839542388916,
      "step": 97
    },
    {
      "KL/chosen_KL_mean": -2.4375991821289062,
      "KL/mean": -5.3795623779296875,
      "KL/rejected_KL_mean": -8.321525573730469,
      "KL/std": 5.74554967880249,
      "epoch": 0.14390602055800295,
      "fcm_dpo/beta": 0.07720876485109329,
      "fcm_dpo/delta": -0.06543175131082535,
      "fcm_dpo/margin": 5.883923530578613,
      "fcm_dpo/q_t": 0.39461973309516907,
      "grad_norm": 21.371925354003906,
      "learning_rate": 4.974220459770639e-07,
      "logits/chosen": -0.7688867449760437,
      "logits/rejected": -0.5065837502479553,
      "logps/chosen": -94.68009948730469,
      "logps/ref_chosen": -92.24249267578125,
      "logps/ref_rejected": -101.51948547363281,
      "logps/rejected": -109.84101104736328,
      "loss": 1.0417,
      "margin_dpo/margin_mean": 5.883923530578613,
      "margin_dpo/margin_std": 6.466315746307373,
      "step": 98
    },
    {
      "KL/chosen_KL_mean": -1.875823974609375,
      "KL/mean": -5.103307723999023,
      "KL/rejected_KL_mean": -8.330795288085938,
      "KL/std": 6.870201110839844,
      "epoch": 0.14537444933920704,
      "fcm_dpo/beta": 0.07417511940002441,
      "fcm_dpo/delta": -0.09249608218669891,
      "fcm_dpo/margin": 6.454975128173828,
      "fcm_dpo/q_t": 0.3947933316230774,
      "grad_norm": 20.892988204956055,
      "learning_rate": 4.972349644343108e-07,
      "logits/chosen": -0.45883697271347046,
      "logits/rejected": -0.24587397277355194,
      "logps/chosen": -74.06047058105469,
      "logps/ref_chosen": -72.18464660644531,
      "logps/ref_rejected": -91.88131713867188,
      "logps/rejected": -100.21211242675781,
      "loss": 1.0511,
      "margin_dpo/margin_mean": 6.454975128173828,
      "margin_dpo/margin_std": 8.757984161376953,
      "step": 99
    },
    {
      "KL/chosen_KL_mean": -2.1724319458007812,
      "KL/mean": -4.510197639465332,
      "KL/rejected_KL_mean": -6.847969055175781,
      "KL/std": 5.260673522949219,
      "epoch": 0.14684287812041116,
      "fcm_dpo/beta": 0.07463701069355011,
      "fcm_dpo/delta": 0.05699891969561577,
      "fcm_dpo/margin": 4.675529956817627,
      "fcm_dpo/q_t": 0.419418066740036,
      "grad_norm": 17.873132705688477,
      "learning_rate": 4.970413680203148e-07,
      "logits/chosen": -0.8066288232803345,
      "logits/rejected": -0.33307552337646484,
      "logps/chosen": -91.68626403808594,
      "logps/ref_chosen": -89.51382446289062,
      "logps/ref_rejected": -81.21713256835938,
      "logps/rejected": -88.06509399414062,
      "loss": 1.1085,
      "margin_dpo/margin_mean": 4.675529479980469,
      "margin_dpo/margin_std": 5.542096138000488,
      "step": 100
    },
    {
      "KL/chosen_KL_mean": -3.5872535705566406,
      "KL/mean": -5.653433322906494,
      "KL/rejected_KL_mean": -7.719612121582031,
      "KL/std": 6.771468639373779,
      "epoch": 0.14831130690161526,
      "fcm_dpo/beta": 0.07795991003513336,
      "fcm_dpo/delta": 0.08620306104421616,
      "fcm_dpo/margin": 4.132352352142334,
      "fcm_dpo/q_t": 0.42760327458381653,
      "grad_norm": 22.06405258178711,
      "learning_rate": 4.968412618365215e-07,
      "logits/chosen": -0.8927627205848694,
      "logits/rejected": -0.6300774812698364,
      "logps/chosen": -97.99044799804688,
      "logps/ref_chosen": -94.4031982421875,
      "logps/ref_rejected": -89.91001892089844,
      "logps/rejected": -97.62962341308594,
      "loss": 1.1698,
      "margin_dpo/margin_mean": 4.132352828979492,
      "margin_dpo/margin_std": 7.339719772338867,
      "step": 101
    },
    {
      "KL/chosen_KL_mean": -4.060203552246094,
      "KL/mean": -5.888674736022949,
      "KL/rejected_KL_mean": -7.717136383056641,
      "KL/std": 6.498498916625977,
      "epoch": 0.14977973568281938,
      "fcm_dpo/beta": 0.07927267998456955,
      "fcm_dpo/delta": -0.028191443532705307,
      "fcm_dpo/margin": 3.6569347381591797,
      "fcm_dpo/q_t": 0.43367448449134827,
      "grad_norm": 26.45783233642578,
      "learning_rate": 4.966346511559149e-07,
      "logits/chosen": -0.8144804835319519,
      "logits/rejected": -0.4419545829296112,
      "logps/chosen": -95.66963195800781,
      "logps/ref_chosen": -91.60942077636719,
      "logps/ref_rejected": -75.7689208984375,
      "logps/rejected": -83.48605346679688,
      "loss": 1.2013,
      "margin_dpo/margin_mean": 3.6569347381591797,
      "margin_dpo/margin_std": 7.378411769866943,
      "step": 102
    },
    {
      "KL/chosen_KL_mean": -3.1652565002441406,
      "KL/mean": -6.207461833953857,
      "KL/rejected_KL_mean": -9.249671936035156,
      "KL/std": 6.868796348571777,
      "epoch": 0.1512481644640235,
      "fcm_dpo/beta": 0.07570649683475494,
      "fcm_dpo/delta": -0.07109224051237106,
      "fcm_dpo/margin": 6.084412574768066,
      "fcm_dpo/q_t": 0.3936630189418793,
      "grad_norm": 21.005929946899414,
      "learning_rate": 4.964215414228785e-07,
      "logits/chosen": -0.8720337152481079,
      "logits/rejected": -0.4868471026420593,
      "logps/chosen": -100.56764221191406,
      "logps/ref_chosen": -97.40238952636719,
      "logps/ref_rejected": -98.981689453125,
      "logps/rejected": -108.23135375976562,
      "loss": 1.057,
      "margin_dpo/margin_mean": 6.084413528442383,
      "margin_dpo/margin_std": 7.694538116455078,
      "step": 103
    },
    {
      "KL/chosen_KL_mean": -2.447490692138672,
      "KL/mean": -5.729905128479004,
      "KL/rejected_KL_mean": -9.012313842773438,
      "KL/std": 7.457200050354004,
      "epoch": 0.1527165932452276,
      "fcm_dpo/beta": 0.07210113108158112,
      "fcm_dpo/delta": -0.08622559905052185,
      "fcm_dpo/margin": 6.564826011657715,
      "fcm_dpo/q_t": 0.3946932256221771,
      "grad_norm": 17.50535774230957,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": -0.8785950541496277,
      "logits/rejected": -0.5176867842674255,
      "logps/chosen": -97.30384826660156,
      "logps/ref_chosen": -94.85636138916016,
      "logps/ref_rejected": -97.02669525146484,
      "logps/rejected": -106.03900909423828,
      "loss": 1.0521,
      "margin_dpo/margin_mean": 6.564826965332031,
      "margin_dpo/margin_std": 8.817479133605957,
      "step": 104
    },
    {
      "KL/chosen_KL_mean": -3.232940673828125,
      "KL/mean": -6.498128890991211,
      "KL/rejected_KL_mean": -9.763324737548828,
      "KL/std": 6.746476173400879,
      "epoch": 0.15418502202643172,
      "fcm_dpo/beta": 0.07029742002487183,
      "fcm_dpo/delta": -0.06878115236759186,
      "fcm_dpo/margin": 6.530384063720703,
      "fcm_dpo/q_t": 0.39464184641838074,
      "grad_norm": 20.49209976196289,
      "learning_rate": 4.959758474331832e-07,
      "logits/chosen": -1.096966028213501,
      "logits/rejected": -0.6280566453933716,
      "logps/chosen": -99.9908447265625,
      "logps/ref_chosen": -96.75790405273438,
      "logps/ref_rejected": -101.5774917602539,
      "logps/rejected": -111.3408203125,
      "loss": 1.0616,
      "margin_dpo/margin_mean": 6.530384063720703,
      "margin_dpo/margin_std": 8.472076416015625,
      "step": 105
    },
    {
      "KL/chosen_KL_mean": -3.2045021057128906,
      "KL/mean": -6.337413311004639,
      "KL/rejected_KL_mean": -9.470321655273438,
      "KL/std": 7.035835266113281,
      "epoch": 0.15565345080763582,
      "fcm_dpo/beta": 0.06823772192001343,
      "fcm_dpo/delta": -0.03183862566947937,
      "fcm_dpo/margin": 6.265822410583496,
      "fcm_dpo/q_t": 0.40417781472206116,
      "grad_norm": 17.713918685913086,
      "learning_rate": 4.957432749209755e-07,
      "logits/chosen": -0.9206595420837402,
      "logits/rejected": -0.49735307693481445,
      "logps/chosen": -95.64241027832031,
      "logps/ref_chosen": -92.43791198730469,
      "logps/ref_rejected": -91.00970458984375,
      "logps/rejected": -100.48002624511719,
      "loss": 1.0803,
      "margin_dpo/margin_mean": 6.265821933746338,
      "margin_dpo/margin_std": 8.557239532470703,
      "step": 106
    },
    {
      "KL/chosen_KL_mean": -2.4019546508789062,
      "KL/mean": -6.033327579498291,
      "KL/rejected_KL_mean": -9.664703369140625,
      "KL/std": 8.967880249023438,
      "epoch": 0.15712187958883994,
      "fcm_dpo/beta": 0.06483877450227737,
      "fcm_dpo/delta": -0.08271554112434387,
      "fcm_dpo/margin": 7.262753009796143,
      "fcm_dpo/q_t": 0.40016061067581177,
      "grad_norm": 17.89908790588379,
      "learning_rate": 4.955042268449307e-07,
      "logits/chosen": -0.8961023092269897,
      "logits/rejected": -0.3874198794364929,
      "logps/chosen": -107.52696228027344,
      "logps/ref_chosen": -105.12500762939453,
      "logps/ref_rejected": -101.1824951171875,
      "logps/rejected": -110.84719848632812,
      "loss": 1.0717,
      "margin_dpo/margin_mean": 7.262753009796143,
      "margin_dpo/margin_std": 11.115553855895996,
      "step": 107
    },
    {
      "KL/chosen_KL_mean": -3.526538848876953,
      "KL/mean": -6.974469184875488,
      "KL/rejected_KL_mean": -10.42239761352539,
      "KL/std": 8.302928924560547,
      "epoch": 0.15859030837004406,
      "fcm_dpo/beta": 0.062272265553474426,
      "fcm_dpo/delta": -0.035427432507276535,
      "fcm_dpo/margin": 6.89585018157959,
      "fcm_dpo/q_t": 0.403268963098526,
      "grad_norm": 18.105493545532227,
      "learning_rate": 4.952587095041881e-07,
      "logits/chosen": -1.1066762208938599,
      "logits/rejected": -0.6103397011756897,
      "logps/chosen": -92.73590087890625,
      "logps/ref_chosen": -89.20936584472656,
      "logps/ref_rejected": -101.83784484863281,
      "logps/rejected": -112.26024627685547,
      "loss": 1.0778,
      "margin_dpo/margin_mean": 6.89585018157959,
      "margin_dpo/margin_std": 9.143634796142578,
      "step": 108
    },
    {
      "KL/chosen_KL_mean": -2.875499725341797,
      "KL/mean": -7.1359052658081055,
      "KL/rejected_KL_mean": -11.396312713623047,
      "KL/std": 8.405050277709961,
      "epoch": 0.16005873715124816,
      "fcm_dpo/beta": 0.05889669805765152,
      "fcm_dpo/delta": -0.12176138907670975,
      "fcm_dpo/margin": 8.520807266235352,
      "fcm_dpo/q_t": 0.3847600817680359,
      "grad_norm": 16.884319305419922,
      "learning_rate": 4.95006729368358e-07,
      "logits/chosen": -0.6510393023490906,
      "logits/rejected": -0.41912877559661865,
      "logps/chosen": -103.06259155273438,
      "logps/ref_chosen": -100.18708801269531,
      "logps/ref_rejected": -103.53739929199219,
      "logps/rejected": -114.93370819091797,
      "loss": 1.0278,
      "margin_dpo/margin_mean": 8.520808219909668,
      "margin_dpo/margin_std": 10.38311767578125,
      "step": 109
    },
    {
      "KL/chosen_KL_mean": -3.773151397705078,
      "KL/mean": -6.2924957275390625,
      "KL/rejected_KL_mean": -8.811847686767578,
      "KL/std": 7.046458721160889,
      "epoch": 0.16152716593245228,
      "fcm_dpo/beta": 0.05855482816696167,
      "fcm_dpo/delta": 0.007113963831216097,
      "fcm_dpo/margin": 5.03869104385376,
      "fcm_dpo/q_t": 0.4315415024757385,
      "grad_norm": 17.309476852416992,
      "learning_rate": 4.947482930773511e-07,
      "logits/chosen": -0.2093043029308319,
      "logits/rejected": 0.21902790665626526,
      "logps/chosen": -110.66886901855469,
      "logps/ref_chosen": -106.89572143554688,
      "logps/ref_rejected": -85.81120300292969,
      "logps/rejected": -94.623046875,
      "loss": 1.1662,
      "margin_dpo/margin_mean": 5.038690567016602,
      "margin_dpo/margin_std": 7.983441352844238,
      "step": 110
    },
    {
      "KL/chosen_KL_mean": -3.2819442749023438,
      "KL/mean": -8.302156448364258,
      "KL/rejected_KL_mean": -13.322372436523438,
      "KL/std": 9.848615646362305,
      "epoch": 0.16299559471365638,
      "fcm_dpo/beta": 0.05347032845020294,
      "fcm_dpo/delta": -0.16825544834136963,
      "fcm_dpo/margin": 10.040424346923828,
      "fcm_dpo/q_t": 0.3742956817150116,
      "grad_norm": 15.72132682800293,
      "learning_rate": 4.944834074412042e-07,
      "logits/chosen": -0.9423152208328247,
      "logits/rejected": -0.6501681804656982,
      "logps/chosen": -96.61131286621094,
      "logps/ref_chosen": -93.3293685913086,
      "logps/ref_rejected": -104.177490234375,
      "logps/rejected": -117.49986267089844,
      "loss": 0.9989,
      "margin_dpo/margin_mean": 10.040424346923828,
      "margin_dpo/margin_std": 10.975632667541504,
      "step": 111
    },
    {
      "KL/chosen_KL_mean": -3.8012313842773438,
      "KL/mean": -6.321389198303223,
      "KL/rejected_KL_mean": -8.841545104980469,
      "KL/std": 5.659070014953613,
      "epoch": 0.1644640234948605,
      "fcm_dpo/beta": 0.056000009179115295,
      "fcm_dpo/delta": 0.12875151634216309,
      "fcm_dpo/margin": 5.040307998657227,
      "fcm_dpo/q_t": 0.4358397126197815,
      "grad_norm": 14.867339134216309,
      "learning_rate": 4.942120794399002e-07,
      "logits/chosen": -1.2282984256744385,
      "logits/rejected": -0.716357409954071,
      "logps/chosen": -86.54827880859375,
      "logps/ref_chosen": -82.74703979492188,
      "logps/ref_rejected": -71.5584487915039,
      "logps/rejected": -80.39999389648438,
      "loss": 1.1687,
      "margin_dpo/margin_mean": 5.040308952331543,
      "margin_dpo/margin_std": 7.694023609161377,
      "step": 112
    },
    {
      "KL/chosen_KL_mean": -4.963401794433594,
      "KL/mean": -7.6679534912109375,
      "KL/rejected_KL_mean": -10.372505187988281,
      "KL/std": 7.751860618591309,
      "epoch": 0.16593245227606462,
      "fcm_dpo/beta": 0.05958745628595352,
      "fcm_dpo/delta": 0.08564908802509308,
      "fcm_dpo/margin": 5.4091033935546875,
      "fcm_dpo/q_t": 0.4267967939376831,
      "grad_norm": 19.366432189941406,
      "learning_rate": 4.939343162231841e-07,
      "logits/chosen": -1.0437860488891602,
      "logits/rejected": -0.5100374817848206,
      "logps/chosen": -102.86289978027344,
      "logps/ref_chosen": -97.89949035644531,
      "logps/ref_rejected": -84.41960144042969,
      "logps/rejected": -94.79209899902344,
      "loss": 1.1402,
      "margin_dpo/margin_mean": 5.409102439880371,
      "margin_dpo/margin_std": 7.510247230529785,
      "step": 113
    },
    {
      "KL/chosen_KL_mean": -4.6693572998046875,
      "KL/mean": -8.545271873474121,
      "KL/rejected_KL_mean": -12.421180725097656,
      "KL/std": 8.949610710144043,
      "epoch": 0.16740088105726872,
      "fcm_dpo/beta": 0.058130957186222076,
      "fcm_dpo/delta": -0.059609554708004,
      "fcm_dpo/margin": 7.7518205642700195,
      "fcm_dpo/q_t": 0.40076667070388794,
      "grad_norm": 18.454483032226562,
      "learning_rate": 4.936501251103751e-07,
      "logits/chosen": -1.1910678148269653,
      "logits/rejected": -0.6140797138214111,
      "logps/chosen": -101.16447448730469,
      "logps/ref_chosen": -96.4951171875,
      "logps/ref_rejected": -92.01536560058594,
      "logps/rejected": -104.43653869628906,
      "loss": 1.0882,
      "margin_dpo/margin_mean": 7.7518205642700195,
      "margin_dpo/margin_std": 11.66125774383545,
      "step": 114
    },
    {
      "KL/chosen_KL_mean": -4.6862640380859375,
      "KL/mean": -8.651681900024414,
      "KL/rejected_KL_mean": -12.61709976196289,
      "KL/std": 9.52731704711914,
      "epoch": 0.16886930983847284,
      "fcm_dpo/beta": 0.056715719401836395,
      "fcm_dpo/delta": -0.057506807148456573,
      "fcm_dpo/margin": 7.930840015411377,
      "fcm_dpo/q_t": 0.39849019050598145,
      "grad_norm": 16.16478729248047,
      "learning_rate": 4.933595135901732e-07,
      "logits/chosen": -1.0249786376953125,
      "logits/rejected": -0.6119292974472046,
      "logps/chosen": -107.86639404296875,
      "logps/ref_chosen": -103.18013000488281,
      "logps/ref_rejected": -105.71595764160156,
      "logps/rejected": -118.33305358886719,
      "loss": 1.0683,
      "margin_dpo/margin_mean": 7.930840015411377,
      "margin_dpo/margin_std": 10.48808765411377,
      "step": 115
    },
    {
      "KL/chosen_KL_mean": -4.230365753173828,
      "KL/mean": -7.527860164642334,
      "KL/rejected_KL_mean": -10.825347900390625,
      "KL/std": 7.498115539550781,
      "epoch": 0.17033773861967694,
      "fcm_dpo/beta": 0.05732718110084534,
      "fcm_dpo/delta": 0.023624097928404808,
      "fcm_dpo/margin": 6.594982147216797,
      "fcm_dpo/q_t": 0.4133802354335785,
      "grad_norm": 15.86227035522461,
      "learning_rate": 4.930624893204624e-07,
      "logits/chosen": -1.0650649070739746,
      "logits/rejected": -0.6310024857521057,
      "logps/chosen": -85.38874053955078,
      "logps/ref_chosen": -81.15837097167969,
      "logps/ref_rejected": -86.30586242675781,
      "logps/rejected": -97.13121032714844,
      "loss": 1.1005,
      "margin_dpo/margin_mean": 6.594982147216797,
      "margin_dpo/margin_std": 8.52669620513916,
      "step": 116
    },
    {
      "KL/chosen_KL_mean": -5.192481994628906,
      "KL/mean": -7.798206806182861,
      "KL/rejected_KL_mean": -10.403934478759766,
      "KL/std": 7.633020401000977,
      "epoch": 0.17180616740088106,
      "fcm_dpo/beta": 0.05689983069896698,
      "fcm_dpo/delta": -0.00043817266123369336,
      "fcm_dpo/margin": 5.211452960968018,
      "fcm_dpo/q_t": 0.4314284920692444,
      "grad_norm": 22.394062042236328,
      "learning_rate": 4.927590601281083e-07,
      "logits/chosen": -0.9912221431732178,
      "logits/rejected": -0.5871477127075195,
      "logps/chosen": -102.2734375,
      "logps/ref_chosen": -97.0809555053711,
      "logps/ref_rejected": -75.1399154663086,
      "logps/rejected": -85.54385375976562,
      "loss": 1.1817,
      "margin_dpo/margin_mean": 5.211452484130859,
      "margin_dpo/margin_std": 9.47552490234375,
      "step": 117
    },
    {
      "KL/chosen_KL_mean": -3.7480430603027344,
      "KL/mean": -7.561947822570801,
      "KL/rejected_KL_mean": -11.375850677490234,
      "KL/std": 8.336671829223633,
      "epoch": 0.17327459618208516,
      "fcm_dpo/beta": 0.05593114346265793,
      "fcm_dpo/delta": -0.030748508870601654,
      "fcm_dpo/margin": 7.627808570861816,
      "fcm_dpo/q_t": 0.40055352449417114,
      "grad_norm": 15.08022403717041,
      "learning_rate": 4.924492340087524e-07,
      "logits/chosen": -1.1526316404342651,
      "logits/rejected": -0.8116798400878906,
      "logps/chosen": -90.2842788696289,
      "logps/ref_chosen": -86.53623962402344,
      "logps/ref_rejected": -82.93563842773438,
      "logps/rejected": -94.31149291992188,
      "loss": 1.0602,
      "margin_dpo/margin_mean": 7.627808094024658,
      "margin_dpo/margin_std": 8.794744491577148,
      "step": 118
    },
    {
      "KL/chosen_KL_mean": -5.666313171386719,
      "KL/mean": -9.672168731689453,
      "KL/rejected_KL_mean": -13.67801284790039,
      "KL/std": 9.912199020385742,
      "epoch": 0.17474302496328928,
      "fcm_dpo/beta": 0.054463207721710205,
      "fcm_dpo/delta": -0.042418867349624634,
      "fcm_dpo/margin": 8.011697769165039,
      "fcm_dpo/q_t": 0.4003477096557617,
      "grad_norm": 16.720144271850586,
      "learning_rate": 4.92133019126601e-07,
      "logits/chosen": -1.1205474138259888,
      "logits/rejected": -0.7846848964691162,
      "logps/chosen": -110.275390625,
      "logps/ref_chosen": -104.60908508300781,
      "logps/ref_rejected": -109.15780639648438,
      "logps/rejected": -122.83582305908203,
      "loss": 1.1017,
      "margin_dpo/margin_mean": 8.011697769165039,
      "margin_dpo/margin_std": 12.553667068481445,
      "step": 119
    },
    {
      "KL/chosen_KL_mean": -4.6512908935546875,
      "KL/mean": -9.906889915466309,
      "KL/rejected_KL_mean": -15.162490844726562,
      "KL/std": 11.882814407348633,
      "epoch": 0.1762114537444934,
      "fcm_dpo/beta": 0.05060321092605591,
      "fcm_dpo/delta": -0.1578252613544464,
      "fcm_dpo/margin": 10.511198043823242,
      "fcm_dpo/q_t": 0.38162821531295776,
      "grad_norm": 18.220991134643555,
      "learning_rate": 4.918104238142103e-07,
      "logits/chosen": -1.432316541671753,
      "logits/rejected": -0.8882876634597778,
      "logps/chosen": -119.33871459960938,
      "logps/ref_chosen": -114.68742370605469,
      "logps/ref_rejected": -111.18586730957031,
      "logps/rejected": -126.34835815429688,
      "loss": 1.0496,
      "margin_dpo/margin_mean": 10.511198043823242,
      "margin_dpo/margin_std": 15.135066986083984,
      "step": 120
    },
    {
      "KL/chosen_KL_mean": -4.210033416748047,
      "KL/mean": -9.018806457519531,
      "KL/rejected_KL_mean": -13.827583312988281,
      "KL/std": 9.263875961303711,
      "epoch": 0.1776798825256975,
      "fcm_dpo/beta": 0.047578584402799606,
      "fcm_dpo/delta": -0.06823188066482544,
      "fcm_dpo/margin": 9.617552757263184,
      "fcm_dpo/q_t": 0.392587274312973,
      "grad_norm": 16.459047317504883,
      "learning_rate": 4.91481456572267e-07,
      "logits/chosen": -1.1184295415878296,
      "logits/rejected": -0.7862751483917236,
      "logps/chosen": -96.32669067382812,
      "logps/ref_chosen": -92.11666107177734,
      "logps/ref_rejected": -114.972900390625,
      "logps/rejected": -128.80047607421875,
      "loss": 1.0357,
      "margin_dpo/margin_mean": 9.617552757263184,
      "margin_dpo/margin_std": 10.331459045410156,
      "step": 121
    },
    {
      "KL/chosen_KL_mean": -4.514045715332031,
      "KL/mean": -10.108457565307617,
      "KL/rejected_KL_mean": -15.702865600585938,
      "KL/std": 10.149539947509766,
      "epoch": 0.17914831130690162,
      "fcm_dpo/beta": 0.04544368386268616,
      "fcm_dpo/delta": -0.12743055820465088,
      "fcm_dpo/margin": 11.188825607299805,
      "fcm_dpo/q_t": 0.38001757860183716,
      "grad_norm": 16.65300941467285,
      "learning_rate": 4.911461260693638e-07,
      "logits/chosen": -0.8921973705291748,
      "logits/rejected": -0.8024289608001709,
      "logps/chosen": -89.39161682128906,
      "logps/ref_chosen": -84.87757110595703,
      "logps/ref_rejected": -111.0829849243164,
      "logps/rejected": -126.78585052490234,
      "loss": 1.0104,
      "margin_dpo/margin_mean": 11.188825607299805,
      "margin_dpo/margin_std": 11.900983810424805,
      "step": 122
    },
    {
      "KL/chosen_KL_mean": -5.10479736328125,
      "KL/mean": -9.099157333374023,
      "KL/rejected_KL_mean": -13.093513488769531,
      "KL/std": 9.339581489562988,
      "epoch": 0.18061674008810572,
      "fcm_dpo/beta": 0.04434993118047714,
      "fcm_dpo/delta": 0.050193607807159424,
      "fcm_dpo/margin": 7.988714218139648,
      "fcm_dpo/q_t": 0.41870957612991333,
      "grad_norm": 13.792718887329102,
      "learning_rate": 4.908044411417711e-07,
      "logits/chosen": -1.3204900026321411,
      "logits/rejected": -1.004927158355713,
      "logps/chosen": -103.58354187011719,
      "logps/ref_chosen": -98.47874450683594,
      "logps/ref_rejected": -93.85417175292969,
      "logps/rejected": -106.94769287109375,
      "loss": 1.1221,
      "margin_dpo/margin_mean": 7.988714218139648,
      "margin_dpo/margin_std": 10.89352035522461,
      "step": 123
    },
    {
      "KL/chosen_KL_mean": -4.932331085205078,
      "KL/mean": -9.885183334350586,
      "KL/rejected_KL_mean": -14.838035583496094,
      "KL/std": 11.709821701049805,
      "epoch": 0.18208516886930984,
      "fcm_dpo/beta": 0.044523365795612335,
      "fcm_dpo/delta": -0.04724256694316864,
      "fcm_dpo/margin": 9.90570068359375,
      "fcm_dpo/q_t": 0.40448686480522156,
      "grad_norm": 14.288955688476562,
      "learning_rate": 4.904564107932048e-07,
      "logits/chosen": -1.2180962562561035,
      "logits/rejected": -0.9537180662155151,
      "logps/chosen": -107.69523620605469,
      "logps/ref_chosen": -102.76290130615234,
      "logps/ref_rejected": -123.005615234375,
      "logps/rejected": -137.84365844726562,
      "loss": 1.0925,
      "margin_dpo/margin_mean": 9.90570068359375,
      "margin_dpo/margin_std": 15.246376037597656,
      "step": 124
    },
    {
      "KL/chosen_KL_mean": -6.442775726318359,
      "KL/mean": -9.954595565795898,
      "KL/rejected_KL_mean": -13.466423034667969,
      "KL/std": 9.589518547058105,
      "epoch": 0.18355359765051396,
      "fcm_dpo/beta": 0.04580872878432274,
      "fcm_dpo/delta": 0.08669982850551605,
      "fcm_dpo/margin": 7.023641109466553,
      "fcm_dpo/q_t": 0.4280632734298706,
      "grad_norm": 14.656176567077637,
      "learning_rate": 4.90102044194588e-07,
      "logits/chosen": -0.933864176273346,
      "logits/rejected": -0.7078385949134827,
      "logps/chosen": -96.86904907226562,
      "logps/ref_chosen": -90.42627716064453,
      "logps/ref_rejected": -89.24901580810547,
      "logps/rejected": -102.71543884277344,
      "loss": 1.1606,
      "margin_dpo/margin_mean": 7.023641586303711,
      "margin_dpo/margin_std": 11.856927871704102,
      "step": 125
    },
    {
      "KL/chosen_KL_mean": -5.463809967041016,
      "KL/mean": -10.606616973876953,
      "KL/rejected_KL_mean": -15.749427795410156,
      "KL/std": 10.39920711517334,
      "epoch": 0.18502202643171806,
      "fcm_dpo/beta": 0.045005638152360916,
      "fcm_dpo/delta": -0.07307875156402588,
      "fcm_dpo/margin": 10.285622596740723,
      "fcm_dpo/q_t": 0.3906862735748291,
      "grad_norm": 14.416794776916504,
      "learning_rate": 4.897413506838102e-07,
      "logits/chosen": -0.9192144274711609,
      "logits/rejected": -0.6277397871017456,
      "logps/chosen": -99.88668823242188,
      "logps/ref_chosen": -94.42288208007812,
      "logps/ref_rejected": -103.88156127929688,
      "logps/rejected": -119.6309814453125,
      "loss": 1.0329,
      "margin_dpo/margin_mean": 10.285621643066406,
      "margin_dpo/margin_std": 11.010307312011719,
      "step": 126
    },
    {
      "KL/chosen_KL_mean": -5.504604339599609,
      "KL/mean": -9.279115676879883,
      "KL/rejected_KL_mean": -13.053627014160156,
      "KL/std": 9.495840072631836,
      "epoch": 0.18649045521292218,
      "fcm_dpo/beta": 0.04601006954908371,
      "fcm_dpo/delta": 0.05787166580557823,
      "fcm_dpo/margin": 7.549020767211914,
      "fcm_dpo/q_t": 0.4224029779434204,
      "grad_norm": 14.988826751708984,
      "learning_rate": 4.89374339765481e-07,
      "logits/chosen": -1.029428482055664,
      "logits/rejected": -0.6788974404335022,
      "logps/chosen": -90.25566101074219,
      "logps/ref_chosen": -84.75105285644531,
      "logps/ref_rejected": -83.54523468017578,
      "logps/rejected": -96.59886169433594,
      "loss": 1.1392,
      "margin_dpo/margin_mean": 7.5490217208862305,
      "margin_dpo/margin_std": 12.000608444213867,
      "step": 127
    },
    {
      "KL/chosen_KL_mean": -5.394142150878906,
      "KL/mean": -9.42244815826416,
      "KL/rejected_KL_mean": -13.450750350952148,
      "KL/std": 11.895051956176758,
      "epoch": 0.18795888399412627,
      "fcm_dpo/beta": 0.047204382717609406,
      "fcm_dpo/delta": 0.02020137757062912,
      "fcm_dpo/margin": 8.056609153747559,
      "fcm_dpo/q_t": 0.41477301716804504,
      "grad_norm": 17.16905975341797,
      "learning_rate": 4.890010211106795e-07,
      "logits/chosen": -0.6343677043914795,
      "logits/rejected": -0.12595103681087494,
      "logps/chosen": -96.01378631591797,
      "logps/ref_chosen": -90.61964416503906,
      "logps/ref_rejected": -83.10347747802734,
      "logps/rejected": -96.55422973632812,
      "loss": 1.1477,
      "margin_dpo/margin_mean": 8.056609153747559,
      "margin_dpo/margin_std": 14.928243637084961,
      "step": 128
    },
    {
      "KL/chosen_KL_mean": -5.975078582763672,
      "KL/mean": -10.194120407104492,
      "KL/rejected_KL_mean": -14.413169860839844,
      "KL/std": 9.71413803100586,
      "epoch": 0.1894273127753304,
      "fcm_dpo/beta": 0.047083113342523575,
      "fcm_dpo/delta": 0.002515943720936775,
      "fcm_dpo/margin": 8.438093185424805,
      "fcm_dpo/q_t": 0.4134944677352905,
      "grad_norm": 13.965925216674805,
      "learning_rate": 4.88621404556699e-07,
      "logits/chosen": -0.9298146963119507,
      "logits/rejected": -0.6197824478149414,
      "logps/chosen": -104.89378356933594,
      "logps/ref_chosen": -98.91870880126953,
      "logps/ref_rejected": -103.28419494628906,
      "logps/rejected": -117.6973648071289,
      "loss": 1.1207,
      "margin_dpo/margin_mean": 8.438095092773438,
      "margin_dpo/margin_std": 13.391377449035645,
      "step": 129
    },
    {
      "KL/chosen_KL_mean": -4.772876739501953,
      "KL/mean": -10.183002471923828,
      "KL/rejected_KL_mean": -15.593124389648438,
      "KL/std": 11.027229309082031,
      "epoch": 0.19089574155653452,
      "fcm_dpo/beta": 0.045498207211494446,
      "fcm_dpo/delta": -0.10921850055456161,
      "fcm_dpo/margin": 10.820249557495117,
      "fcm_dpo/q_t": 0.38842514157295227,
      "grad_norm": 13.041718482971191,
      "learning_rate": 4.882355001067891e-07,
      "logits/chosen": -1.2939610481262207,
      "logits/rejected": -1.0548527240753174,
      "logps/chosen": -77.9565658569336,
      "logps/ref_chosen": -73.18368530273438,
      "logps/ref_rejected": -88.02154541015625,
      "logps/rejected": -103.61466979980469,
      "loss": 1.0416,
      "margin_dpo/margin_mean": 10.8202486038208,
      "margin_dpo/margin_std": 13.232869148254395,
      "step": 130
    },
    {
      "KL/chosen_KL_mean": -4.214378356933594,
      "KL/mean": -9.228597640991211,
      "KL/rejected_KL_mean": -14.24282455444336,
      "KL/std": 11.164024353027344,
      "epoch": 0.19236417033773862,
      "fcm_dpo/beta": 0.04362526535987854,
      "fcm_dpo/delta": -0.04443546012043953,
      "fcm_dpo/margin": 10.0284423828125,
      "fcm_dpo/q_t": 0.39964014291763306,
      "grad_norm": 15.600025177001953,
      "learning_rate": 4.878433179298909e-07,
      "logits/chosen": -1.0939481258392334,
      "logits/rejected": -0.8807210922241211,
      "logps/chosen": -78.17082214355469,
      "logps/ref_chosen": -73.9564437866211,
      "logps/ref_rejected": -94.45582580566406,
      "logps/rejected": -108.69865417480469,
      "loss": 1.0724,
      "margin_dpo/margin_mean": 10.028443336486816,
      "margin_dpo/margin_std": 12.84974193572998,
      "step": 131
    },
    {
      "KL/chosen_KL_mean": -5.9134368896484375,
      "KL/mean": -10.863350868225098,
      "KL/rejected_KL_mean": -15.813255310058594,
      "KL/std": 11.732865333557129,
      "epoch": 0.19383259911894274,
      "fcm_dpo/beta": 0.04180599004030228,
      "fcm_dpo/delta": -0.016881762072443962,
      "fcm_dpo/margin": 9.899813652038574,
      "fcm_dpo/q_t": 0.4048902690410614,
      "grad_norm": 14.749571800231934,
      "learning_rate": 4.874448683603694e-07,
      "logits/chosen": -1.3559077978134155,
      "logits/rejected": -0.9298880100250244,
      "logps/chosen": -98.35292053222656,
      "logps/ref_chosen": -92.43948364257812,
      "logps/ref_rejected": -94.13252258300781,
      "logps/rejected": -109.94577026367188,
      "loss": 1.0903,
      "margin_dpo/margin_mean": 9.89981460571289,
      "margin_dpo/margin_std": 13.646930694580078,
      "step": 132
    },
    {
      "KL/chosen_KL_mean": -5.117412567138672,
      "KL/mean": -10.266277313232422,
      "KL/rejected_KL_mean": -15.415138244628906,
      "KL/std": 11.674213409423828,
      "epoch": 0.19530102790014683,
      "fcm_dpo/beta": 0.041627101600170135,
      "fcm_dpo/delta": -0.032858047634363174,
      "fcm_dpo/margin": 10.297725677490234,
      "fcm_dpo/q_t": 0.4015202522277832,
      "grad_norm": 13.647047996520996,
      "learning_rate": 4.870401618977415e-07,
      "logits/chosen": -1.7883720397949219,
      "logits/rejected": -1.4036619663238525,
      "logps/chosen": -98.23406982421875,
      "logps/ref_chosen": -93.11666107177734,
      "logps/ref_rejected": -102.09490966796875,
      "logps/rejected": -117.51005554199219,
      "loss": 1.0632,
      "margin_dpo/margin_mean": 10.297725677490234,
      "margin_dpo/margin_std": 12.359915733337402,
      "step": 133
    },
    {
      "KL/chosen_KL_mean": -5.297447204589844,
      "KL/mean": -10.87514877319336,
      "KL/rejected_KL_mean": -16.452842712402344,
      "KL/std": 11.027538299560547,
      "epoch": 0.19676945668135096,
      "fcm_dpo/beta": 0.040983647108078,
      "fcm_dpo/delta": -0.06800971925258636,
      "fcm_dpo/margin": 11.155399322509766,
      "fcm_dpo/q_t": 0.39273083209991455,
      "grad_norm": 13.334856986999512,
      "learning_rate": 4.866292092063986e-07,
      "logits/chosen": -1.6543853282928467,
      "logits/rejected": -1.3142082691192627,
      "logps/chosen": -88.84757232666016,
      "logps/ref_chosen": -83.55012512207031,
      "logps/ref_rejected": -93.8815689086914,
      "logps/rejected": -110.33441162109375,
      "loss": 1.0435,
      "margin_dpo/margin_mean": 11.155399322509766,
      "margin_dpo/margin_std": 12.00726318359375,
      "step": 134
    },
    {
      "KL/chosen_KL_mean": -4.90049934387207,
      "KL/mean": -10.526739120483398,
      "KL/rejected_KL_mean": -16.15298080444336,
      "KL/std": 12.15073013305664,
      "epoch": 0.19823788546255505,
      "fcm_dpo/beta": 0.03911972790956497,
      "fcm_dpo/delta": -0.04629451036453247,
      "fcm_dpo/margin": 11.252479553222656,
      "fcm_dpo/q_t": 0.40019702911376953,
      "grad_norm": 17.035648345947266,
      "learning_rate": 4.862120211153265e-07,
      "logits/chosen": -1.273064374923706,
      "logits/rejected": -1.1649422645568848,
      "logps/chosen": -82.48312377929688,
      "logps/ref_chosen": -77.58262634277344,
      "logps/ref_rejected": -121.05648040771484,
      "logps/rejected": -137.20945739746094,
      "loss": 1.0611,
      "margin_dpo/margin_mean": 11.252479553222656,
      "margin_dpo/margin_std": 13.655558586120605,
      "step": 135
    },
    {
      "KL/chosen_KL_mean": -5.8282012939453125,
      "KL/mean": -11.22076416015625,
      "KL/rejected_KL_mean": -16.613323211669922,
      "KL/std": 12.443155288696289,
      "epoch": 0.19970631424375918,
      "fcm_dpo/beta": 0.03825946897268295,
      "fcm_dpo/delta": -0.014484753832221031,
      "fcm_dpo/margin": 10.78512191772461,
      "fcm_dpo/q_t": 0.4047594964504242,
      "grad_norm": 12.717401504516602,
      "learning_rate": 4.857886086178193e-07,
      "logits/chosen": -1.7243549823760986,
      "logits/rejected": -1.3009737730026245,
      "logps/chosen": -100.00093078613281,
      "logps/ref_chosen": -94.1727294921875,
      "logps/ref_rejected": -103.26428985595703,
      "logps/rejected": -119.87760925292969,
      "loss": 1.0691,
      "margin_dpo/margin_mean": 10.78512191772461,
      "margin_dpo/margin_std": 12.7401762008667,
      "step": 136
    },
    {
      "KL/chosen_KL_mean": -4.572509765625,
      "KL/mean": -13.217981338500977,
      "KL/rejected_KL_mean": -21.863452911376953,
      "KL/std": 16.098283767700195,
      "epoch": 0.2011747430249633,
      "fcm_dpo/beta": 0.033853743225336075,
      "fcm_dpo/delta": -0.23291651904582977,
      "fcm_dpo/margin": 17.29094696044922,
      "fcm_dpo/q_t": 0.36216065287590027,
      "grad_norm": 15.15427017211914,
      "learning_rate": 4.853589828711902e-07,
      "logits/chosen": -1.389699935913086,
      "logits/rejected": -1.1790175437927246,
      "logps/chosen": -82.69125366210938,
      "logps/ref_chosen": -78.11874389648438,
      "logps/ref_rejected": -117.92237091064453,
      "logps/rejected": -139.78582763671875,
      "loss": 0.9609,
      "margin_dpo/margin_mean": 17.290945053100586,
      "margin_dpo/margin_std": 17.719701766967773,
      "step": 137
    },
    {
      "KL/chosen_KL_mean": -7.490550994873047,
      "KL/mean": -12.749855041503906,
      "KL/rejected_KL_mean": -18.009159088134766,
      "KL/std": 11.803364753723145,
      "epoch": 0.2026431718061674,
      "fcm_dpo/beta": 0.03386121988296509,
      "fcm_dpo/delta": 0.04915328696370125,
      "fcm_dpo/margin": 10.518611907958984,
      "fcm_dpo/q_t": 0.41804957389831543,
      "grad_norm": 11.44312858581543,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": -0.9466790556907654,
      "logits/rejected": -0.6698113679885864,
      "logps/chosen": -100.32049560546875,
      "logps/ref_chosen": -92.82994079589844,
      "logps/ref_rejected": -98.64840698242188,
      "logps/rejected": -116.65756225585938,
      "loss": 1.1099,
      "margin_dpo/margin_mean": 10.518611907958984,
      "margin_dpo/margin_std": 13.372552871704102,
      "step": 138
    },
    {
      "KL/chosen_KL_mean": -5.184123992919922,
      "KL/mean": -11.333059310913086,
      "KL/rejected_KL_mean": -17.481998443603516,
      "KL/std": 12.761141777038574,
      "epoch": 0.20411160058737152,
      "fcm_dpo/beta": 0.03380892053246498,
      "fcm_dpo/delta": -0.018319128081202507,
      "fcm_dpo/margin": 12.297881126403809,
      "fcm_dpo/q_t": 0.4018271267414093,
      "grad_norm": 11.832630157470703,
      "learning_rate": 4.844811370781446e-07,
      "logits/chosen": -1.250072717666626,
      "logits/rejected": -0.9115183353424072,
      "logps/chosen": -84.1871337890625,
      "logps/ref_chosen": -79.00301361083984,
      "logps/ref_rejected": -85.82733154296875,
      "logps/rejected": -103.309326171875,
      "loss": 1.0747,
      "margin_dpo/margin_mean": 12.297881126403809,
      "margin_dpo/margin_std": 15.293699264526367,
      "step": 139
    },
    {
      "KL/chosen_KL_mean": -5.788917541503906,
      "KL/mean": -11.353538513183594,
      "KL/rejected_KL_mean": -16.91815948486328,
      "KL/std": 11.981672286987305,
      "epoch": 0.2055800293685756,
      "fcm_dpo/beta": 0.03426872938871384,
      "fcm_dpo/delta": 0.020951729267835617,
      "fcm_dpo/margin": 11.129243850708008,
      "fcm_dpo/q_t": 0.4110473394393921,
      "grad_norm": 11.290325164794922,
      "learning_rate": 4.840329401637809e-07,
      "logits/chosen": -1.8390710353851318,
      "logits/rejected": -1.4361215829849243,
      "logps/chosen": -94.14136505126953,
      "logps/ref_chosen": -88.35244750976562,
      "logps/ref_rejected": -90.57476806640625,
      "logps/rejected": -107.49293518066406,
      "loss": 1.0869,
      "margin_dpo/margin_mean": 11.129243850708008,
      "margin_dpo/margin_std": 12.899505615234375,
      "step": 140
    },
    {
      "KL/chosen_KL_mean": -7.938182830810547,
      "KL/mean": -14.176923751831055,
      "KL/rejected_KL_mean": -20.415668487548828,
      "KL/std": 16.11138153076172,
      "epoch": 0.20704845814977973,
      "fcm_dpo/beta": 0.033963218331336975,
      "fcm_dpo/delta": -0.027213044464588165,
      "fcm_dpo/margin": 12.477484703063965,
      "fcm_dpo/q_t": 0.4011157751083374,
      "grad_norm": 14.097794532775879,
      "learning_rate": 4.83578576263792e-07,
      "logits/chosen": -1.4166722297668457,
      "logits/rejected": -1.0177409648895264,
      "logps/chosen": -115.15834045410156,
      "logps/ref_chosen": -107.22016143798828,
      "logps/ref_rejected": -104.71223449707031,
      "logps/rejected": -125.12789916992188,
      "loss": 1.0787,
      "margin_dpo/margin_mean": 12.477485656738281,
      "margin_dpo/margin_std": 16.479755401611328,
      "step": 141
    },
    {
      "KL/chosen_KL_mean": -7.795650482177734,
      "KL/mean": -14.022761344909668,
      "KL/rejected_KL_mean": -20.249866485595703,
      "KL/std": 14.792800903320312,
      "epoch": 0.20851688693098386,
      "fcm_dpo/beta": 0.033346209675073624,
      "fcm_dpo/delta": -0.017706559970974922,
      "fcm_dpo/margin": 12.454215049743652,
      "fcm_dpo/q_t": 0.40544143319129944,
      "grad_norm": 11.234152793884277,
      "learning_rate": 4.83118057351089e-07,
      "logits/chosen": -1.2594969272613525,
      "logits/rejected": -0.997650146484375,
      "logps/chosen": -96.52459716796875,
      "logps/ref_chosen": -88.72894287109375,
      "logps/ref_rejected": -99.73038482666016,
      "logps/rejected": -119.98025512695312,
      "loss": 1.0725,
      "margin_dpo/margin_mean": 12.454214096069336,
      "margin_dpo/margin_std": 15.454838752746582,
      "step": 142
    },
    {
      "KL/chosen_KL_mean": -6.1115875244140625,
      "KL/mean": -11.10864543914795,
      "KL/rejected_KL_mean": -16.105701446533203,
      "KL/std": 12.111422538757324,
      "epoch": 0.20998531571218795,
      "fcm_dpo/beta": 0.034042838960886,
      "fcm_dpo/delta": 0.06647498160600662,
      "fcm_dpo/margin": 9.994112968444824,
      "fcm_dpo/q_t": 0.4216306805610657,
      "grad_norm": 11.751716613769531,
      "learning_rate": 4.826513955607734e-07,
      "logits/chosen": -1.8371599912643433,
      "logits/rejected": -1.391351342201233,
      "logps/chosen": -91.61616516113281,
      "logps/ref_chosen": -85.50457763671875,
      "logps/ref_rejected": -85.8056640625,
      "logps/rejected": -101.91136169433594,
      "loss": 1.1244,
      "margin_dpo/margin_mean": 9.99411392211914,
      "margin_dpo/margin_std": 13.143512725830078,
      "step": 143
    },
    {
      "KL/chosen_KL_mean": -7.506839752197266,
      "KL/mean": -12.467695236206055,
      "KL/rejected_KL_mean": -17.428550720214844,
      "KL/std": 12.970579147338867,
      "epoch": 0.21145374449339208,
      "fcm_dpo/beta": 0.036071110516786575,
      "fcm_dpo/delta": 0.04428183659911156,
      "fcm_dpo/margin": 9.921716690063477,
      "fcm_dpo/q_t": 0.41863852739334106,
      "grad_norm": 15.846207618713379,
      "learning_rate": 4.821786031898176e-07,
      "logits/chosen": -1.580657720565796,
      "logits/rejected": -1.0297203063964844,
      "logps/chosen": -112.19393157958984,
      "logps/ref_chosen": -104.68708801269531,
      "logps/ref_rejected": -87.71908569335938,
      "logps/rejected": -105.14762878417969,
      "loss": 1.1397,
      "margin_dpo/margin_mean": 9.921716690063477,
      "margin_dpo/margin_std": 15.972646713256836,
      "step": 144
    },
    {
      "KL/chosen_KL_mean": -6.184413909912109,
      "KL/mean": -12.14926528930664,
      "KL/rejected_KL_mean": -18.114120483398438,
      "KL/std": 12.288372039794922,
      "epoch": 0.21292217327459617,
      "fcm_dpo/beta": 0.03530154749751091,
      "fcm_dpo/delta": -0.024208884686231613,
      "fcm_dpo/margin": 11.929705619812012,
      "fcm_dpo/q_t": 0.40174782276153564,
      "grad_norm": 12.480833053588867,
      "learning_rate": 4.816996926967401e-07,
      "logits/chosen": -0.7980265021324158,
      "logits/rejected": -0.17590448260307312,
      "logps/chosen": -94.70429992675781,
      "logps/ref_chosen": -88.51988983154297,
      "logps/ref_rejected": -83.76513671875,
      "logps/rejected": -101.87925720214844,
      "loss": 1.0645,
      "margin_dpo/margin_mean": 11.929706573486328,
      "margin_dpo/margin_std": 13.840070724487305,
      "step": 145
    },
    {
      "KL/chosen_KL_mean": -6.675304412841797,
      "KL/mean": -13.28018856048584,
      "KL/rejected_KL_mean": -19.885066986083984,
      "KL/std": 13.478302001953125,
      "epoch": 0.2143906020558003,
      "fcm_dpo/beta": 0.03382384032011032,
      "fcm_dpo/delta": -0.056416697800159454,
      "fcm_dpo/margin": 13.209756851196289,
      "fcm_dpo/q_t": 0.3947208523750305,
      "grad_norm": 12.085694313049316,
      "learning_rate": 4.812146767012779e-07,
      "logits/chosen": -1.716130018234253,
      "logits/rejected": -1.0607819557189941,
      "logps/chosen": -109.58116912841797,
      "logps/ref_chosen": -102.90586853027344,
      "logps/ref_rejected": -88.9493637084961,
      "logps/rejected": -108.83442687988281,
      "loss": 1.0492,
      "margin_dpo/margin_mean": 13.209756851196289,
      "margin_dpo/margin_std": 14.951105117797852,
      "step": 146
    },
    {
      "KL/chosen_KL_mean": -6.427024841308594,
      "KL/mean": -13.252843856811523,
      "KL/rejected_KL_mean": -20.078662872314453,
      "KL/std": 12.861209869384766,
      "epoch": 0.21585903083700442,
      "fcm_dpo/beta": 0.03315863758325577,
      "fcm_dpo/delta": -0.06088149547576904,
      "fcm_dpo/margin": 13.651636123657227,
      "fcm_dpo/q_t": 0.3933600187301636,
      "grad_norm": 11.349029541015625,
      "learning_rate": 4.807235679840536e-07,
      "logits/chosen": -1.7110228538513184,
      "logits/rejected": -1.1393932104110718,
      "logps/chosen": -95.689208984375,
      "logps/ref_chosen": -89.26219177246094,
      "logps/ref_rejected": -76.50011444091797,
      "logps/rejected": -96.57878112792969,
      "loss": 1.0336,
      "margin_dpo/margin_mean": 13.651636123657227,
      "margin_dpo/margin_std": 14.031829833984375,
      "step": 147
    },
    {
      "KL/chosen_KL_mean": -4.773429870605469,
      "KL/mean": -11.224148750305176,
      "KL/rejected_KL_mean": -17.67486572265625,
      "KL/std": 13.604753494262695,
      "epoch": 0.2173274596182085,
      "fcm_dpo/beta": 0.0330842025578022,
      "fcm_dpo/delta": -0.03387654200196266,
      "fcm_dpo/margin": 12.90142822265625,
      "fcm_dpo/q_t": 0.4030953645706177,
      "grad_norm": 11.805949211120605,
      "learning_rate": 4.802263794862384e-07,
      "logits/chosen": -1.4924992322921753,
      "logits/rejected": -1.0896402597427368,
      "logps/chosen": -96.53631591796875,
      "logps/ref_chosen": -91.76287841796875,
      "logps/ref_rejected": -110.38552856445312,
      "logps/rejected": -128.06039428710938,
      "loss": 1.0732,
      "margin_dpo/margin_mean": 12.90142822265625,
      "margin_dpo/margin_std": 15.44575309753418,
      "step": 148
    },
    {
      "KL/chosen_KL_mean": -8.096477508544922,
      "KL/mean": -14.154094696044922,
      "KL/rejected_KL_mean": -20.21170425415039,
      "KL/std": 14.432670593261719,
      "epoch": 0.21879588839941264,
      "fcm_dpo/beta": 0.03199036419391632,
      "fcm_dpo/delta": 0.013430111110210419,
      "fcm_dpo/margin": 12.11522102355957,
      "fcm_dpo/q_t": 0.414778470993042,
      "grad_norm": 11.838626861572266,
      "learning_rate": 4.797231243092118e-07,
      "logits/chosen": -1.3947621583938599,
      "logits/rejected": -1.0567431449890137,
      "logps/chosen": -101.58049011230469,
      "logps/ref_chosen": -93.48401641845703,
      "logps/ref_rejected": -105.6436996459961,
      "logps/rejected": -125.85540771484375,
      "loss": 1.1141,
      "margin_dpo/margin_mean": 12.115221977233887,
      "margin_dpo/margin_std": 18.258501052856445,
      "step": 149
    },
    {
      "KL/chosen_KL_mean": -6.4923553466796875,
      "KL/mean": -12.948942184448242,
      "KL/rejected_KL_mean": -19.405529022216797,
      "KL/std": 15.358789443969727,
      "epoch": 0.22026431718061673,
      "fcm_dpo/beta": 0.031942784786224365,
      "fcm_dpo/delta": -0.014758754521608353,
      "fcm_dpo/margin": 12.91317367553711,
      "fcm_dpo/q_t": 0.4060860872268677,
      "grad_norm": 10.461620330810547,
      "learning_rate": 4.792138157142157e-07,
      "logits/chosen": -1.4785902500152588,
      "logits/rejected": -1.2209328413009644,
      "logps/chosen": -82.27104187011719,
      "logps/ref_chosen": -75.7786865234375,
      "logps/ref_rejected": -90.02676391601562,
      "logps/rejected": -109.43229675292969,
      "loss": 1.0835,
      "margin_dpo/margin_mean": 12.913172721862793,
      "margin_dpo/margin_std": 17.16702651977539,
      "step": 150
    },
    {
      "KL/chosen_KL_mean": -8.545989990234375,
      "KL/mean": -14.715003967285156,
      "KL/rejected_KL_mean": -20.884010314941406,
      "KL/std": 14.726766586303711,
      "epoch": 0.22173274596182085,
      "fcm_dpo/beta": 0.03212989121675491,
      "fcm_dpo/delta": 0.0040601445361971855,
      "fcm_dpo/margin": 12.338018417358398,
      "fcm_dpo/q_t": 0.4097358286380768,
      "grad_norm": 12.853394508361816,
      "learning_rate": 4.786984671220053e-07,
      "logits/chosen": -2.0970449447631836,
      "logits/rejected": -1.4967824220657349,
      "logps/chosen": -111.28021240234375,
      "logps/ref_chosen": -102.73422241210938,
      "logps/ref_rejected": -106.03781127929688,
      "logps/rejected": -126.92182159423828,
      "loss": 1.0947,
      "margin_dpo/margin_mean": 12.338018417358398,
      "margin_dpo/margin_std": 16.881481170654297,
      "step": 151
    },
    {
      "KL/chosen_KL_mean": -5.853679656982422,
      "KL/mean": -13.376672744750977,
      "KL/rejected_KL_mean": -20.89966583251953,
      "KL/std": 14.780817031860352,
      "epoch": 0.22320117474302498,
      "fcm_dpo/beta": 0.03093091771006584,
      "fcm_dpo/delta": -0.07673415541648865,
      "fcm_dpo/margin": 15.045978546142578,
      "fcm_dpo/q_t": 0.3908926248550415,
      "grad_norm": 11.406817436218262,
      "learning_rate": 4.78177092112495e-07,
      "logits/chosen": -1.8672168254852295,
      "logits/rejected": -1.4914605617523193,
      "logps/chosen": -103.44442749023438,
      "logps/ref_chosen": -97.59074401855469,
      "logps/ref_rejected": -110.49913024902344,
      "logps/rejected": -131.3988037109375,
      "loss": 1.0417,
      "margin_dpo/margin_mean": 15.045978546142578,
      "margin_dpo/margin_std": 17.616579055786133,
      "step": 152
    },
    {
      "KL/chosen_KL_mean": -8.187274932861328,
      "KL/mean": -13.57938289642334,
      "KL/rejected_KL_mean": -18.97149658203125,
      "KL/std": 14.400384902954102,
      "epoch": 0.22466960352422907,
      "fcm_dpo/beta": 0.03168682008981705,
      "fcm_dpo/delta": 0.06480428576469421,
      "fcm_dpo/margin": 10.784226417541504,
      "fcm_dpo/q_t": 0.4224597215652466,
      "grad_norm": 11.61185359954834,
      "learning_rate": 4.776497044244016e-07,
      "logits/chosen": -1.967718243598938,
      "logits/rejected": -1.652695894241333,
      "logps/chosen": -95.21287536621094,
      "logps/ref_chosen": -87.02560424804688,
      "logps/ref_rejected": -91.06854248046875,
      "logps/rejected": -110.0400390625,
      "loss": 1.1412,
      "margin_dpo/margin_mean": 10.784225463867188,
      "margin_dpo/margin_std": 16.810392379760742,
      "step": 153
    },
    {
      "KL/chosen_KL_mean": -6.4344482421875,
      "KL/mean": -15.175057411193848,
      "KL/rejected_KL_mean": -23.915664672851562,
      "KL/std": 15.646356582641602,
      "epoch": 0.2261380323054332,
      "fcm_dpo/beta": 0.030415097251534462,
      "fcm_dpo/delta": -0.1562114804983139,
      "fcm_dpo/margin": 17.48122787475586,
      "fcm_dpo/q_t": 0.37412792444229126,
      "grad_norm": 11.175724029541016,
      "learning_rate": 4.771163179548808e-07,
      "logits/chosen": -1.5038012266159058,
      "logits/rejected": -1.1805346012115479,
      "logps/chosen": -97.54011535644531,
      "logps/ref_chosen": -91.10566711425781,
      "logps/ref_rejected": -111.37238311767578,
      "logps/rejected": -135.28805541992188,
      "loss": 0.9905,
      "margin_dpo/margin_mean": 17.48122787475586,
      "margin_dpo/margin_std": 16.929115295410156,
      "step": 154
    },
    {
      "KL/chosen_KL_mean": -8.67160415649414,
      "KL/mean": -15.020164489746094,
      "KL/rejected_KL_mean": -21.36872100830078,
      "KL/std": 16.1307430267334,
      "epoch": 0.2276064610866373,
      "fcm_dpo/beta": 0.029160354286432266,
      "fcm_dpo/delta": 0.032775528728961945,
      "fcm_dpo/margin": 12.697118759155273,
      "fcm_dpo/q_t": 0.4128928780555725,
      "grad_norm": 13.086478233337402,
      "learning_rate": 4.7657694675916247e-07,
      "logits/chosen": -1.4173638820648193,
      "logits/rejected": -1.0179321765899658,
      "logps/chosen": -103.23257446289062,
      "logps/ref_chosen": -94.56097412109375,
      "logps/ref_rejected": -84.77359008789062,
      "logps/rejected": -106.1423110961914,
      "loss": 1.1311,
      "margin_dpo/margin_mean": 12.69711685180664,
      "margin_dpo/margin_std": 20.211891174316406,
      "step": 155
    },
    {
      "KL/chosen_KL_mean": -10.346954345703125,
      "KL/mean": -16.114011764526367,
      "KL/rejected_KL_mean": -21.881072998046875,
      "KL/std": 14.47014045715332,
      "epoch": 0.2290748898678414,
      "fcm_dpo/beta": 0.030057359486818314,
      "fcm_dpo/delta": 0.059010110795497894,
      "fcm_dpo/margin": 11.534120559692383,
      "fcm_dpo/q_t": 0.42289891839027405,
      "grad_norm": 12.281519889831543,
      "learning_rate": 4.7603160505017893e-07,
      "logits/chosen": -1.6307940483093262,
      "logits/rejected": -1.1204171180725098,
      "logps/chosen": -105.0023422241211,
      "logps/ref_chosen": -94.65538787841797,
      "logps/ref_rejected": -84.41934204101562,
      "logps/rejected": -106.3004150390625,
      "loss": 1.1406,
      "margin_dpo/margin_mean": 11.534119606018066,
      "margin_dpo/margin_std": 17.978801727294922,
      "step": 156
    },
    {
      "KL/chosen_KL_mean": -9.692138671875,
      "KL/mean": -17.97943115234375,
      "KL/rejected_KL_mean": -26.266727447509766,
      "KL/std": 15.651881217956543,
      "epoch": 0.2305433186490455,
      "fcm_dpo/beta": 0.02944299206137657,
      "fcm_dpo/delta": -0.10294470191001892,
      "fcm_dpo/margin": 16.574586868286133,
      "fcm_dpo/q_t": 0.38414299488067627,
      "grad_norm": 12.18120288848877,
      "learning_rate": 4.7548030719819154e-07,
      "logits/chosen": -2.265202045440674,
      "logits/rejected": -1.8204944133758545,
      "logps/chosen": -106.70452880859375,
      "logps/ref_chosen": -97.01239013671875,
      "logps/ref_rejected": -111.70133972167969,
      "logps/rejected": -137.96807861328125,
      "loss": 1.0132,
      "margin_dpo/margin_mean": 16.574586868286133,
      "margin_dpo/margin_std": 16.712081909179688,
      "step": 157
    },
    {
      "KL/chosen_KL_mean": -8.852519989013672,
      "KL/mean": -17.462451934814453,
      "KL/rejected_KL_mean": -26.07238006591797,
      "KL/std": 16.914749145507812,
      "epoch": 0.23201174743024963,
      "fcm_dpo/beta": 0.02785329334437847,
      "fcm_dpo/delta": -0.09277564287185669,
      "fcm_dpo/margin": 17.219865798950195,
      "fcm_dpo/q_t": 0.3878876864910126,
      "grad_norm": 10.896947860717773,
      "learning_rate": 4.7492306773041136e-07,
      "logits/chosen": -1.6021058559417725,
      "logits/rejected": -1.3430471420288086,
      "logps/chosen": -103.56548309326172,
      "logps/ref_chosen": -94.71296691894531,
      "logps/ref_rejected": -120.93733978271484,
      "logps/rejected": -147.0097198486328,
      "loss": 1.0228,
      "margin_dpo/margin_mean": 17.219865798950195,
      "margin_dpo/margin_std": 18.44536590576172,
      "step": 158
    },
    {
      "KL/chosen_KL_mean": -8.561744689941406,
      "KL/mean": -16.119964599609375,
      "KL/rejected_KL_mean": -23.678180694580078,
      "KL/std": 16.2584285736084,
      "epoch": 0.23348017621145375,
      "fcm_dpo/beta": 0.027177296578884125,
      "fcm_dpo/delta": -0.01235281303524971,
      "fcm_dpo/margin": 15.116445541381836,
      "fcm_dpo/q_t": 0.4047987163066864,
      "grad_norm": 10.092301368713379,
      "learning_rate": 4.743599013306165e-07,
      "logits/chosen": -2.097383737564087,
      "logits/rejected": -1.6021504402160645,
      "logps/chosen": -116.77998352050781,
      "logps/ref_chosen": -108.2182388305664,
      "logps/ref_rejected": -94.83901977539062,
      "logps/rejected": -118.51720428466797,
      "loss": 1.0762,
      "margin_dpo/margin_mean": 15.116445541381836,
      "margin_dpo/margin_std": 18.637964248657227,
      "step": 159
    },
    {
      "KL/chosen_KL_mean": -7.186309814453125,
      "KL/mean": -14.701943397521973,
      "KL/rejected_KL_mean": -22.217578887939453,
      "KL/std": 14.780736923217773,
      "epoch": 0.23494860499265785,
      "fcm_dpo/beta": 0.026917602866888046,
      "fcm_dpo/delta": -0.005502127110958099,
      "fcm_dpo/margin": 15.031262397766113,
      "fcm_dpo/q_t": 0.40443798899650574,
      "grad_norm": 10.275331497192383,
      "learning_rate": 4.737908228387656e-07,
      "logits/chosen": -1.6396293640136719,
      "logits/rejected": -1.0860886573791504,
      "logps/chosen": -105.34971618652344,
      "logps/ref_chosen": -98.16340637207031,
      "logps/ref_rejected": -102.28097534179688,
      "logps/rejected": -124.4985580444336,
      "loss": 1.0655,
      "margin_dpo/margin_mean": 15.03126335144043,
      "margin_dpo/margin_std": 15.932374954223633,
      "step": 160
    },
    {
      "KL/chosen_KL_mean": -9.278522491455078,
      "KL/mean": -15.968920707702637,
      "KL/rejected_KL_mean": -22.659313201904297,
      "KL/std": 13.560185432434082,
      "epoch": 0.23641703377386197,
      "fcm_dpo/beta": 0.027259081602096558,
      "fcm_dpo/delta": 0.039303503930568695,
      "fcm_dpo/margin": 13.380792617797852,
      "fcm_dpo/q_t": 0.4158288240432739,
      "grad_norm": 10.434248924255371,
      "learning_rate": 4.7321584725060594e-07,
      "logits/chosen": -1.995617389678955,
      "logits/rejected": -1.5088104009628296,
      "logps/chosen": -95.87039184570312,
      "logps/ref_chosen": -86.59187316894531,
      "logps/ref_rejected": -89.01643371582031,
      "logps/rejected": -111.67575073242188,
      "loss": 1.1121,
      "margin_dpo/margin_mean": 13.380792617797852,
      "margin_dpo/margin_std": 17.837596893310547,
      "step": 161
    },
    {
      "KL/chosen_KL_mean": -8.321937561035156,
      "KL/mean": -15.890300750732422,
      "KL/rejected_KL_mean": -23.458663940429688,
      "KL/std": 15.898338317871094,
      "epoch": 0.23788546255506607,
      "fcm_dpo/beta": 0.02725927159190178,
      "fcm_dpo/delta": -0.015128863975405693,
      "fcm_dpo/margin": 15.136725425720215,
      "fcm_dpo/q_t": 0.4029352366924286,
      "grad_norm": 10.096059799194336,
      "learning_rate": 4.7263498971727905e-07,
      "logits/chosen": -1.2063941955566406,
      "logits/rejected": -0.7894168496131897,
      "logps/chosen": -105.72482299804688,
      "logps/ref_chosen": -97.40288543701172,
      "logps/ref_rejected": -95.86923217773438,
      "logps/rejected": -119.32789611816406,
      "loss": 1.0671,
      "margin_dpo/margin_mean": 15.136726379394531,
      "margin_dpo/margin_std": 17.1365909576416,
      "step": 162
    },
    {
      "KL/chosen_KL_mean": -8.10141372680664,
      "KL/mean": -16.938617706298828,
      "KL/rejected_KL_mean": -25.775829315185547,
      "KL/std": 17.88212013244629,
      "epoch": 0.2393538913362702,
      "fcm_dpo/beta": 0.026312246918678284,
      "fcm_dpo/delta": -0.07674652338027954,
      "fcm_dpo/margin": 17.674421310424805,
      "fcm_dpo/q_t": 0.39159178733825684,
      "grad_norm": 9.052946090698242,
      "learning_rate": 4.720482655449212e-07,
      "logits/chosen": -1.8747472763061523,
      "logits/rejected": -1.4856879711151123,
      "logps/chosen": -90.27680969238281,
      "logps/ref_chosen": -82.1753921508789,
      "logps/ref_rejected": -82.86473846435547,
      "logps/rejected": -108.64056396484375,
      "loss": 1.0323,
      "margin_dpo/margin_mean": 17.674419403076172,
      "margin_dpo/margin_std": 19.34854507446289,
      "step": 163
    },
    {
      "KL/chosen_KL_mean": -11.943096160888672,
      "KL/mean": -18.04058074951172,
      "KL/rejected_KL_mean": -24.1380615234375,
      "KL/std": 14.12021255493164,
      "epoch": 0.24082232011747431,
      "fcm_dpo/beta": 0.025882616639137268,
      "fcm_dpo/delta": -0.020743507891893387,
      "fcm_dpo/margin": 12.194963455200195,
      "fcm_dpo/q_t": 0.42447391152381897,
      "grad_norm": 10.523982048034668,
      "learning_rate": 4.714556901942599e-07,
      "logits/chosen": -1.8004100322723389,
      "logits/rejected": -1.3511395454406738,
      "logps/chosen": -102.41736602783203,
      "logps/ref_chosen": -90.47427368164062,
      "logps/ref_rejected": -87.02809143066406,
      "logps/rejected": -111.16615295410156,
      "loss": 1.1392,
      "margin_dpo/margin_mean": 12.194963455200195,
      "margin_dpo/margin_std": 15.892459869384766,
      "step": 164
    },
    {
      "KL/chosen_KL_mean": -8.22634506225586,
      "KL/mean": -16.965126037597656,
      "KL/rejected_KL_mean": -25.703899383544922,
      "KL/std": 16.611644744873047,
      "epoch": 0.2422907488986784,
      "fcm_dpo/beta": 0.0254361629486084,
      "fcm_dpo/delta": -0.051343657076358795,
      "fcm_dpo/margin": 17.477561950683594,
      "fcm_dpo/q_t": 0.39710259437561035,
      "grad_norm": 8.581192970275879,
      "learning_rate": 4.708572792802069e-07,
      "logits/chosen": -1.6798717975616455,
      "logits/rejected": -1.2335255146026611,
      "logps/chosen": -95.5395736694336,
      "logps/ref_chosen": -87.313232421875,
      "logps/ref_rejected": -81.902587890625,
      "logps/rejected": -107.60648345947266,
      "loss": 1.0432,
      "margin_dpo/margin_mean": 17.477561950683594,
      "margin_dpo/margin_std": 19.050025939941406,
      "step": 165
    },
    {
      "KL/chosen_KL_mean": -12.971107482910156,
      "KL/mean": -21.66787338256836,
      "KL/rejected_KL_mean": -30.364639282226562,
      "KL/std": 18.517845153808594,
      "epoch": 0.24375917767988253,
      "fcm_dpo/beta": 0.02470836229622364,
      "fcm_dpo/delta": -0.034194283187389374,
      "fcm_dpo/margin": 17.393524169921875,
      "fcm_dpo/q_t": 0.40338605642318726,
      "grad_norm": 9.83923625946045,
      "learning_rate": 4.702530485714461e-07,
      "logits/chosen": -1.5370171070098877,
      "logits/rejected": -1.336988925933838,
      "logps/chosen": -94.55717468261719,
      "logps/ref_chosen": -81.58606719970703,
      "logps/ref_rejected": -103.16517639160156,
      "logps/rejected": -133.52981567382812,
      "loss": 1.0687,
      "margin_dpo/margin_mean": 17.393524169921875,
      "margin_dpo/margin_std": 22.524951934814453,
      "step": 166
    },
    {
      "KL/chosen_KL_mean": -11.536540985107422,
      "KL/mean": -20.804237365722656,
      "KL/rejected_KL_mean": -30.07193374633789,
      "KL/std": 16.96955680847168,
      "epoch": 0.24522760646108663,
      "fcm_dpo/beta": 0.023719962686300278,
      "fcm_dpo/delta": -0.04798275604844093,
      "fcm_dpo/margin": 18.535383224487305,
      "fcm_dpo/q_t": 0.39537909626960754,
      "grad_norm": 8.977136611938477,
      "learning_rate": 4.6964301399001877e-07,
      "logits/chosen": -1.213284969329834,
      "logits/rejected": -1.0452625751495361,
      "logps/chosen": -95.09821319580078,
      "logps/ref_chosen": -83.56167602539062,
      "logps/ref_rejected": -99.05305480957031,
      "logps/rejected": -129.12498474121094,
      "loss": 1.0363,
      "margin_dpo/margin_mean": 18.535383224487305,
      "margin_dpo/margin_std": 17.580799102783203,
      "step": 167
    },
    {
      "KL/chosen_KL_mean": -11.119564056396484,
      "KL/mean": -19.085857391357422,
      "KL/rejected_KL_mean": -27.05215835571289,
      "KL/std": 16.772336959838867,
      "epoch": 0.24669603524229075,
      "fcm_dpo/beta": 0.024177230894565582,
      "fcm_dpo/delta": 0.0164943877607584,
      "fcm_dpo/margin": 15.932589530944824,
      "fcm_dpo/q_t": 0.40960603952407837,
      "grad_norm": 9.742465019226074,
      "learning_rate": 4.690271916109034e-07,
      "logits/chosen": -1.699075698852539,
      "logits/rejected": -1.3574926853179932,
      "logps/chosen": -91.05079650878906,
      "logps/ref_chosen": -79.93123626708984,
      "logps/ref_rejected": -80.10216522216797,
      "logps/rejected": -107.15432739257812,
      "loss": 1.0781,
      "margin_dpo/margin_mean": 15.932589530944824,
      "margin_dpo/margin_std": 17.02577018737793,
      "step": 168
    },
    {
      "KL/chosen_KL_mean": -9.513259887695312,
      "KL/mean": -18.790782928466797,
      "KL/rejected_KL_mean": -28.06829833984375,
      "KL/std": 18.255535125732422,
      "epoch": 0.24816446402349487,
      "fcm_dpo/beta": 0.02360670268535614,
      "fcm_dpo/delta": -0.04393249377608299,
      "fcm_dpo/margin": 18.555034637451172,
      "fcm_dpo/q_t": 0.39742740988731384,
      "grad_norm": 9.469151496887207,
      "learning_rate": 4.6840559766159235e-07,
      "logits/chosen": -1.5672073364257812,
      "logits/rejected": -1.341023325920105,
      "logps/chosen": -92.41696166992188,
      "logps/ref_chosen": -82.90370178222656,
      "logps/ref_rejected": -90.12191009521484,
      "logps/rejected": -118.1902084350586,
      "loss": 1.0446,
      "margin_dpo/margin_mean": 18.555034637451172,
      "margin_dpo/margin_std": 19.543724060058594,
      "step": 169
    },
    {
      "KL/chosen_KL_mean": -11.192089080810547,
      "KL/mean": -19.163105010986328,
      "KL/rejected_KL_mean": -27.13412857055664,
      "KL/std": 15.107439041137695,
      "epoch": 0.24963289280469897,
      "fcm_dpo/beta": 0.02349795028567314,
      "fcm_dpo/delta": 0.02784838154911995,
      "fcm_dpo/margin": 15.94204330444336,
      "fcm_dpo/q_t": 0.41101014614105225,
      "grad_norm": 9.897601127624512,
      "learning_rate": 4.6777824852166437e-07,
      "logits/chosen": -1.3097617626190186,
      "logits/rejected": -1.0885812044143677,
      "logps/chosen": -86.17405700683594,
      "logps/ref_chosen": -74.98197174072266,
      "logps/ref_rejected": -82.16560363769531,
      "logps/rejected": -109.29972839355469,
      "loss": 1.079,
      "margin_dpo/margin_mean": 15.94204330444336,
      "margin_dpo/margin_std": 15.093128204345703,
      "step": 170
    },
    {
      "KL/chosen_KL_mean": -14.526031494140625,
      "KL/mean": -23.3891544342041,
      "KL/rejected_KL_mean": -32.25229263305664,
      "KL/std": 22.09073257446289,
      "epoch": 0.2511013215859031,
      "fcm_dpo/beta": 0.0237274207174778,
      "fcm_dpo/delta": -0.023572197183966637,
      "fcm_dpo/margin": 17.726261138916016,
      "fcm_dpo/q_t": 0.4076903462409973,
      "grad_norm": 11.539567947387695,
      "learning_rate": 4.6714516072235273e-07,
      "logits/chosen": -1.6438771486282349,
      "logits/rejected": -1.07921302318573,
      "logps/chosen": -133.5045623779297,
      "logps/ref_chosen": -118.97853088378906,
      "logps/ref_rejected": -115.76930236816406,
      "logps/rejected": -148.0216064453125,
      "loss": 1.0958,
      "margin_dpo/margin_mean": 17.72626304626465,
      "margin_dpo/margin_std": 26.787620544433594,
      "step": 171
    },
    {
      "KL/chosen_KL_mean": -15.447242736816406,
      "KL/mean": -23.207347869873047,
      "KL/rejected_KL_mean": -30.967453002929688,
      "KL/std": 15.670121192932129,
      "epoch": 0.2525697503671072,
      "fcm_dpo/beta": 0.023777689784765244,
      "fcm_dpo/delta": 0.034762054681777954,
      "fcm_dpo/margin": 15.52021598815918,
      "fcm_dpo/q_t": 0.41189804673194885,
      "grad_norm": 9.62450885772705,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": -2.414691686630249,
      "logits/rejected": -1.649678349494934,
      "logps/chosen": -116.60102844238281,
      "logps/ref_chosen": -101.15379333496094,
      "logps/ref_rejected": -91.00568389892578,
      "logps/rejected": -121.97313690185547,
      "loss": 1.0748,
      "margin_dpo/margin_mean": 15.52021598815918,
      "margin_dpo/margin_std": 12.934565544128418,
      "step": 172
    },
    {
      "KL/chosen_KL_mean": -13.753116607666016,
      "KL/mean": -20.682374954223633,
      "KL/rejected_KL_mean": -27.611629486083984,
      "KL/std": 14.901544570922852,
      "epoch": 0.2540381791483113,
      "fcm_dpo/beta": 0.024844897910952568,
      "fcm_dpo/delta": 0.06161680445075035,
      "fcm_dpo/margin": 13.8585205078125,
      "fcm_dpo/q_t": 0.41939958930015564,
      "grad_norm": 9.406453132629395,
      "learning_rate": 4.6586183602616687e-07,
      "logits/chosen": -1.6135804653167725,
      "logits/rejected": -1.1224783658981323,
      "logps/chosen": -112.80979919433594,
      "logps/ref_chosen": -99.05668640136719,
      "logps/ref_rejected": -83.8507308959961,
      "logps/rejected": -111.46235656738281,
      "loss": 1.1079,
      "margin_dpo/margin_mean": 13.8585205078125,
      "margin_dpo/margin_std": 15.738492965698242,
      "step": 173
    },
    {
      "KL/chosen_KL_mean": -13.360185623168945,
      "KL/mean": -22.990327835083008,
      "KL/rejected_KL_mean": -32.62047576904297,
      "KL/std": 20.518863677978516,
      "epoch": 0.2555066079295154,
      "fcm_dpo/beta": 0.0242212675511837,
      "fcm_dpo/delta": -0.07714719325304031,
      "fcm_dpo/margin": 19.26028823852539,
      "fcm_dpo/q_t": 0.39044389128685,
      "grad_norm": 10.690933227539062,
      "learning_rate": 4.652116329460919e-07,
      "logits/chosen": -1.9662153720855713,
      "logits/rejected": -1.9288229942321777,
      "logps/chosen": -93.76908874511719,
      "logps/ref_chosen": -80.40890502929688,
      "logps/ref_rejected": -108.20171356201172,
      "logps/rejected": -140.8221893310547,
      "loss": 1.0317,
      "margin_dpo/margin_mean": 19.260286331176758,
      "margin_dpo/margin_std": 20.92588996887207,
      "step": 174
    },
    {
      "KL/chosen_KL_mean": -17.011600494384766,
      "KL/mean": -26.152978897094727,
      "KL/rejected_KL_mean": -35.29435729980469,
      "KL/std": 18.071067810058594,
      "epoch": 0.25697503671071953,
      "fcm_dpo/beta": 0.02361335977911949,
      "fcm_dpo/delta": -0.03651190176606178,
      "fcm_dpo/margin": 18.282752990722656,
      "fcm_dpo/q_t": 0.3970140814781189,
      "grad_norm": 10.488755226135254,
      "learning_rate": 4.645557588393406e-07,
      "logits/chosen": -1.2624796628952026,
      "logits/rejected": -0.993411660194397,
      "logps/chosen": -99.72808837890625,
      "logps/ref_chosen": -82.71649169921875,
      "logps/ref_rejected": -96.66712951660156,
      "logps/rejected": -131.96148681640625,
      "loss": 1.0421,
      "margin_dpo/margin_mean": 18.282752990722656,
      "margin_dpo/margin_std": 17.389225006103516,
      "step": 175
    },
    {
      "KL/chosen_KL_mean": -13.982173919677734,
      "KL/mean": -23.53622055053711,
      "KL/rejected_KL_mean": -33.090267181396484,
      "KL/std": 19.722393035888672,
      "epoch": 0.25844346549192365,
      "fcm_dpo/beta": 0.022852081805467606,
      "fcm_dpo/delta": -0.042271070182323456,
      "fcm_dpo/margin": 19.10809326171875,
      "fcm_dpo/q_t": 0.39874207973480225,
      "grad_norm": 10.21567153930664,
      "learning_rate": 4.638942309888058e-07,
      "logits/chosen": -1.5761849880218506,
      "logits/rejected": -1.4209538698196411,
      "logps/chosen": -93.61148071289062,
      "logps/ref_chosen": -79.62930297851562,
      "logps/ref_rejected": -102.6885986328125,
      "logps/rejected": -135.77886962890625,
      "loss": 1.0598,
      "margin_dpo/margin_mean": 19.10809326171875,
      "margin_dpo/margin_std": 22.90863800048828,
      "step": 176
    },
    {
      "KL/chosen_KL_mean": -15.822185516357422,
      "KL/mean": -26.90675926208496,
      "KL/rejected_KL_mean": -37.991336822509766,
      "KL/std": 22.678550720214844,
      "epoch": 0.2599118942731278,
      "fcm_dpo/beta": 0.02197520062327385,
      "fcm_dpo/delta": -0.10180935263633728,
      "fcm_dpo/margin": 22.169139862060547,
      "fcm_dpo/q_t": 0.38743624091148376,
      "grad_norm": 9.18872356414795,
      "learning_rate": 4.6322706682636137e-07,
      "logits/chosen": -1.774768352508545,
      "logits/rejected": -1.4427521228790283,
      "logps/chosen": -108.38328552246094,
      "logps/ref_chosen": -92.56109619140625,
      "logps/ref_rejected": -103.33839416503906,
      "logps/rejected": -141.32972717285156,
      "loss": 1.0182,
      "margin_dpo/margin_mean": 22.16914176940918,
      "margin_dpo/margin_std": 23.64037322998047,
      "step": 177
    },
    {
      "KL/chosen_KL_mean": -14.50729751586914,
      "KL/mean": -27.773235321044922,
      "KL/rejected_KL_mean": -41.03916931152344,
      "KL/std": 22.108051300048828,
      "epoch": 0.26138032305433184,
      "fcm_dpo/beta": 0.019915781915187836,
      "fcm_dpo/delta": -0.15340155363082886,
      "fcm_dpo/margin": 26.53186798095703,
      "fcm_dpo/q_t": 0.37354332208633423,
      "grad_norm": 9.731657981872559,
      "learning_rate": 4.6255428393240354e-07,
      "logits/chosen": -1.6195755004882812,
      "logits/rejected": -1.4086883068084717,
      "logps/chosen": -98.7730941772461,
      "logps/ref_chosen": -84.26579284667969,
      "logps/ref_rejected": -110.79730987548828,
      "logps/rejected": -151.83648681640625,
      "loss": 0.9767,
      "margin_dpo/margin_mean": 26.53186798095703,
      "margin_dpo/margin_std": 23.904035568237305,
      "step": 178
    },
    {
      "KL/chosen_KL_mean": -18.583770751953125,
      "KL/mean": -26.902341842651367,
      "KL/rejected_KL_mean": -35.220909118652344,
      "KL/std": 21.268850326538086,
      "epoch": 0.26284875183553597,
      "fcm_dpo/beta": 0.02000836282968521,
      "fcm_dpo/delta": 0.0747188851237297,
      "fcm_dpo/margin": 16.63713836669922,
      "fcm_dpo/q_t": 0.42435377836227417,
      "grad_norm": 9.259207725524902,
      "learning_rate": 4.6187590003538724e-07,
      "logits/chosen": -1.4939193725585938,
      "logits/rejected": -1.0608761310577393,
      "logps/chosen": -112.03174591064453,
      "logps/ref_chosen": -93.4479751586914,
      "logps/ref_rejected": -94.25541687011719,
      "logps/rejected": -129.476318359375,
      "loss": 1.1409,
      "margin_dpo/margin_mean": 16.63713836669922,
      "margin_dpo/margin_std": 24.870399475097656,
      "step": 179
    },
    {
      "KL/chosen_KL_mean": -15.475288391113281,
      "KL/mean": -26.449047088623047,
      "KL/rejected_KL_mean": -37.42280578613281,
      "KL/std": 20.493576049804688,
      "epoch": 0.2643171806167401,
      "fcm_dpo/beta": 0.01986520364880562,
      "fcm_dpo/delta": -0.042206570506095886,
      "fcm_dpo/margin": 21.94751739501953,
      "fcm_dpo/q_t": 0.39686936140060425,
      "grad_norm": 11.2254056930542,
      "learning_rate": 4.611919330113591e-07,
      "logits/chosen": -1.3857073783874512,
      "logits/rejected": -1.1516025066375732,
      "logps/chosen": -104.31082916259766,
      "logps/ref_chosen": -88.83554077148438,
      "logps/ref_rejected": -103.90696716308594,
      "logps/rejected": -141.32977294921875,
      "loss": 1.0371,
      "margin_dpo/margin_mean": 21.94751739501953,
      "margin_dpo/margin_std": 21.128055572509766,
      "step": 180
    },
    {
      "KL/chosen_KL_mean": -15.315860748291016,
      "KL/mean": -23.612628936767578,
      "KL/rejected_KL_mean": -31.90939712524414,
      "KL/std": 19.501216888427734,
      "epoch": 0.2657856093979442,
      "fcm_dpo/beta": 0.02054513245820999,
      "fcm_dpo/delta": 0.065505750477314,
      "fcm_dpo/margin": 16.593536376953125,
      "fcm_dpo/q_t": 0.42178016901016235,
      "grad_norm": 8.678281784057617,
      "learning_rate": 4.605024008834863e-07,
      "logits/chosen": -1.6980277299880981,
      "logits/rejected": -1.2378581762313843,
      "logps/chosen": -94.54981994628906,
      "logps/ref_chosen": -79.23396301269531,
      "logps/ref_rejected": -69.17442321777344,
      "logps/rejected": -101.08381652832031,
      "loss": 1.117,
      "margin_dpo/margin_mean": 16.593538284301758,
      "margin_dpo/margin_std": 20.945758819580078,
      "step": 181
    },
    {
      "KL/chosen_KL_mean": -19.225135803222656,
      "KL/mean": -30.076480865478516,
      "KL/rejected_KL_mean": -40.92782211303711,
      "KL/std": 22.64492416381836,
      "epoch": 0.26725403817914833,
      "fcm_dpo/beta": 0.02011769451200962,
      "fcm_dpo/delta": -0.0430900976061821,
      "fcm_dpo/margin": 21.702682495117188,
      "fcm_dpo/q_t": 0.39815235137939453,
      "grad_norm": 9.340291976928711,
      "learning_rate": 4.598073218215817e-07,
      "logits/chosen": -1.3511741161346436,
      "logits/rejected": -1.140545129776001,
      "logps/chosen": -95.30731964111328,
      "logps/ref_chosen": -76.08218383789062,
      "logps/ref_rejected": -94.9686050415039,
      "logps/rejected": -135.89642333984375,
      "loss": 1.0551,
      "margin_dpo/margin_mean": 21.702680587768555,
      "margin_dpo/margin_std": 24.976882934570312,
      "step": 182
    },
    {
      "KL/chosen_KL_mean": -20.244537353515625,
      "KL/mean": -30.627281188964844,
      "KL/rejected_KL_mean": -41.01002502441406,
      "KL/std": 20.59008026123047,
      "epoch": 0.2687224669603524,
      "fcm_dpo/beta": 0.019819848239421844,
      "fcm_dpo/delta": -0.014179060235619545,
      "fcm_dpo/margin": 20.765483856201172,
      "fcm_dpo/q_t": 0.40265128016471863,
      "grad_norm": 9.769152641296387,
      "learning_rate": 4.5910671414162484e-07,
      "logits/chosen": -1.519278645515442,
      "logits/rejected": -0.9016001224517822,
      "logps/chosen": -114.1524429321289,
      "logps/ref_chosen": -93.90790557861328,
      "logps/ref_rejected": -83.56061553955078,
      "logps/rejected": -124.57064056396484,
      "loss": 1.0597,
      "margin_dpo/margin_mean": 20.765483856201172,
      "margin_dpo/margin_std": 21.18230438232422,
      "step": 183
    },
    {
      "KL/chosen_KL_mean": -18.036354064941406,
      "KL/mean": -25.206161499023438,
      "KL/rejected_KL_mean": -32.37596130371094,
      "KL/std": 21.281497955322266,
      "epoch": 0.2701908957415565,
      "fcm_dpo/beta": 0.021052077412605286,
      "fcm_dpo/delta": 0.10731098055839539,
      "fcm_dpo/margin": 14.339606285095215,
      "fcm_dpo/q_t": 0.43046483397483826,
      "grad_norm": 9.190556526184082,
      "learning_rate": 4.5840059630527985e-07,
      "logits/chosen": -1.7955892086029053,
      "logits/rejected": -1.5171985626220703,
      "logps/chosen": -104.90426635742188,
      "logps/ref_chosen": -86.86790466308594,
      "logps/ref_rejected": -84.28456115722656,
      "logps/rejected": -116.6605224609375,
      "loss": 1.1632,
      "margin_dpo/margin_mean": 14.339605331420898,
      "margin_dpo/margin_std": 23.11536407470703,
      "step": 184
    },
    {
      "KL/chosen_KL_mean": -19.128517150878906,
      "KL/mean": -27.713497161865234,
      "KL/rejected_KL_mean": -36.29847717285156,
      "KL/std": 20.91175079345703,
      "epoch": 0.27165932452276065,
      "fcm_dpo/beta": 0.021779756993055344,
      "fcm_dpo/delta": 0.02852557972073555,
      "fcm_dpo/margin": 17.169960021972656,
      "fcm_dpo/q_t": 0.4146445095539093,
      "grad_norm": 10.656231880187988,
      "learning_rate": 4.5768898691940836e-07,
      "logits/chosen": -2.2485578060150146,
      "logits/rejected": -1.6333786249160767,
      "logps/chosen": -113.14994049072266,
      "logps/ref_chosen": -94.02142333984375,
      "logps/ref_rejected": -81.45668029785156,
      "logps/rejected": -117.75515747070312,
      "loss": 1.1105,
      "margin_dpo/margin_mean": 17.169960021972656,
      "margin_dpo/margin_std": 23.958045959472656,
      "step": 185
    },
    {
      "KL/chosen_KL_mean": -23.863224029541016,
      "KL/mean": -33.149139404296875,
      "KL/rejected_KL_mean": -42.43505859375,
      "KL/std": 20.411338806152344,
      "epoch": 0.27312775330396477,
      "fcm_dpo/beta": 0.02149161323904991,
      "fcm_dpo/delta": 0.00037893839180469513,
      "fcm_dpo/margin": 18.571834564208984,
      "fcm_dpo/q_t": 0.4067208468914032,
      "grad_norm": 11.348291397094727,
      "learning_rate": 4.5697190473557947e-07,
      "logits/chosen": -2.2882018089294434,
      "logits/rejected": -1.3611664772033691,
      "logps/chosen": -138.05490112304688,
      "logps/ref_chosen": -114.19168090820312,
      "logps/ref_rejected": -93.33912658691406,
      "logps/rejected": -135.77418518066406,
      "loss": 1.0808,
      "margin_dpo/margin_mean": 18.571834564208984,
      "margin_dpo/margin_std": 22.185989379882812,
      "step": 186
    },
    {
      "KL/chosen_KL_mean": -20.118228912353516,
      "KL/mean": -29.168453216552734,
      "KL/rejected_KL_mean": -38.21868133544922,
      "KL/std": 22.799646377563477,
      "epoch": 0.2745961820851689,
      "fcm_dpo/beta": 0.021936997771263123,
      "fcm_dpo/delta": 0.0025730356574058533,
      "fcm_dpo/margin": 18.100452423095703,
      "fcm_dpo/q_t": 0.40917423367500305,
      "grad_norm": 10.019723892211914,
      "learning_rate": 4.5624936864957555e-07,
      "logits/chosen": -1.8618402481079102,
      "logits/rejected": -1.628331184387207,
      "logps/chosen": -96.98777770996094,
      "logps/ref_chosen": -76.86955261230469,
      "logps/ref_rejected": -88.16494750976562,
      "logps/rejected": -126.38362121582031,
      "loss": 1.0984,
      "margin_dpo/margin_mean": 18.100452423095703,
      "margin_dpo/margin_std": 24.92947006225586,
      "step": 187
    },
    {
      "KL/chosen_KL_mean": -15.444210052490234,
      "KL/mean": -25.727336883544922,
      "KL/rejected_KL_mean": -36.010467529296875,
      "KL/std": 22.331035614013672,
      "epoch": 0.27606461086637296,
      "fcm_dpo/beta": 0.02109968289732933,
      "fcm_dpo/delta": -0.04041576385498047,
      "fcm_dpo/margin": 20.56625747680664,
      "fcm_dpo/q_t": 0.396384060382843,
      "grad_norm": 10.285928726196289,
      "learning_rate": 4.5552139770089454e-07,
      "logits/chosen": -2.1993632316589355,
      "logits/rejected": -1.962559461593628,
      "logps/chosen": -94.63900756835938,
      "logps/ref_chosen": -79.1948013305664,
      "logps/ref_rejected": -96.82889556884766,
      "logps/rejected": -132.83935546875,
      "loss": 1.0492,
      "margin_dpo/margin_mean": 20.566259384155273,
      "margin_dpo/margin_std": 21.20244598388672,
      "step": 188
    },
    {
      "KL/chosen_KL_mean": -20.65337371826172,
      "KL/mean": -31.079914093017578,
      "KL/rejected_KL_mean": -41.50645446777344,
      "KL/std": 22.278610229492188,
      "epoch": 0.2775330396475771,
      "fcm_dpo/beta": 0.021100571379065514,
      "fcm_dpo/delta": -0.04709731042385101,
      "fcm_dpo/margin": 20.85308837890625,
      "fcm_dpo/q_t": 0.3989672362804413,
      "grad_norm": 9.282942771911621,
      "learning_rate": 4.5478801107224794e-07,
      "logits/chosen": -1.8345025777816772,
      "logits/rejected": -1.1924097537994385,
      "logps/chosen": -104.54617309570312,
      "logps/ref_chosen": -83.8927993774414,
      "logps/ref_rejected": -79.46589660644531,
      "logps/rejected": -120.97235107421875,
      "loss": 1.0586,
      "margin_dpo/margin_mean": 20.853086471557617,
      "margin_dpo/margin_std": 24.852012634277344,
      "step": 189
    },
    {
      "KL/chosen_KL_mean": -16.529399871826172,
      "KL/mean": -29.170495986938477,
      "KL/rejected_KL_mean": -41.81159591674805,
      "KL/std": 24.83116912841797,
      "epoch": 0.2790014684287812,
      "fcm_dpo/beta": 0.01958942599594593,
      "fcm_dpo/delta": -0.1121324747800827,
      "fcm_dpo/margin": 25.28219985961914,
      "fcm_dpo/q_t": 0.3817041516304016,
      "grad_norm": 9.480283737182617,
      "learning_rate": 4.5404922808905543e-07,
      "logits/chosen": -1.8151490688323975,
      "logits/rejected": -1.2111976146697998,
      "logps/chosen": -119.97087097167969,
      "logps/ref_chosen": -103.44146728515625,
      "logps/ref_rejected": -109.21502685546875,
      "logps/rejected": -151.02662658691406,
      "loss": 1.0012,
      "margin_dpo/margin_mean": 25.282201766967773,
      "margin_dpo/margin_std": 23.71310043334961,
      "step": 190
    },
    {
      "KL/chosen_KL_mean": -15.965904235839844,
      "KL/mean": -30.40115737915039,
      "KL/rejected_KL_mean": -44.83641815185547,
      "KL/std": 27.055679321289062,
      "epoch": 0.28046989720998533,
      "fcm_dpo/beta": 0.018127698451280594,
      "fcm_dpo/delta": -0.1460278332233429,
      "fcm_dpo/margin": 28.870512008666992,
      "fcm_dpo/q_t": 0.3758046627044678,
      "grad_norm": 10.0971040725708,
      "learning_rate": 4.5330506821893565e-07,
      "logits/chosen": -1.5365405082702637,
      "logits/rejected": -1.2251192331314087,
      "logps/chosen": -115.75141906738281,
      "logps/ref_chosen": -99.78551483154297,
      "logps/ref_rejected": -116.00028228759766,
      "logps/rejected": -160.83670043945312,
      "loss": 0.9867,
      "margin_dpo/margin_mean": 28.870513916015625,
      "margin_dpo/margin_std": 27.582794189453125,
      "step": 191
    },
    {
      "KL/chosen_KL_mean": -25.582469940185547,
      "KL/mean": -35.57929992675781,
      "KL/rejected_KL_mean": -45.57612228393555,
      "KL/std": 21.630931854248047,
      "epoch": 0.28193832599118945,
      "fcm_dpo/beta": 0.017978977411985397,
      "fcm_dpo/delta": 0.045495737344026566,
      "fcm_dpo/margin": 19.993656158447266,
      "fcm_dpo/q_t": 0.41715818643569946,
      "grad_norm": 10.230315208435059,
      "learning_rate": 4.5255555107119336e-07,
      "logits/chosen": -1.548901081085205,
      "logits/rejected": -1.2951288223266602,
      "logps/chosen": -126.44279479980469,
      "logps/ref_chosen": -100.86032104492188,
      "logps/ref_rejected": -109.44500732421875,
      "logps/rejected": -155.0211181640625,
      "loss": 1.108,
      "margin_dpo/margin_mean": 19.993654251098633,
      "margin_dpo/margin_std": 25.413257598876953,
      "step": 192
    },
    {
      "KL/chosen_KL_mean": -21.23882293701172,
      "KL/mean": -29.310688018798828,
      "KL/rejected_KL_mean": -37.3825569152832,
      "KL/std": 23.102760314941406,
      "epoch": 0.2834067547723935,
      "fcm_dpo/beta": 0.019126176834106445,
      "fcm_dpo/delta": 0.09912265092134476,
      "fcm_dpo/margin": 16.14373016357422,
      "fcm_dpo/q_t": 0.4296215772628784,
      "grad_norm": 10.184418678283691,
      "learning_rate": 4.5180069639630236e-07,
      "logits/chosen": -2.1451730728149414,
      "logits/rejected": -1.5630059242248535,
      "logps/chosen": -114.17713928222656,
      "logps/ref_chosen": -92.93831634521484,
      "logps/ref_rejected": -83.10917663574219,
      "logps/rejected": -120.49173736572266,
      "loss": 1.1701,
      "margin_dpo/margin_mean": 16.143728256225586,
      "margin_dpo/margin_std": 27.94436264038086,
      "step": 193
    },
    {
      "KL/chosen_KL_mean": -21.4451904296875,
      "KL/mean": -30.904666900634766,
      "KL/rejected_KL_mean": -40.36414337158203,
      "KL/std": 23.349491119384766,
      "epoch": 0.28487518355359764,
      "fcm_dpo/beta": 0.01969415694475174,
      "fcm_dpo/delta": 0.029964370653033257,
      "fcm_dpo/margin": 18.918954849243164,
      "fcm_dpo/q_t": 0.41413891315460205,
      "grad_norm": 8.797778129577637,
      "learning_rate": 4.510405240853854e-07,
      "logits/chosen": -2.36493182182312,
      "logits/rejected": -1.980708122253418,
      "logps/chosen": -85.23676300048828,
      "logps/ref_chosen": -63.79157257080078,
      "logps/ref_rejected": -68.72665405273438,
      "logps/rejected": -109.0907974243164,
      "loss": 1.1036,
      "margin_dpo/margin_mean": 18.91895294189453,
      "margin_dpo/margin_std": 24.611190795898438,
      "step": 194
    },
    {
      "KL/chosen_KL_mean": -20.660598754882812,
      "KL/mean": -33.4468879699707,
      "KL/rejected_KL_mean": -46.23317337036133,
      "KL/std": 23.869831085205078,
      "epoch": 0.28634361233480177,
      "fcm_dpo/beta": 0.018912335857748985,
      "fcm_dpo/delta": -0.09750112891197205,
      "fcm_dpo/margin": 25.57257080078125,
      "fcm_dpo/q_t": 0.38532236218452454,
      "grad_norm": 8.938124656677246,
      "learning_rate": 4.5027505416968985e-07,
      "logits/chosen": -1.2560853958129883,
      "logits/rejected": -1.1438567638397217,
      "logps/chosen": -104.54605865478516,
      "logps/ref_chosen": -83.88545989990234,
      "logps/ref_rejected": -104.00125122070312,
      "logps/rejected": -150.23443603515625,
      "loss": 1.0226,
      "margin_dpo/margin_mean": 25.572572708129883,
      "margin_dpo/margin_std": 27.215011596679688,
      "step": 195
    },
    {
      "KL/chosen_KL_mean": -24.244754791259766,
      "KL/mean": -36.06471252441406,
      "KL/rejected_KL_mean": -47.88466262817383,
      "KL/std": 25.440845489501953,
      "epoch": 0.2878120411160059,
      "fcm_dpo/beta": 0.01819492131471634,
      "fcm_dpo/delta": -0.03462740033864975,
      "fcm_dpo/margin": 23.63991355895996,
      "fcm_dpo/q_t": 0.39716991782188416,
      "grad_norm": 8.601868629455566,
      "learning_rate": 4.495043068200599e-07,
      "logits/chosen": -1.408975601196289,
      "logits/rejected": -0.9613739252090454,
      "logps/chosen": -113.6461181640625,
      "logps/ref_chosen": -89.4013671875,
      "logps/ref_rejected": -83.34735107421875,
      "logps/rejected": -131.2320098876953,
      "loss": 1.0432,
      "margin_dpo/margin_mean": 23.639911651611328,
      "margin_dpo/margin_std": 22.9451904296875,
      "step": 196
    },
    {
      "KL/chosen_KL_mean": -24.4058837890625,
      "KL/mean": -31.727996826171875,
      "KL/rejected_KL_mean": -39.05010986328125,
      "KL/std": 20.573314666748047,
      "epoch": 0.28928046989721,
      "fcm_dpo/beta": 0.019214363768696785,
      "fcm_dpo/delta": 0.12932348251342773,
      "fcm_dpo/margin": 14.644214630126953,
      "fcm_dpo/q_t": 0.4352540969848633,
      "grad_norm": 9.155078887939453,
      "learning_rate": 4.4872830234640493e-07,
      "logits/chosen": -1.5159151554107666,
      "logits/rejected": -1.186545968055725,
      "logps/chosen": -111.24658966064453,
      "logps/ref_chosen": -86.84070587158203,
      "logps/ref_rejected": -82.79289245605469,
      "logps/rejected": -121.84300231933594,
      "loss": 1.1669,
      "margin_dpo/margin_mean": 14.644214630126953,
      "margin_dpo/margin_std": 21.971435546875,
      "step": 197
    },
    {
      "KL/chosen_KL_mean": -28.047325134277344,
      "KL/mean": -40.09437942504883,
      "KL/rejected_KL_mean": -52.14143371582031,
      "KL/std": 24.55077362060547,
      "epoch": 0.2907488986784141,
      "fcm_dpo/beta": 0.01901492103934288,
      "fcm_dpo/delta": -0.06737254559993744,
      "fcm_dpo/margin": 24.09410858154297,
      "fcm_dpo/q_t": 0.3923225998878479,
      "grad_norm": 10.7271146774292,
      "learning_rate": 4.479470611971645e-07,
      "logits/chosen": -1.7548937797546387,
      "logits/rejected": -1.5861167907714844,
      "logps/chosen": -113.09556579589844,
      "logps/ref_chosen": -85.04824829101562,
      "logps/ref_rejected": -104.10365295410156,
      "logps/rejected": -156.24508666992188,
      "loss": 1.0324,
      "margin_dpo/margin_mean": 24.0941104888916,
      "margin_dpo/margin_std": 25.29715347290039,
      "step": 198
    },
    {
      "KL/chosen_KL_mean": -27.40906524658203,
      "KL/mean": -37.24967956542969,
      "KL/rejected_KL_mean": -47.09029769897461,
      "KL/std": 23.825267791748047,
      "epoch": 0.2922173274596182,
      "fcm_dpo/beta": 0.01909640058875084,
      "fcm_dpo/delta": 0.02701462246477604,
      "fcm_dpo/margin": 19.68123435974121,
      "fcm_dpo/q_t": 0.41291409730911255,
      "grad_norm": 9.398530006408691,
      "learning_rate": 4.471606039587695e-07,
      "logits/chosen": -1.637481689453125,
      "logits/rejected": -1.3767681121826172,
      "logps/chosen": -109.19576263427734,
      "logps/ref_chosen": -81.78669738769531,
      "logps/ref_rejected": -90.790771484375,
      "logps/rejected": -137.88107299804688,
      "loss": 1.0998,
      "margin_dpo/margin_mean": 19.68123435974121,
      "margin_dpo/margin_std": 25.091045379638672,
      "step": 199
    },
    {
      "KL/chosen_KL_mean": -26.993839263916016,
      "KL/mean": -38.638427734375,
      "KL/rejected_KL_mean": -50.283016204833984,
      "KL/std": 25.430805206298828,
      "epoch": 0.2936857562408223,
      "fcm_dpo/beta": 0.01853042095899582,
      "fcm_dpo/delta": -0.03812400996685028,
      "fcm_dpo/margin": 23.28917694091797,
      "fcm_dpo/q_t": 0.39991119503974915,
      "grad_norm": 9.394206047058105,
      "learning_rate": 4.4636895135509966e-07,
      "logits/chosen": -1.7743488550186157,
      "logits/rejected": -1.235065221786499,
      "logps/chosen": -111.26193237304688,
      "logps/ref_chosen": -84.26809692382812,
      "logps/ref_rejected": -86.96786499023438,
      "logps/rejected": -137.25088500976562,
      "loss": 1.0723,
      "margin_dpo/margin_mean": 23.289175033569336,
      "margin_dpo/margin_std": 29.54176902770996,
      "step": 200
    },
    {
      "KL/chosen_KL_mean": -29.017475128173828,
      "KL/mean": -40.28607940673828,
      "KL/rejected_KL_mean": -51.5546875,
      "KL/std": 27.740219116210938,
      "epoch": 0.29515418502202645,
      "fcm_dpo/beta": 0.018636617809534073,
      "fcm_dpo/delta": -0.023199351504445076,
      "fcm_dpo/margin": 22.537216186523438,
      "fcm_dpo/q_t": 0.4033128023147583,
      "grad_norm": 9.672564506530762,
      "learning_rate": 4.455721242469372e-07,
      "logits/chosen": -1.3853214979171753,
      "logits/rejected": -1.0000929832458496,
      "logps/chosen": -136.78997802734375,
      "logps/ref_chosen": -107.77249145507812,
      "logps/ref_rejected": -119.79248046875,
      "logps/rejected": -171.34716796875,
      "loss": 1.0838,
      "margin_dpo/margin_mean": 22.537216186523438,
      "margin_dpo/margin_std": 30.315513610839844,
      "step": 201
    },
    {
      "KL/chosen_KL_mean": -26.92308807373047,
      "KL/mean": -37.41119384765625,
      "KL/rejected_KL_mean": -47.89931106567383,
      "KL/std": 22.869792938232422,
      "epoch": 0.2966226138032305,
      "fcm_dpo/beta": 0.018575526773929596,
      "fcm_dpo/delta": 0.01158919557929039,
      "fcm_dpo/margin": 20.976211547851562,
      "fcm_dpo/q_t": 0.4101453423500061,
      "grad_norm": 9.843216896057129,
      "learning_rate": 4.4477014363141755e-07,
      "logits/chosen": -1.914925456047058,
      "logits/rejected": -1.7928508520126343,
      "logps/chosen": -102.89553833007812,
      "logps/ref_chosen": -75.97245788574219,
      "logps/ref_rejected": -94.4599838256836,
      "logps/rejected": -142.3592987060547,
      "loss": 1.091,
      "margin_dpo/margin_mean": 20.976211547851562,
      "margin_dpo/margin_std": 26.031604766845703,
      "step": 202
    },
    {
      "KL/chosen_KL_mean": -24.172698974609375,
      "KL/mean": -35.94194030761719,
      "KL/rejected_KL_mean": -47.71118927001953,
      "KL/std": 23.85287857055664,
      "epoch": 0.29809104258443464,
      "fcm_dpo/beta": 0.01833461970090866,
      "fcm_dpo/delta": -0.03625010699033737,
      "fcm_dpo/margin": 23.53848648071289,
      "fcm_dpo/q_t": 0.3980984687805176,
      "grad_norm": 8.905462265014648,
      "learning_rate": 4.439630306414758e-07,
      "logits/chosen": -2.019613742828369,
      "logits/rejected": -1.5727707147598267,
      "logps/chosen": -119.13985443115234,
      "logps/ref_chosen": -94.96715545654297,
      "logps/ref_rejected": -92.8876724243164,
      "logps/rejected": -140.59886169433594,
      "loss": 1.0504,
      "margin_dpo/margin_mean": 23.53848648071289,
      "margin_dpo/margin_std": 25.049537658691406,
      "step": 203
    },
    {
      "KL/chosen_KL_mean": -30.267471313476562,
      "KL/mean": -40.416202545166016,
      "KL/rejected_KL_mean": -50.56492614746094,
      "KL/std": 23.931123733520508,
      "epoch": 0.29955947136563876,
      "fcm_dpo/beta": 0.0185072161257267,
      "fcm_dpo/delta": 0.026766132563352585,
      "fcm_dpo/margin": 20.297460556030273,
      "fcm_dpo/q_t": 0.41372233629226685,
      "grad_norm": 9.759594917297363,
      "learning_rate": 4.431508065452897e-07,
      "logits/chosen": -1.82832932472229,
      "logits/rejected": -1.1557576656341553,
      "logps/chosen": -146.62466430664062,
      "logps/ref_chosen": -116.35719299316406,
      "logps/ref_rejected": -93.39759063720703,
      "logps/rejected": -143.9625244140625,
      "loss": 1.1103,
      "margin_dpo/margin_mean": 20.297460556030273,
      "margin_dpo/margin_std": 27.815555572509766,
      "step": 204
    },
    {
      "KL/chosen_KL_mean": -28.356292724609375,
      "KL/mean": -41.444149017333984,
      "KL/rejected_KL_mean": -54.532005310058594,
      "KL/std": 26.368846893310547,
      "epoch": 0.3010279001468429,
      "fcm_dpo/beta": 0.017708495259284973,
      "fcm_dpo/delta": -0.07476004213094711,
      "fcm_dpo/margin": 26.175704956054688,
      "fcm_dpo/q_t": 0.3879004716873169,
      "grad_norm": 8.642118453979492,
      "learning_rate": 4.4233349274571974e-07,
      "logits/chosen": -2.3132309913635254,
      "logits/rejected": -1.8835896253585815,
      "logps/chosen": -117.21563720703125,
      "logps/ref_chosen": -88.85934448242188,
      "logps/ref_rejected": -91.8544921875,
      "logps/rejected": -146.38650512695312,
      "loss": 1.0129,
      "margin_dpo/margin_mean": 26.175704956054688,
      "margin_dpo/margin_std": 22.430309295654297,
      "step": 205
    },
    {
      "KL/chosen_KL_mean": -28.77259063720703,
      "KL/mean": -40.91282653808594,
      "KL/rejected_KL_mean": -53.05305862426758,
      "KL/std": 25.56645965576172,
      "epoch": 0.302496328928047,
      "fcm_dpo/beta": 0.017275433987379074,
      "fcm_dpo/delta": -0.023204969242215157,
      "fcm_dpo/margin": 24.280471801757812,
      "fcm_dpo/q_t": 0.39931872487068176,
      "grad_norm": 9.73973274230957,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": -1.5572476387023926,
      "logits/rejected": -1.4145678281784058,
      "logps/chosen": -105.31893920898438,
      "logps/ref_chosen": -76.54634857177734,
      "logps/ref_rejected": -102.95314025878906,
      "logps/rejected": -156.00619506835938,
      "loss": 1.0451,
      "margin_dpo/margin_mean": 24.280471801757812,
      "margin_dpo/margin_std": 21.76795768737793,
      "step": 206
    },
    {
      "KL/chosen_KL_mean": -30.284927368164062,
      "KL/mean": -43.926151275634766,
      "KL/rejected_KL_mean": -57.5673713684082,
      "KL/std": 31.49856185913086,
      "epoch": 0.3039647577092511,
      "fcm_dpo/beta": 0.01686319150030613,
      "fcm_dpo/delta": -0.06968116760253906,
      "fcm_dpo/margin": 27.282451629638672,
      "fcm_dpo/q_t": 0.395307719707489,
      "grad_norm": 9.654642105102539,
      "learning_rate": 4.4068368231789365e-07,
      "logits/chosen": -2.052151918411255,
      "logits/rejected": -1.6251635551452637,
      "logps/chosen": -116.51657104492188,
      "logps/ref_chosen": -86.23164367675781,
      "logps/ref_rejected": -90.65512084960938,
      "logps/rejected": -148.2224884033203,
      "loss": 1.0604,
      "margin_dpo/margin_mean": 27.282451629638672,
      "margin_dpo/margin_std": 35.864410400390625,
      "step": 207
    },
    {
      "KL/chosen_KL_mean": -31.235599517822266,
      "KL/mean": -45.22512435913086,
      "KL/rejected_KL_mean": -59.21466827392578,
      "KL/std": 28.580568313598633,
      "epoch": 0.3054331864904552,
      "fcm_dpo/beta": 0.016369763761758804,
      "fcm_dpo/delta": -0.06738057732582092,
      "fcm_dpo/margin": 27.97906494140625,
      "fcm_dpo/q_t": 0.38994133472442627,
      "grad_norm": 9.559148788452148,
      "learning_rate": 4.398512291636768e-07,
      "logits/chosen": -1.8809267282485962,
      "logits/rejected": -1.5325149297714233,
      "logps/chosen": -125.3951416015625,
      "logps/ref_chosen": -94.1595458984375,
      "logps/ref_rejected": -100.96233367919922,
      "logps/rejected": -160.177001953125,
      "loss": 1.025,
      "margin_dpo/margin_mean": 27.97906494140625,
      "margin_dpo/margin_std": 26.043319702148438,
      "step": 208
    },
    {
      "KL/chosen_KL_mean": -26.680591583251953,
      "KL/mean": -38.953834533691406,
      "KL/rejected_KL_mean": -51.22707748413086,
      "KL/std": 25.580793380737305,
      "epoch": 0.3069016152716593,
      "fcm_dpo/beta": 0.015925925225019455,
      "fcm_dpo/delta": 0.00970291905105114,
      "fcm_dpo/margin": 24.546480178833008,
      "fcm_dpo/q_t": 0.4087965488433838,
      "grad_norm": 8.436522483825684,
      "learning_rate": 4.3901377325300857e-07,
      "logits/chosen": -2.100515127182007,
      "logits/rejected": -1.6220874786376953,
      "logps/chosen": -110.85115814208984,
      "logps/ref_chosen": -84.17056274414062,
      "logps/ref_rejected": -87.61955261230469,
      "logps/rejected": -138.84661865234375,
      "loss": 1.0845,
      "margin_dpo/margin_mean": 24.54648208618164,
      "margin_dpo/margin_std": 28.904224395751953,
      "step": 209
    },
    {
      "KL/chosen_KL_mean": -30.13021469116211,
      "KL/mean": -41.827919006347656,
      "KL/rejected_KL_mean": -53.525630950927734,
      "KL/std": 26.984102249145508,
      "epoch": 0.30837004405286345,
      "fcm_dpo/beta": 0.016214735805988312,
      "fcm_dpo/delta": 0.023295193910598755,
      "fcm_dpo/margin": 23.395416259765625,
      "fcm_dpo/q_t": 0.4115968346595764,
      "grad_norm": 7.990538120269775,
      "learning_rate": 4.381713366536311e-07,
      "logits/chosen": -2.0413215160369873,
      "logits/rejected": -1.6589241027832031,
      "logps/chosen": -111.3013916015625,
      "logps/ref_chosen": -81.17117309570312,
      "logps/ref_rejected": -84.17478942871094,
      "logps/rejected": -137.70040893554688,
      "loss": 1.0883,
      "margin_dpo/margin_mean": 23.395416259765625,
      "margin_dpo/margin_std": 27.115039825439453,
      "step": 210
    },
    {
      "KL/chosen_KL_mean": -34.16080093383789,
      "KL/mean": -45.40283203125,
      "KL/rejected_KL_mean": -56.64486312866211,
      "KL/std": 28.512554168701172,
      "epoch": 0.30983847283406757,
      "fcm_dpo/beta": 0.016649527475237846,
      "fcm_dpo/delta": 0.02838345244526863,
      "fcm_dpo/margin": 22.484066009521484,
      "fcm_dpo/q_t": 0.4136359393596649,
      "grad_norm": 9.996211051940918,
      "learning_rate": 4.373239415645323e-07,
      "logits/chosen": -2.1738691329956055,
      "logits/rejected": -1.5534462928771973,
      "logps/chosen": -142.87350463867188,
      "logps/ref_chosen": -108.71270751953125,
      "logps/ref_rejected": -93.55564880371094,
      "logps/rejected": -150.2005157470703,
      "loss": 1.1005,
      "margin_dpo/margin_mean": 22.484066009521484,
      "margin_dpo/margin_std": 28.859760284423828,
      "step": 211
    },
    {
      "KL/chosen_KL_mean": -24.03048324584961,
      "KL/mean": -40.84147644042969,
      "KL/rejected_KL_mean": -57.65247344970703,
      "KL/std": 32.73773193359375,
      "epoch": 0.31130690161527164,
      "fcm_dpo/beta": 0.015460281632840633,
      "fcm_dpo/delta": -0.143840491771698,
      "fcm_dpo/margin": 33.621986389160156,
      "fcm_dpo/q_t": 0.3763201832771301,
      "grad_norm": 8.443184852600098,
      "learning_rate": 4.3647161031536086e-07,
      "logits/chosen": -1.7805800437927246,
      "logits/rejected": -1.3593270778656006,
      "logps/chosen": -122.3924331665039,
      "logps/ref_chosen": -98.36195373535156,
      "logps/ref_rejected": -109.88999938964844,
      "logps/rejected": -167.54248046875,
      "loss": 0.9908,
      "margin_dpo/margin_mean": 33.621986389160156,
      "margin_dpo/margin_std": 32.816261291503906,
      "step": 212
    },
    {
      "KL/chosen_KL_mean": -28.061565399169922,
      "KL/mean": -40.63105773925781,
      "KL/rejected_KL_mean": -53.200538635253906,
      "KL/std": 29.762836456298828,
      "epoch": 0.31277533039647576,
      "fcm_dpo/beta": 0.01527487300336361,
      "fcm_dpo/delta": 0.017967110499739647,
      "fcm_dpo/margin": 25.138973236083984,
      "fcm_dpo/q_t": 0.41061243414878845,
      "grad_norm": 10.423155784606934,
      "learning_rate": 4.3561436536583774e-07,
      "logits/chosen": -1.917766809463501,
      "logits/rejected": -1.1178702116012573,
      "logps/chosen": -136.11688232421875,
      "logps/ref_chosen": -108.05531311035156,
      "logps/ref_rejected": -100.14414978027344,
      "logps/rejected": -153.34469604492188,
      "loss": 1.098,
      "margin_dpo/margin_mean": 25.138973236083984,
      "margin_dpo/margin_std": 32.73542785644531,
      "step": 213
    },
    {
      "KL/chosen_KL_mean": -29.71451187133789,
      "KL/mean": -41.49960708618164,
      "KL/rejected_KL_mean": -53.28471374511719,
      "KL/std": 31.67058563232422,
      "epoch": 0.3142437591776799,
      "fcm_dpo/beta": 0.01568576693534851,
      "fcm_dpo/delta": 0.033882588148117065,
      "fcm_dpo/margin": 23.570201873779297,
      "fcm_dpo/q_t": 0.416045218706131,
      "grad_norm": 8.338690757751465,
      "learning_rate": 4.3475222930516473e-07,
      "logits/chosen": -1.944061517715454,
      "logits/rejected": -1.8242771625518799,
      "logps/chosen": -107.51924896240234,
      "logps/ref_chosen": -77.80473327636719,
      "logps/ref_rejected": -84.27578735351562,
      "logps/rejected": -137.5605010986328,
      "loss": 1.1198,
      "margin_dpo/margin_mean": 23.570199966430664,
      "margin_dpo/margin_std": 34.795780181884766,
      "step": 214
    },
    {
      "KL/chosen_KL_mean": -25.644088745117188,
      "KL/mean": -41.779258728027344,
      "KL/rejected_KL_mean": -57.9144172668457,
      "KL/std": 31.298419952392578,
      "epoch": 0.315712187958884,
      "fcm_dpo/beta": 0.015044927597045898,
      "fcm_dpo/delta": -0.10009916871786118,
      "fcm_dpo/margin": 32.27032470703125,
      "fcm_dpo/q_t": 0.38470157980918884,
      "grad_norm": 8.165526390075684,
      "learning_rate": 4.3388522485142885e-07,
      "logits/chosen": -1.8447377681732178,
      "logits/rejected": -1.3776376247406006,
      "logps/chosen": -110.75798034667969,
      "logps/ref_chosen": -85.1138916015625,
      "logps/ref_rejected": -96.86151123046875,
      "logps/rejected": -154.77593994140625,
      "loss": 1.0082,
      "margin_dpo/margin_mean": 32.27032470703125,
      "margin_dpo/margin_std": 30.968067169189453,
      "step": 215
    },
    {
      "KL/chosen_KL_mean": -28.003318786621094,
      "KL/mean": -41.966148376464844,
      "KL/rejected_KL_mean": -55.928977966308594,
      "KL/std": 28.097248077392578,
      "epoch": 0.31718061674008813,
      "fcm_dpo/beta": 0.014866422861814499,
      "fcm_dpo/delta": -0.018131621181964874,
      "fcm_dpo/margin": 27.925662994384766,
      "fcm_dpo/q_t": 0.4023159146308899,
      "grad_norm": 7.755163669586182,
      "learning_rate": 4.330133748510036e-07,
      "logits/chosen": -1.7314649820327759,
      "logits/rejected": -1.3154144287109375,
      "logps/chosen": -108.59562683105469,
      "logps/ref_chosen": -80.59230041503906,
      "logps/ref_rejected": -81.41983795166016,
      "logps/rejected": -137.34881591796875,
      "loss": 1.0596,
      "margin_dpo/margin_mean": 27.925662994384766,
      "margin_dpo/margin_std": 29.12470245361328,
      "step": 216
    },
    {
      "KL/chosen_KL_mean": -34.197086334228516,
      "KL/mean": -45.59093475341797,
      "KL/rejected_KL_mean": -56.98478317260742,
      "KL/std": 29.171340942382812,
      "epoch": 0.3186490455212922,
      "fcm_dpo/beta": 0.015132774598896503,
      "fcm_dpo/delta": 0.06091568246483803,
      "fcm_dpo/margin": 22.787704467773438,
      "fcm_dpo/q_t": 0.4217386841773987,
      "grad_norm": 9.653948783874512,
      "learning_rate": 4.3213670227794757e-07,
      "logits/chosen": -1.9670844078063965,
      "logits/rejected": -1.658888816833496,
      "logps/chosen": -127.66967010498047,
      "logps/ref_chosen": -93.47257995605469,
      "logps/ref_rejected": -103.488525390625,
      "logps/rejected": -160.47329711914062,
      "loss": 1.1304,
      "margin_dpo/margin_mean": 22.787704467773438,
      "margin_dpo/margin_std": 33.411964416503906,
      "step": 217
    },
    {
      "KL/chosen_KL_mean": -33.47174835205078,
      "KL/mean": -47.44684982299805,
      "KL/rejected_KL_mean": -61.421958923339844,
      "KL/std": 29.559141159057617,
      "epoch": 0.3201174743024963,
      "fcm_dpo/beta": 0.01493634283542633,
      "fcm_dpo/delta": -0.020562421530485153,
      "fcm_dpo/margin": 27.950206756591797,
      "fcm_dpo/q_t": 0.40323537588119507,
      "grad_norm": 8.294123649597168,
      "learning_rate": 4.3125523023339815e-07,
      "logits/chosen": -1.4402577877044678,
      "logits/rejected": -1.0586003065109253,
      "logps/chosen": -122.53057861328125,
      "logps/ref_chosen": -89.05883026123047,
      "logps/ref_rejected": -94.30680847167969,
      "logps/rejected": -155.728759765625,
      "loss": 1.0787,
      "margin_dpo/margin_mean": 27.950206756591797,
      "margin_dpo/margin_std": 35.67589569091797,
      "step": 218
    },
    {
      "KL/chosen_KL_mean": -30.220890045166016,
      "KL/mean": -42.61649703979492,
      "KL/rejected_KL_mean": -55.012107849121094,
      "KL/std": 34.28545379638672,
      "epoch": 0.32158590308370044,
      "fcm_dpo/beta": 0.015388960018754005,
      "fcm_dpo/delta": 0.01846860721707344,
      "fcm_dpo/margin": 24.79121208190918,
      "fcm_dpo/q_t": 0.41252005100250244,
      "grad_norm": 9.504846572875977,
      "learning_rate": 4.303689819449636e-07,
      "logits/chosen": -1.7772912979125977,
      "logits/rejected": -1.3169386386871338,
      "logps/chosen": -131.2282257080078,
      "logps/ref_chosen": -101.00733947753906,
      "logps/ref_rejected": -92.46794128417969,
      "logps/rejected": -147.48004150390625,
      "loss": 1.1171,
      "margin_dpo/margin_mean": 24.79121208190918,
      "margin_dpo/margin_std": 36.60302734375,
      "step": 219
    },
    {
      "KL/chosen_KL_mean": -31.079143524169922,
      "KL/mean": -42.7591552734375,
      "KL/rejected_KL_mean": -54.43916320800781,
      "KL/std": 28.457765579223633,
      "epoch": 0.32305433186490456,
      "fcm_dpo/beta": 0.015493739396333694,
      "fcm_dpo/delta": 0.04256540909409523,
      "fcm_dpo/margin": 23.360031127929688,
      "fcm_dpo/q_t": 0.4155496656894684,
      "grad_norm": 9.988616943359375,
      "learning_rate": 4.2947798076611047e-07,
      "logits/chosen": -1.4417502880096436,
      "logits/rejected": -1.09086012840271,
      "logps/chosen": -126.61636352539062,
      "logps/ref_chosen": -95.53721618652344,
      "logps/ref_rejected": -94.30703735351562,
      "logps/rejected": -148.74620056152344,
      "loss": 1.1023,
      "margin_dpo/margin_mean": 23.360031127929688,
      "margin_dpo/margin_std": 27.967208862304688,
      "step": 220
    },
    {
      "KL/chosen_KL_mean": -29.144927978515625,
      "KL/mean": -47.1540412902832,
      "KL/rejected_KL_mean": -65.16316223144531,
      "KL/std": 35.544464111328125,
      "epoch": 0.3245227606461087,
      "fcm_dpo/beta": 0.014790689572691917,
      "fcm_dpo/delta": -0.15740202367305756,
      "fcm_dpo/margin": 36.018226623535156,
      "fcm_dpo/q_t": 0.3727129101753235,
      "grad_norm": 9.548389434814453,
      "learning_rate": 4.285822501755485e-07,
      "logits/chosen": -1.2715402841567993,
      "logits/rejected": -1.1378960609436035,
      "logps/chosen": -111.98979187011719,
      "logps/ref_chosen": -82.84486389160156,
      "logps/ref_rejected": -110.81179809570312,
      "logps/rejected": -175.97496032714844,
      "loss": 0.9779,
      "margin_dpo/margin_mean": 36.018226623535156,
      "margin_dpo/margin_std": 31.168209075927734,
      "step": 221
    },
    {
      "KL/chosen_KL_mean": -37.16849136352539,
      "KL/mean": -51.853267669677734,
      "KL/rejected_KL_mean": -66.53804779052734,
      "KL/std": 34.08497619628906,
      "epoch": 0.32599118942731276,
      "fcm_dpo/beta": 0.014209168031811714,
      "fcm_dpo/delta": -0.020641878247261047,
      "fcm_dpo/margin": 29.369548797607422,
      "fcm_dpo/q_t": 0.4034920334815979,
      "grad_norm": 9.18616008758545,
      "learning_rate": 4.276818137766118e-07,
      "logits/chosen": -1.944960117340088,
      "logits/rejected": -1.5129632949829102,
      "logps/chosen": -132.3104705810547,
      "logps/ref_chosen": -95.14198303222656,
      "logps/ref_rejected": -106.80441284179688,
      "logps/rejected": -173.34246826171875,
      "loss": 1.0891,
      "margin_dpo/margin_mean": 29.369544982910156,
      "margin_dpo/margin_std": 39.782310485839844,
      "step": 222
    },
    {
      "KL/chosen_KL_mean": -36.30195236206055,
      "KL/mean": -46.663185119628906,
      "KL/rejected_KL_mean": -57.0244255065918,
      "KL/std": 30.015953063964844,
      "epoch": 0.3274596182085169,
      "fcm_dpo/beta": 0.014624722301959991,
      "fcm_dpo/delta": 0.10688076913356781,
      "fcm_dpo/margin": 20.72247314453125,
      "fcm_dpo/q_t": 0.4310515820980072,
      "grad_norm": 9.187801361083984,
      "learning_rate": 4.2677669529663686e-07,
      "logits/chosen": -2.1677494049072266,
      "logits/rejected": -1.776566982269287,
      "logps/chosen": -121.87705993652344,
      "logps/ref_chosen": -85.57511138916016,
      "logps/ref_rejected": -86.45237731933594,
      "logps/rejected": -143.476806640625,
      "loss": 1.1616,
      "margin_dpo/margin_mean": 20.722471237182617,
      "margin_dpo/margin_std": 32.90885925292969,
      "step": 223
    },
    {
      "KL/chosen_KL_mean": -33.19157791137695,
      "KL/mean": -48.01884460449219,
      "KL/rejected_KL_mean": -62.84612274169922,
      "KL/std": 33.21797180175781,
      "epoch": 0.328928046989721,
      "fcm_dpo/beta": 0.014619097113609314,
      "fcm_dpo/delta": -0.03870069980621338,
      "fcm_dpo/margin": 29.65454864501953,
      "fcm_dpo/q_t": 0.3990010619163513,
      "grad_norm": 8.813560485839844,
      "learning_rate": 4.2586691858633747e-07,
      "logits/chosen": -1.7668473720550537,
      "logits/rejected": -1.3054616451263428,
      "logps/chosen": -115.9153823852539,
      "logps/ref_chosen": -82.72380065917969,
      "logps/ref_rejected": -82.59538269042969,
      "logps/rejected": -145.44149780273438,
      "loss": 1.06,
      "margin_dpo/margin_mean": 29.65454864501953,
      "margin_dpo/margin_std": 34.95264434814453,
      "step": 224
    },
    {
      "KL/chosen_KL_mean": -37.32892990112305,
      "KL/mean": -52.60606384277344,
      "KL/rejected_KL_mean": -67.88320922851562,
      "KL/std": 35.007789611816406,
      "epoch": 0.3303964757709251,
      "fcm_dpo/beta": 0.014427173882722855,
      "fcm_dpo/delta": -0.047357380390167236,
      "fcm_dpo/margin": 30.55428123474121,
      "fcm_dpo/q_t": 0.3986132740974426,
      "grad_norm": 9.688596725463867,
      "learning_rate": 4.249525076191759e-07,
      "logits/chosen": -1.7345938682556152,
      "logits/rejected": -1.2354097366333008,
      "logps/chosen": -133.00662231445312,
      "logps/ref_chosen": -95.67768859863281,
      "logps/ref_rejected": -105.09687805175781,
      "logps/rejected": -172.98008728027344,
      "loss": 1.0682,
      "margin_dpo/margin_mean": 30.554279327392578,
      "margin_dpo/margin_std": 39.50019073486328,
      "step": 225
    },
    {
      "KL/chosen_KL_mean": -30.840145111083984,
      "KL/mean": -45.54393768310547,
      "KL/rejected_KL_mean": -60.247718811035156,
      "KL/std": 34.222442626953125,
      "epoch": 0.33186490455212925,
      "fcm_dpo/beta": 0.013879513368010521,
      "fcm_dpo/delta": -0.011140517890453339,
      "fcm_dpo/margin": 29.407577514648438,
      "fcm_dpo/q_t": 0.4036685824394226,
      "grad_norm": 7.947854995727539,
      "learning_rate": 4.2403348649073167e-07,
      "logits/chosen": -1.8271138668060303,
      "logits/rejected": -1.2873154878616333,
      "logps/chosen": -124.30107116699219,
      "logps/ref_chosen": -93.46092224121094,
      "logps/ref_rejected": -86.7017593383789,
      "logps/rejected": -146.94947814941406,
      "loss": 1.0638,
      "margin_dpo/margin_mean": 29.407577514648438,
      "margin_dpo/margin_std": 30.40665054321289,
      "step": 226
    },
    {
      "KL/chosen_KL_mean": -41.63078689575195,
      "KL/mean": -56.82681655883789,
      "KL/rejected_KL_mean": -72.02284240722656,
      "KL/std": 33.21185302734375,
      "epoch": 0.3333333333333333,
      "fcm_dpo/beta": 0.013919282704591751,
      "fcm_dpo/delta": -0.026364684104919434,
      "fcm_dpo/margin": 30.392061233520508,
      "fcm_dpo/q_t": 0.40231481194496155,
      "grad_norm": 8.901263236999512,
      "learning_rate": 4.2310987941806615e-07,
      "logits/chosen": -1.4442360401153564,
      "logits/rejected": -1.0912678241729736,
      "logps/chosen": -134.44505310058594,
      "logps/ref_chosen": -92.81427001953125,
      "logps/ref_rejected": -104.73693084716797,
      "logps/rejected": -176.759765625,
      "loss": 1.078,
      "margin_dpo/margin_mean": 30.39206314086914,
      "margin_dpo/margin_std": 39.389835357666016,
      "step": 227
    },
    {
      "KL/chosen_KL_mean": -39.0935173034668,
      "KL/mean": -50.57151794433594,
      "KL/rejected_KL_mean": -62.04951095581055,
      "KL/std": 31.196022033691406,
      "epoch": 0.33480176211453744,
      "fcm_dpo/beta": 0.014328522607684135,
      "fcm_dpo/delta": 0.078910693526268,
      "fcm_dpo/margin": 22.95599365234375,
      "fcm_dpo/q_t": 0.42444005608558655,
      "grad_norm": 9.298930168151855,
      "learning_rate": 4.2218171073908463e-07,
      "logits/chosen": -1.7088861465454102,
      "logits/rejected": -1.264207363128662,
      "logps/chosen": -133.13064575195312,
      "logps/ref_chosen": -94.03712463378906,
      "logps/ref_rejected": -96.02151489257812,
      "logps/rejected": -158.07102966308594,
      "loss": 1.1457,
      "margin_dpo/margin_mean": 22.95599365234375,
      "margin_dpo/margin_std": 35.35384750366211,
      "step": 228
    },
    {
      "KL/chosen_KL_mean": -34.79833221435547,
      "KL/mean": -53.803077697753906,
      "KL/rejected_KL_mean": -72.80783081054688,
      "KL/std": 37.62078857421875,
      "epoch": 0.33627019089574156,
      "fcm_dpo/beta": 0.013645105995237827,
      "fcm_dpo/delta": -0.14060872793197632,
      "fcm_dpo/margin": 38.00949478149414,
      "fcm_dpo/q_t": 0.3789459764957428,
      "grad_norm": 8.890701293945312,
      "learning_rate": 4.212490049118951e-07,
      "logits/chosen": -1.7620396614074707,
      "logits/rejected": -1.1189091205596924,
      "logps/chosen": -130.37600708007812,
      "logps/ref_chosen": -95.57766723632812,
      "logps/ref_rejected": -89.17379760742188,
      "logps/rejected": -161.98162841796875,
      "loss": 1.0048,
      "margin_dpo/margin_mean": 38.00949478149414,
      "margin_dpo/margin_std": 41.31869888305664,
      "step": 229
    },
    {
      "KL/chosen_KL_mean": -35.97259521484375,
      "KL/mean": -55.644981384277344,
      "KL/rejected_KL_mean": -75.31736755371094,
      "KL/std": 37.3564453125,
      "epoch": 0.3377386196769457,
      "fcm_dpo/beta": 0.012655312195420265,
      "fcm_dpo/delta": -0.11544807255268097,
      "fcm_dpo/margin": 39.34477233886719,
      "fcm_dpo/q_t": 0.38065895438194275,
      "grad_norm": 9.806679725646973,
      "learning_rate": 4.203117865141635e-07,
      "logits/chosen": -1.6973414421081543,
      "logits/rejected": -1.6399433612823486,
      "logps/chosen": -99.68621826171875,
      "logps/ref_chosen": -63.713626861572266,
      "logps/ref_rejected": -91.90870666503906,
      "logps/rejected": -167.22607421875,
      "loss": 1.0008,
      "margin_dpo/margin_mean": 39.34477233886719,
      "margin_dpo/margin_std": 37.32190704345703,
      "step": 230
    },
    {
      "KL/chosen_KL_mean": -39.92816162109375,
      "KL/mean": -57.00890350341797,
      "KL/rejected_KL_mean": -74.08964538574219,
      "KL/std": 36.08980178833008,
      "epoch": 0.3392070484581498,
      "fcm_dpo/beta": 0.01218702457845211,
      "fcm_dpo/delta": -0.020040269941091537,
      "fcm_dpo/margin": 34.16147994995117,
      "fcm_dpo/q_t": 0.4005749821662903,
      "grad_norm": 7.506011962890625,
      "learning_rate": 4.1937008024246625e-07,
      "logits/chosen": -1.6800830364227295,
      "logits/rejected": -1.1010432243347168,
      "logps/chosen": -135.3838348388672,
      "logps/ref_chosen": -95.45567321777344,
      "logps/ref_rejected": -80.95568084716797,
      "logps/rejected": -155.04531860351562,
      "loss": 1.0486,
      "margin_dpo/margin_mean": 34.16147994995117,
      "margin_dpo/margin_std": 31.218984603881836,
      "step": 231
    },
    {
      "KL/chosen_KL_mean": -42.26312255859375,
      "KL/mean": -56.344459533691406,
      "KL/rejected_KL_mean": -70.42581176757812,
      "KL/std": 33.302879333496094,
      "epoch": 0.3406754772393539,
      "fcm_dpo/beta": 0.012570271268486977,
      "fcm_dpo/delta": 0.05150187015533447,
      "fcm_dpo/margin": 28.16268539428711,
      "fcm_dpo/q_t": 0.4172869920730591,
      "grad_norm": 8.701995849609375,
      "learning_rate": 4.1842391091163933e-07,
      "logits/chosen": -1.3089271783828735,
      "logits/rejected": -0.8069263696670532,
      "logps/chosen": -139.16038513183594,
      "logps/ref_chosen": -96.89726257324219,
      "logps/ref_rejected": -89.76461791992188,
      "logps/rejected": -160.1904296875,
      "loss": 1.1053,
      "margin_dpo/margin_mean": 28.16268539428711,
      "margin_dpo/margin_std": 33.205421447753906,
      "step": 232
    },
    {
      "KL/chosen_KL_mean": -43.592529296875,
      "KL/mean": -63.530338287353516,
      "KL/rejected_KL_mean": -83.46813201904297,
      "KL/std": 42.281585693359375,
      "epoch": 0.342143906020558,
      "fcm_dpo/beta": 0.01209480781108141,
      "fcm_dpo/delta": -0.0969400629401207,
      "fcm_dpo/margin": 39.8756103515625,
      "fcm_dpo/q_t": 0.3854675889015198,
      "grad_norm": 8.98547077178955,
      "learning_rate": 4.174733034541245e-07,
      "logits/chosen": -1.6700687408447266,
      "logits/rejected": -1.471588373184204,
      "logps/chosen": -132.64285278320312,
      "logps/ref_chosen": -89.05032348632812,
      "logps/ref_rejected": -112.75917053222656,
      "logps/rejected": -196.227294921875,
      "loss": 1.0181,
      "margin_dpo/margin_mean": 39.8756103515625,
      "margin_dpo/margin_std": 40.97986602783203,
      "step": 233
    },
    {
      "KL/chosen_KL_mean": -43.398006439208984,
      "KL/mean": -64.55957794189453,
      "KL/rejected_KL_mean": -85.72115325927734,
      "KL/std": 40.232666015625,
      "epoch": 0.3436123348017621,
      "fcm_dpo/beta": 0.011244862340390682,
      "fcm_dpo/delta": -0.09386735409498215,
      "fcm_dpo/margin": 42.323150634765625,
      "fcm_dpo/q_t": 0.38708919286727905,
      "grad_norm": 7.830517292022705,
      "learning_rate": 4.165182829193126e-07,
      "logits/chosen": -1.2696139812469482,
      "logits/rejected": -1.2068569660186768,
      "logps/chosen": -117.71636962890625,
      "logps/ref_chosen": -74.318359375,
      "logps/ref_rejected": -106.38758850097656,
      "logps/rejected": -192.10873413085938,
      "loss": 1.0258,
      "margin_dpo/margin_mean": 42.323150634765625,
      "margin_dpo/margin_std": 44.20882034301758,
      "step": 234
    },
    {
      "KL/chosen_KL_mean": -52.462406158447266,
      "KL/mean": -68.36253356933594,
      "KL/rejected_KL_mean": -84.26265716552734,
      "KL/std": 36.256805419921875,
      "epoch": 0.34508076358296624,
      "fcm_dpo/beta": 0.011547401547431946,
      "fcm_dpo/delta": 0.03657087683677673,
      "fcm_dpo/margin": 31.800249099731445,
      "fcm_dpo/q_t": 0.41362643241882324,
      "grad_norm": 8.045166015625,
      "learning_rate": 4.1555887447288255e-07,
      "logits/chosen": -1.658717155456543,
      "logits/rejected": -1.1033270359039307,
      "logps/chosen": -150.679443359375,
      "logps/ref_chosen": -98.217041015625,
      "logps/ref_rejected": -97.24677276611328,
      "logps/rejected": -181.50942993164062,
      "loss": 1.09,
      "margin_dpo/margin_mean": 31.800249099731445,
      "margin_dpo/margin_std": 34.342552185058594,
      "step": 235
    },
    {
      "KL/chosen_KL_mean": -43.885406494140625,
      "KL/mean": -64.615966796875,
      "KL/rejected_KL_mean": -85.34652709960938,
      "KL/std": 45.61574935913086,
      "epoch": 0.3465491923641703,
      "fcm_dpo/beta": 0.011235845275223255,
      "fcm_dpo/delta": -0.0763515904545784,
      "fcm_dpo/margin": 41.46112060546875,
      "fcm_dpo/q_t": 0.39434897899627686,
      "grad_norm": 8.695735931396484,
      "learning_rate": 4.1459510339613946e-07,
      "logits/chosen": -1.5149106979370117,
      "logits/rejected": -1.5225255489349365,
      "logps/chosen": -122.72313690185547,
      "logps/ref_chosen": -78.83773040771484,
      "logps/ref_rejected": -109.06343078613281,
      "logps/rejected": -194.40997314453125,
      "loss": 1.041,
      "margin_dpo/margin_mean": 41.46112060546875,
      "margin_dpo/margin_std": 49.4439697265625,
      "step": 236
    },
    {
      "KL/chosen_KL_mean": -52.6256217956543,
      "KL/mean": -67.5654525756836,
      "KL/rejected_KL_mean": -82.5052719116211,
      "KL/std": 40.421905517578125,
      "epoch": 0.34801762114537443,
      "fcm_dpo/beta": 0.011394113302230835,
      "fcm_dpo/delta": 0.06620240956544876,
      "fcm_dpo/margin": 29.87965202331543,
      "fcm_dpo/q_t": 0.4248065650463104,
      "grad_norm": 8.090413093566895,
      "learning_rate": 4.136269950853473e-07,
      "logits/chosen": -1.5859911441802979,
      "logits/rejected": -1.2399260997772217,
      "logps/chosen": -137.8369140625,
      "logps/ref_chosen": -85.21128845214844,
      "logps/ref_rejected": -99.90999603271484,
      "logps/rejected": -182.41526794433594,
      "loss": 1.1564,
      "margin_dpo/margin_mean": 29.879650115966797,
      "margin_dpo/margin_std": 52.36406707763672,
      "step": 237
    },
    {
      "KL/chosen_KL_mean": -57.59463119506836,
      "KL/mean": -75.26953125,
      "KL/rejected_KL_mean": -92.9444351196289,
      "KL/std": 41.95830535888672,
      "epoch": 0.34948604992657856,
      "fcm_dpo/beta": 0.011433232575654984,
      "fcm_dpo/delta": -0.004832110367715359,
      "fcm_dpo/margin": 35.349815368652344,
      "fcm_dpo/q_t": 0.4066636264324188,
      "grad_norm": 7.66969633102417,
      "learning_rate": 4.126545750510605e-07,
      "logits/chosen": -1.3973877429962158,
      "logits/rejected": -1.2512876987457275,
      "logps/chosen": -136.32586669921875,
      "logps/ref_chosen": -78.73123168945312,
      "logps/ref_rejected": -95.41840362548828,
      "logps/rejected": -188.3628387451172,
      "loss": 1.0801,
      "margin_dpo/margin_mean": 35.349815368652344,
      "margin_dpo/margin_std": 43.66181945800781,
      "step": 238
    },
    {
      "KL/chosen_KL_mean": -52.46268081665039,
      "KL/mean": -68.91459655761719,
      "KL/rejected_KL_mean": -85.36651611328125,
      "KL/std": 37.3505859375,
      "epoch": 0.3509544787077827,
      "fcm_dpo/beta": 0.011404107324779034,
      "fcm_dpo/delta": 0.026313740760087967,
      "fcm_dpo/margin": 32.903839111328125,
      "fcm_dpo/q_t": 0.4129123091697693,
      "grad_norm": 8.50284194946289,
      "learning_rate": 4.116778689174514e-07,
      "logits/chosen": -1.8405203819274902,
      "logits/rejected": -1.3631144762039185,
      "logps/chosen": -145.06362915039062,
      "logps/ref_chosen": -92.60093688964844,
      "logps/ref_rejected": -100.51769256591797,
      "logps/rejected": -185.88421630859375,
      "loss": 1.1043,
      "margin_dpo/margin_mean": 32.90384292602539,
      "margin_dpo/margin_std": 41.64423751831055,
      "step": 239
    },
    {
      "KL/chosen_KL_mean": -54.8105583190918,
      "KL/mean": -69.31118774414062,
      "KL/rejected_KL_mean": -83.81182861328125,
      "KL/std": 37.696876525878906,
      "epoch": 0.3524229074889868,
      "fcm_dpo/beta": 0.011896757408976555,
      "fcm_dpo/delta": 0.061443787068128586,
      "fcm_dpo/margin": 29.00127410888672,
      "fcm_dpo/q_t": 0.4205732047557831,
      "grad_norm": 9.52718448638916,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -2.0782198905944824,
      "logits/rejected": -1.7133290767669678,
      "logps/chosen": -140.9703369140625,
      "logps/ref_chosen": -86.15977478027344,
      "logps/ref_rejected": -80.45567321777344,
      "logps/rejected": -164.26748657226562,
      "loss": 1.1276,
      "margin_dpo/margin_mean": 29.00127410888672,
      "margin_dpo/margin_std": 40.73919677734375,
      "step": 240
    },
    {
      "KL/chosen_KL_mean": -50.630062103271484,
      "KL/mean": -76.37762451171875,
      "KL/rejected_KL_mean": -102.12518310546875,
      "KL/std": 42.83787155151367,
      "epoch": 0.35389133627019087,
      "fcm_dpo/beta": 0.011156737804412842,
      "fcm_dpo/delta": -0.2094455510377884,
      "fcm_dpo/margin": 51.4951171875,
      "fcm_dpo/q_t": 0.3629229962825775,
      "grad_norm": 8.158084869384766,
      "learning_rate": 4.097117014129903e-07,
      "logits/chosen": -1.8233071565628052,
      "logits/rejected": -1.1455518007278442,
      "logps/chosen": -151.67599487304688,
      "logps/ref_chosen": -101.04594421386719,
      "logps/ref_rejected": -94.04934692382812,
      "logps/rejected": -196.17453002929688,
      "loss": 0.9598,
      "margin_dpo/margin_mean": 51.495121002197266,
      "margin_dpo/margin_std": 47.947509765625,
      "step": 241
    },
    {
      "KL/chosen_KL_mean": -59.671627044677734,
      "KL/mean": -78.28520202636719,
      "KL/rejected_KL_mean": -96.89878845214844,
      "KL/std": 38.88921356201172,
      "epoch": 0.355359765051395,
      "fcm_dpo/beta": 0.010744722560048103,
      "fcm_dpo/delta": -0.0005094613879919052,
      "fcm_dpo/margin": 37.22716522216797,
      "fcm_dpo/q_t": 0.405732661485672,
      "grad_norm": 8.448933601379395,
      "learning_rate": 4.087222918524807e-07,
      "logits/chosen": -2.0030977725982666,
      "logits/rejected": -1.351003646850586,
      "logps/chosen": -155.34429931640625,
      "logps/ref_chosen": -95.67266845703125,
      "logps/ref_rejected": -90.65454864501953,
      "logps/rejected": -187.5533447265625,
      "loss": 1.0783,
      "margin_dpo/margin_mean": 37.22716522216797,
      "margin_dpo/margin_std": 43.3050651550293,
      "step": 242
    },
    {
      "KL/chosen_KL_mean": -59.35002899169922,
      "KL/mean": -77.89738464355469,
      "KL/rejected_KL_mean": -96.44474029541016,
      "KL/std": 41.74457550048828,
      "epoch": 0.3568281938325991,
      "fcm_dpo/beta": 0.01086503453552723,
      "fcm_dpo/delta": -0.006921149790287018,
      "fcm_dpo/margin": 37.09471130371094,
      "fcm_dpo/q_t": 0.40688997507095337,
      "grad_norm": 8.28245735168457,
      "learning_rate": 4.07728699811968e-07,
      "logits/chosen": -1.951370358467102,
      "logits/rejected": -1.1294584274291992,
      "logps/chosen": -157.38143920898438,
      "logps/ref_chosen": -98.03140258789062,
      "logps/ref_rejected": -83.18806457519531,
      "logps/rejected": -179.6328125,
      "loss": 1.0985,
      "margin_dpo/margin_mean": 37.09471130371094,
      "margin_dpo/margin_std": 49.90336608886719,
      "step": 243
    },
    {
      "KL/chosen_KL_mean": -61.74498748779297,
      "KL/mean": -81.40054321289062,
      "KL/rejected_KL_mean": -101.05609130859375,
      "KL/std": 45.7300910949707,
      "epoch": 0.35829662261380324,
      "fcm_dpo/beta": 0.010491423308849335,
      "fcm_dpo/delta": -0.014410645700991154,
      "fcm_dpo/margin": 39.31110382080078,
      "fcm_dpo/q_t": 0.40364331007003784,
      "grad_norm": 7.970212936401367,
      "learning_rate": 4.067309514735267e-07,
      "logits/chosen": -2.1783878803253174,
      "logits/rejected": -1.8497850894927979,
      "logps/chosen": -150.638916015625,
      "logps/ref_chosen": -88.8939208984375,
      "logps/ref_rejected": -102.57278442382812,
      "logps/rejected": -203.62887573242188,
      "loss": 1.0689,
      "margin_dpo/margin_mean": 39.31110382080078,
      "margin_dpo/margin_std": 45.46587371826172,
      "step": 244
    },
    {
      "KL/chosen_KL_mean": -52.77909469604492,
      "KL/mean": -73.41931915283203,
      "KL/rejected_KL_mean": -94.05953979492188,
      "KL/std": 42.075531005859375,
      "epoch": 0.35976505139500736,
      "fcm_dpo/beta": 0.010407022200524807,
      "fcm_dpo/delta": -0.03397602587938309,
      "fcm_dpo/margin": 41.28044509887695,
      "fcm_dpo/q_t": 0.3987758159637451,
      "grad_norm": 7.489794731140137,
      "learning_rate": 4.057290731287531e-07,
      "logits/chosen": -2.377868175506592,
      "logits/rejected": -1.5516254901885986,
      "logps/chosen": -156.97308349609375,
      "logps/ref_chosen": -104.19400024414062,
      "logps/ref_rejected": -92.65645599365234,
      "logps/rejected": -186.71600341796875,
      "loss": 1.0509,
      "margin_dpo/margin_mean": 41.28044128417969,
      "margin_dpo/margin_std": 43.60002899169922,
      "step": 245
    },
    {
      "KL/chosen_KL_mean": -68.64227294921875,
      "KL/mean": -84.95735931396484,
      "KL/rejected_KL_mean": -101.27244567871094,
      "KL/std": 41.553619384765625,
      "epoch": 0.36123348017621143,
      "fcm_dpo/beta": 0.010507388971745968,
      "fcm_dpo/delta": 0.0637056827545166,
      "fcm_dpo/margin": 32.630184173583984,
      "fcm_dpo/q_t": 0.42146989703178406,
      "grad_norm": 7.7393927574157715,
      "learning_rate": 4.047230911780736e-07,
      "logits/chosen": -1.8429794311523438,
      "logits/rejected": -1.294196367263794,
      "logps/chosen": -171.861328125,
      "logps/ref_chosen": -103.21904754638672,
      "logps/ref_rejected": -90.99221801757812,
      "logps/rejected": -192.26466369628906,
      "loss": 1.1273,
      "margin_dpo/margin_mean": 32.63018035888672,
      "margin_dpo/margin_std": 45.164329528808594,
      "step": 246
    },
    {
      "KL/chosen_KL_mean": -62.59090042114258,
      "KL/mean": -89.53938293457031,
      "KL/rejected_KL_mean": -116.48786926269531,
      "KL/std": 46.51542282104492,
      "epoch": 0.36270190895741555,
      "fcm_dpo/beta": 0.009828225709497929,
      "fcm_dpo/delta": -0.15870153903961182,
      "fcm_dpo/margin": 53.89698028564453,
      "fcm_dpo/q_t": 0.37447887659072876,
      "grad_norm": 7.965665817260742,
      "learning_rate": 4.0371303213004814e-07,
      "logits/chosen": -2.1885366439819336,
      "logits/rejected": -2.017482280731201,
      "logps/chosen": -149.58526611328125,
      "logps/ref_chosen": -86.99437713623047,
      "logps/ref_rejected": -111.33802795410156,
      "logps/rejected": -227.82589721679688,
      "loss": 0.9885,
      "margin_dpo/margin_mean": 53.896976470947266,
      "margin_dpo/margin_std": 52.99579620361328,
      "step": 247
    },
    {
      "KL/chosen_KL_mean": -72.08421325683594,
      "KL/mean": -89.9249267578125,
      "KL/rejected_KL_mean": -107.76563262939453,
      "KL/std": 40.858116149902344,
      "epoch": 0.3641703377386197,
      "fcm_dpo/beta": 0.010024044662714005,
      "fcm_dpo/delta": 0.046441417187452316,
      "fcm_dpo/margin": 35.681419372558594,
      "fcm_dpo/q_t": 0.41639524698257446,
      "grad_norm": 8.185891151428223,
      "learning_rate": 4.0269892260067197e-07,
      "logits/chosen": -1.8590974807739258,
      "logits/rejected": -1.8369181156158447,
      "logps/chosen": -146.86973571777344,
      "logps/ref_chosen": -74.7855224609375,
      "logps/ref_rejected": -98.27689361572266,
      "logps/rejected": -206.0425262451172,
      "loss": 1.1031,
      "margin_dpo/margin_mean": 35.681419372558594,
      "margin_dpo/margin_std": 42.02518844604492,
      "step": 248
    },
    {
      "KL/chosen_KL_mean": -79.96482849121094,
      "KL/mean": -96.26017761230469,
      "KL/rejected_KL_mean": -112.55552673339844,
      "KL/std": 44.281288146972656,
      "epoch": 0.3656387665198238,
      "fcm_dpo/beta": 0.010237883776426315,
      "fcm_dpo/delta": 0.07373890280723572,
      "fcm_dpo/margin": 32.5907096862793,
      "fcm_dpo/q_t": 0.424102783203125,
      "grad_norm": 9.586993217468262,
      "learning_rate": 4.0168078931267426e-07,
      "logits/chosen": -2.182391405105591,
      "logits/rejected": -1.6364858150482178,
      "logps/chosen": -175.6686248779297,
      "logps/ref_chosen": -95.70379638671875,
      "logps/ref_rejected": -87.14646911621094,
      "logps/rejected": -199.70199584960938,
      "loss": 1.1472,
      "margin_dpo/margin_mean": 32.59070587158203,
      "margin_dpo/margin_std": 50.90509033203125,
      "step": 249
    },
    {
      "KL/chosen_KL_mean": -70.88127136230469,
      "KL/mean": -89.37921905517578,
      "KL/rejected_KL_mean": -107.87718200683594,
      "KL/std": 44.8576774597168,
      "epoch": 0.3671071953010279,
      "fcm_dpo/beta": 0.010392602533102036,
      "fcm_dpo/delta": 0.016385085880756378,
      "fcm_dpo/margin": 36.99591827392578,
      "fcm_dpo/q_t": 0.41025781631469727,
      "grad_norm": 8.78945255279541,
      "learning_rate": 4.006586590948141e-07,
      "logits/chosen": -2.0525963306427,
      "logits/rejected": -1.1221344470977783,
      "logps/chosen": -184.9334716796875,
      "logps/ref_chosen": -114.05221557617188,
      "logps/ref_rejected": -81.08767700195312,
      "logps/rejected": -188.96485900878906,
      "loss": 1.103,
      "margin_dpo/margin_mean": 36.99591827392578,
      "margin_dpo/margin_std": 48.388427734375,
      "step": 250
    },
    {
      "KL/chosen_KL_mean": -71.29917907714844,
      "KL/mean": -87.50176239013672,
      "KL/rejected_KL_mean": -103.704345703125,
      "KL/std": 41.62684631347656,
      "epoch": 0.368575624082232,
      "fcm_dpo/beta": 0.010676702484488487,
      "fcm_dpo/delta": 0.05941471457481384,
      "fcm_dpo/margin": 32.405174255371094,
      "fcm_dpo/q_t": 0.42178577184677124,
      "grad_norm": 9.779589653015137,
      "learning_rate": 3.9963255888117325e-07,
      "logits/chosen": -1.9642109870910645,
      "logits/rejected": -1.270952582359314,
      "logps/chosen": -169.01046752929688,
      "logps/ref_chosen": -97.71128845214844,
      "logps/ref_rejected": -83.52742004394531,
      "logps/rejected": -187.2317657470703,
      "loss": 1.1429,
      "margin_dpo/margin_mean": 32.405174255371094,
      "margin_dpo/margin_std": 50.124114990234375,
      "step": 251
    },
    {
      "KL/chosen_KL_mean": -71.8251724243164,
      "KL/mean": -88.14775085449219,
      "KL/rejected_KL_mean": -104.4703369140625,
      "KL/std": 44.61165237426758,
      "epoch": 0.3700440528634361,
      "fcm_dpo/beta": 0.01120392419397831,
      "fcm_dpo/delta": 0.03793972358107567,
      "fcm_dpo/margin": 32.645164489746094,
      "fcm_dpo/q_t": 0.4155373275279999,
      "grad_norm": 9.808162689208984,
      "learning_rate": 3.9860251571044666e-07,
      "logits/chosen": -2.201120615005493,
      "logits/rejected": -1.6690587997436523,
      "logps/chosen": -180.811279296875,
      "logps/ref_chosen": -108.9861068725586,
      "logps/ref_rejected": -91.56423950195312,
      "logps/rejected": -196.03457641601562,
      "loss": 1.122,
      "margin_dpo/margin_mean": 32.645164489746094,
      "margin_dpo/margin_std": 47.26023864746094,
      "step": 252
    },
    {
      "KL/chosen_KL_mean": -78.63085174560547,
      "KL/mean": -93.84303283691406,
      "KL/rejected_KL_mean": -109.05521392822266,
      "KL/std": 48.15412902832031,
      "epoch": 0.37151248164464024,
      "fcm_dpo/beta": 0.011657394468784332,
      "fcm_dpo/delta": 0.04725554585456848,
      "fcm_dpo/margin": 30.424348831176758,
      "fcm_dpo/q_t": 0.41730812191963196,
      "grad_norm": 10.1517915725708,
      "learning_rate": 3.9756855672522986e-07,
      "logits/chosen": -2.1958975791931152,
      "logits/rejected": -1.9949252605438232,
      "logps/chosen": -178.84715270996094,
      "logps/ref_chosen": -100.21630096435547,
      "logps/ref_rejected": -105.67670440673828,
      "logps/rejected": -214.73191833496094,
      "loss": 1.1224,
      "margin_dpo/margin_mean": 30.42435073852539,
      "margin_dpo/margin_std": 41.66051483154297,
      "step": 253
    },
    {
      "KL/chosen_KL_mean": -76.11441040039062,
      "KL/mean": -91.86878967285156,
      "KL/rejected_KL_mean": -107.62316131591797,
      "KL/std": 45.60261535644531,
      "epoch": 0.37298091042584436,
      "fcm_dpo/beta": 0.011582508683204651,
      "fcm_dpo/delta": 0.03879503160715103,
      "fcm_dpo/margin": 31.50873374938965,
      "fcm_dpo/q_t": 0.41573745012283325,
      "grad_norm": 10.070737838745117,
      "learning_rate": 3.965307091713037e-07,
      "logits/chosen": -2.1188697814941406,
      "logits/rejected": -1.5979440212249756,
      "logps/chosen": -174.849609375,
      "logps/ref_chosen": -98.73518371582031,
      "logps/ref_rejected": -93.73825073242188,
      "logps/rejected": -201.36141967773438,
      "loss": 1.1327,
      "margin_dpo/margin_mean": 31.50873565673828,
      "margin_dpo/margin_std": 48.739234924316406,
      "step": 254
    },
    {
      "KL/chosen_KL_mean": -77.07427978515625,
      "KL/mean": -101.38314819335938,
      "KL/rejected_KL_mean": -125.6920166015625,
      "KL/std": 51.08869552612305,
      "epoch": 0.3744493392070485,
      "fcm_dpo/beta": 0.011007426306605339,
      "fcm_dpo/delta": -0.16085678339004517,
      "fcm_dpo/margin": 48.617740631103516,
      "fcm_dpo/q_t": 0.3735049366950989,
      "grad_norm": 9.528095245361328,
      "learning_rate": 3.954890003969163e-07,
      "logits/chosen": -1.8923178911209106,
      "logits/rejected": -1.5514780282974243,
      "logps/chosen": -167.45684814453125,
      "logps/ref_chosen": -90.382568359375,
      "logps/ref_rejected": -97.07624816894531,
      "logps/rejected": -222.7682647705078,
      "loss": 0.9975,
      "margin_dpo/margin_mean": 48.617740631103516,
      "margin_dpo/margin_std": 52.2093505859375,
      "step": 255
    },
    {
      "KL/chosen_KL_mean": -75.69412231445312,
      "KL/mean": -96.9531478881836,
      "KL/rejected_KL_mean": -118.21217346191406,
      "KL/std": 53.2116584777832,
      "epoch": 0.37591776798825255,
      "fcm_dpo/beta": 0.010526902973651886,
      "fcm_dpo/delta": -0.054925475269556046,
      "fcm_dpo/margin": 42.51805114746094,
      "fcm_dpo/q_t": 0.39832502603530884,
      "grad_norm": 9.652372360229492,
      "learning_rate": 3.944434578520628e-07,
      "logits/chosen": -1.997702956199646,
      "logits/rejected": -1.6668697595596313,
      "logps/chosen": -164.44692993164062,
      "logps/ref_chosen": -88.7528076171875,
      "logps/ref_rejected": -98.49383544921875,
      "logps/rejected": -216.7060089111328,
      "loss": 1.0892,
      "margin_dpo/margin_mean": 42.51805114746094,
      "margin_dpo/margin_std": 62.773353576660156,
      "step": 256
    },
    {
      "KL/chosen_KL_mean": -76.6502685546875,
      "KL/mean": -98.52722930908203,
      "KL/rejected_KL_mean": -120.40419006347656,
      "KL/std": 54.69975280761719,
      "epoch": 0.37738619676945667,
      "fcm_dpo/beta": 0.009892786853015423,
      "fcm_dpo/delta": -0.04418522119522095,
      "fcm_dpo/margin": 43.75391387939453,
      "fcm_dpo/q_t": 0.3986341655254364,
      "grad_norm": 8.415376663208008,
      "learning_rate": 3.933941090877615e-07,
      "logits/chosen": -1.919945240020752,
      "logits/rejected": -1.5170255899429321,
      "logps/chosen": -159.45379638671875,
      "logps/ref_chosen": -82.80352020263672,
      "logps/ref_rejected": -85.8677978515625,
      "logps/rejected": -206.27197265625,
      "loss": 1.0712,
      "margin_dpo/margin_mean": 43.75391387939453,
      "margin_dpo/margin_std": 52.838470458984375,
      "step": 257
    },
    {
      "KL/chosen_KL_mean": -80.43136596679688,
      "KL/mean": -100.30524444580078,
      "KL/rejected_KL_mean": -120.17913055419922,
      "KL/std": 52.00364685058594,
      "epoch": 0.3788546255506608,
      "fcm_dpo/beta": 0.01023812498897314,
      "fcm_dpo/delta": -0.010244850069284439,
      "fcm_dpo/margin": 39.74775695800781,
      "fcm_dpo/q_t": 0.40473923087120056,
      "grad_norm": 9.368532180786133,
      "learning_rate": 3.923409817553284e-07,
      "logits/chosen": -2.083334445953369,
      "logits/rejected": -1.7254438400268555,
      "logps/chosen": -170.61911010742188,
      "logps/ref_chosen": -90.187744140625,
      "logps/ref_rejected": -103.47068786621094,
      "logps/rejected": -223.64981079101562,
      "loss": 1.086,
      "margin_dpo/margin_mean": 39.74775695800781,
      "margin_dpo/margin_std": 49.641822814941406,
      "step": 258
    },
    {
      "KL/chosen_KL_mean": -86.82758331298828,
      "KL/mean": -103.84414672851562,
      "KL/rejected_KL_mean": -120.86073303222656,
      "KL/std": 52.240234375,
      "epoch": 0.3803230543318649,
      "fcm_dpo/beta": 0.010130513459444046,
      "fcm_dpo/delta": 0.060587089508771896,
      "fcm_dpo/margin": 34.033164978027344,
      "fcm_dpo/q_t": 0.42115500569343567,
      "grad_norm": 9.14433479309082,
      "learning_rate": 3.9128410360564793e-07,
      "logits/chosen": -1.3621164560317993,
      "logits/rejected": -1.1089414358139038,
      "logps/chosen": -177.60012817382812,
      "logps/ref_chosen": -90.77254486083984,
      "logps/ref_rejected": -94.58816528320312,
      "logps/rejected": -215.4488983154297,
      "loss": 1.1605,
      "margin_dpo/margin_mean": 34.03316116333008,
      "margin_dpo/margin_std": 59.304046630859375,
      "step": 259
    },
    {
      "KL/chosen_KL_mean": -88.6685562133789,
      "KL/mean": -103.02587890625,
      "KL/rejected_KL_mean": -117.3832015991211,
      "KL/std": 52.84949493408203,
      "epoch": 0.38179148311306904,
      "fcm_dpo/beta": 0.010855580680072308,
      "fcm_dpo/delta": 0.09742642939090729,
      "fcm_dpo/margin": 28.714643478393555,
      "fcm_dpo/q_t": 0.429966539144516,
      "grad_norm": 11.215559005737305,
      "learning_rate": 3.9022350248844246e-07,
      "logits/chosen": -1.5558514595031738,
      "logits/rejected": -1.5917694568634033,
      "logps/chosen": -164.26126098632812,
      "logps/ref_chosen": -75.59269714355469,
      "logps/ref_rejected": -100.84554290771484,
      "logps/rejected": -218.22874450683594,
      "loss": 1.1912,
      "margin_dpo/margin_mean": 28.714645385742188,
      "margin_dpo/margin_std": 57.67557144165039,
      "step": 260
    },
    {
      "KL/chosen_KL_mean": -68.3658447265625,
      "KL/mean": -93.65855407714844,
      "KL/rejected_KL_mean": -118.95125579833984,
      "KL/std": 53.31805419921875,
      "epoch": 0.3832599118942731,
      "fcm_dpo/beta": 0.01015451643615961,
      "fcm_dpo/delta": -0.1404636949300766,
      "fcm_dpo/margin": 50.585426330566406,
      "fcm_dpo/q_t": 0.3778427541255951,
      "grad_norm": 8.560873985290527,
      "learning_rate": 3.891592063515376e-07,
      "logits/chosen": -2.0605859756469727,
      "logits/rejected": -1.695174217224121,
      "logps/chosen": -162.62075805664062,
      "logps/ref_chosen": -94.25491333007812,
      "logps/ref_rejected": -93.65699005126953,
      "logps/rejected": -212.60824584960938,
      "loss": 1.0024,
      "margin_dpo/margin_mean": 50.585426330566406,
      "margin_dpo/margin_std": 51.927154541015625,
      "step": 261
    },
    {
      "KL/chosen_KL_mean": -74.50924682617188,
      "KL/mean": -98.14222717285156,
      "KL/rejected_KL_mean": -121.77520751953125,
      "KL/std": 56.24950408935547,
      "epoch": 0.38472834067547723,
      "fcm_dpo/beta": 0.009707055985927582,
      "fcm_dpo/delta": -0.07015504688024521,
      "fcm_dpo/margin": 47.265960693359375,
      "fcm_dpo/q_t": 0.39253637194633484,
      "grad_norm": 8.391780853271484,
      "learning_rate": 3.880912432401264e-07,
      "logits/chosen": -1.7772690057754517,
      "logits/rejected": -1.2426416873931885,
      "logps/chosen": -159.77655029296875,
      "logps/ref_chosen": -85.26730346679688,
      "logps/ref_rejected": -90.82609558105469,
      "logps/rejected": -212.60130310058594,
      "loss": 1.0494,
      "margin_dpo/margin_mean": 47.26596450805664,
      "margin_dpo/margin_std": 56.38022994995117,
      "step": 262
    },
    {
      "KL/chosen_KL_mean": -60.36378479003906,
      "KL/mean": -92.2585220336914,
      "KL/rejected_KL_mean": -124.15325927734375,
      "KL/std": 61.01824188232422,
      "epoch": 0.38619676945668135,
      "fcm_dpo/beta": 0.00891412328928709,
      "fcm_dpo/delta": -0.20297911763191223,
      "fcm_dpo/margin": 63.78947448730469,
      "fcm_dpo/q_t": 0.36425143480300903,
      "grad_norm": 7.882530689239502,
      "learning_rate": 3.870196412960302e-07,
      "logits/chosen": -2.161959648132324,
      "logits/rejected": -1.6139254570007324,
      "logps/chosen": -156.12168884277344,
      "logps/ref_chosen": -95.75790405273438,
      "logps/ref_rejected": -101.83377075195312,
      "logps/rejected": -225.98703002929688,
      "loss": 0.9749,
      "margin_dpo/margin_mean": 63.78948211669922,
      "margin_dpo/margin_std": 65.34928894042969,
      "step": 263
    },
    {
      "KL/chosen_KL_mean": -69.69477844238281,
      "KL/mean": -96.63699340820312,
      "KL/rejected_KL_mean": -123.57920837402344,
      "KL/std": 57.46876525878906,
      "epoch": 0.3876651982378855,
      "fcm_dpo/beta": 0.008347106166183949,
      "fcm_dpo/delta": -0.057976607233285904,
      "fcm_dpo/margin": 53.884422302246094,
      "fcm_dpo/q_t": 0.39360958337783813,
      "grad_norm": 7.61600923538208,
      "learning_rate": 3.8594442875695665e-07,
      "logits/chosen": -2.0812058448791504,
      "logits/rejected": -1.7955950498580933,
      "logps/chosen": -160.31744384765625,
      "logps/ref_chosen": -90.6226577758789,
      "logps/ref_rejected": -100.32554626464844,
      "logps/rejected": -223.90475463867188,
      "loss": 1.0424,
      "margin_dpo/margin_mean": 53.884422302246094,
      "margin_dpo/margin_std": 58.4178581237793,
      "step": 264
    },
    {
      "KL/chosen_KL_mean": -73.62472534179688,
      "KL/mean": -97.24623107910156,
      "KL/rejected_KL_mean": -120.86775207519531,
      "KL/std": 58.86333465576172,
      "epoch": 0.3891336270190896,
      "fcm_dpo/beta": 0.00825489405542612,
      "fcm_dpo/delta": 0.010562878102064133,
      "fcm_dpo/margin": 47.2430305480957,
      "fcm_dpo/q_t": 0.4102952480316162,
      "grad_norm": 7.844132900238037,
      "learning_rate": 3.848656339557562e-07,
      "logits/chosen": -1.8757095336914062,
      "logits/rejected": -1.4512426853179932,
      "logps/chosen": -165.99705505371094,
      "logps/ref_chosen": -92.37232971191406,
      "logps/ref_rejected": -94.62757873535156,
      "logps/rejected": -215.49533081054688,
      "loss": 1.1007,
      "margin_dpo/margin_mean": 47.2430305480957,
      "margin_dpo/margin_std": 63.77228546142578,
      "step": 265
    },
    {
      "KL/chosen_KL_mean": -85.43109130859375,
      "KL/mean": -106.81117248535156,
      "KL/rejected_KL_mean": -128.1912384033203,
      "KL/std": 56.95308303833008,
      "epoch": 0.39060205580029367,
      "fcm_dpo/beta": 0.008480701595544815,
      "fcm_dpo/delta": 0.04197200760245323,
      "fcm_dpo/margin": 42.76014709472656,
      "fcm_dpo/q_t": 0.41702723503112793,
      "grad_norm": 9.463756561279297,
      "learning_rate": 3.8378328531967507e-07,
      "logits/chosen": -2.26895809173584,
      "logits/rejected": -1.4377670288085938,
      "logps/chosen": -187.63111877441406,
      "logps/ref_chosen": -102.20002746582031,
      "logps/ref_rejected": -74.36642456054688,
      "logps/rejected": -202.5576629638672,
      "loss": 1.1205,
      "margin_dpo/margin_mean": 42.7601432800293,
      "margin_dpo/margin_std": 60.7507438659668,
      "step": 266
    },
    {
      "KL/chosen_KL_mean": -75.14224243164062,
      "KL/mean": -100.17218017578125,
      "KL/rejected_KL_mean": -125.20210266113281,
      "KL/std": 57.106597900390625,
      "epoch": 0.3920704845814978,
      "fcm_dpo/beta": 0.008391467854380608,
      "fcm_dpo/delta": -0.024104202166199684,
      "fcm_dpo/margin": 50.059852600097656,
      "fcm_dpo/q_t": 0.40140455961227417,
      "grad_norm": 8.36542797088623,
      "learning_rate": 3.8269741136960646e-07,
      "logits/chosen": -1.6418241262435913,
      "logits/rejected": -0.9979358911514282,
      "logps/chosen": -179.42825317382812,
      "logps/ref_chosen": -104.28599548339844,
      "logps/ref_rejected": -95.98719024658203,
      "logps/rejected": -221.18930053710938,
      "loss": 1.0614,
      "margin_dpo/margin_mean": 50.059852600097656,
      "margin_dpo/margin_std": 55.06150817871094,
      "step": 267
    },
    {
      "KL/chosen_KL_mean": -74.90093994140625,
      "KL/mean": -104.31741333007812,
      "KL/rejected_KL_mean": -133.73387145996094,
      "KL/std": 57.37654495239258,
      "epoch": 0.3935389133627019,
      "fcm_dpo/beta": 0.00801922008395195,
      "fcm_dpo/delta": -0.08642961084842682,
      "fcm_dpo/margin": 58.83293914794922,
      "fcm_dpo/q_t": 0.38677024841308594,
      "grad_norm": 8.478584289550781,
      "learning_rate": 3.8160804071933894e-07,
      "logits/chosen": -1.494148850440979,
      "logits/rejected": -1.3418071269989014,
      "logps/chosen": -161.59716796875,
      "logps/ref_chosen": -86.69622039794922,
      "logps/ref_rejected": -109.19184112548828,
      "logps/rejected": -242.92572021484375,
      "loss": 1.0195,
      "margin_dpo/margin_mean": 58.83293914794922,
      "margin_dpo/margin_std": 56.98937225341797,
      "step": 268
    },
    {
      "KL/chosen_KL_mean": -79.40711975097656,
      "KL/mean": -108.07966613769531,
      "KL/rejected_KL_mean": -136.752197265625,
      "KL/std": 61.669960021972656,
      "epoch": 0.39500734214390604,
      "fcm_dpo/beta": 0.007872671820223331,
      "fcm_dpo/delta": -0.059450414031744,
      "fcm_dpo/margin": 57.34508514404297,
      "fcm_dpo/q_t": 0.393041729927063,
      "grad_norm": 7.5475358963012695,
      "learning_rate": 3.8051520207480204e-07,
      "logits/chosen": -1.7454595565795898,
      "logits/rejected": -1.2871346473693848,
      "logps/chosen": -184.37893676757812,
      "logps/ref_chosen": -104.97181701660156,
      "logps/ref_rejected": -112.47647094726562,
      "logps/rejected": -249.22866821289062,
      "loss": 1.0358,
      "margin_dpo/margin_mean": 57.34508514404297,
      "margin_dpo/margin_std": 58.669677734375,
      "step": 269
    },
    {
      "KL/chosen_KL_mean": -88.43389892578125,
      "KL/mean": -112.35294342041016,
      "KL/rejected_KL_mean": -136.27198791503906,
      "KL/std": 60.552581787109375,
      "epoch": 0.3964757709251101,
      "fcm_dpo/beta": 0.007900059223175049,
      "fcm_dpo/delta": 0.024042436853051186,
      "fcm_dpo/margin": 47.83807373046875,
      "fcm_dpo/q_t": 0.4123604893684387,
      "grad_norm": 8.408806800842285,
      "learning_rate": 3.794189242333106e-07,
      "logits/chosen": -2.554713249206543,
      "logits/rejected": -2.151423215866089,
      "logps/chosen": -189.5077362060547,
      "logps/ref_chosen": -101.07383728027344,
      "logps/ref_rejected": -117.75289154052734,
      "logps/rejected": -254.02487182617188,
      "loss": 1.1153,
      "margin_dpo/margin_mean": 47.83807373046875,
      "margin_dpo/margin_std": 68.95150756835938,
      "step": 270
    },
    {
      "KL/chosen_KL_mean": -72.96852111816406,
      "KL/mean": -102.7685546875,
      "KL/rejected_KL_mean": -132.568603515625,
      "KL/std": 56.65791702270508,
      "epoch": 0.39794419970631423,
      "fcm_dpo/beta": 0.007607318926602602,
      "fcm_dpo/delta": -0.062269166111946106,
      "fcm_dpo/margin": 59.600059509277344,
      "fcm_dpo/q_t": 0.3916059136390686,
      "grad_norm": 8.392607688903809,
      "learning_rate": 3.7831923608280514e-07,
      "logits/chosen": -1.51436448097229,
      "logits/rejected": -1.0603737831115723,
      "logps/chosen": -169.693115234375,
      "logps/ref_chosen": -96.72459411621094,
      "logps/ref_rejected": -98.5244140625,
      "logps/rejected": -231.093017578125,
      "loss": 1.0423,
      "margin_dpo/margin_mean": 59.60005569458008,
      "margin_dpo/margin_std": 65.11143493652344,
      "step": 271
    },
    {
      "KL/chosen_KL_mean": -63.12590789794922,
      "KL/mean": -91.66281127929688,
      "KL/rejected_KL_mean": -120.19970703125,
      "KL/std": 62.88972473144531,
      "epoch": 0.39941262848751835,
      "fcm_dpo/beta": 0.007465401664376259,
      "fcm_dpo/delta": -0.029916198924183846,
      "fcm_dpo/margin": 57.07378387451172,
      "fcm_dpo/q_t": 0.40060439705848694,
      "grad_norm": 7.5345869064331055,
      "learning_rate": 3.772161666010912e-07,
      "logits/chosen": -1.5145647525787354,
      "logits/rejected": -1.4082553386688232,
      "logps/chosen": -144.10311889648438,
      "logps/ref_chosen": -80.97721862792969,
      "logps/ref_rejected": -108.55535888671875,
      "logps/rejected": -228.75506591796875,
      "loss": 1.0672,
      "margin_dpo/margin_mean": 57.07378387451172,
      "margin_dpo/margin_std": 68.7159423828125,
      "step": 272
    },
    {
      "KL/chosen_KL_mean": -85.36793518066406,
      "KL/mean": -114.81698608398438,
      "KL/rejected_KL_mean": -144.2660369873047,
      "KL/std": 64.58415222167969,
      "epoch": 0.4008810572687225,
      "fcm_dpo/beta": 0.007184908725321293,
      "fcm_dpo/delta": -0.030260521918535233,
      "fcm_dpo/margin": 58.89809799194336,
      "fcm_dpo/q_t": 0.40028858184814453,
      "grad_norm": 7.51796293258667,
      "learning_rate": 3.761097448550755e-07,
      "logits/chosen": -1.7702538967132568,
      "logits/rejected": -1.3680752515792847,
      "logps/chosen": -177.592529296875,
      "logps/ref_chosen": -92.22460174560547,
      "logps/ref_rejected": -97.3630599975586,
      "logps/rejected": -241.62908935546875,
      "loss": 1.0778,
      "margin_dpo/margin_mean": 58.89809799194336,
      "margin_dpo/margin_std": 73.6016845703125,
      "step": 273
    },
    {
      "KL/chosen_KL_mean": -87.55693054199219,
      "KL/mean": -110.18521118164062,
      "KL/rejected_KL_mean": -132.8134765625,
      "KL/std": 63.21394348144531,
      "epoch": 0.4023494860499266,
      "fcm_dpo/beta": 0.0074812667444348335,
      "fcm_dpo/delta": 0.06852783262729645,
      "fcm_dpo/margin": 45.25654983520508,
      "fcm_dpo/q_t": 0.421749472618103,
      "grad_norm": 11.842967987060547,
      "learning_rate": 3.75e-07,
      "logits/chosen": -2.0553009510040283,
      "logits/rejected": -1.6684143543243408,
      "logps/chosen": -169.81781005859375,
      "logps/ref_chosen": -82.26089477539062,
      "logps/ref_rejected": -83.87699890136719,
      "logps/rejected": -216.6904754638672,
      "loss": 1.1226,
      "margin_dpo/margin_mean": 45.25654602050781,
      "margin_dpo/margin_std": 58.3599853515625,
      "step": 274
    },
    {
      "KL/chosen_KL_mean": -73.90571594238281,
      "KL/mean": -99.8382568359375,
      "KL/rejected_KL_mean": -125.77079010009766,
      "KL/std": 62.196311950683594,
      "epoch": 0.40381791483113066,
      "fcm_dpo/beta": 0.007723146118223667,
      "fcm_dpo/delta": -0.0024840757250785828,
      "fcm_dpo/margin": 51.865074157714844,
      "fcm_dpo/q_t": 0.4069976210594177,
      "grad_norm": 8.829025268554688,
      "learning_rate": 3.738869612786737e-07,
      "logits/chosen": -2.2572052478790283,
      "logits/rejected": -1.9413707256317139,
      "logps/chosen": -153.59268188476562,
      "logps/ref_chosen": -79.68696594238281,
      "logps/ref_rejected": -98.7509765625,
      "logps/rejected": -224.52175903320312,
      "loss": 1.1066,
      "margin_dpo/margin_mean": 51.86506652832031,
      "margin_dpo/margin_std": 74.0754623413086,
      "step": 275
    },
    {
      "KL/chosen_KL_mean": -88.08319091796875,
      "KL/mean": -115.8038330078125,
      "KL/rejected_KL_mean": -143.52447509765625,
      "KL/std": 62.582916259765625,
      "epoch": 0.4052863436123348,
      "fcm_dpo/beta": 0.007404758594930172,
      "fcm_dpo/delta": -0.014834661036729813,
      "fcm_dpo/margin": 55.44127655029297,
      "fcm_dpo/q_t": 0.40468674898147583,
      "grad_norm": 8.779338836669922,
      "learning_rate": 3.7277065802070204e-07,
      "logits/chosen": -2.047200918197632,
      "logits/rejected": -1.4640905857086182,
      "logps/chosen": -174.62289428710938,
      "logps/ref_chosen": -86.53970336914062,
      "logps/ref_rejected": -77.85394287109375,
      "logps/rejected": -221.37841796875,
      "loss": 1.087,
      "margin_dpo/margin_mean": 55.44127655029297,
      "margin_dpo/margin_std": 71.59327697753906,
      "step": 276
    },
    {
      "KL/chosen_KL_mean": -95.25180053710938,
      "KL/mean": -115.22438049316406,
      "KL/rejected_KL_mean": -135.1969451904297,
      "KL/std": 60.897117614746094,
      "epoch": 0.4067547723935389,
      "fcm_dpo/beta": 0.007797634229063988,
      "fcm_dpo/delta": 0.09797467291355133,
      "fcm_dpo/margin": 39.945152282714844,
      "fcm_dpo/q_t": 0.42905524373054504,
      "grad_norm": 8.754385948181152,
      "learning_rate": 3.71651119641714e-07,
      "logits/chosen": -1.9803802967071533,
      "logits/rejected": -1.5600274801254272,
      "logps/chosen": -179.49591064453125,
      "logps/ref_chosen": -84.24411010742188,
      "logps/ref_rejected": -98.83421325683594,
      "logps/rejected": -234.03115844726562,
      "loss": 1.1578,
      "margin_dpo/margin_mean": 39.945152282714844,
      "margin_dpo/margin_std": 62.69588088989258,
      "step": 277
    },
    {
      "KL/chosen_KL_mean": -90.74461364746094,
      "KL/mean": -116.53231811523438,
      "KL/rejected_KL_mean": -142.32003784179688,
      "KL/std": 65.10066223144531,
      "epoch": 0.40822320117474303,
      "fcm_dpo/beta": 0.007905353792011738,
      "fcm_dpo/delta": -0.00908732321113348,
      "fcm_dpo/margin": 51.57542419433594,
      "fcm_dpo/q_t": 0.40429818630218506,
      "grad_norm": 11.133872985839844,
      "learning_rate": 3.705283756425872e-07,
      "logits/chosen": -2.2811732292175293,
      "logits/rejected": -2.071383476257324,
      "logps/chosen": -173.17649841308594,
      "logps/ref_chosen": -82.431884765625,
      "logps/ref_rejected": -97.85691833496094,
      "logps/rejected": -240.17694091796875,
      "loss": 1.0845,
      "margin_dpo/margin_mean": 51.57542419433594,
      "margin_dpo/margin_std": 65.3988265991211,
      "step": 278
    },
    {
      "KL/chosen_KL_mean": -93.3064956665039,
      "KL/mean": -125.59779357910156,
      "KL/rejected_KL_mean": -157.8890838623047,
      "KL/std": 69.32545471191406,
      "epoch": 0.40969162995594716,
      "fcm_dpo/beta": 0.00750393234193325,
      "fcm_dpo/delta": -0.10128327459096909,
      "fcm_dpo/margin": 64.58258819580078,
      "fcm_dpo/q_t": 0.3853558897972107,
      "grad_norm": 9.97189712524414,
      "learning_rate": 3.6940245560867e-07,
      "logits/chosen": -1.8420437574386597,
      "logits/rejected": -1.6304829120635986,
      "logps/chosen": -178.4744873046875,
      "logps/ref_chosen": -85.16799926757812,
      "logps/ref_rejected": -94.12664794921875,
      "logps/rejected": -252.01573181152344,
      "loss": 1.0224,
      "margin_dpo/margin_mean": 64.58258819580078,
      "margin_dpo/margin_std": 69.48948669433594,
      "step": 279
    },
    {
      "KL/chosen_KL_mean": -96.58975982666016,
      "KL/mean": -128.3885498046875,
      "KL/rejected_KL_mean": -160.18734741210938,
      "KL/std": 63.400474548339844,
      "epoch": 0.4111600587371512,
      "fcm_dpo/beta": 0.007273231633007526,
      "fcm_dpo/delta": -0.07245732098817825,
      "fcm_dpo/margin": 63.597564697265625,
      "fcm_dpo/q_t": 0.390281081199646,
      "grad_norm": 8.236285209655762,
      "learning_rate": 3.6827338920900253e-07,
      "logits/chosen": -1.8263084888458252,
      "logits/rejected": -1.6298155784606934,
      "logps/chosen": -182.4461669921875,
      "logps/ref_chosen": -85.85641479492188,
      "logps/ref_rejected": -104.11859130859375,
      "logps/rejected": -264.3059387207031,
      "loss": 1.0322,
      "margin_dpo/margin_mean": 63.597564697265625,
      "margin_dpo/margin_std": 66.3111572265625,
      "step": 280
    },
    {
      "KL/chosen_KL_mean": -102.53993225097656,
      "KL/mean": -134.12139892578125,
      "KL/rejected_KL_mean": -165.70286560058594,
      "KL/std": 73.78829956054688,
      "epoch": 0.41262848751835535,
      "fcm_dpo/beta": 0.00695809256285429,
      "fcm_dpo/delta": -0.04622301831841469,
      "fcm_dpo/margin": 63.162925720214844,
      "fcm_dpo/q_t": 0.3972835838794708,
      "grad_norm": 6.577522277832031,
      "learning_rate": 3.6714120619553435e-07,
      "logits/chosen": -2.0334692001342773,
      "logits/rejected": -1.286896824836731,
      "logps/chosen": -189.09075927734375,
      "logps/ref_chosen": -86.55082702636719,
      "logps/ref_rejected": -88.62866973876953,
      "logps/rejected": -254.33154296875,
      "loss": 1.0574,
      "margin_dpo/margin_mean": 63.162925720214844,
      "margin_dpo/margin_std": 74.52091217041016,
      "step": 281
    },
    {
      "KL/chosen_KL_mean": -101.2602767944336,
      "KL/mean": -140.8326416015625,
      "KL/rejected_KL_mean": -180.40499877929688,
      "KL/std": 71.6164779663086,
      "epoch": 0.41409691629955947,
      "fcm_dpo/beta": 0.006522033829241991,
      "fcm_dpo/delta": -0.13821499049663544,
      "fcm_dpo/margin": 79.14472198486328,
      "fcm_dpo/q_t": 0.3756367564201355,
      "grad_norm": 8.263315200805664,
      "learning_rate": 3.660059364023408e-07,
      "logits/chosen": -1.8664900064468384,
      "logits/rejected": -1.4884825944900513,
      "logps/chosen": -206.36538696289062,
      "logps/ref_chosen": -105.10511016845703,
      "logps/ref_rejected": -102.85336303710938,
      "logps/rejected": -283.25836181640625,
      "loss": 0.986,
      "margin_dpo/margin_mean": 79.14472198486328,
      "margin_dpo/margin_std": 72.08448791503906,
      "step": 282
    },
    {
      "KL/chosen_KL_mean": -116.74232482910156,
      "KL/mean": -144.504150390625,
      "KL/rejected_KL_mean": -172.26602172851562,
      "KL/std": 70.56391906738281,
      "epoch": 0.4155653450807636,
      "fcm_dpo/beta": 0.006524321623146534,
      "fcm_dpo/delta": 0.04235255718231201,
      "fcm_dpo/margin": 55.52368927001953,
      "fcm_dpo/q_t": 0.41640666127204895,
      "grad_norm": 9.038834571838379,
      "learning_rate": 3.6486760974483685e-07,
      "logits/chosen": -2.2727789878845215,
      "logits/rejected": -1.9427587985992432,
      "logps/chosen": -211.794921875,
      "logps/ref_chosen": -95.05259704589844,
      "logps/ref_rejected": -103.54454803466797,
      "logps/rejected": -275.810546875,
      "loss": 1.1193,
      "margin_dpo/margin_mean": 55.52368927001953,
      "margin_dpo/margin_std": 78.76528930664062,
      "step": 283
    },
    {
      "KL/chosen_KL_mean": -106.45089721679688,
      "KL/mean": -141.6618194580078,
      "KL/rejected_KL_mean": -176.8727569580078,
      "KL/std": 69.63705444335938,
      "epoch": 0.4170337738619677,
      "fcm_dpo/beta": 0.006338733248412609,
      "fcm_dpo/delta": -0.055130213499069214,
      "fcm_dpo/margin": 70.42183685302734,
      "fcm_dpo/q_t": 0.3925381302833557,
      "grad_norm": 9.065511703491211,
      "learning_rate": 3.6372625621898863e-07,
      "logits/chosen": -1.8039182424545288,
      "logits/rejected": -1.4675073623657227,
      "logps/chosen": -194.11737060546875,
      "logps/ref_chosen": -87.66647338867188,
      "logps/ref_rejected": -98.75103759765625,
      "logps/rejected": -275.623779296875,
      "loss": 1.0271,
      "margin_dpo/margin_mean": 70.42183685302734,
      "margin_dpo/margin_std": 62.957969665527344,
      "step": 284
    },
    {
      "KL/chosen_KL_mean": -118.67031860351562,
      "KL/mean": -143.33200073242188,
      "KL/rejected_KL_mean": -167.9936981201172,
      "KL/std": 64.45108032226562,
      "epoch": 0.4185022026431718,
      "fcm_dpo/beta": 0.006630629301071167,
      "fcm_dpo/delta": 0.0799785703420639,
      "fcm_dpo/margin": 49.323394775390625,
      "fcm_dpo/q_t": 0.42332184314727783,
      "grad_norm": 9.297537803649902,
      "learning_rate": 3.625819059005228e-07,
      "logits/chosen": -2.3449583053588867,
      "logits/rejected": -2.0632729530334473,
      "logps/chosen": -213.10336303710938,
      "logps/ref_chosen": -94.43303680419922,
      "logps/ref_rejected": -104.07194519042969,
      "logps/rejected": -272.0656433105469,
      "loss": 1.1263,
      "margin_dpo/margin_mean": 49.323394775390625,
      "margin_dpo/margin_std": 61.639041900634766,
      "step": 285
    },
    {
      "KL/chosen_KL_mean": -122.80055236816406,
      "KL/mean": -156.54180908203125,
      "KL/rejected_KL_mean": -190.28305053710938,
      "KL/std": 68.81938171386719,
      "epoch": 0.4199706314243759,
      "fcm_dpo/beta": 0.006507739890366793,
      "fcm_dpo/delta": -0.045428819954395294,
      "fcm_dpo/margin": 67.48250579833984,
      "fcm_dpo/q_t": 0.3958500623703003,
      "grad_norm": 7.646583080291748,
      "learning_rate": 3.614345889441346e-07,
      "logits/chosen": -2.051436424255371,
      "logits/rejected": -1.5935051441192627,
      "logps/chosen": -226.5209503173828,
      "logps/ref_chosen": -103.72039794921875,
      "logps/ref_rejected": -96.25775909423828,
      "logps/rejected": -286.54083251953125,
      "loss": 1.0469,
      "margin_dpo/margin_mean": 67.48251342773438,
      "margin_dpo/margin_std": 71.55256652832031,
      "step": 286
    },
    {
      "KL/chosen_KL_mean": -141.04295349121094,
      "KL/mean": -160.23431396484375,
      "KL/rejected_KL_mean": -179.4256591796875,
      "KL/std": 62.030670166015625,
      "epoch": 0.42143906020558003,
      "fcm_dpo/beta": 0.006554480642080307,
      "fcm_dpo/delta": 0.03366393968462944,
      "fcm_dpo/margin": 38.3826904296875,
      "fcm_dpo/q_t": 0.44209378957748413,
      "grad_norm": 7.571416854858398,
      "learning_rate": 3.6028433558269275e-07,
      "logits/chosen": -2.204502582550049,
      "logits/rejected": -1.6015360355377197,
      "logps/chosen": -234.93284606933594,
      "logps/ref_chosen": -93.88988494873047,
      "logps/ref_rejected": -83.33365631103516,
      "logps/rejected": -262.7593078613281,
      "loss": 1.2075,
      "margin_dpo/margin_mean": 38.3826904296875,
      "margin_dpo/margin_std": 72.74533081054688,
      "step": 287
    },
    {
      "KL/chosen_KL_mean": -133.0811767578125,
      "KL/mean": -168.0272674560547,
      "KL/rejected_KL_mean": -202.9733428955078,
      "KL/std": 70.45852661132812,
      "epoch": 0.42290748898678415,
      "fcm_dpo/beta": 0.006418304052203894,
      "fcm_dpo/delta": -0.05676087364554405,
      "fcm_dpo/margin": 69.89216613769531,
      "fcm_dpo/q_t": 0.39527344703674316,
      "grad_norm": 7.141501426696777,
      "learning_rate": 3.5913117612644327e-07,
      "logits/chosen": -1.918578863143921,
      "logits/rejected": -1.508875846862793,
      "logps/chosen": -221.23719787597656,
      "logps/ref_chosen": -88.15602111816406,
      "logps/ref_rejected": -93.28195190429688,
      "logps/rejected": -296.25531005859375,
      "loss": 1.0466,
      "margin_dpo/margin_mean": 69.89216613769531,
      "margin_dpo/margin_std": 78.99122619628906,
      "step": 288
    },
    {
      "KL/chosen_KL_mean": -129.7563018798828,
      "KL/mean": -164.956298828125,
      "KL/rejected_KL_mean": -200.1563262939453,
      "KL/std": 73.34017944335938,
      "epoch": 0.4243759177679883,
      "fcm_dpo/beta": 0.006293187383562326,
      "fcm_dpo/delta": -0.04956952482461929,
      "fcm_dpo/margin": 70.4000015258789,
      "fcm_dpo/q_t": 0.39797332882881165,
      "grad_norm": 6.7478227615356445,
      "learning_rate": 3.5797514096221024e-07,
      "logits/chosen": -2.166922092437744,
      "logits/rejected": -1.9699561595916748,
      "logps/chosen": -205.14923095703125,
      "logps/ref_chosen": -75.39292907714844,
      "logps/ref_rejected": -93.15428161621094,
      "logps/rejected": -293.31060791015625,
      "loss": 1.0573,
      "margin_dpo/margin_mean": 70.40000915527344,
      "margin_dpo/margin_std": 84.62669372558594,
      "step": 289
    },
    {
      "KL/chosen_KL_mean": -138.880126953125,
      "KL/mean": -175.57286071777344,
      "KL/rejected_KL_mean": -212.26559448242188,
      "KL/std": 78.25411987304688,
      "epoch": 0.42584434654919234,
      "fcm_dpo/beta": 0.006154117174446583,
      "fcm_dpo/delta": -0.060762763023376465,
      "fcm_dpo/margin": 73.38548278808594,
      "fcm_dpo/q_t": 0.3950348496437073,
      "grad_norm": 7.53394079208374,
      "learning_rate": 3.568162605525952e-07,
      "logits/chosen": -1.3265564441680908,
      "logits/rejected": -1.3745806217193604,
      "logps/chosen": -226.92208862304688,
      "logps/ref_chosen": -88.04197692871094,
      "logps/ref_rejected": -123.212158203125,
      "logps/rejected": -335.4777526855469,
      "loss": 1.0478,
      "margin_dpo/margin_mean": 73.38548278808594,
      "margin_dpo/margin_std": 82.68341827392578,
      "step": 290
    },
    {
      "KL/chosen_KL_mean": -130.85845947265625,
      "KL/mean": -162.75054931640625,
      "KL/rejected_KL_mean": -194.64260864257812,
      "KL/std": 74.35317993164062,
      "epoch": 0.42731277533039647,
      "fcm_dpo/beta": 0.006055745761841536,
      "fcm_dpo/delta": 0.015342654660344124,
      "fcm_dpo/margin": 63.78415298461914,
      "fcm_dpo/q_t": 0.40906083583831787,
      "grad_norm": 8.594744682312012,
      "learning_rate": 3.5565456543517485e-07,
      "logits/chosen": -2.429558277130127,
      "logits/rejected": -1.9684524536132812,
      "logps/chosen": -224.95370483398438,
      "logps/ref_chosen": -94.09524536132812,
      "logps/ref_rejected": -96.05006408691406,
      "logps/rejected": -290.69268798828125,
      "loss": 1.0856,
      "margin_dpo/margin_mean": 63.78415298461914,
      "margin_dpo/margin_std": 74.5076904296875,
      "step": 291
    },
    {
      "KL/chosen_KL_mean": -135.24456787109375,
      "KL/mean": -167.4720001220703,
      "KL/rejected_KL_mean": -199.69943237304688,
      "KL/std": 72.06605529785156,
      "epoch": 0.4287812041116006,
      "fcm_dpo/beta": 0.006069207563996315,
      "fcm_dpo/delta": 0.00998375378549099,
      "fcm_dpo/margin": 64.45488739013672,
      "fcm_dpo/q_t": 0.40778690576553345,
      "grad_norm": 7.70608377456665,
      "learning_rate": 3.5449008622169583e-07,
      "logits/chosen": -1.9899282455444336,
      "logits/rejected": -1.5528864860534668,
      "logps/chosen": -223.49496459960938,
      "logps/ref_chosen": -88.25041198730469,
      "logps/ref_rejected": -96.41764068603516,
      "logps/rejected": -296.1170654296875,
      "loss": 1.0723,
      "margin_dpo/margin_mean": 64.45488739013672,
      "margin_dpo/margin_std": 67.4219741821289,
      "step": 292
    },
    {
      "KL/chosen_KL_mean": -134.42678833007812,
      "KL/mean": -160.657470703125,
      "KL/rejected_KL_mean": -186.88815307617188,
      "KL/std": 76.31157684326172,
      "epoch": 0.4302496328928047,
      "fcm_dpo/beta": 0.006354843731969595,
      "fcm_dpo/delta": 0.07208885997533798,
      "fcm_dpo/margin": 52.461395263671875,
      "fcm_dpo/q_t": 0.4244605302810669,
      "grad_norm": 7.5019073486328125,
      "learning_rate": 3.5332285359726846e-07,
      "logits/chosen": -2.3708455562591553,
      "logits/rejected": -1.8030140399932861,
      "logps/chosen": -221.80332946777344,
      "logps/ref_chosen": -87.37654113769531,
      "logps/ref_rejected": -85.75579833984375,
      "logps/rejected": -272.64398193359375,
      "loss": 1.1422,
      "margin_dpo/margin_mean": 52.461395263671875,
      "margin_dpo/margin_std": 80.07009887695312,
      "step": 293
    },
    {
      "KL/chosen_KL_mean": -137.2183380126953,
      "KL/mean": -158.7799072265625,
      "KL/rejected_KL_mean": -180.3414764404297,
      "KL/std": 71.8897476196289,
      "epoch": 0.43171806167400884,
      "fcm_dpo/beta": 0.0067512234672904015,
      "fcm_dpo/delta": 0.117617167532444,
      "fcm_dpo/margin": 43.123146057128906,
      "fcm_dpo/q_t": 0.43462103605270386,
      "grad_norm": 8.334159851074219,
      "learning_rate": 3.5215289831955786e-07,
      "logits/chosen": -1.9056549072265625,
      "logits/rejected": -1.743308424949646,
      "logps/chosen": -210.72628784179688,
      "logps/ref_chosen": -73.50794982910156,
      "logps/ref_rejected": -88.08877563476562,
      "logps/rejected": -268.43023681640625,
      "loss": 1.194,
      "margin_dpo/margin_mean": 43.123146057128906,
      "margin_dpo/margin_std": 83.74362182617188,
      "step": 294
    },
    {
      "KL/chosen_KL_mean": -130.05218505859375,
      "KL/mean": -163.19882202148438,
      "KL/rejected_KL_mean": -196.345458984375,
      "KL/std": 74.06254577636719,
      "epoch": 0.4331864904552129,
      "fcm_dpo/beta": 0.006595575716346502,
      "fcm_dpo/delta": -0.04464585334062576,
      "fcm_dpo/margin": 66.29325866699219,
      "fcm_dpo/q_t": 0.39814111590385437,
      "grad_norm": 8.410390853881836,
      "learning_rate": 3.509802512179737e-07,
      "logits/chosen": -1.6742653846740723,
      "logits/rejected": -1.5180573463439941,
      "logps/chosen": -207.81768798828125,
      "logps/ref_chosen": -77.76548767089844,
      "logps/ref_rejected": -94.24726867675781,
      "logps/rejected": -290.59271240234375,
      "loss": 1.0618,
      "margin_dpo/margin_mean": 66.29325866699219,
      "margin_dpo/margin_std": 80.11978149414062,
      "step": 295
    },
    {
      "KL/chosen_KL_mean": -139.48240661621094,
      "KL/mean": -174.98089599609375,
      "KL/rejected_KL_mean": -210.4793701171875,
      "KL/std": 74.1700210571289,
      "epoch": 0.434654919236417,
      "fcm_dpo/beta": 0.006379299331456423,
      "fcm_dpo/delta": -0.06290261447429657,
      "fcm_dpo/margin": 70.9969711303711,
      "fcm_dpo/q_t": 0.3920992612838745,
      "grad_norm": 11.928153991699219,
      "learning_rate": 3.498049431928577e-07,
      "logits/chosen": -2.6360886096954346,
      "logits/rejected": -2.1600401401519775,
      "logps/chosen": -237.3388214111328,
      "logps/ref_chosen": -97.85641479492188,
      "logps/ref_rejected": -100.81631469726562,
      "logps/rejected": -311.2956848144531,
      "loss": 1.0381,
      "margin_dpo/margin_mean": 70.9969711303711,
      "margin_dpo/margin_std": 74.63200378417969,
      "step": 296
    },
    {
      "KL/chosen_KL_mean": -140.5662841796875,
      "KL/mean": -170.31793212890625,
      "KL/rejected_KL_mean": -200.0696258544922,
      "KL/std": 74.49140167236328,
      "epoch": 0.43612334801762115,
      "fcm_dpo/beta": 0.0064747268334031105,
      "fcm_dpo/delta": 0.01627337373793125,
      "fcm_dpo/margin": 59.503353118896484,
      "fcm_dpo/q_t": 0.4116967022418976,
      "grad_norm": 10.533320426940918,
      "learning_rate": 3.486270052146694e-07,
      "logits/chosen": -1.011125922203064,
      "logits/rejected": -0.6938551664352417,
      "logps/chosen": -229.13211059570312,
      "logps/ref_chosen": -88.56583404541016,
      "logps/ref_rejected": -101.55656433105469,
      "logps/rejected": -301.6261901855469,
      "loss": 1.0988,
      "margin_dpo/margin_mean": 59.503353118896484,
      "margin_dpo/margin_std": 79.04351806640625,
      "step": 297
    },
    {
      "KL/chosen_KL_mean": -118.03746795654297,
      "KL/mean": -156.47598266601562,
      "KL/rejected_KL_mean": -194.9145050048828,
      "KL/std": 80.84974670410156,
      "epoch": 0.43759177679882527,
      "fcm_dpo/beta": 0.006235489621758461,
      "fcm_dpo/delta": -0.09239931404590607,
      "fcm_dpo/margin": 76.87702178955078,
      "fcm_dpo/q_t": 0.387935996055603,
      "grad_norm": 10.68265438079834,
      "learning_rate": 3.474464683231698e-07,
      "logits/chosen": -1.947884440422058,
      "logits/rejected": -1.932731032371521,
      "logps/chosen": -212.91790771484375,
      "logps/ref_chosen": -94.88043975830078,
      "logps/ref_rejected": -122.31101989746094,
      "logps/rejected": -317.22552490234375,
      "loss": 1.0228,
      "margin_dpo/margin_mean": 76.87701416015625,
      "margin_dpo/margin_std": 81.3737564086914,
      "step": 298
    },
    {
      "KL/chosen_KL_mean": -122.97752380371094,
      "KL/mean": -160.4781494140625,
      "KL/rejected_KL_mean": -197.97879028320312,
      "KL/std": 77.99992370605469,
      "epoch": 0.4390602055800294,
      "fcm_dpo/beta": 0.005952715408056974,
      "fcm_dpo/delta": -0.05373072624206543,
      "fcm_dpo/margin": 75.00125122070312,
      "fcm_dpo/q_t": 0.39469024538993835,
      "grad_norm": 10.22171688079834,
      "learning_rate": 3.462633636266041e-07,
      "logits/chosen": -1.1066675186157227,
      "logits/rejected": -0.8851231336593628,
      "logps/chosen": -203.38587951660156,
      "logps/ref_chosen": -80.40835571289062,
      "logps/ref_rejected": -89.53717041015625,
      "logps/rejected": -287.51593017578125,
      "loss": 1.0431,
      "margin_dpo/margin_mean": 75.00125122070312,
      "margin_dpo/margin_std": 80.80252838134766,
      "step": 299
    },
    {
      "KL/chosen_KL_mean": -129.96456909179688,
      "KL/mean": -161.49423217773438,
      "KL/rejected_KL_mean": -193.02386474609375,
      "KL/std": 81.32210540771484,
      "epoch": 0.44052863436123346,
      "fcm_dpo/beta": 0.005922247655689716,
      "fcm_dpo/delta": 0.02937682718038559,
      "fcm_dpo/margin": 63.059303283691406,
      "fcm_dpo/q_t": 0.413620263338089,
      "grad_norm": 7.8086748123168945,
      "learning_rate": 3.4507772230088147e-07,
      "logits/chosen": -1.6738898754119873,
      "logits/rejected": -1.465260624885559,
      "logps/chosen": -218.12347412109375,
      "logps/ref_chosen": -88.15890502929688,
      "logps/ref_rejected": -100.93919372558594,
      "logps/rejected": -293.96307373046875,
      "loss": 1.1131,
      "margin_dpo/margin_mean": 63.059303283691406,
      "margin_dpo/margin_std": 87.58981323242188,
      "step": 300
    },
    {
      "KL/chosen_KL_mean": -120.4306640625,
      "KL/mean": -159.427490234375,
      "KL/rejected_KL_mean": -198.42431640625,
      "KL/std": 79.54063415527344,
      "epoch": 0.4419970631424376,
      "fcm_dpo/beta": 0.005772819742560387,
      "fcm_dpo/delta": -0.060459867119789124,
      "fcm_dpo/margin": 77.99364471435547,
      "fcm_dpo/q_t": 0.3941025733947754,
      "grad_norm": 8.89098072052002,
      "learning_rate": 3.4388957558875316e-07,
      "logits/chosen": -1.8011213541030884,
      "logits/rejected": -1.3748443126678467,
      "logps/chosen": -205.93557739257812,
      "logps/ref_chosen": -85.50491333007812,
      "logps/ref_rejected": -98.91683959960938,
      "logps/rejected": -297.3411560058594,
      "loss": 1.0495,
      "margin_dpo/margin_mean": 77.99365234375,
      "margin_dpo/margin_std": 89.60458374023438,
      "step": 301
    },
    {
      "KL/chosen_KL_mean": -115.5346450805664,
      "KL/mean": -149.16921997070312,
      "KL/rejected_KL_mean": -182.80377197265625,
      "KL/std": 81.80726623535156,
      "epoch": 0.4434654919236417,
      "fcm_dpo/beta": 0.005801204591989517,
      "fcm_dpo/delta": 0.010949157178401947,
      "fcm_dpo/margin": 67.26913452148438,
      "fcm_dpo/q_t": 0.40831226110458374,
      "grad_norm": 7.933813571929932,
      "learning_rate": 3.426989547989902e-07,
      "logits/chosen": -1.4428355693817139,
      "logits/rejected": -1.269667148590088,
      "logps/chosen": -194.98504638671875,
      "logps/ref_chosen": -79.45040893554688,
      "logps/ref_rejected": -103.0591049194336,
      "logps/rejected": -285.86285400390625,
      "loss": 1.0852,
      "margin_dpo/margin_mean": 67.26913452148438,
      "margin_dpo/margin_std": 80.04563903808594,
      "step": 302
    },
    {
      "KL/chosen_KL_mean": -109.11972045898438,
      "KL/mean": -145.62786865234375,
      "KL/rejected_KL_mean": -182.135986328125,
      "KL/std": 76.15892791748047,
      "epoch": 0.44493392070484583,
      "fcm_dpo/beta": 0.005806837230920792,
      "fcm_dpo/delta": -0.027625154703855515,
      "fcm_dpo/margin": 73.01626586914062,
      "fcm_dpo/q_t": 0.39976799488067627,
      "grad_norm": 7.943498611450195,
      "learning_rate": 3.4150589130555773e-07,
      "logits/chosen": -1.9334867000579834,
      "logits/rejected": -1.4778800010681152,
      "logps/chosen": -204.718017578125,
      "logps/ref_chosen": -95.59829711914062,
      "logps/ref_rejected": -92.0645751953125,
      "logps/rejected": -274.2005615234375,
      "loss": 1.0613,
      "margin_dpo/margin_mean": 73.01626586914062,
      "margin_dpo/margin_std": 81.57154846191406,
      "step": 303
    },
    {
      "KL/chosen_KL_mean": -108.46159362792969,
      "KL/mean": -146.30368041992188,
      "KL/rejected_KL_mean": -184.14578247070312,
      "KL/std": 78.64433288574219,
      "epoch": 0.44640234948604995,
      "fcm_dpo/beta": 0.005660864524543285,
      "fcm_dpo/delta": -0.03263058885931969,
      "fcm_dpo/margin": 75.68418884277344,
      "fcm_dpo/q_t": 0.3986150026321411,
      "grad_norm": 8.445808410644531,
      "learning_rate": 3.403104165467883e-07,
      "logits/chosen": -2.1140055656433105,
      "logits/rejected": -1.6925418376922607,
      "logps/chosen": -207.91473388671875,
      "logps/ref_chosen": -99.4531478881836,
      "logps/ref_rejected": -98.08226013183594,
      "logps/rejected": -282.2280578613281,
      "loss": 1.0537,
      "margin_dpo/margin_mean": 75.68418884277344,
      "margin_dpo/margin_std": 81.82255554199219,
      "step": 304
    },
    {
      "KL/chosen_KL_mean": -119.05268096923828,
      "KL/mean": -154.907470703125,
      "KL/rejected_KL_mean": -190.76226806640625,
      "KL/std": 82.18238830566406,
      "epoch": 0.447870778267254,
      "fcm_dpo/beta": 0.005550110712647438,
      "fcm_dpo/delta": 0.0007077492773532867,
      "fcm_dpo/margin": 71.70960235595703,
      "fcm_dpo/q_t": 0.406512975692749,
      "grad_norm": 7.312972545623779,
      "learning_rate": 3.391125620245535e-07,
      "logits/chosen": -1.9195187091827393,
      "logits/rejected": -1.4169200658798218,
      "logps/chosen": -222.31607055664062,
      "logps/ref_chosen": -103.26339721679688,
      "logps/ref_rejected": -97.82897186279297,
      "logps/rejected": -288.59124755859375,
      "loss": 1.0897,
      "margin_dpo/margin_mean": 71.70960235595703,
      "margin_dpo/margin_std": 89.65907287597656,
      "step": 305
    },
    {
      "KL/chosen_KL_mean": -106.9560546875,
      "KL/mean": -148.65618896484375,
      "KL/rejected_KL_mean": -190.35626220703125,
      "KL/std": 77.5550537109375,
      "epoch": 0.44933920704845814,
      "fcm_dpo/beta": 0.005435375031083822,
      "fcm_dpo/delta": -0.06270827353000641,
      "fcm_dpo/margin": 83.40020751953125,
      "fcm_dpo/q_t": 0.3903319537639618,
      "grad_norm": 9.386154174804688,
      "learning_rate": 3.3791235930343417e-07,
      "logits/chosen": -2.170140266418457,
      "logits/rejected": -1.605558156967163,
      "logps/chosen": -207.2598876953125,
      "logps/ref_chosen": -100.3038330078125,
      "logps/ref_rejected": -92.23086547851562,
      "logps/rejected": -282.587158203125,
      "loss": 1.0208,
      "margin_dpo/margin_mean": 83.40020751953125,
      "margin_dpo/margin_std": 72.05933380126953,
      "step": 306
    },
    {
      "KL/chosen_KL_mean": -111.76624298095703,
      "KL/mean": -143.470703125,
      "KL/rejected_KL_mean": -175.17515563964844,
      "KL/std": 78.60736083984375,
      "epoch": 0.45080763582966227,
      "fcm_dpo/beta": 0.005511650815606117,
      "fcm_dpo/delta": 0.056343525648117065,
      "fcm_dpo/margin": 63.408905029296875,
      "fcm_dpo/q_t": 0.4198901653289795,
      "grad_norm": 8.682507514953613,
      "learning_rate": 3.367098400098881e-07,
      "logits/chosen": -1.7767791748046875,
      "logits/rejected": -1.4172430038452148,
      "logps/chosen": -208.416015625,
      "logps/ref_chosen": -96.64976501464844,
      "logps/ref_rejected": -93.34860229492188,
      "logps/rejected": -268.52374267578125,
      "loss": 1.1321,
      "margin_dpo/margin_mean": 63.408905029296875,
      "margin_dpo/margin_std": 93.45675659179688,
      "step": 307
    },
    {
      "KL/chosen_KL_mean": -117.28561401367188,
      "KL/mean": -148.8955078125,
      "KL/rejected_KL_mean": -180.50540161132812,
      "KL/std": 80.15508270263672,
      "epoch": 0.4522760646108664,
      "fcm_dpo/beta": 0.005636774003505707,
      "fcm_dpo/delta": 0.047651227563619614,
      "fcm_dpo/margin": 63.219764709472656,
      "fcm_dpo/q_t": 0.4186602234840393,
      "grad_norm": 9.909399032592773,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": -1.778223991394043,
      "logits/rejected": -1.4973942041397095,
      "logps/chosen": -203.01028442382812,
      "logps/ref_chosen": -85.72467041015625,
      "logps/ref_rejected": -86.42317962646484,
      "logps/rejected": -266.9285888671875,
      "loss": 1.1333,
      "margin_dpo/margin_mean": 63.219764709472656,
      "margin_dpo/margin_std": 94.22024536132812,
      "step": 308
    },
    {
      "KL/chosen_KL_mean": -131.90509033203125,
      "KL/mean": -160.4676513671875,
      "KL/rejected_KL_mean": -189.03021240234375,
      "KL/std": 86.04443359375,
      "epoch": 0.45374449339207046,
      "fcm_dpo/beta": 0.005960534326732159,
      "fcm_dpo/delta": 0.06606906652450562,
      "fcm_dpo/margin": 57.1251220703125,
      "fcm_dpo/q_t": 0.42200562357902527,
      "grad_norm": 11.773571014404297,
      "learning_rate": 3.3429797851573183e-07,
      "logits/chosen": -1.8637316226959229,
      "logits/rejected": -1.4413548707962036,
      "logps/chosen": -215.05654907226562,
      "logps/ref_chosen": -83.15145874023438,
      "logps/ref_rejected": -86.58602142333984,
      "logps/rejected": -275.6162109375,
      "loss": 1.141,
      "margin_dpo/margin_mean": 57.1251220703125,
      "margin_dpo/margin_std": 86.46903991699219,
      "step": 309
    },
    {
      "KL/chosen_KL_mean": -120.67080688476562,
      "KL/mean": -152.20919799804688,
      "KL/rejected_KL_mean": -183.74761962890625,
      "KL/std": 77.71434020996094,
      "epoch": 0.4552129221732746,
      "fcm_dpo/beta": 0.006051789037883282,
      "fcm_dpo/delta": 0.020649230107665062,
      "fcm_dpo/margin": 63.07681655883789,
      "fcm_dpo/q_t": 0.411434143781662,
      "grad_norm": 9.134157180786133,
      "learning_rate": 3.3308869986991487e-07,
      "logits/chosen": -2.2631711959838867,
      "logits/rejected": -1.849839687347412,
      "logps/chosen": -210.40879821777344,
      "logps/ref_chosen": -89.73799133300781,
      "logps/ref_rejected": -87.07083129882812,
      "logps/rejected": -270.8184509277344,
      "loss": 1.0992,
      "margin_dpo/margin_mean": 63.076812744140625,
      "margin_dpo/margin_std": 82.10831451416016,
      "step": 310
    },
    {
      "KL/chosen_KL_mean": -139.718017578125,
      "KL/mean": -180.35275268554688,
      "KL/rejected_KL_mean": -220.9874725341797,
      "KL/std": 79.03710174560547,
      "epoch": 0.4566813509544787,
      "fcm_dpo/beta": 0.005852034781128168,
      "fcm_dpo/delta": -0.088184654712677,
      "fcm_dpo/margin": 81.26944732666016,
      "fcm_dpo/q_t": 0.3874503970146179,
      "grad_norm": 9.236626625061035,
      "learning_rate": 3.3187723175958346e-07,
      "logits/chosen": -1.53908371925354,
      "logits/rejected": -0.7962143421173096,
      "logps/chosen": -228.2444610595703,
      "logps/ref_chosen": -88.52644348144531,
      "logps/ref_rejected": -82.26608276367188,
      "logps/rejected": -303.2535400390625,
      "loss": 1.0176,
      "margin_dpo/margin_mean": 81.26944732666016,
      "margin_dpo/margin_std": 80.77706909179688,
      "step": 311
    },
    {
      "KL/chosen_KL_mean": -135.0360107421875,
      "KL/mean": -169.1075439453125,
      "KL/rejected_KL_mean": -203.1790771484375,
      "KL/std": 86.574462890625,
      "epoch": 0.4581497797356828,
      "fcm_dpo/beta": 0.005831355229020119,
      "fcm_dpo/delta": 0.002313617616891861,
      "fcm_dpo/margin": 68.14306640625,
      "fcm_dpo/q_t": 0.40707600116729736,
      "grad_norm": 9.257375717163086,
      "learning_rate": 3.306636061080487e-07,
      "logits/chosen": -1.3446286916732788,
      "logits/rejected": -0.7982256412506104,
      "logps/chosen": -215.43670654296875,
      "logps/ref_chosen": -80.40069580078125,
      "logps/ref_rejected": -82.4471435546875,
      "logps/rejected": -285.626220703125,
      "loss": 1.0848,
      "margin_dpo/margin_mean": 68.14306640625,
      "margin_dpo/margin_std": 83.34016418457031,
      "step": 312
    },
    {
      "KL/chosen_KL_mean": -129.87913513183594,
      "KL/mean": -162.72952270507812,
      "KL/rejected_KL_mean": -195.57994079589844,
      "KL/std": 85.09794616699219,
      "epoch": 0.45961820851688695,
      "fcm_dpo/beta": 0.005879998207092285,
      "fcm_dpo/delta": 0.01442498154938221,
      "fcm_dpo/margin": 65.70079803466797,
      "fcm_dpo/q_t": 0.4107208847999573,
      "grad_norm": 9.782100677490234,
      "learning_rate": 3.2944785489547537e-07,
      "logits/chosen": -2.2331438064575195,
      "logits/rejected": -1.8172311782836914,
      "logps/chosen": -214.02102661132812,
      "logps/ref_chosen": -84.14190673828125,
      "logps/ref_rejected": -93.42684173583984,
      "logps/rejected": -289.00677490234375,
      "loss": 1.1084,
      "margin_dpo/margin_mean": 65.7008056640625,
      "margin_dpo/margin_std": 92.86199188232422,
      "step": 313
    },
    {
      "KL/chosen_KL_mean": -131.8540802001953,
      "KL/mean": -164.32131958007812,
      "KL/rejected_KL_mean": -196.78857421875,
      "KL/std": 85.54974365234375,
      "epoch": 0.461086637298091,
      "fcm_dpo/beta": 0.005839771591126919,
      "fcm_dpo/delta": 0.023175854235887527,
      "fcm_dpo/margin": 64.93448638916016,
      "fcm_dpo/q_t": 0.4131675362586975,
      "grad_norm": 10.565170288085938,
      "learning_rate": 3.2823001015803857e-07,
      "logits/chosen": -2.0716934204101562,
      "logits/rejected": -1.8579692840576172,
      "logps/chosen": -220.66380310058594,
      "logps/ref_chosen": -88.80972290039062,
      "logps/ref_rejected": -104.15892028808594,
      "logps/rejected": -300.947509765625,
      "loss": 1.1049,
      "margin_dpo/margin_mean": 64.93448638916016,
      "margin_dpo/margin_std": 87.54901123046875,
      "step": 314
    },
    {
      "KL/chosen_KL_mean": -127.70088195800781,
      "KL/mean": -160.9679412841797,
      "KL/rejected_KL_mean": -194.23501586914062,
      "KL/std": 89.96272277832031,
      "epoch": 0.46255506607929514,
      "fcm_dpo/beta": 0.005913122557103634,
      "fcm_dpo/delta": 0.0074251494370400906,
      "fcm_dpo/margin": 66.53411865234375,
      "fcm_dpo/q_t": 0.41073668003082275,
      "grad_norm": 9.670418739318848,
      "learning_rate": 3.270101039870797e-07,
      "logits/chosen": -1.2981984615325928,
      "logits/rejected": -1.1226776838302612,
      "logps/chosen": -208.4945831298828,
      "logps/ref_chosen": -80.793701171875,
      "logps/ref_rejected": -92.13711547851562,
      "logps/rejected": -286.37213134765625,
      "loss": 1.1205,
      "margin_dpo/margin_mean": 66.53411865234375,
      "margin_dpo/margin_std": 104.60867309570312,
      "step": 315
    },
    {
      "KL/chosen_KL_mean": -129.9150848388672,
      "KL/mean": -166.31155395507812,
      "KL/rejected_KL_mean": -202.70802307128906,
      "KL/std": 88.19239807128906,
      "epoch": 0.46402349486049926,
      "fcm_dpo/beta": 0.005875320639461279,
      "fcm_dpo/delta": -0.03173623979091644,
      "fcm_dpo/margin": 72.79293823242188,
      "fcm_dpo/q_t": 0.40155383944511414,
      "grad_norm": 13.702407836914062,
      "learning_rate": 3.2578816852826086e-07,
      "logits/chosen": -1.808872938156128,
      "logits/rejected": -1.8094680309295654,
      "logps/chosen": -214.09933471679688,
      "logps/ref_chosen": -84.18424987792969,
      "logps/ref_rejected": -105.6908187866211,
      "logps/rejected": -308.3988342285156,
      "loss": 1.0731,
      "margin_dpo/margin_mean": 72.79293823242188,
      "margin_dpo/margin_std": 93.36244201660156,
      "step": 316
    },
    {
      "KL/chosen_KL_mean": -134.52671813964844,
      "KL/mean": -179.03346252441406,
      "KL/rejected_KL_mean": -223.54017639160156,
      "KL/std": 97.07418060302734,
      "epoch": 0.4654919236417034,
      "fcm_dpo/beta": 0.005589790642261505,
      "fcm_dpo/delta": -0.11423610150814056,
      "fcm_dpo/margin": 89.01347351074219,
      "fcm_dpo/q_t": 0.38411399722099304,
      "grad_norm": 10.648103713989258,
      "learning_rate": 3.2456423598071783e-07,
      "logits/chosen": -2.04435396194458,
      "logits/rejected": -1.5773963928222656,
      "logps/chosen": -222.40020751953125,
      "logps/ref_chosen": -87.87348937988281,
      "logps/ref_rejected": -108.26519775390625,
      "logps/rejected": -331.80535888671875,
      "loss": 1.0294,
      "margin_dpo/margin_mean": 89.01347351074219,
      "margin_dpo/margin_std": 104.09390258789062,
      "step": 317
    },
    {
      "KL/chosen_KL_mean": -130.67420959472656,
      "KL/mean": -169.90048217773438,
      "KL/rejected_KL_mean": -209.12672424316406,
      "KL/std": 81.33235168457031,
      "epoch": 0.4669603524229075,
      "fcm_dpo/beta": 0.005364990793168545,
      "fcm_dpo/delta": -0.02404986135661602,
      "fcm_dpo/margin": 78.45250701904297,
      "fcm_dpo/q_t": 0.40114617347717285,
      "grad_norm": 9.91274356842041,
      "learning_rate": 3.233383385962115e-07,
      "logits/chosen": -2.442930221557617,
      "logits/rejected": -1.7546088695526123,
      "logps/chosen": -228.87974548339844,
      "logps/ref_chosen": -98.20553588867188,
      "logps/ref_rejected": -88.13629150390625,
      "logps/rejected": -297.26300048828125,
      "loss": 1.0645,
      "margin_dpo/margin_mean": 78.45250701904297,
      "margin_dpo/margin_std": 90.48660278320312,
      "step": 318
    },
    {
      "KL/chosen_KL_mean": -111.87690734863281,
      "KL/mean": -155.62767028808594,
      "KL/rejected_KL_mean": -199.37843322753906,
      "KL/std": 84.97116088867188,
      "epoch": 0.4684287812041116,
      "fcm_dpo/beta": 0.005186549387872219,
      "fcm_dpo/delta": -0.06285504251718521,
      "fcm_dpo/margin": 87.50152587890625,
      "fcm_dpo/q_t": 0.39274078607559204,
      "grad_norm": 9.458187103271484,
      "learning_rate": 3.2211050867827805e-07,
      "logits/chosen": -1.7007707357406616,
      "logits/rejected": -1.6355392932891846,
      "logps/chosen": -194.38027954101562,
      "logps/ref_chosen": -82.50337219238281,
      "logps/ref_rejected": -118.65068054199219,
      "logps/rejected": -318.02911376953125,
      "loss": 1.0408,
      "margin_dpo/margin_mean": 87.50152587890625,
      "margin_dpo/margin_std": 95.77153015136719,
      "step": 319
    },
    {
      "KL/chosen_KL_mean": -114.88970947265625,
      "KL/mean": -150.27049255371094,
      "KL/rejected_KL_mean": -185.65127563476562,
      "KL/std": 86.50686645507812,
      "epoch": 0.4698972099853157,
      "fcm_dpo/beta": 0.005218931008130312,
      "fcm_dpo/delta": 0.03454342111945152,
      "fcm_dpo/margin": 70.7615737915039,
      "fcm_dpo/q_t": 0.41401463747024536,
      "grad_norm": 8.241997718811035,
      "learning_rate": 3.208807785813777e-07,
      "logits/chosen": -2.0784120559692383,
      "logits/rejected": -1.892157793045044,
      "logps/chosen": -202.0269317626953,
      "logps/ref_chosen": -87.13722229003906,
      "logps/ref_rejected": -103.22412109375,
      "logps/rejected": -288.8753967285156,
      "loss": 1.0969,
      "margin_dpo/margin_mean": 70.7615737915039,
      "margin_dpo/margin_std": 84.33700561523438,
      "step": 320
    },
    {
      "KL/chosen_KL_mean": -128.28750610351562,
      "KL/mean": -169.54470825195312,
      "KL/rejected_KL_mean": -210.80191040039062,
      "KL/std": 88.56735229492188,
      "epoch": 0.4713656387665198,
      "fcm_dpo/beta": 0.005148157477378845,
      "fcm_dpo/delta": -0.029500113800168037,
      "fcm_dpo/margin": 82.514404296875,
      "fcm_dpo/q_t": 0.3987131118774414,
      "grad_norm": 8.36191177368164,
      "learning_rate": 3.1964918071004217e-07,
      "logits/chosen": -1.9422032833099365,
      "logits/rejected": -1.551262378692627,
      "logps/chosen": -221.3921661376953,
      "logps/ref_chosen": -93.10466766357422,
      "logps/ref_rejected": -97.4681396484375,
      "logps/rejected": -308.2700500488281,
      "loss": 1.0565,
      "margin_dpo/margin_mean": 82.514404296875,
      "margin_dpo/margin_std": 88.83772277832031,
      "step": 321
    },
    {
      "KL/chosen_KL_mean": -118.99695587158203,
      "KL/mean": -163.9141845703125,
      "KL/rejected_KL_mean": -208.8314208984375,
      "KL/std": 87.69303894042969,
      "epoch": 0.47283406754772395,
      "fcm_dpo/beta": 0.004954569973051548,
      "fcm_dpo/delta": -0.0558592714369297,
      "fcm_dpo/margin": 89.83447265625,
      "fcm_dpo/q_t": 0.3938998281955719,
      "grad_norm": 13.05055046081543,
      "learning_rate": 3.184157475180207e-07,
      "logits/chosen": -1.7690379619598389,
      "logits/rejected": -1.4130442142486572,
      "logps/chosen": -211.1898651123047,
      "logps/ref_chosen": -92.19291687011719,
      "logps/ref_rejected": -100.89321899414062,
      "logps/rejected": -309.7246398925781,
      "loss": 1.0451,
      "margin_dpo/margin_mean": 89.83448028564453,
      "margin_dpo/margin_std": 94.42408752441406,
      "step": 322
    },
    {
      "KL/chosen_KL_mean": -138.12551879882812,
      "KL/mean": -175.12896728515625,
      "KL/rejected_KL_mean": -212.1324005126953,
      "KL/std": 88.34961700439453,
      "epoch": 0.47430249632892807,
      "fcm_dpo/beta": 0.005048529244959354,
      "fcm_dpo/delta": 0.02968144416809082,
      "fcm_dpo/margin": 74.00686645507812,
      "fcm_dpo/q_t": 0.41369736194610596,
      "grad_norm": 10.7435884475708,
      "learning_rate": 3.171805115074251e-07,
      "logits/chosen": -2.0187454223632812,
      "logits/rejected": -1.6394858360290527,
      "logps/chosen": -218.3976287841797,
      "logps/ref_chosen": -80.27210235595703,
      "logps/ref_rejected": -80.3033447265625,
      "logps/rejected": -292.43572998046875,
      "loss": 1.0994,
      "margin_dpo/margin_mean": 74.00686645507812,
      "margin_dpo/margin_std": 92.57878112792969,
      "step": 323
    },
    {
      "KL/chosen_KL_mean": -136.56396484375,
      "KL/mean": -175.2288360595703,
      "KL/rejected_KL_mean": -213.89370727539062,
      "KL/std": 88.37493133544922,
      "epoch": 0.47577092511013214,
      "fcm_dpo/beta": 0.005127800162881613,
      "fcm_dpo/delta": 0.0037819761782884598,
      "fcm_dpo/margin": 77.32975006103516,
      "fcm_dpo/q_t": 0.4061412513256073,
      "grad_norm": 8.304902076721191,
      "learning_rate": 3.1594350522787295e-07,
      "logits/chosen": -1.5456180572509766,
      "logits/rejected": -0.8840495347976685,
      "logps/chosen": -232.29331970214844,
      "logps/ref_chosen": -95.72935485839844,
      "logps/ref_rejected": -94.16924285888672,
      "logps/rejected": -308.06292724609375,
      "loss": 1.0728,
      "margin_dpo/margin_mean": 77.32974243164062,
      "margin_dpo/margin_std": 83.725830078125,
      "step": 324
    },
    {
      "KL/chosen_KL_mean": -125.18402099609375,
      "KL/mean": -167.2108154296875,
      "KL/rejected_KL_mean": -209.23760986328125,
      "KL/std": 84.26690673828125,
      "epoch": 0.47723935389133626,
      "fcm_dpo/beta": 0.005056754685938358,
      "fcm_dpo/delta": -0.028670165687799454,
      "fcm_dpo/margin": 84.05357360839844,
      "fcm_dpo/q_t": 0.39906448125839233,
      "grad_norm": 7.499780178070068,
      "learning_rate": 3.147047612756302e-07,
      "logits/chosen": -1.6622391939163208,
      "logits/rejected": -0.9101150035858154,
      "logps/chosen": -223.89089965820312,
      "logps/ref_chosen": -98.70686340332031,
      "logps/ref_rejected": -76.40809631347656,
      "logps/rejected": -285.64569091796875,
      "loss": 1.0532,
      "margin_dpo/margin_mean": 84.05357360839844,
      "margin_dpo/margin_std": 87.6025619506836,
      "step": 325
    },
    {
      "KL/chosen_KL_mean": -140.07901000976562,
      "KL/mean": -177.63111877441406,
      "KL/rejected_KL_mean": -215.1832275390625,
      "KL/std": 83.72927856445312,
      "epoch": 0.4787077826725404,
      "fcm_dpo/beta": 0.005064009688794613,
      "fcm_dpo/delta": 0.022222068160772324,
      "fcm_dpo/margin": 75.10420989990234,
      "fcm_dpo/q_t": 0.41064396500587463,
      "grad_norm": 7.244331359863281,
      "learning_rate": 3.134643122927519e-07,
      "logits/chosen": -2.392141103744507,
      "logits/rejected": -1.7113181352615356,
      "logps/chosen": -244.74032592773438,
      "logps/ref_chosen": -104.66130828857422,
      "logps/ref_rejected": -86.82537841796875,
      "logps/rejected": -302.00860595703125,
      "loss": 1.0925,
      "margin_dpo/margin_mean": 75.10420989990234,
      "margin_dpo/margin_std": 90.8126220703125,
      "step": 326
    },
    {
      "KL/chosen_KL_mean": -134.6155548095703,
      "KL/mean": -170.80613708496094,
      "KL/rejected_KL_mean": -206.99673461914062,
      "KL/std": 76.85627746582031,
      "epoch": 0.4801762114537445,
      "fcm_dpo/beta": 0.005125164054334164,
      "fcm_dpo/delta": 0.03253272920846939,
      "fcm_dpo/margin": 72.38117218017578,
      "fcm_dpo/q_t": 0.41401445865631104,
      "grad_norm": 8.244698524475098,
      "learning_rate": 3.1222219096622264e-07,
      "logits/chosen": -1.67340886592865,
      "logits/rejected": -1.078773021697998,
      "logps/chosen": -242.0557861328125,
      "logps/ref_chosen": -107.44023895263672,
      "logps/ref_rejected": -117.05323028564453,
      "logps/rejected": -324.0499572753906,
      "loss": 1.0969,
      "margin_dpo/margin_mean": 72.38117218017578,
      "margin_dpo/margin_std": 87.19709014892578,
      "step": 327
    },
    {
      "KL/chosen_KL_mean": -130.78688049316406,
      "KL/mean": -178.60931396484375,
      "KL/rejected_KL_mean": -226.43173217773438,
      "KL/std": 91.47960662841797,
      "epoch": 0.48164464023494863,
      "fcm_dpo/beta": 0.00499006500467658,
      "fcm_dpo/delta": -0.09015590697526932,
      "fcm_dpo/margin": 95.64486694335938,
      "fcm_dpo/q_t": 0.3855307102203369,
      "grad_norm": 9.748368263244629,
      "learning_rate": 3.1097843002709427e-07,
      "logits/chosen": -1.7337027788162231,
      "logits/rejected": -1.4375680685043335,
      "logps/chosen": -217.3975830078125,
      "logps/ref_chosen": -86.6106948852539,
      "logps/ref_rejected": -97.40188598632812,
      "logps/rejected": -323.8336181640625,
      "loss": 1.0113,
      "margin_dpo/margin_mean": 95.64486694335938,
      "margin_dpo/margin_std": 88.78097534179688,
      "step": 328
    },
    {
      "KL/chosen_KL_mean": -137.16033935546875,
      "KL/mean": -174.50672912597656,
      "KL/rejected_KL_mean": -211.85311889648438,
      "KL/std": 88.2598876953125,
      "epoch": 0.4831130690161527,
      "fcm_dpo/beta": 0.00493753794580698,
      "fcm_dpo/delta": 0.03461095690727234,
      "fcm_dpo/margin": 74.6927719116211,
      "fcm_dpo/q_t": 0.4146957993507385,
      "grad_norm": 9.653441429138184,
      "learning_rate": 3.0973306224962437e-07,
      "logits/chosen": -1.6301474571228027,
      "logits/rejected": -1.1383018493652344,
      "logps/chosen": -237.77340698242188,
      "logps/ref_chosen": -100.61308288574219,
      "logps/ref_rejected": -104.95370483398438,
      "logps/rejected": -316.80682373046875,
      "loss": 1.1052,
      "margin_dpo/margin_mean": 74.6927719116211,
      "margin_dpo/margin_std": 95.19576263427734,
      "step": 329
    },
    {
      "KL/chosen_KL_mean": -116.28146362304688,
      "KL/mean": -156.14154052734375,
      "KL/rejected_KL_mean": -196.0016326904297,
      "KL/std": 85.76058197021484,
      "epoch": 0.4845814977973568,
      "fcm_dpo/beta": 0.005006903782486916,
      "fcm_dpo/delta": 0.0009117983281612396,
      "fcm_dpo/margin": 79.72016906738281,
      "fcm_dpo/q_t": 0.4044761657714844,
      "grad_norm": 8.044327735900879,
      "learning_rate": 3.084861204504122e-07,
      "logits/chosen": -1.8007159233093262,
      "logits/rejected": -1.7222074270248413,
      "logps/chosen": -193.23989868164062,
      "logps/ref_chosen": -76.95843505859375,
      "logps/ref_rejected": -92.81494140625,
      "logps/rejected": -288.81658935546875,
      "loss": 1.0636,
      "margin_dpo/margin_mean": 79.72016906738281,
      "margin_dpo/margin_std": 76.9364242553711,
      "step": 330
    },
    {
      "KL/chosen_KL_mean": -124.7020263671875,
      "KL/mean": -169.58425903320312,
      "KL/rejected_KL_mean": -214.4664764404297,
      "KL/std": 84.02842712402344,
      "epoch": 0.48604992657856094,
      "fcm_dpo/beta": 0.004954406060278416,
      "fcm_dpo/delta": -0.05174029618501663,
      "fcm_dpo/margin": 89.76446533203125,
      "fcm_dpo/q_t": 0.39385104179382324,
      "grad_norm": 9.423430442810059,
      "learning_rate": 3.072376374875335e-07,
      "logits/chosen": -1.8290188312530518,
      "logits/rejected": -1.5233136415481567,
      "logps/chosen": -207.76144409179688,
      "logps/ref_chosen": -83.05941772460938,
      "logps/ref_rejected": -94.13113403320312,
      "logps/rejected": -308.59759521484375,
      "loss": 1.0338,
      "margin_dpo/margin_mean": 89.76446533203125,
      "margin_dpo/margin_std": 85.22640991210938,
      "step": 331
    },
    {
      "KL/chosen_KL_mean": -133.5218505859375,
      "KL/mean": -164.142333984375,
      "KL/rejected_KL_mean": -194.7628173828125,
      "KL/std": 92.26820373535156,
      "epoch": 0.48751835535976507,
      "fcm_dpo/beta": 0.005103833973407745,
      "fcm_dpo/delta": 0.09585808217525482,
      "fcm_dpo/margin": 61.24098205566406,
      "fcm_dpo/q_t": 0.4279337227344513,
      "grad_norm": 7.890995979309082,
      "learning_rate": 3.059876462596758e-07,
      "logits/chosen": -2.0985238552093506,
      "logits/rejected": -1.6672172546386719,
      "logps/chosen": -215.45274353027344,
      "logps/ref_chosen": -81.93089294433594,
      "logps/ref_rejected": -82.84590148925781,
      "logps/rejected": -277.6087341308594,
      "loss": 1.1494,
      "margin_dpo/margin_mean": 61.24098205566406,
      "margin_dpo/margin_std": 90.88442993164062,
      "step": 332
    },
    {
      "KL/chosen_KL_mean": -153.3829345703125,
      "KL/mean": -196.7569580078125,
      "KL/rejected_KL_mean": -240.13099670410156,
      "KL/std": 97.24162292480469,
      "epoch": 0.4889867841409692,
      "fcm_dpo/beta": 0.004981888458132744,
      "fcm_dpo/delta": -0.03970459848642349,
      "fcm_dpo/margin": 86.74806213378906,
      "fcm_dpo/q_t": 0.39884936809539795,
      "grad_norm": 7.898747444152832,
      "learning_rate": 3.0473617970527015e-07,
      "logits/chosen": -1.6321008205413818,
      "logits/rejected": -1.294562816619873,
      "logps/chosen": -241.91551208496094,
      "logps/ref_chosen": -88.53257751464844,
      "logps/ref_rejected": -102.46601867675781,
      "logps/rejected": -342.5970153808594,
      "loss": 1.0661,
      "margin_dpo/margin_mean": 86.74806213378906,
      "margin_dpo/margin_std": 104.07328033447266,
      "step": 333
    },
    {
      "KL/chosen_KL_mean": -139.67495727539062,
      "KL/mean": -178.35337829589844,
      "KL/rejected_KL_mean": -217.0318145751953,
      "KL/std": 91.70336151123047,
      "epoch": 0.49045521292217326,
      "fcm_dpo/beta": 0.004985345993191004,
      "fcm_dpo/delta": 0.014539476484060287,
      "fcm_dpo/margin": 77.35685729980469,
      "fcm_dpo/q_t": 0.4093141555786133,
      "grad_norm": 8.955771446228027,
      "learning_rate": 3.034832708016243e-07,
      "logits/chosen": -1.3317067623138428,
      "logits/rejected": -1.0311429500579834,
      "logps/chosen": -237.4327392578125,
      "logps/ref_chosen": -97.75778198242188,
      "logps/ref_rejected": -100.0058364868164,
      "logps/rejected": -317.03765869140625,
      "loss": 1.1034,
      "margin_dpo/margin_mean": 77.35685729980469,
      "margin_dpo/margin_std": 102.12141418457031,
      "step": 334
    },
    {
      "KL/chosen_KL_mean": -139.32772827148438,
      "KL/mean": -172.41163635253906,
      "KL/rejected_KL_mean": -205.49554443359375,
      "KL/std": 90.82597351074219,
      "epoch": 0.4919236417033774,
      "fcm_dpo/beta": 0.005224327556788921,
      "fcm_dpo/delta": 0.0604991614818573,
      "fcm_dpo/margin": 66.16781616210938,
      "fcm_dpo/q_t": 0.42059406638145447,
      "grad_norm": 12.870566368103027,
      "learning_rate": 3.022289525640531e-07,
      "logits/chosen": -2.116569995880127,
      "logits/rejected": -1.6753369569778442,
      "logps/chosen": -235.70376586914062,
      "logps/ref_chosen": -96.37603759765625,
      "logps/ref_rejected": -94.92401123046875,
      "logps/rejected": -300.4195556640625,
      "loss": 1.1353,
      "margin_dpo/margin_mean": 66.1678237915039,
      "margin_dpo/margin_std": 100.04443359375,
      "step": 335
    },
    {
      "KL/chosen_KL_mean": -141.91519165039062,
      "KL/mean": -184.43634033203125,
      "KL/rejected_KL_mean": -226.95748901367188,
      "KL/std": 93.50979614257812,
      "epoch": 0.4933920704845815,
      "fcm_dpo/beta": 0.005189797841012478,
      "fcm_dpo/delta": -0.04763997718691826,
      "fcm_dpo/margin": 85.04228210449219,
      "fcm_dpo/q_t": 0.39659538865089417,
      "grad_norm": 10.544360160827637,
      "learning_rate": 3.009732580450086e-07,
      "logits/chosen": -1.5550687313079834,
      "logits/rejected": -1.4038405418395996,
      "logps/chosen": -230.4447021484375,
      "logps/ref_chosen": -88.52952575683594,
      "logps/ref_rejected": -109.5557861328125,
      "logps/rejected": -336.5132751464844,
      "loss": 1.0535,
      "margin_dpo/margin_mean": 85.04228210449219,
      "margin_dpo/margin_std": 98.87345886230469,
      "step": 336
    },
    {
      "KL/chosen_KL_mean": -140.6769561767578,
      "KL/mean": -181.1038360595703,
      "KL/rejected_KL_mean": -221.53070068359375,
      "KL/std": 94.43257904052734,
      "epoch": 0.4948604992657856,
      "fcm_dpo/beta": 0.0051058391109108925,
      "fcm_dpo/delta": -0.014694934710860252,
      "fcm_dpo/margin": 80.85374450683594,
      "fcm_dpo/q_t": 0.4050077795982361,
      "grad_norm": 8.686392784118652,
      "learning_rate": 2.9971622033320914e-07,
      "logits/chosen": -2.475356101989746,
      "logits/rejected": -2.1970982551574707,
      "logps/chosen": -237.98934936523438,
      "logps/ref_chosen": -97.3123779296875,
      "logps/ref_rejected": -110.13134765625,
      "logps/rejected": -331.66204833984375,
      "loss": 1.0911,
      "margin_dpo/margin_mean": 80.85375213623047,
      "margin_dpo/margin_std": 111.23469543457031,
      "step": 337
    },
    {
      "KL/chosen_KL_mean": -145.9003448486328,
      "KL/mean": -182.98727416992188,
      "KL/rejected_KL_mean": -220.07418823242188,
      "KL/std": 95.34557342529297,
      "epoch": 0.49632892804698975,
      "fcm_dpo/beta": 0.005116141401231289,
      "fcm_dpo/delta": 0.02317037060856819,
      "fcm_dpo/margin": 74.17385864257812,
      "fcm_dpo/q_t": 0.4110550582408905,
      "grad_norm": 8.950212478637695,
      "learning_rate": 2.984578725527675e-07,
      "logits/chosen": -1.6864533424377441,
      "logits/rejected": -1.5310287475585938,
      "logps/chosen": -229.53045654296875,
      "logps/ref_chosen": -83.63011169433594,
      "logps/ref_rejected": -93.69235229492188,
      "logps/rejected": -313.76654052734375,
      "loss": 1.0977,
      "margin_dpo/margin_mean": 74.17385864257812,
      "margin_dpo/margin_std": 93.90530395507812,
      "step": 338
    },
    {
      "KL/chosen_KL_mean": -155.6811065673828,
      "KL/mean": -195.48480224609375,
      "KL/rejected_KL_mean": -235.28848266601562,
      "KL/std": 94.69602966308594,
      "epoch": 0.4977973568281938,
      "fcm_dpo/beta": 0.005210503935813904,
      "fcm_dpo/delta": -0.019667256623506546,
      "fcm_dpo/margin": 79.60739135742188,
      "fcm_dpo/q_t": 0.40322262048721313,
      "grad_norm": 8.093759536743164,
      "learning_rate": 2.9719824786231796e-07,
      "logits/chosen": -2.8836655616760254,
      "logits/rejected": -2.4521937370300293,
      "logps/chosen": -255.53292846679688,
      "logps/ref_chosen": -99.85182189941406,
      "logps/ref_rejected": -109.0743637084961,
      "logps/rejected": -344.36285400390625,
      "loss": 1.0909,
      "margin_dpo/margin_mean": 79.60739135742188,
      "margin_dpo/margin_std": 105.40576934814453,
      "step": 339
    },
    {
      "KL/chosen_KL_mean": -166.615478515625,
      "KL/mean": -201.19976806640625,
      "KL/rejected_KL_mean": -235.78408813476562,
      "KL/std": 87.04264068603516,
      "epoch": 0.49926578560939794,
      "fcm_dpo/beta": 0.00519480649381876,
      "fcm_dpo/delta": 0.04551742970943451,
      "fcm_dpo/margin": 69.16860961914062,
      "fcm_dpo/q_t": 0.41657695174217224,
      "grad_norm": 9.71963119506836,
      "learning_rate": 2.959373794541426e-07,
      "logits/chosen": -1.2762441635131836,
      "logits/rejected": -0.7800500392913818,
      "logps/chosen": -251.58944702148438,
      "logps/ref_chosen": -84.97396850585938,
      "logps/ref_rejected": -78.80033874511719,
      "logps/rejected": -314.58441162109375,
      "loss": 1.1165,
      "margin_dpo/margin_mean": 69.16860961914062,
      "margin_dpo/margin_std": 95.58157348632812,
      "step": 340
    },
    {
      "KL/chosen_KL_mean": -173.88502502441406,
      "KL/mean": -207.6162109375,
      "KL/rejected_KL_mean": -241.34738159179688,
      "KL/std": 91.3199234008789,
      "epoch": 0.5007342143906021,
      "fcm_dpo/beta": 0.005300821736454964,
      "fcm_dpo/delta": 0.04749292880296707,
      "fcm_dpo/margin": 67.46232604980469,
      "fcm_dpo/q_t": 0.41867220401763916,
      "grad_norm": 9.459185600280762,
      "learning_rate": 2.946753005532965e-07,
      "logits/chosen": -1.35988187789917,
      "logits/rejected": -1.1077429056167603,
      "logps/chosen": -259.1490173339844,
      "logps/ref_chosen": -85.26399230957031,
      "logps/ref_rejected": -106.08595275878906,
      "logps/rejected": -347.433349609375,
      "loss": 1.1354,
      "margin_dpo/margin_mean": 67.46232604980469,
      "margin_dpo/margin_std": 105.12226104736328,
      "step": 341
    },
    {
      "KL/chosen_KL_mean": -139.9071044921875,
      "KL/mean": -188.6729736328125,
      "KL/rejected_KL_mean": -237.43885803222656,
      "KL/std": 91.36782836914062,
      "epoch": 0.5022026431718062,
      "fcm_dpo/beta": 0.005023137666285038,
      "fcm_dpo/delta": -0.11031514406204224,
      "fcm_dpo/margin": 97.53175354003906,
      "fcm_dpo/q_t": 0.3810012936592102,
      "grad_norm": 7.5685625076293945,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -1.4242944717407227,
      "logits/rejected": -0.6633044481277466,
      "logps/chosen": -240.94570922851562,
      "logps/ref_chosen": -101.03860473632812,
      "logps/ref_rejected": -98.89851379394531,
      "logps/rejected": -336.3373718261719,
      "loss": 1.0171,
      "margin_dpo/margin_mean": 97.53175354003906,
      "margin_dpo/margin_std": 97.68126678466797,
      "step": 342
    },
    {
      "KL/chosen_KL_mean": -152.2353973388672,
      "KL/mean": -193.00601196289062,
      "KL/rejected_KL_mean": -233.77662658691406,
      "KL/std": 95.3868637084961,
      "epoch": 0.5036710719530103,
      "fcm_dpo/beta": 0.005086997523903847,
      "fcm_dpo/delta": -0.01915212720632553,
      "fcm_dpo/margin": 81.54122161865234,
      "fcm_dpo/q_t": 0.4028986692428589,
      "grad_norm": 9.840593338012695,
      "learning_rate": 2.9214764433242476e-07,
      "logits/chosen": -2.028923749923706,
      "logits/rejected": -1.9199120998382568,
      "logps/chosen": -235.98275756835938,
      "logps/ref_chosen": -83.74736022949219,
      "logps/ref_rejected": -116.57908630371094,
      "logps/rejected": -350.355712890625,
      "loss": 1.0805,
      "margin_dpo/margin_mean": 81.54122161865234,
      "margin_dpo/margin_std": 100.23389434814453,
      "step": 343
    },
    {
      "KL/chosen_KL_mean": -151.87216186523438,
      "KL/mean": -195.52398681640625,
      "KL/rejected_KL_mean": -239.17581176757812,
      "KL/std": 101.1781997680664,
      "epoch": 0.5051395007342144,
      "fcm_dpo/beta": 0.004966346547007561,
      "fcm_dpo/delta": -0.03973696753382683,
      "fcm_dpo/margin": 87.30367279052734,
      "fcm_dpo/q_t": 0.3990062177181244,
      "grad_norm": 9.641231536865234,
      "learning_rate": 2.9088213361849126e-07,
      "logits/chosen": -1.5786139965057373,
      "logits/rejected": -1.3297735452651978,
      "logps/chosen": -230.12612915039062,
      "logps/ref_chosen": -78.25396728515625,
      "logps/ref_rejected": -95.42870330810547,
      "logps/rejected": -334.6045227050781,
      "loss": 1.0686,
      "margin_dpo/margin_mean": 87.30368041992188,
      "margin_dpo/margin_std": 107.09952545166016,
      "step": 344
    },
    {
      "KL/chosen_KL_mean": -157.1041259765625,
      "KL/mean": -202.9161376953125,
      "KL/rejected_KL_mean": -248.7281494140625,
      "KL/std": 101.22309875488281,
      "epoch": 0.5066079295154186,
      "fcm_dpo/beta": 0.0047477614134550095,
      "fcm_dpo/delta": -0.04063037782907486,
      "fcm_dpo/margin": 91.6240234375,
      "fcm_dpo/q_t": 0.3980073928833008,
      "grad_norm": 9.357902526855469,
      "learning_rate": 2.896155456223163e-07,
      "logits/chosen": -1.462306261062622,
      "logits/rejected": -1.1634180545806885,
      "logps/chosen": -255.18362426757812,
      "logps/ref_chosen": -98.0794906616211,
      "logps/ref_rejected": -104.03477478027344,
      "logps/rejected": -352.762939453125,
      "loss": 1.0559,
      "margin_dpo/margin_mean": 91.6240234375,
      "margin_dpo/margin_std": 104.84120178222656,
      "step": 345
    },
    {
      "KL/chosen_KL_mean": -174.54006958007812,
      "KL/mean": -210.2685546875,
      "KL/rejected_KL_mean": -245.99703979492188,
      "KL/std": 94.14483642578125,
      "epoch": 0.5080763582966226,
      "fcm_dpo/beta": 0.004862795118242502,
      "fcm_dpo/delta": 0.058651164174079895,
      "fcm_dpo/margin": 71.45698547363281,
      "fcm_dpo/q_t": 0.4206523597240448,
      "grad_norm": 7.706295013427734,
      "learning_rate": 2.883479137196714e-07,
      "logits/chosen": -1.6456987857818604,
      "logits/rejected": -1.3242378234863281,
      "logps/chosen": -255.2235565185547,
      "logps/ref_chosen": -80.68348693847656,
      "logps/ref_rejected": -85.89260864257812,
      "logps/rejected": -331.8896484375,
      "loss": 1.1334,
      "margin_dpo/margin_mean": 71.45698547363281,
      "margin_dpo/margin_std": 107.6796875,
      "step": 346
    },
    {
      "KL/chosen_KL_mean": -164.8995361328125,
      "KL/mean": -198.92141723632812,
      "KL/rejected_KL_mean": -232.9432830810547,
      "KL/std": 93.32675170898438,
      "epoch": 0.5095447870778267,
      "fcm_dpo/beta": 0.005095065105706453,
      "fcm_dpo/delta": 0.0579054057598114,
      "fcm_dpo/margin": 68.04373168945312,
      "fcm_dpo/q_t": 0.42063087224960327,
      "grad_norm": 8.546566009521484,
      "learning_rate": 2.8707927131383614e-07,
      "logits/chosen": -1.700346827507019,
      "logits/rejected": -1.369409203529358,
      "logps/chosen": -258.57891845703125,
      "logps/ref_chosen": -93.67938232421875,
      "logps/ref_rejected": -97.54251098632812,
      "logps/rejected": -330.48577880859375,
      "loss": 1.1332,
      "margin_dpo/margin_mean": 68.04373168945312,
      "margin_dpo/margin_std": 101.03919219970703,
      "step": 347
    },
    {
      "KL/chosen_KL_mean": -142.93927001953125,
      "KL/mean": -181.62413024902344,
      "KL/rejected_KL_mean": -220.3090362548828,
      "KL/std": 87.82957458496094,
      "epoch": 0.5110132158590308,
      "fcm_dpo/beta": 0.004983104299753904,
      "fcm_dpo/delta": 0.012012869119644165,
      "fcm_dpo/margin": 77.3697738647461,
      "fcm_dpo/q_t": 0.4088042676448822,
      "grad_norm": 8.113774299621582,
      "learning_rate": 2.858096518347179e-07,
      "logits/chosen": -2.0552661418914795,
      "logits/rejected": -1.7874932289123535,
      "logps/chosen": -233.19149780273438,
      "logps/ref_chosen": -90.25224304199219,
      "logps/ref_rejected": -95.77027893066406,
      "logps/rejected": -316.0793151855469,
      "loss": 1.0939,
      "margin_dpo/margin_mean": 77.36976623535156,
      "margin_dpo/margin_std": 89.05610656738281,
      "step": 348
    },
    {
      "KL/chosen_KL_mean": -155.17347717285156,
      "KL/mean": -191.27239990234375,
      "KL/rejected_KL_mean": -227.37130737304688,
      "KL/std": 96.73709106445312,
      "epoch": 0.5124816446402349,
      "fcm_dpo/beta": 0.0052385032176971436,
      "fcm_dpo/delta": 0.023156480863690376,
      "fcm_dpo/margin": 72.19783020019531,
      "fcm_dpo/q_t": 0.4132113754749298,
      "grad_norm": 9.926619529724121,
      "learning_rate": 2.845390887379706e-07,
      "logits/chosen": -1.9355063438415527,
      "logits/rejected": -1.7069427967071533,
      "logps/chosen": -247.2315673828125,
      "logps/ref_chosen": -92.05809020996094,
      "logps/ref_rejected": -103.96527099609375,
      "logps/rejected": -331.3365783691406,
      "loss": 1.1113,
      "margin_dpo/margin_mean": 72.19783020019531,
      "margin_dpo/margin_std": 100.7230453491211,
      "step": 349
    },
    {
      "KL/chosen_KL_mean": -163.97439575195312,
      "KL/mean": -194.77862548828125,
      "KL/rejected_KL_mean": -225.58285522460938,
      "KL/std": 88.69832611083984,
      "epoch": 0.5139500734214391,
      "fcm_dpo/beta": 0.005359445698559284,
      "fcm_dpo/delta": 0.07764375954866409,
      "fcm_dpo/margin": 61.608455657958984,
      "fcm_dpo/q_t": 0.4247250556945801,
      "grad_norm": 11.116376876831055,
      "learning_rate": 2.8326761550411346e-07,
      "logits/chosen": -1.60597825050354,
      "logits/rejected": -1.3707542419433594,
      "logps/chosen": -251.02859497070312,
      "logps/ref_chosen": -87.05419921875,
      "logps/ref_rejected": -95.72193145751953,
      "logps/rejected": -321.3047790527344,
      "loss": 1.1481,
      "margin_dpo/margin_mean": 61.60845947265625,
      "margin_dpo/margin_std": 96.5386962890625,
      "step": 350
    },
    {
      "KL/chosen_KL_mean": -164.55801391601562,
      "KL/mean": -207.79859924316406,
      "KL/rejected_KL_mean": -251.0392303466797,
      "KL/std": 99.10594177246094,
      "epoch": 0.5154185022026432,
      "fcm_dpo/beta": 0.005304581485688686,
      "fcm_dpo/delta": -0.0679606944322586,
      "fcm_dpo/margin": 86.481201171875,
      "fcm_dpo/q_t": 0.3925182819366455,
      "grad_norm": 8.592435836791992,
      "learning_rate": 2.819952656376487e-07,
      "logits/chosen": -1.5428069829940796,
      "logits/rejected": -1.2364004850387573,
      "logps/chosen": -253.72705078125,
      "logps/ref_chosen": -89.16903686523438,
      "logps/ref_rejected": -108.29801940917969,
      "logps/rejected": -359.3372497558594,
      "loss": 1.0534,
      "margin_dpo/margin_mean": 86.481201171875,
      "margin_dpo/margin_std": 105.57600402832031,
      "step": 351
    },
    {
      "KL/chosen_KL_mean": -151.49264526367188,
      "KL/mean": -192.4200897216797,
      "KL/rejected_KL_mean": -233.34756469726562,
      "KL/std": 97.57020568847656,
      "epoch": 0.5168869309838473,
      "fcm_dpo/beta": 0.005156824365258217,
      "fcm_dpo/delta": -0.02536623179912567,
      "fcm_dpo/margin": 81.85490417480469,
      "fcm_dpo/q_t": 0.40344762802124023,
      "grad_norm": 8.403756141662598,
      "learning_rate": 2.8072207266617854e-07,
      "logits/chosen": -1.7808645963668823,
      "logits/rejected": -1.221010684967041,
      "logps/chosen": -248.3041229248047,
      "logps/ref_chosen": -96.81147766113281,
      "logps/ref_rejected": -84.40483093261719,
      "logps/rejected": -317.75238037109375,
      "loss": 1.0788,
      "margin_dpo/margin_mean": 81.85491180419922,
      "margin_dpo/margin_std": 105.71539306640625,
      "step": 352
    },
    {
      "KL/chosen_KL_mean": -173.2101593017578,
      "KL/mean": -199.38922119140625,
      "KL/rejected_KL_mean": -225.56826782226562,
      "KL/std": 86.2092514038086,
      "epoch": 0.5183553597650514,
      "fcm_dpo/beta": 0.005235383752733469,
      "fcm_dpo/delta": 0.029666248708963394,
      "fcm_dpo/margin": 52.35810089111328,
      "fcm_dpo/q_t": 0.43592923879623413,
      "grad_norm": 10.03467082977295,
      "learning_rate": 2.794480701395219e-07,
      "logits/chosen": -2.093709707260132,
      "logits/rejected": -1.703459620475769,
      "logps/chosen": -258.5726318359375,
      "logps/ref_chosen": -85.36248779296875,
      "logps/ref_rejected": -87.36732482910156,
      "logps/rejected": -312.93560791015625,
      "loss": 1.1958,
      "margin_dpo/margin_mean": 52.35810089111328,
      "margin_dpo/margin_std": 97.62669372558594,
      "step": 353
    },
    {
      "KL/chosen_KL_mean": -152.18714904785156,
      "KL/mean": -193.5396728515625,
      "KL/rejected_KL_mean": -234.8922119140625,
      "KL/std": 87.56482696533203,
      "epoch": 0.5198237885462555,
      "fcm_dpo/beta": 0.005189601331949234,
      "fcm_dpo/delta": -0.03357437998056412,
      "fcm_dpo/margin": 82.7050552368164,
      "fcm_dpo/q_t": 0.3983612358570099,
      "grad_norm": 7.9278645515441895,
      "learning_rate": 2.781732916288303e-07,
      "logits/chosen": -1.8331456184387207,
      "logits/rejected": -1.358760952949524,
      "logps/chosen": -247.98831176757812,
      "logps/ref_chosen": -95.8011703491211,
      "logps/ref_rejected": -94.86614227294922,
      "logps/rejected": -329.75836181640625,
      "loss": 1.0455,
      "margin_dpo/margin_mean": 82.7050552368164,
      "margin_dpo/margin_std": 82.12599182128906,
      "step": 354
    },
    {
      "KL/chosen_KL_mean": -150.40118408203125,
      "KL/mean": -188.6392364501953,
      "KL/rejected_KL_mean": -226.8773193359375,
      "KL/std": 89.09295654296875,
      "epoch": 0.5212922173274597,
      "fcm_dpo/beta": 0.005103080999106169,
      "fcm_dpo/delta": 0.010406162589788437,
      "fcm_dpo/margin": 76.47613525390625,
      "fcm_dpo/q_t": 0.4086816608905792,
      "grad_norm": 10.251924514770508,
      "learning_rate": 2.7689777072570284e-07,
      "logits/chosen": -2.472856044769287,
      "logits/rejected": -1.9909342527389526,
      "logps/chosen": -236.58673095703125,
      "logps/ref_chosen": -86.185546875,
      "logps/ref_rejected": -88.4615707397461,
      "logps/rejected": -315.3388671875,
      "loss": 1.0917,
      "margin_dpo/margin_mean": 76.47613525390625,
      "margin_dpo/margin_std": 94.29251098632812,
      "step": 355
    },
    {
      "KL/chosen_KL_mean": -177.50701904296875,
      "KL/mean": -206.95785522460938,
      "KL/rejected_KL_mean": -236.40869140625,
      "KL/std": 86.53642272949219,
      "epoch": 0.5227606461086637,
      "fcm_dpo/beta": 0.005416443571448326,
      "fcm_dpo/delta": 0.08819755911827087,
      "fcm_dpo/margin": 58.901649475097656,
      "fcm_dpo/q_t": 0.42743033170700073,
      "grad_norm": 8.92505168914795,
      "learning_rate": 2.7562154104130176e-07,
      "logits/chosen": -1.7475285530090332,
      "logits/rejected": -1.2726366519927979,
      "logps/chosen": -268.32513427734375,
      "logps/ref_chosen": -90.818115234375,
      "logps/ref_rejected": -82.1273193359375,
      "logps/rejected": -318.5360107421875,
      "loss": 1.1615,
      "margin_dpo/margin_mean": 58.90165710449219,
      "margin_dpo/margin_std": 99.18927001953125,
      "step": 356
    },
    {
      "KL/chosen_KL_mean": -149.11685180664062,
      "KL/mean": -195.63888549804688,
      "KL/rejected_KL_mean": -242.16094970703125,
      "KL/std": 95.26244354248047,
      "epoch": 0.5242290748898678,
      "fcm_dpo/beta": 0.005114290397614241,
      "fcm_dpo/delta": -0.09376128017902374,
      "fcm_dpo/margin": 93.04409790039062,
      "fcm_dpo/q_t": 0.38555341958999634,
      "grad_norm": 8.263771057128906,
      "learning_rate": 2.7434463620546594e-07,
      "logits/chosen": -1.7292366027832031,
      "logits/rejected": -1.3480415344238281,
      "logps/chosen": -236.62159729003906,
      "logps/ref_chosen": -87.5047607421875,
      "logps/ref_rejected": -92.24937438964844,
      "logps/rejected": -334.4103088378906,
      "loss": 1.018,
      "margin_dpo/margin_mean": 93.0440902709961,
      "margin_dpo/margin_std": 88.94567108154297,
      "step": 357
    },
    {
      "KL/chosen_KL_mean": -159.5251007080078,
      "KL/mean": -194.75816345214844,
      "KL/rejected_KL_mean": -229.9912109375,
      "KL/std": 88.70181274414062,
      "epoch": 0.5256975036710719,
      "fcm_dpo/beta": 0.005223399959504604,
      "fcm_dpo/delta": 0.03591739758849144,
      "fcm_dpo/margin": 70.46611785888672,
      "fcm_dpo/q_t": 0.41615432500839233,
      "grad_norm": 8.52690601348877,
      "learning_rate": 2.730670898658255e-07,
      "logits/chosen": -1.6409136056900024,
      "logits/rejected": -1.1719400882720947,
      "logps/chosen": -254.58056640625,
      "logps/ref_chosen": -95.05546569824219,
      "logps/ref_rejected": -96.47611999511719,
      "logps/rejected": -326.46734619140625,
      "loss": 1.108,
      "margin_dpo/margin_mean": 70.46611022949219,
      "margin_dpo/margin_std": 94.36090087890625,
      "step": 358
    },
    {
      "KL/chosen_KL_mean": -179.85806274414062,
      "KL/mean": -211.88221740722656,
      "KL/rejected_KL_mean": -243.9063720703125,
      "KL/std": 82.75933837890625,
      "epoch": 0.527165932452276,
      "fcm_dpo/beta": 0.005327449180185795,
      "fcm_dpo/delta": 0.06474639475345612,
      "fcm_dpo/margin": 64.04832458496094,
      "fcm_dpo/q_t": 0.4224433898925781,
      "grad_norm": 9.924165725708008,
      "learning_rate": 2.717889356869146e-07,
      "logits/chosen": -1.5319398641586304,
      "logits/rejected": -0.9600592851638794,
      "logps/chosen": -267.23138427734375,
      "logps/ref_chosen": -87.37332916259766,
      "logps/ref_rejected": -88.77188110351562,
      "logps/rejected": -332.6782531738281,
      "loss": 1.1419,
      "margin_dpo/margin_mean": 64.04832458496094,
      "margin_dpo/margin_std": 97.3951416015625,
      "step": 359
    },
    {
      "KL/chosen_KL_mean": -165.70233154296875,
      "KL/mean": -206.47323608398438,
      "KL/rejected_KL_mean": -247.24415588378906,
      "KL/std": 83.04244995117188,
      "epoch": 0.5286343612334802,
      "fcm_dpo/beta": 0.005343434400856495,
      "fcm_dpo/delta": -0.041418079286813736,
      "fcm_dpo/margin": 81.54183197021484,
      "fcm_dpo/q_t": 0.396697461605072,
      "grad_norm": 14.163802146911621,
      "learning_rate": 2.7051020734928443e-07,
      "logits/chosen": -1.6066391468048096,
      "logits/rejected": -1.263431429862976,
      "logps/chosen": -240.92578125,
      "logps/ref_chosen": -75.22344970703125,
      "logps/ref_rejected": -76.09400177001953,
      "logps/rejected": -323.338134765625,
      "loss": 1.0522,
      "margin_dpo/margin_mean": 81.54183959960938,
      "margin_dpo/margin_std": 90.12895202636719,
      "step": 360
    },
    {
      "KL/chosen_KL_mean": -167.71771240234375,
      "KL/mean": -203.894287109375,
      "KL/rejected_KL_mean": -240.0708465576172,
      "KL/std": 95.26480102539062,
      "epoch": 0.5301027900146843,
      "fcm_dpo/beta": 0.005372712388634682,
      "fcm_dpo/delta": 0.012701474130153656,
      "fcm_dpo/margin": 72.3531494140625,
      "fcm_dpo/q_t": 0.41137343645095825,
      "grad_norm": 10.442744255065918,
      "learning_rate": 2.6923093854861593e-07,
      "logits/chosen": -1.6200389862060547,
      "logits/rejected": -1.3751978874206543,
      "logps/chosen": -249.67103576660156,
      "logps/ref_chosen": -81.95333099365234,
      "logps/ref_rejected": -96.67919921875,
      "logps/rejected": -336.75006103515625,
      "loss": 1.1039,
      "margin_dpo/margin_mean": 72.3531494140625,
      "margin_dpo/margin_std": 101.13809204101562,
      "step": 361
    },
    {
      "KL/chosen_KL_mean": -172.676025390625,
      "KL/mean": -211.38510131835938,
      "KL/rejected_KL_mean": -250.0941925048828,
      "KL/std": 90.53765869140625,
      "epoch": 0.5315712187958884,
      "fcm_dpo/beta": 0.005303596146404743,
      "fcm_dpo/delta": -0.012574443593621254,
      "fcm_dpo/margin": 77.41818237304688,
      "fcm_dpo/q_t": 0.40402132272720337,
      "grad_norm": 9.30694580078125,
      "learning_rate": 2.679511629948319e-07,
      "logits/chosen": -1.9246060848236084,
      "logits/rejected": -1.7267541885375977,
      "logps/chosen": -262.0157470703125,
      "logps/ref_chosen": -89.33971405029297,
      "logps/ref_rejected": -110.50196838378906,
      "logps/rejected": -360.5961608886719,
      "loss": 1.0821,
      "margin_dpo/margin_mean": 77.41818237304688,
      "margin_dpo/margin_std": 97.41252899169922,
      "step": 362
    },
    {
      "KL/chosen_KL_mean": -160.71774291992188,
      "KL/mean": -209.76309204101562,
      "KL/rejected_KL_mean": -258.8084411621094,
      "KL/std": 94.84002685546875,
      "epoch": 0.5330396475770925,
      "fcm_dpo/beta": 0.005120113492012024,
      "fcm_dpo/delta": -0.11994121968746185,
      "fcm_dpo/margin": 98.09066772460938,
      "fcm_dpo/q_t": 0.38176798820495605,
      "grad_norm": 11.345287322998047,
      "learning_rate": 2.6667091441120816e-07,
      "logits/chosen": -1.8147861957550049,
      "logits/rejected": -1.382467269897461,
      "logps/chosen": -241.49273681640625,
      "logps/ref_chosen": -80.77499389648438,
      "logps/ref_rejected": -79.43267822265625,
      "logps/rejected": -338.2410888671875,
      "loss": 1.0133,
      "margin_dpo/margin_mean": 98.0906753540039,
      "margin_dpo/margin_std": 103.7646255493164,
      "step": 363
    },
    {
      "KL/chosen_KL_mean": -160.16314697265625,
      "KL/mean": -199.41665649414062,
      "KL/rejected_KL_mean": -238.670166015625,
      "KL/std": 87.71381378173828,
      "epoch": 0.5345080763582967,
      "fcm_dpo/beta": 0.0050638094544410706,
      "fcm_dpo/delta": -8.411705493927002e-05,
      "fcm_dpo/margin": 78.50701904296875,
      "fcm_dpo/q_t": 0.40565919876098633,
      "grad_norm": 11.35587215423584,
      "learning_rate": 2.6539022653348575e-07,
      "logits/chosen": -2.100440263748169,
      "logits/rejected": -1.9600975513458252,
      "logps/chosen": -236.6927490234375,
      "logps/ref_chosen": -76.52960205078125,
      "logps/ref_rejected": -97.56173706054688,
      "logps/rejected": -336.2319030761719,
      "loss": 1.0764,
      "margin_dpo/margin_mean": 78.50701141357422,
      "margin_dpo/margin_std": 84.19218444824219,
      "step": 364
    },
    {
      "KL/chosen_KL_mean": -156.20028686523438,
      "KL/mean": -199.5833282470703,
      "KL/rejected_KL_mean": -242.9663543701172,
      "KL/std": 94.86885833740234,
      "epoch": 0.5359765051395007,
      "fcm_dpo/beta": 0.004943717271089554,
      "fcm_dpo/delta": -0.03371645510196686,
      "fcm_dpo/margin": 86.76606750488281,
      "fcm_dpo/q_t": 0.4009535312652588,
      "grad_norm": 8.578414916992188,
      "learning_rate": 2.641091331089811e-07,
      "logits/chosen": -1.7433940172195435,
      "logits/rejected": -1.619950771331787,
      "logps/chosen": -236.74517822265625,
      "logps/ref_chosen": -80.54489135742188,
      "logps/ref_rejected": -99.32316589355469,
      "logps/rejected": -342.2895202636719,
      "loss": 1.0699,
      "margin_dpo/margin_mean": 86.76606750488281,
      "margin_dpo/margin_std": 107.68307495117188,
      "step": 365
    },
    {
      "KL/chosen_KL_mean": -156.23974609375,
      "KL/mean": -192.7197265625,
      "KL/rejected_KL_mean": -229.19970703125,
      "KL/std": 98.94192504882812,
      "epoch": 0.5374449339207048,
      "fcm_dpo/beta": 0.004954167176038027,
      "fcm_dpo/delta": 0.043250784277915955,
      "fcm_dpo/margin": 72.95995330810547,
      "fcm_dpo/q_t": 0.4182211458683014,
      "grad_norm": 8.367152214050293,
      "learning_rate": 2.6282766789569736e-07,
      "logits/chosen": -1.6623716354370117,
      "logits/rejected": -1.698833703994751,
      "logps/chosen": -228.4778289794922,
      "logps/ref_chosen": -72.23806762695312,
      "logps/ref_rejected": -91.67182922363281,
      "logps/rejected": -320.87152099609375,
      "loss": 1.1114,
      "margin_dpo/margin_mean": 72.95995330810547,
      "margin_dpo/margin_std": 98.265869140625,
      "step": 366
    },
    {
      "KL/chosen_KL_mean": -142.04977416992188,
      "KL/mean": -186.07168579101562,
      "KL/rejected_KL_mean": -230.09361267089844,
      "KL/std": 86.89035034179688,
      "epoch": 0.5389133627019089,
      "fcm_dpo/beta": 0.004913915880024433,
      "fcm_dpo/delta": -0.037478044629096985,
      "fcm_dpo/margin": 88.04383087158203,
      "fcm_dpo/q_t": 0.3974781632423401,
      "grad_norm": 8.02967357635498,
      "learning_rate": 2.615458646614349e-07,
      "logits/chosen": -1.8027875423431396,
      "logits/rejected": -1.4547054767608643,
      "logps/chosen": -227.4718017578125,
      "logps/ref_chosen": -85.42201232910156,
      "logps/ref_rejected": -83.51779174804688,
      "logps/rejected": -313.61138916015625,
      "loss": 1.0447,
      "margin_dpo/margin_mean": 88.04383087158203,
      "margin_dpo/margin_std": 89.31350708007812,
      "step": 367
    },
    {
      "KL/chosen_KL_mean": -132.62530517578125,
      "KL/mean": -181.16256713867188,
      "KL/rejected_KL_mean": -229.6998291015625,
      "KL/std": 95.87257385253906,
      "epoch": 0.540381791483113,
      "fcm_dpo/beta": 0.0047770533710718155,
      "fcm_dpo/delta": -0.07408590614795685,
      "fcm_dpo/margin": 97.07452392578125,
      "fcm_dpo/q_t": 0.388897180557251,
      "grad_norm": 8.453216552734375,
      "learning_rate": 2.6026375718290083e-07,
      "logits/chosen": -1.4581232070922852,
      "logits/rejected": -1.4274537563323975,
      "logps/chosen": -206.4664306640625,
      "logps/ref_chosen": -73.84112548828125,
      "logps/ref_rejected": -104.69528198242188,
      "logps/rejected": -334.3951110839844,
      "loss": 1.0288,
      "margin_dpo/margin_mean": 97.07452392578125,
      "margin_dpo/margin_std": 95.85379028320312,
      "step": 368
    },
    {
      "KL/chosen_KL_mean": -147.20144653320312,
      "KL/mean": -192.07115173339844,
      "KL/rejected_KL_mean": -236.94085693359375,
      "KL/std": 88.7171401977539,
      "epoch": 0.5418502202643172,
      "fcm_dpo/beta": 0.004645342007279396,
      "fcm_dpo/delta": -0.019309505820274353,
      "fcm_dpo/margin": 89.73941802978516,
      "fcm_dpo/q_t": 0.40136176347732544,
      "grad_norm": 8.308738708496094,
      "learning_rate": 2.589813792448196e-07,
      "logits/chosen": -1.7041356563568115,
      "logits/rejected": -1.338580846786499,
      "logps/chosen": -245.66339111328125,
      "logps/ref_chosen": -98.46195983886719,
      "logps/ref_rejected": -99.15672302246094,
      "logps/rejected": -336.09759521484375,
      "loss": 1.0535,
      "margin_dpo/margin_mean": 89.73941802978516,
      "margin_dpo/margin_std": 89.92587280273438,
      "step": 369
    },
    {
      "KL/chosen_KL_mean": -162.91748046875,
      "KL/mean": -192.98562622070312,
      "KL/rejected_KL_mean": -223.05374145507812,
      "KL/std": 84.96009063720703,
      "epoch": 0.5433186490455213,
      "fcm_dpo/beta": 0.004875605925917625,
      "fcm_dpo/delta": 0.11661145836114883,
      "fcm_dpo/margin": 60.13625717163086,
      "fcm_dpo/q_t": 0.43256524205207825,
      "grad_norm": 8.406270980834961,
      "learning_rate": 2.5769876463904263e-07,
      "logits/chosen": -1.9047750234603882,
      "logits/rejected": -1.7059738636016846,
      "logps/chosen": -259.6932067871094,
      "logps/ref_chosen": -96.77574157714844,
      "logps/ref_rejected": -102.74411010742188,
      "logps/rejected": -325.7978515625,
      "loss": 1.1707,
      "margin_dpo/margin_mean": 60.136260986328125,
      "margin_dpo/margin_std": 99.20993041992188,
      "step": 370
    },
    {
      "KL/chosen_KL_mean": -154.72763061523438,
      "KL/mean": -188.0284423828125,
      "KL/rejected_KL_mean": -221.32925415039062,
      "KL/std": 86.06187438964844,
      "epoch": 0.5447870778267254,
      "fcm_dpo/beta": 0.005136905703693628,
      "fcm_dpo/delta": 0.0632498487830162,
      "fcm_dpo/margin": 66.60163116455078,
      "fcm_dpo/q_t": 0.4204738736152649,
      "grad_norm": 12.082716941833496,
      "learning_rate": 2.5641594716365744e-07,
      "logits/chosen": -1.8970531225204468,
      "logits/rejected": -1.5319273471832275,
      "logps/chosen": -257.71783447265625,
      "logps/ref_chosen": -102.99020385742188,
      "logps/ref_rejected": -105.35062408447266,
      "logps/rejected": -326.67987060546875,
      "loss": 1.1301,
      "margin_dpo/margin_mean": 66.60163116455078,
      "margin_dpo/margin_std": 94.04608917236328,
      "step": 371
    },
    {
      "KL/chosen_KL_mean": -131.47531127929688,
      "KL/mean": -181.46875,
      "KL/rejected_KL_mean": -231.4622039794922,
      "KL/std": 100.22373962402344,
      "epoch": 0.5462555066079295,
      "fcm_dpo/beta": 0.0049166493117809296,
      "fcm_dpo/delta": -0.10722362995147705,
      "fcm_dpo/margin": 99.98688507080078,
      "fcm_dpo/q_t": 0.38375526666641235,
      "grad_norm": 7.368245601654053,
      "learning_rate": 2.551329606220976e-07,
      "logits/chosen": -1.9922466278076172,
      "logits/rejected": -1.3694287538528442,
      "logps/chosen": -222.50155639648438,
      "logps/ref_chosen": -91.02623748779297,
      "logps/ref_rejected": -85.6102294921875,
      "logps/rejected": -317.07244873046875,
      "loss": 1.0139,
      "margin_dpo/margin_mean": 99.98688507080078,
      "margin_dpo/margin_std": 102.93263244628906,
      "step": 372
    },
    {
      "KL/chosen_KL_mean": -152.26698303222656,
      "KL/mean": -189.22152709960938,
      "KL/rejected_KL_mean": -226.1761016845703,
      "KL/std": 95.74559020996094,
      "epoch": 0.5477239353891337,
      "fcm_dpo/beta": 0.004897077567875385,
      "fcm_dpo/delta": 0.04276173561811447,
      "fcm_dpo/margin": 73.90913391113281,
      "fcm_dpo/q_t": 0.4164084792137146,
      "grad_norm": 10.621423721313477,
      "learning_rate": 2.538498388222517e-07,
      "logits/chosen": -2.208800792694092,
      "logits/rejected": -1.6963143348693848,
      "logps/chosen": -252.34005737304688,
      "logps/ref_chosen": -100.07307434082031,
      "logps/ref_rejected": -91.11634826660156,
      "logps/rejected": -317.2924499511719,
      "loss": 1.1179,
      "margin_dpo/margin_mean": 73.90912628173828,
      "margin_dpo/margin_std": 103.107177734375,
      "step": 373
    },
    {
      "KL/chosen_KL_mean": -132.01058959960938,
      "KL/mean": -172.6668243408203,
      "KL/rejected_KL_mean": -213.3230743408203,
      "KL/std": 90.5032958984375,
      "epoch": 0.5491923641703378,
      "fcm_dpo/beta": 0.00490906834602356,
      "fcm_dpo/delta": 0.0005551986396312714,
      "fcm_dpo/margin": 81.31248474121094,
      "fcm_dpo/q_t": 0.40638357400894165,
      "grad_norm": 8.346116065979004,
      "learning_rate": 2.525666155755725e-07,
      "logits/chosen": -2.2254552841186523,
      "logits/rejected": -1.8356719017028809,
      "logps/chosen": -230.02291870117188,
      "logps/ref_chosen": -98.01234436035156,
      "logps/ref_rejected": -100.75224304199219,
      "logps/rejected": -314.0753173828125,
      "loss": 1.0707,
      "margin_dpo/margin_mean": 81.31248474121094,
      "margin_dpo/margin_std": 87.73213195800781,
      "step": 374
    },
    {
      "KL/chosen_KL_mean": -134.2064971923828,
      "KL/mean": -180.77426147460938,
      "KL/rejected_KL_mean": -227.34201049804688,
      "KL/std": 87.02664184570312,
      "epoch": 0.5506607929515418,
      "fcm_dpo/beta": 0.004742823541164398,
      "fcm_dpo/delta": -0.05175640061497688,
      "fcm_dpo/margin": 93.1355209350586,
      "fcm_dpo/q_t": 0.3939504623413086,
      "grad_norm": 9.099494934082031,
      "learning_rate": 2.512833246961859e-07,
      "logits/chosen": -1.581992506980896,
      "logits/rejected": -1.347760796546936,
      "logps/chosen": -218.602783203125,
      "logps/ref_chosen": -84.39627838134766,
      "logps/ref_rejected": -95.7188491821289,
      "logps/rejected": -323.06085205078125,
      "loss": 1.0417,
      "margin_dpo/margin_mean": 93.13552856445312,
      "margin_dpo/margin_std": 92.43021392822266,
      "step": 375
    },
    {
      "KL/chosen_KL_mean": -128.52577209472656,
      "KL/mean": -180.23915100097656,
      "KL/rejected_KL_mean": -231.95254516601562,
      "KL/std": 100.8866195678711,
      "epoch": 0.5521292217327459,
      "fcm_dpo/beta": 0.004590433090925217,
      "fcm_dpo/delta": -0.08782419562339783,
      "fcm_dpo/margin": 103.42677307128906,
      "fcm_dpo/q_t": 0.3871370553970337,
      "grad_norm": 8.796163558959961,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.649522066116333,
      "logits/rejected": -1.3302950859069824,
      "logps/chosen": -224.2634735107422,
      "logps/ref_chosen": -95.73770141601562,
      "logps/ref_rejected": -111.97331237792969,
      "logps/rejected": -343.92584228515625,
      "loss": 1.0238,
      "margin_dpo/margin_mean": 103.42677307128906,
      "margin_dpo/margin_std": 107.49687194824219,
      "step": 376
    },
    {
      "KL/chosen_KL_mean": -136.33412170410156,
      "KL/mean": -189.6724853515625,
      "KL/rejected_KL_mean": -243.01084899902344,
      "KL/std": 101.03268432617188,
      "epoch": 0.55359765051395,
      "fcm_dpo/beta": 0.0043496256694197655,
      "fcm_dpo/delta": -0.07614002376794815,
      "fcm_dpo/margin": 106.67672729492188,
      "fcm_dpo/q_t": 0.38841545581817627,
      "grad_norm": 7.670997142791748,
      "learning_rate": 2.487166753038141e-07,
      "logits/chosen": -1.659024953842163,
      "logits/rejected": -1.545137882232666,
      "logps/chosen": -215.26107788085938,
      "logps/ref_chosen": -78.92694854736328,
      "logps/ref_rejected": -105.89892578125,
      "logps/rejected": -348.9097900390625,
      "loss": 1.0204,
      "margin_dpo/margin_mean": 106.67672729492188,
      "margin_dpo/margin_std": 100.0009765625,
      "step": 377
    },
    {
      "KL/chosen_KL_mean": -132.797607421875,
      "KL/mean": -175.29139709472656,
      "KL/rejected_KL_mean": -217.78521728515625,
      "KL/std": 95.90017700195312,
      "epoch": 0.5550660792951542,
      "fcm_dpo/beta": 0.004448545165359974,
      "fcm_dpo/delta": 0.02320113778114319,
      "fcm_dpo/margin": 84.98760986328125,
      "fcm_dpo/q_t": 0.4103398323059082,
      "grad_norm": 8.615662574768066,
      "learning_rate": 2.4743338442442754e-07,
      "logits/chosen": -1.9198946952819824,
      "logits/rejected": -1.638253092765808,
      "logps/chosen": -208.1699981689453,
      "logps/ref_chosen": -75.37240600585938,
      "logps/ref_rejected": -93.70409393310547,
      "logps/rejected": -311.48931884765625,
      "loss": 1.0917,
      "margin_dpo/margin_mean": 84.98760986328125,
      "margin_dpo/margin_std": 98.5930404663086,
      "step": 378
    },
    {
      "KL/chosen_KL_mean": -156.29623413085938,
      "KL/mean": -193.61988830566406,
      "KL/rejected_KL_mean": -230.94354248046875,
      "KL/std": 99.18685913085938,
      "epoch": 0.5565345080763583,
      "fcm_dpo/beta": 0.00453660823404789,
      "fcm_dpo/delta": 0.06836271286010742,
      "fcm_dpo/margin": 74.64730834960938,
      "fcm_dpo/q_t": 0.4230104386806488,
      "grad_norm": 8.433028221130371,
      "learning_rate": 2.461501611777483e-07,
      "logits/chosen": -1.9431078433990479,
      "logits/rejected": -1.869607925415039,
      "logps/chosen": -245.15191650390625,
      "logps/ref_chosen": -88.85568237304688,
      "logps/ref_rejected": -121.38114166259766,
      "logps/rejected": -352.3246765136719,
      "loss": 1.1394,
      "margin_dpo/margin_mean": 74.64730834960938,
      "margin_dpo/margin_std": 113.87494659423828,
      "step": 379
    },
    {
      "KL/chosen_KL_mean": -170.65826416015625,
      "KL/mean": -216.55682373046875,
      "KL/rejected_KL_mean": -262.4554138183594,
      "KL/std": 101.9696273803711,
      "epoch": 0.5580029368575624,
      "fcm_dpo/beta": 0.004547636490315199,
      "fcm_dpo/delta": -0.019977018237113953,
      "fcm_dpo/margin": 91.79715728759766,
      "fcm_dpo/q_t": 0.40375804901123047,
      "grad_norm": 11.459606170654297,
      "learning_rate": 2.4486703937790243e-07,
      "logits/chosen": -1.1940205097198486,
      "logits/rejected": -1.1698651313781738,
      "logps/chosen": -250.49147033691406,
      "logps/ref_chosen": -79.83321380615234,
      "logps/ref_rejected": -110.7447509765625,
      "logps/rejected": -373.2001647949219,
      "loss": 1.0752,
      "margin_dpo/margin_mean": 91.79714965820312,
      "margin_dpo/margin_std": 115.78707122802734,
      "step": 380
    },
    {
      "KL/chosen_KL_mean": -149.60897827148438,
      "KL/mean": -189.39556884765625,
      "KL/rejected_KL_mean": -229.18215942382812,
      "KL/std": 97.41273498535156,
      "epoch": 0.5594713656387665,
      "fcm_dpo/beta": 0.004618371836841106,
      "fcm_dpo/delta": 0.036436621099710464,
      "fcm_dpo/margin": 79.57316589355469,
      "fcm_dpo/q_t": 0.4140639901161194,
      "grad_norm": 9.16002368927002,
      "learning_rate": 2.435840528363426e-07,
      "logits/chosen": -1.8925800323486328,
      "logits/rejected": -1.3510448932647705,
      "logps/chosen": -236.17581176757812,
      "logps/ref_chosen": -86.56683349609375,
      "logps/ref_rejected": -84.416748046875,
      "logps/rejected": -313.5989074707031,
      "loss": 1.1053,
      "margin_dpo/margin_mean": 79.57316589355469,
      "margin_dpo/margin_std": 102.04167938232422,
      "step": 381
    },
    {
      "KL/chosen_KL_mean": -147.51422119140625,
      "KL/mean": -193.20059204101562,
      "KL/rejected_KL_mean": -238.886962890625,
      "KL/std": 96.06451416015625,
      "epoch": 0.5609397944199707,
      "fcm_dpo/beta": 0.004525131080299616,
      "fcm_dpo/delta": -0.017651356756687164,
      "fcm_dpo/margin": 91.37274932861328,
      "fcm_dpo/q_t": 0.40209949016571045,
      "grad_norm": 8.559001922607422,
      "learning_rate": 2.4230123536095745e-07,
      "logits/chosen": -1.8999464511871338,
      "logits/rejected": -1.6363697052001953,
      "logps/chosen": -251.97601318359375,
      "logps/ref_chosen": -104.46180725097656,
      "logps/ref_rejected": -114.60223388671875,
      "logps/rejected": -353.48919677734375,
      "loss": 1.0647,
      "margin_dpo/margin_mean": 91.37275695800781,
      "margin_dpo/margin_std": 98.10073852539062,
      "step": 382
    },
    {
      "KL/chosen_KL_mean": -160.88543701171875,
      "KL/mean": -199.69662475585938,
      "KL/rejected_KL_mean": -238.50782775878906,
      "KL/std": 99.33993530273438,
      "epoch": 0.5624082232011748,
      "fcm_dpo/beta": 0.00465422123670578,
      "fcm_dpo/delta": 0.043479882180690765,
      "fcm_dpo/margin": 77.6224136352539,
      "fcm_dpo/q_t": 0.4160575270652771,
      "grad_norm": 8.87612533569336,
      "learning_rate": 2.4101862075518037e-07,
      "logits/chosen": -1.673201322555542,
      "logits/rejected": -1.5514789819717407,
      "logps/chosen": -243.00149536132812,
      "logps/ref_chosen": -82.11607360839844,
      "logps/ref_rejected": -98.77716064453125,
      "logps/rejected": -337.28497314453125,
      "loss": 1.1091,
      "margin_dpo/margin_mean": 77.6224136352539,
      "margin_dpo/margin_std": 98.67461395263672,
      "step": 383
    },
    {
      "KL/chosen_KL_mean": -172.54315185546875,
      "KL/mean": -210.253662109375,
      "KL/rejected_KL_mean": -247.9641876220703,
      "KL/std": 102.98550415039062,
      "epoch": 0.5638766519823789,
      "fcm_dpo/beta": 0.004876617342233658,
      "fcm_dpo/delta": 0.03254539892077446,
      "fcm_dpo/margin": 75.42105102539062,
      "fcm_dpo/q_t": 0.4145125150680542,
      "grad_norm": 11.82705307006836,
      "learning_rate": 2.397362428170992e-07,
      "logits/chosen": -1.6093180179595947,
      "logits/rejected": -1.3807873725891113,
      "logps/chosen": -253.12840270996094,
      "logps/ref_chosen": -80.58525085449219,
      "logps/ref_rejected": -91.44789123535156,
      "logps/rejected": -339.4120788574219,
      "loss": 1.1349,
      "margin_dpo/margin_mean": 75.42105102539062,
      "margin_dpo/margin_std": 115.34616088867188,
      "step": 384
    },
    {
      "KL/chosen_KL_mean": -143.10995483398438,
      "KL/mean": -192.3440399169922,
      "KL/rejected_KL_mean": -241.578125,
      "KL/std": 92.57405853271484,
      "epoch": 0.5653450807635829,
      "fcm_dpo/beta": 0.004593793302774429,
      "fcm_dpo/delta": -0.0635334700345993,
      "fcm_dpo/margin": 98.46817016601562,
      "fcm_dpo/q_t": 0.3930342197418213,
      "grad_norm": 10.031228065490723,
      "learning_rate": 2.3845413533856514e-07,
      "logits/chosen": -2.176541805267334,
      "logits/rejected": -1.562623143196106,
      "logps/chosen": -235.8701171875,
      "logps/ref_chosen": -92.76016235351562,
      "logps/ref_rejected": -84.45997619628906,
      "logps/rejected": -326.0380859375,
      "loss": 1.0426,
      "margin_dpo/margin_mean": 98.46817016601562,
      "margin_dpo/margin_std": 106.56240844726562,
      "step": 385
    },
    {
      "KL/chosen_KL_mean": -150.96910095214844,
      "KL/mean": -198.31097412109375,
      "KL/rejected_KL_mean": -245.6528778076172,
      "KL/std": 94.69892883300781,
      "epoch": 0.566813509544787,
      "fcm_dpo/beta": 0.004548883531242609,
      "fcm_dpo/delta": -0.03528036177158356,
      "fcm_dpo/margin": 94.68376159667969,
      "fcm_dpo/q_t": 0.3970075249671936,
      "grad_norm": 7.46783447265625,
      "learning_rate": 2.3717233210430254e-07,
      "logits/chosen": -1.550349235534668,
      "logits/rejected": -1.2616389989852905,
      "logps/chosen": -236.32806396484375,
      "logps/ref_chosen": -85.35896301269531,
      "logps/ref_rejected": -96.99065399169922,
      "logps/rejected": -342.6435241699219,
      "loss": 1.0543,
      "margin_dpo/margin_mean": 94.68376159667969,
      "margin_dpo/margin_std": 103.30970764160156,
      "step": 386
    },
    {
      "KL/chosen_KL_mean": -175.87057495117188,
      "KL/mean": -218.05178833007812,
      "KL/rejected_KL_mean": -260.2330017089844,
      "KL/std": 92.22168731689453,
      "epoch": 0.5682819383259912,
      "fcm_dpo/beta": 0.0045783547684550285,
      "fcm_dpo/delta": 0.015317270532250404,
      "fcm_dpo/margin": 84.36245727539062,
      "fcm_dpo/q_t": 0.4091179668903351,
      "grad_norm": 10.308074951171875,
      "learning_rate": 2.3589086689101889e-07,
      "logits/chosen": -2.467068672180176,
      "logits/rejected": -1.7861285209655762,
      "logps/chosen": -278.5587158203125,
      "logps/ref_chosen": -102.68814086914062,
      "logps/ref_rejected": -99.42858123779297,
      "logps/rejected": -359.6615905761719,
      "loss": 1.0815,
      "margin_dpo/margin_mean": 84.36245727539062,
      "margin_dpo/margin_std": 94.29981994628906,
      "step": 387
    },
    {
      "KL/chosen_KL_mean": -176.7794952392578,
      "KL/mean": -223.70068359375,
      "KL/rejected_KL_mean": -270.62188720703125,
      "KL/std": 102.20881652832031,
      "epoch": 0.5697503671071953,
      "fcm_dpo/beta": 0.004502397961914539,
      "fcm_dpo/delta": -0.025917744264006615,
      "fcm_dpo/margin": 93.84237670898438,
      "fcm_dpo/q_t": 0.40009352564811707,
      "grad_norm": 8.445474624633789,
      "learning_rate": 2.3460977346651428e-07,
      "logits/chosen": -1.6725871562957764,
      "logits/rejected": -1.5555288791656494,
      "logps/chosen": -260.36474609375,
      "logps/ref_chosen": -83.58523559570312,
      "logps/ref_rejected": -110.10037231445312,
      "logps/rejected": -380.7222595214844,
      "loss": 1.0604,
      "margin_dpo/margin_mean": 93.84237670898438,
      "margin_dpo/margin_std": 104.47183227539062,
      "step": 388
    },
    {
      "KL/chosen_KL_mean": -183.3534393310547,
      "KL/mean": -222.88482666015625,
      "KL/rejected_KL_mean": -262.416259765625,
      "KL/std": 103.7096939086914,
      "epoch": 0.5712187958883994,
      "fcm_dpo/beta": 0.004575867671519518,
      "fcm_dpo/delta": 0.042920876294374466,
      "fcm_dpo/margin": 79.06282043457031,
      "fcm_dpo/q_t": 0.41663143038749695,
      "grad_norm": 10.971561431884766,
      "learning_rate": 2.3332908558879177e-07,
      "logits/chosen": -2.4969429969787598,
      "logits/rejected": -2.057525634765625,
      "logps/chosen": -273.17572021484375,
      "logps/ref_chosen": -89.822265625,
      "logps/ref_rejected": -84.85376739501953,
      "logps/rejected": -347.27001953125,
      "loss": 1.1189,
      "margin_dpo/margin_mean": 79.06282043457031,
      "margin_dpo/margin_std": 111.94400024414062,
      "step": 389
    },
    {
      "KL/chosen_KL_mean": -182.88601684570312,
      "KL/mean": -221.65008544921875,
      "KL/rejected_KL_mean": -260.4141540527344,
      "KL/std": 101.7956771850586,
      "epoch": 0.5726872246696035,
      "fcm_dpo/beta": 0.00463997945189476,
      "fcm_dpo/delta": 0.04434869810938835,
      "fcm_dpo/margin": 77.52813720703125,
      "fcm_dpo/q_t": 0.41907984018325806,
      "grad_norm": 8.570809364318848,
      "learning_rate": 2.320488370051681e-07,
      "logits/chosen": -1.5085253715515137,
      "logits/rejected": -1.2996234893798828,
      "logps/chosen": -260.41845703125,
      "logps/ref_chosen": -77.53244018554688,
      "logps/ref_rejected": -92.58392333984375,
      "logps/rejected": -352.9980773925781,
      "loss": 1.1294,
      "margin_dpo/margin_mean": 77.52812957763672,
      "margin_dpo/margin_std": 116.91828918457031,
      "step": 390
    },
    {
      "KL/chosen_KL_mean": -165.90274047851562,
      "KL/mean": -207.5533447265625,
      "KL/rejected_KL_mean": -249.20394897460938,
      "KL/std": 97.32146453857422,
      "epoch": 0.5741556534508077,
      "fcm_dpo/beta": 0.004711050074547529,
      "fcm_dpo/delta": 0.00802275724709034,
      "fcm_dpo/margin": 83.30120849609375,
      "fcm_dpo/q_t": 0.4083816409111023,
      "grad_norm": 9.285392761230469,
      "learning_rate": 2.3076906145138405e-07,
      "logits/chosen": -1.9478373527526855,
      "logits/rejected": -1.497823715209961,
      "logps/chosen": -254.13568115234375,
      "logps/ref_chosen": -88.2329330444336,
      "logps/ref_rejected": -93.00756072998047,
      "logps/rejected": -342.2115173339844,
      "loss": 1.0892,
      "margin_dpo/margin_mean": 83.30120849609375,
      "margin_dpo/margin_std": 102.46261596679688,
      "step": 391
    },
    {
      "KL/chosen_KL_mean": -167.5869903564453,
      "KL/mean": -219.05943298339844,
      "KL/rejected_KL_mean": -270.5318908691406,
      "KL/std": 109.52334594726562,
      "epoch": 0.5756240822320118,
      "fcm_dpo/beta": 0.004619993269443512,
      "fcm_dpo/delta": -0.0879494771361351,
      "fcm_dpo/margin": 102.94487762451172,
      "fcm_dpo/q_t": 0.38761013746261597,
      "grad_norm": 9.217992782592773,
      "learning_rate": 2.294897926507156e-07,
      "logits/chosen": -1.3567733764648438,
      "logits/rejected": -1.2386513948440552,
      "logps/chosen": -255.3314208984375,
      "logps/ref_chosen": -87.74442291259766,
      "logps/ref_rejected": -108.57071685791016,
      "logps/rejected": -379.10260009765625,
      "loss": 1.0364,
      "margin_dpo/margin_mean": 102.94488525390625,
      "margin_dpo/margin_std": 117.41236877441406,
      "step": 392
    },
    {
      "KL/chosen_KL_mean": -149.58218383789062,
      "KL/mean": -195.30120849609375,
      "KL/rejected_KL_mean": -241.02023315429688,
      "KL/std": 102.98703002929688,
      "epoch": 0.5770925110132159,
      "fcm_dpo/beta": 0.004484082572162151,
      "fcm_dpo/delta": -0.011478514410555363,
      "fcm_dpo/margin": 91.43806457519531,
      "fcm_dpo/q_t": 0.40352392196655273,
      "grad_norm": 7.945555210113525,
      "learning_rate": 2.2821106431308543e-07,
      "logits/chosen": -1.5717883110046387,
      "logits/rejected": -1.243028163909912,
      "logps/chosen": -226.14599609375,
      "logps/ref_chosen": -76.56381225585938,
      "logps/ref_rejected": -79.72618103027344,
      "logps/rejected": -320.7464294433594,
      "loss": 1.0663,
      "margin_dpo/margin_mean": 91.43806457519531,
      "margin_dpo/margin_std": 101.05241394042969,
      "step": 393
    },
    {
      "KL/chosen_KL_mean": -159.33502197265625,
      "KL/mean": -209.50881958007812,
      "KL/rejected_KL_mean": -259.6825866699219,
      "KL/std": 98.46173095703125,
      "epoch": 0.57856093979442,
      "fcm_dpo/beta": 0.004391576163470745,
      "fcm_dpo/delta": -0.046911392360925674,
      "fcm_dpo/margin": 100.34756469726562,
      "fcm_dpo/q_t": 0.39457863569259644,
      "grad_norm": 7.893409729003906,
      "learning_rate": 2.2693291013417452e-07,
      "logits/chosen": -1.3690569400787354,
      "logits/rejected": -1.1319715976715088,
      "logps/chosen": -244.03369140625,
      "logps/ref_chosen": -84.69866180419922,
      "logps/ref_rejected": -97.457763671875,
      "logps/rejected": -357.1403503417969,
      "loss": 1.0366,
      "margin_dpo/margin_mean": 100.34756469726562,
      "margin_dpo/margin_std": 96.91682434082031,
      "step": 394
    },
    {
      "KL/chosen_KL_mean": -164.85971069335938,
      "KL/mean": -202.98159790039062,
      "KL/rejected_KL_mean": -241.10348510742188,
      "KL/std": 102.74571228027344,
      "epoch": 0.580029368575624,
      "fcm_dpo/beta": 0.004440673161298037,
      "fcm_dpo/delta": 0.06814345717430115,
      "fcm_dpo/margin": 76.24376678466797,
      "fcm_dpo/q_t": 0.42457613348960876,
      "grad_norm": 8.685137748718262,
      "learning_rate": 2.2565536379453404e-07,
      "logits/chosen": -2.563803195953369,
      "logits/rejected": -2.1636290550231934,
      "logps/chosen": -253.7473602294922,
      "logps/ref_chosen": -88.88763427734375,
      "logps/ref_rejected": -91.0455322265625,
      "logps/rejected": -332.1490173339844,
      "loss": 1.1679,
      "margin_dpo/margin_mean": 76.24376678466797,
      "margin_dpo/margin_std": 138.3853759765625,
      "step": 395
    },
    {
      "KL/chosen_KL_mean": -174.69224548339844,
      "KL/mean": -209.1037139892578,
      "KL/rejected_KL_mean": -243.51519775390625,
      "KL/std": 91.12174987792969,
      "epoch": 0.5814977973568282,
      "fcm_dpo/beta": 0.004716053139418364,
      "fcm_dpo/delta": 0.08353376388549805,
      "fcm_dpo/margin": 68.82295227050781,
      "fcm_dpo/q_t": 0.42582136392593384,
      "grad_norm": 11.227849006652832,
      "learning_rate": 2.2437845895869825e-07,
      "logits/chosen": -1.9563055038452148,
      "logits/rejected": -1.4680548906326294,
      "logps/chosen": -277.16357421875,
      "logps/ref_chosen": -102.47132873535156,
      "logps/ref_rejected": -93.23478698730469,
      "logps/rejected": -336.75,
      "loss": 1.1501,
      "margin_dpo/margin_mean": 68.82295227050781,
      "margin_dpo/margin_std": 108.13592529296875,
      "step": 396
    },
    {
      "KL/chosen_KL_mean": -155.44110107421875,
      "KL/mean": -216.5762481689453,
      "KL/rejected_KL_mean": -277.71136474609375,
      "KL/std": 106.04058837890625,
      "epoch": 0.5829662261380323,
      "fcm_dpo/beta": 0.004405719228088856,
      "fcm_dpo/delta": -0.16720707714557648,
      "fcm_dpo/margin": 122.27027130126953,
      "fcm_dpo/q_t": 0.3710610568523407,
      "grad_norm": 8.183685302734375,
      "learning_rate": 2.2310222927429716e-07,
      "logits/chosen": -2.264683246612549,
      "logits/rejected": -1.6658573150634766,
      "logps/chosen": -256.48553466796875,
      "logps/ref_chosen": -101.04441833496094,
      "logps/ref_rejected": -108.8511734008789,
      "logps/rejected": -386.5625305175781,
      "loss": 0.9904,
      "margin_dpo/margin_mean": 122.27027130126953,
      "margin_dpo/margin_std": 126.36883544921875,
      "step": 397
    },
    {
      "KL/chosen_KL_mean": -164.7427978515625,
      "KL/mean": -208.992431640625,
      "KL/rejected_KL_mean": -253.2420654296875,
      "KL/std": 104.13616943359375,
      "epoch": 0.5844346549192364,
      "fcm_dpo/beta": 0.00433848612010479,
      "fcm_dpo/delta": 0.018099859356880188,
      "fcm_dpo/margin": 88.4992904663086,
      "fcm_dpo/q_t": 0.411425918340683,
      "grad_norm": 7.778429985046387,
      "learning_rate": 2.2182670837116972e-07,
      "logits/chosen": -2.1553726196289062,
      "logits/rejected": -1.8778647184371948,
      "logps/chosen": -259.7920227050781,
      "logps/ref_chosen": -95.04922485351562,
      "logps/ref_rejected": -114.00563049316406,
      "logps/rejected": -367.2476806640625,
      "loss": 1.106,
      "margin_dpo/margin_mean": 88.4992904663086,
      "margin_dpo/margin_std": 123.15249633789062,
      "step": 398
    },
    {
      "KL/chosen_KL_mean": -143.72894287109375,
      "KL/mean": -200.17095947265625,
      "KL/rejected_KL_mean": -256.61297607421875,
      "KL/std": 94.45652770996094,
      "epoch": 0.5859030837004405,
      "fcm_dpo/beta": 0.00421685166656971,
      "fcm_dpo/delta": -0.0884578600525856,
      "fcm_dpo/margin": 112.88401794433594,
      "fcm_dpo/q_t": 0.3856189250946045,
      "grad_norm": 8.197100639343262,
      "learning_rate": 2.2055192986047804e-07,
      "logits/chosen": -1.8911659717559814,
      "logits/rejected": -1.0562729835510254,
      "logps/chosen": -240.36749267578125,
      "logps/ref_chosen": -96.63854217529297,
      "logps/ref_rejected": -83.53645324707031,
      "logps/rejected": -340.1494140625,
      "loss": 1.0098,
      "margin_dpo/margin_mean": 112.88402557373047,
      "margin_dpo/margin_std": 101.98155212402344,
      "step": 399
    },
    {
      "KL/chosen_KL_mean": -150.80177307128906,
      "KL/mean": -208.2266845703125,
      "KL/rejected_KL_mean": -265.6515808105469,
      "KL/std": 103.47392272949219,
      "epoch": 0.5873715124816447,
      "fcm_dpo/beta": 0.004013285972177982,
      "fcm_dpo/delta": -0.07074441760778427,
      "fcm_dpo/margin": 114.84982299804688,
      "fcm_dpo/q_t": 0.3904189169406891,
      "grad_norm": 10.966231346130371,
      "learning_rate": 2.192779273338215e-07,
      "logits/chosen": -2.0582799911499023,
      "logits/rejected": -1.567161202430725,
      "logps/chosen": -256.10491943359375,
      "logps/ref_chosen": -105.30314636230469,
      "logps/ref_rejected": -108.6555404663086,
      "logps/rejected": -374.30712890625,
      "loss": 1.0329,
      "margin_dpo/margin_mean": 114.84982299804688,
      "margin_dpo/margin_std": 118.66209411621094,
      "step": 400
    },
    {
      "KL/chosen_KL_mean": -166.92843627929688,
      "KL/mean": -202.49392700195312,
      "KL/rejected_KL_mean": -238.0594024658203,
      "KL/std": 105.227294921875,
      "epoch": 0.5888399412628488,
      "fcm_dpo/beta": 0.004151647444814444,
      "fcm_dpo/delta": 0.11532101035118103,
      "fcm_dpo/margin": 71.1309585571289,
      "fcm_dpo/q_t": 0.4349185526371002,
      "grad_norm": 10.359856605529785,
      "learning_rate": 2.1800473436235136e-07,
      "logits/chosen": -1.5894618034362793,
      "logits/rejected": -1.3493752479553223,
      "logps/chosen": -251.2176055908203,
      "logps/ref_chosen": -84.28916931152344,
      "logps/ref_rejected": -90.943115234375,
      "logps/rejected": -329.00250244140625,
      "loss": 1.1804,
      "margin_dpo/margin_mean": 71.1309585571289,
      "margin_dpo/margin_std": 128.97247314453125,
      "step": 401
    },
    {
      "KL/chosen_KL_mean": -147.266357421875,
      "KL/mean": -214.3287353515625,
      "KL/rejected_KL_mean": -281.39111328125,
      "KL/std": 112.6015625,
      "epoch": 0.5903083700440529,
      "fcm_dpo/beta": 0.003980084788054228,
      "fcm_dpo/delta": -0.15899497270584106,
      "fcm_dpo/margin": 134.12478637695312,
      "fcm_dpo/q_t": 0.36967307329177856,
      "grad_norm": 7.436604022979736,
      "learning_rate": 2.1673238449588665e-07,
      "logits/chosen": -1.7598283290863037,
      "logits/rejected": -1.2790424823760986,
      "logps/chosen": -230.85948181152344,
      "logps/ref_chosen": -83.59312438964844,
      "logps/ref_rejected": -87.81027221679688,
      "logps/rejected": -369.201416015625,
      "loss": 0.9689,
      "margin_dpo/margin_mean": 134.12478637695312,
      "margin_dpo/margin_std": 110.04307556152344,
      "step": 402
    },
    {
      "KL/chosen_KL_mean": -151.2656707763672,
      "KL/mean": -200.72451782226562,
      "KL/rejected_KL_mean": -250.183349609375,
      "KL/std": 107.93656158447266,
      "epoch": 0.591776798825257,
      "fcm_dpo/beta": 0.0038803061470389366,
      "fcm_dpo/delta": 0.018203644081950188,
      "fcm_dpo/margin": 98.91768646240234,
      "fcm_dpo/q_t": 0.4098866879940033,
      "grad_norm": 8.143406867980957,
      "learning_rate": 2.154609112620295e-07,
      "logits/chosen": -2.086491584777832,
      "logits/rejected": -1.67596435546875,
      "logps/chosen": -225.0187530517578,
      "logps/ref_chosen": -73.75308227539062,
      "logps/ref_rejected": -83.92012786865234,
      "logps/rejected": -334.1034851074219,
      "loss": 1.0875,
      "margin_dpo/margin_mean": 98.91769409179688,
      "margin_dpo/margin_std": 115.61625671386719,
      "step": 403
    },
    {
      "KL/chosen_KL_mean": -178.80975341796875,
      "KL/mean": -221.28366088867188,
      "KL/rejected_KL_mean": -263.7575988769531,
      "KL/std": 104.05540466308594,
      "epoch": 0.593245227606461,
      "fcm_dpo/beta": 0.004003855399787426,
      "fcm_dpo/delta": 0.06683460623025894,
      "fcm_dpo/margin": 84.94783020019531,
      "fcm_dpo/q_t": 0.42199981212615967,
      "grad_norm": 11.307469367980957,
      "learning_rate": 2.1419034816528218e-07,
      "logits/chosen": -2.0690481662750244,
      "logits/rejected": -1.7356536388397217,
      "logps/chosen": -258.4859313964844,
      "logps/ref_chosen": -79.67617797851562,
      "logps/ref_rejected": -84.280517578125,
      "logps/rejected": -348.0381164550781,
      "loss": 1.1372,
      "margin_dpo/margin_mean": 84.94783020019531,
      "margin_dpo/margin_std": 127.27704620361328,
      "step": 404
    },
    {
      "KL/chosen_KL_mean": -182.47357177734375,
      "KL/mean": -225.52978515625,
      "KL/rejected_KL_mean": -268.58599853515625,
      "KL/std": 97.80992126464844,
      "epoch": 0.5947136563876652,
      "fcm_dpo/beta": 0.0042035323567688465,
      "fcm_dpo/delta": 0.040597252547740936,
      "fcm_dpo/margin": 86.11239624023438,
      "fcm_dpo/q_t": 0.41563090682029724,
      "grad_norm": 8.181960105895996,
      "learning_rate": 2.129207286861638e-07,
      "logits/chosen": -1.4764511585235596,
      "logits/rejected": -0.9940147399902344,
      "logps/chosen": -278.9355163574219,
      "logps/ref_chosen": -96.46195220947266,
      "logps/ref_rejected": -92.87071228027344,
      "logps/rejected": -361.4566955566406,
      "loss": 1.1036,
      "margin_dpo/margin_mean": 86.1124038696289,
      "margin_dpo/margin_std": 103.62257385253906,
      "step": 405
    },
    {
      "KL/chosen_KL_mean": -173.99362182617188,
      "KL/mean": -213.15048217773438,
      "KL/rejected_KL_mean": -252.30734252929688,
      "KL/std": 104.57958984375,
      "epoch": 0.5961820851688693,
      "fcm_dpo/beta": 0.004304743837565184,
      "fcm_dpo/delta": 0.06988808512687683,
      "fcm_dpo/margin": 78.31371307373047,
      "fcm_dpo/q_t": 0.42279112339019775,
      "grad_norm": 14.069229125976562,
      "learning_rate": 2.1165208628032861e-07,
      "logits/chosen": -1.802344799041748,
      "logits/rejected": -1.6326732635498047,
      "logps/chosen": -252.12759399414062,
      "logps/ref_chosen": -78.13396453857422,
      "logps/ref_rejected": -98.28359985351562,
      "logps/rejected": -350.5909423828125,
      "loss": 1.1435,
      "margin_dpo/margin_mean": 78.31371307373047,
      "margin_dpo/margin_std": 122.24439239501953,
      "step": 406
    },
    {
      "KL/chosen_KL_mean": -180.4658203125,
      "KL/mean": -214.3982696533203,
      "KL/rejected_KL_mean": -248.33071899414062,
      "KL/std": 102.90857696533203,
      "epoch": 0.5976505139500734,
      "fcm_dpo/beta": 0.004551921971142292,
      "fcm_dpo/delta": 0.10025401413440704,
      "fcm_dpo/margin": 67.8648910522461,
      "fcm_dpo/q_t": 0.4295039176940918,
      "grad_norm": 9.537001609802246,
      "learning_rate": 2.1038445437768375e-07,
      "logits/chosen": -2.0779848098754883,
      "logits/rejected": -1.5485072135925293,
      "logps/chosen": -264.4786376953125,
      "logps/ref_chosen": -84.01283264160156,
      "logps/ref_rejected": -82.78103637695312,
      "logps/rejected": -331.11175537109375,
      "loss": 1.16,
      "margin_dpo/margin_mean": 67.86488342285156,
      "margin_dpo/margin_std": 109.87419128417969,
      "step": 407
    },
    {
      "KL/chosen_KL_mean": -172.74349975585938,
      "KL/mean": -207.26742553710938,
      "KL/rejected_KL_mean": -241.79135131835938,
      "KL/std": 98.70328521728516,
      "epoch": 0.5991189427312775,
      "fcm_dpo/beta": 0.00476740300655365,
      "fcm_dpo/delta": 0.07875862717628479,
      "fcm_dpo/margin": 69.0478515625,
      "fcm_dpo/q_t": 0.42617201805114746,
      "grad_norm": 9.759138107299805,
      "learning_rate": 2.0911786638150872e-07,
      "logits/chosen": -2.270904064178467,
      "logits/rejected": -1.6501350402832031,
      "logps/chosen": -277.20526123046875,
      "logps/ref_chosen": -104.46174621582031,
      "logps/ref_rejected": -96.37217712402344,
      "logps/rejected": -338.1635437011719,
      "loss": 1.1646,
      "margin_dpo/margin_mean": 69.0478515625,
      "margin_dpo/margin_std": 122.5137939453125,
      "step": 408
    },
    {
      "KL/chosen_KL_mean": -184.67181396484375,
      "KL/mean": -220.80307006835938,
      "KL/rejected_KL_mean": -256.9343566894531,
      "KL/std": 100.59230041503906,
      "epoch": 0.6005873715124816,
      "fcm_dpo/beta": 0.004945676773786545,
      "fcm_dpo/delta": 0.04772175848484039,
      "fcm_dpo/margin": 72.26254272460938,
      "fcm_dpo/q_t": 0.41665390133857727,
      "grad_norm": 9.557785034179688,
      "learning_rate": 2.0785235566757517e-07,
      "logits/chosen": -1.8638701438903809,
      "logits/rejected": -1.4491209983825684,
      "logps/chosen": -282.3401184082031,
      "logps/ref_chosen": -97.66830444335938,
      "logps/ref_rejected": -90.04583740234375,
      "logps/rejected": -346.9801940917969,
      "loss": 1.134,
      "margin_dpo/margin_mean": 72.2625503540039,
      "margin_dpo/margin_std": 110.58590698242188,
      "step": 409
    },
    {
      "KL/chosen_KL_mean": -178.375244140625,
      "KL/mean": -222.2113037109375,
      "KL/rejected_KL_mean": -266.04736328125,
      "KL/std": 106.47080993652344,
      "epoch": 0.6020558002936858,
      "fcm_dpo/beta": 0.004898872226476669,
      "fcm_dpo/delta": -0.03395042568445206,
      "fcm_dpo/margin": 87.67211151123047,
      "fcm_dpo/q_t": 0.39849308133125305,
      "grad_norm": 8.681079864501953,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -1.7037760019302368,
      "logits/rejected": -1.5733428001403809,
      "logps/chosen": -254.844482421875,
      "logps/ref_chosen": -76.46923828125,
      "logps/ref_rejected": -88.64064025878906,
      "logps/rejected": -354.68798828125,
      "loss": 1.0666,
      "margin_dpo/margin_mean": 87.67210388183594,
      "margin_dpo/margin_std": 106.7376480102539,
      "step": 410
    },
    {
      "KL/chosen_KL_mean": -191.08126831054688,
      "KL/mean": -231.86575317382812,
      "KL/rejected_KL_mean": -272.6502685546875,
      "KL/std": 99.01495361328125,
      "epoch": 0.6035242290748899,
      "fcm_dpo/beta": 0.004838800989091396,
      "fcm_dpo/delta": 0.00518215075135231,
      "fcm_dpo/margin": 81.56901550292969,
      "fcm_dpo/q_t": 0.4098510146141052,
      "grad_norm": 8.966156005859375,
      "learning_rate": 2.0532469944670343e-07,
      "logits/chosen": -2.2752270698547363,
      "logits/rejected": -1.864394187927246,
      "logps/chosen": -278.24755859375,
      "logps/ref_chosen": -87.16630554199219,
      "logps/ref_rejected": -87.09603118896484,
      "logps/rejected": -359.74627685546875,
      "loss": 1.0952,
      "margin_dpo/margin_mean": 81.56901550292969,
      "margin_dpo/margin_std": 107.6809310913086,
      "step": 411
    },
    {
      "KL/chosen_KL_mean": -180.63238525390625,
      "KL/mean": -221.05010986328125,
      "KL/rejected_KL_mean": -261.46783447265625,
      "KL/std": 99.58441162109375,
      "epoch": 0.604992657856094,
      "fcm_dpo/beta": 0.004875939339399338,
      "fcm_dpo/delta": 0.00631989911198616,
      "fcm_dpo/margin": 80.83546447753906,
      "fcm_dpo/q_t": 0.41139933466911316,
      "grad_norm": 10.640175819396973,
      "learning_rate": 2.0406262054585738e-07,
      "logits/chosen": -2.3790364265441895,
      "logits/rejected": -2.163079261779785,
      "logps/chosen": -259.5797119140625,
      "logps/ref_chosen": -78.94734191894531,
      "logps/ref_rejected": -106.10554504394531,
      "logps/rejected": -367.5733947753906,
      "loss": 1.122,
      "margin_dpo/margin_mean": 80.83546447753906,
      "margin_dpo/margin_std": 128.23597717285156,
      "step": 412
    },
    {
      "KL/chosen_KL_mean": -194.2091827392578,
      "KL/mean": -238.43841552734375,
      "KL/rejected_KL_mean": -282.66766357421875,
      "KL/std": 105.7294692993164,
      "epoch": 0.6064610866372981,
      "fcm_dpo/beta": 0.004801267758011818,
      "fcm_dpo/delta": -0.02943670004606247,
      "fcm_dpo/margin": 88.45846557617188,
      "fcm_dpo/q_t": 0.4005562663078308,
      "grad_norm": 10.302332878112793,
      "learning_rate": 2.0280175213768205e-07,
      "logits/chosen": -1.3096027374267578,
      "logits/rejected": -1.0556010007858276,
      "logps/chosen": -289.9039001464844,
      "logps/ref_chosen": -95.69470977783203,
      "logps/ref_rejected": -107.96085357666016,
      "logps/rejected": -390.6285095214844,
      "loss": 1.0925,
      "margin_dpo/margin_mean": 88.45846557617188,
      "margin_dpo/margin_std": 126.99897766113281,
      "step": 413
    },
    {
      "KL/chosen_KL_mean": -179.16851806640625,
      "KL/mean": -220.7781982421875,
      "KL/rejected_KL_mean": -262.38787841796875,
      "KL/std": 102.31583404541016,
      "epoch": 0.6079295154185022,
      "fcm_dpo/beta": 0.004846329800784588,
      "fcm_dpo/delta": -0.003935467451810837,
      "fcm_dpo/margin": 83.21937561035156,
      "fcm_dpo/q_t": 0.4083821773529053,
      "grad_norm": 13.315023422241211,
      "learning_rate": 2.0154212744723247e-07,
      "logits/chosen": -1.721221685409546,
      "logits/rejected": -1.3731058835983276,
      "logps/chosen": -267.4451904296875,
      "logps/ref_chosen": -88.27667236328125,
      "logps/ref_rejected": -92.87004089355469,
      "logps/rejected": -355.2579345703125,
      "loss": 1.1163,
      "margin_dpo/margin_mean": 83.2193832397461,
      "margin_dpo/margin_std": 130.64285278320312,
      "step": 414
    },
    {
      "KL/chosen_KL_mean": -194.59170532226562,
      "KL/mean": -229.35910034179688,
      "KL/rejected_KL_mean": -264.12652587890625,
      "KL/std": 93.54156494140625,
      "epoch": 0.6093979441997063,
      "fcm_dpo/beta": 0.004968162160366774,
      "fcm_dpo/delta": 0.0605001375079155,
      "fcm_dpo/margin": 69.53480529785156,
      "fcm_dpo/q_t": 0.42171797156333923,
      "grad_norm": 11.560786247253418,
      "learning_rate": 2.002837796667909e-07,
      "logits/chosen": -1.5718086957931519,
      "logits/rejected": -1.190260410308838,
      "logps/chosen": -303.50762939453125,
      "logps/ref_chosen": -108.91590118408203,
      "logps/ref_rejected": -107.47135925292969,
      "logps/rejected": -371.5978698730469,
      "loss": 1.1479,
      "margin_dpo/margin_mean": 69.53480529785156,
      "margin_dpo/margin_std": 114.68606567382812,
      "step": 415
    },
    {
      "KL/chosen_KL_mean": -194.1864013671875,
      "KL/mean": -242.579345703125,
      "KL/rejected_KL_mean": -290.9722595214844,
      "KL/std": 107.55183410644531,
      "epoch": 0.6108663729809104,
      "fcm_dpo/beta": 0.00491677038371563,
      "fcm_dpo/delta": -0.09020207822322845,
      "fcm_dpo/margin": 96.78587341308594,
      "fcm_dpo/q_t": 0.38903918862342834,
      "grad_norm": 9.547143936157227,
      "learning_rate": 1.990267419549914e-07,
      "logits/chosen": -1.87496018409729,
      "logits/rejected": -1.376505970954895,
      "logps/chosen": -287.58526611328125,
      "logps/ref_chosen": -93.39888000488281,
      "logps/ref_rejected": -97.6729736328125,
      "logps/rejected": -388.645263671875,
      "loss": 1.0453,
      "margin_dpo/margin_mean": 96.78585815429688,
      "margin_dpo/margin_std": 112.66618347167969,
      "step": 416
    },
    {
      "KL/chosen_KL_mean": -179.27740478515625,
      "KL/mean": -224.13714599609375,
      "KL/rejected_KL_mean": -268.99688720703125,
      "KL/std": 100.55647277832031,
      "epoch": 0.6123348017621145,
      "fcm_dpo/beta": 0.004694630391895771,
      "fcm_dpo/delta": -0.02425021305680275,
      "fcm_dpo/margin": 89.71949768066406,
      "fcm_dpo/q_t": 0.4011738896369934,
      "grad_norm": 9.072989463806152,
      "learning_rate": 1.9777104743594686e-07,
      "logits/chosen": -2.144528865814209,
      "logits/rejected": -1.2169300317764282,
      "logps/chosen": -262.812744140625,
      "logps/ref_chosen": -83.53533172607422,
      "logps/ref_rejected": -74.44184875488281,
      "logps/rejected": -343.4387512207031,
      "loss": 1.0643,
      "margin_dpo/margin_mean": 89.71949768066406,
      "margin_dpo/margin_std": 102.77764892578125,
      "step": 417
    },
    {
      "KL/chosen_KL_mean": -172.86984252929688,
      "KL/mean": -229.8451690673828,
      "KL/rejected_KL_mean": -286.82049560546875,
      "KL/std": 107.40227508544922,
      "epoch": 0.6138032305433186,
      "fcm_dpo/beta": 0.0045183803886175156,
      "fcm_dpo/delta": -0.1374366134405136,
      "fcm_dpo/margin": 113.95067596435547,
      "fcm_dpo/q_t": 0.3773067593574524,
      "grad_norm": 10.768372535705566,
      "learning_rate": 1.965167291983757e-07,
      "logits/chosen": -2.2745208740234375,
      "logits/rejected": -1.6285841464996338,
      "logps/chosen": -281.09136962890625,
      "logps/ref_chosen": -108.22152709960938,
      "logps/ref_rejected": -111.8646469116211,
      "logps/rejected": -398.6851501464844,
      "loss": 1.0,
      "margin_dpo/margin_mean": 113.95067596435547,
      "margin_dpo/margin_std": 105.04487609863281,
      "step": 418
    },
    {
      "KL/chosen_KL_mean": -179.87644958496094,
      "KL/mean": -229.48301696777344,
      "KL/rejected_KL_mean": -279.089599609375,
      "KL/std": 95.82611083984375,
      "epoch": 0.6152716593245228,
      "fcm_dpo/beta": 0.004291839897632599,
      "fcm_dpo/delta": -0.030301060527563095,
      "fcm_dpo/margin": 99.213134765625,
      "fcm_dpo/q_t": 0.39939647912979126,
      "grad_norm": 11.219310760498047,
      "learning_rate": 1.9526382029472988e-07,
      "logits/chosen": -1.2081050872802734,
      "logits/rejected": -0.742085337638855,
      "logps/chosen": -277.05975341796875,
      "logps/ref_chosen": -97.18328094482422,
      "logps/ref_rejected": -98.18531799316406,
      "logps/rejected": -377.27490234375,
      "loss": 1.0552,
      "margin_dpo/margin_mean": 99.213134765625,
      "margin_dpo/margin_std": 105.02522277832031,
      "step": 419
    },
    {
      "KL/chosen_KL_mean": -184.5465087890625,
      "KL/mean": -228.37522888183594,
      "KL/rejected_KL_mean": -272.2039794921875,
      "KL/std": 100.86854553222656,
      "epoch": 0.6167400881057269,
      "fcm_dpo/beta": 0.0043035587295889854,
      "fcm_dpo/delta": 0.024852512404322624,
      "fcm_dpo/margin": 87.65745544433594,
      "fcm_dpo/q_t": 0.41257575154304504,
      "grad_norm": 13.900484085083008,
      "learning_rate": 1.9401235374032425e-07,
      "logits/chosen": -2.080049753189087,
      "logits/rejected": -0.9489184617996216,
      "logps/chosen": -298.85498046875,
      "logps/ref_chosen": -114.30848693847656,
      "logps/ref_rejected": -75.68355560302734,
      "logps/rejected": -347.88751220703125,
      "loss": 1.1039,
      "margin_dpo/margin_mean": 87.65745544433594,
      "margin_dpo/margin_std": 115.92739868164062,
      "step": 420
    },
    {
      "KL/chosen_KL_mean": -174.69720458984375,
      "KL/mean": -223.8994598388672,
      "KL/rejected_KL_mean": -273.1016845703125,
      "KL/std": 104.28186798095703,
      "epoch": 0.618208516886931,
      "fcm_dpo/beta": 0.004228582605719566,
      "fcm_dpo/delta": -0.018615929409861565,
      "fcm_dpo/margin": 98.40449523925781,
      "fcm_dpo/q_t": 0.40195125341415405,
      "grad_norm": 7.445271015167236,
      "learning_rate": 1.9276236251246653e-07,
      "logits/chosen": -1.8981404304504395,
      "logits/rejected": -1.528111219406128,
      "logps/chosen": -260.5770568847656,
      "logps/ref_chosen": -85.87985229492188,
      "logps/ref_rejected": -96.33648681640625,
      "logps/rejected": -369.43817138671875,
      "loss": 1.0695,
      "margin_dpo/margin_mean": 98.40449523925781,
      "margin_dpo/margin_std": 114.79826354980469,
      "step": 421
    },
    {
      "KL/chosen_KL_mean": -173.000732421875,
      "KL/mean": -219.85577392578125,
      "KL/rejected_KL_mean": -266.7108154296875,
      "KL/std": 96.28413391113281,
      "epoch": 0.6196769456681351,
      "fcm_dpo/beta": 0.004249933175742626,
      "fcm_dpo/delta": 0.0019067898392677307,
      "fcm_dpo/margin": 93.7101058959961,
      "fcm_dpo/q_t": 0.40683144330978394,
      "grad_norm": 11.853196144104004,
      "learning_rate": 1.9151387954958792e-07,
      "logits/chosen": -2.5530638694763184,
      "logits/rejected": -2.198111057281494,
      "logps/chosen": -273.4813232421875,
      "logps/ref_chosen": -100.48060607910156,
      "logps/ref_rejected": -94.40821838378906,
      "logps/rejected": -361.1190490722656,
      "loss": 1.0767,
      "margin_dpo/margin_mean": 93.71011352539062,
      "margin_dpo/margin_std": 107.87059020996094,
      "step": 422
    },
    {
      "KL/chosen_KL_mean": -160.7986602783203,
      "KL/mean": -208.81069946289062,
      "KL/rejected_KL_mean": -256.82275390625,
      "KL/std": 115.53514862060547,
      "epoch": 0.6211453744493393,
      "fcm_dpo/beta": 0.0042663393542170525,
      "fcm_dpo/delta": -0.011852338910102844,
      "fcm_dpo/margin": 96.02410125732422,
      "fcm_dpo/q_t": 0.4059971868991852,
      "grad_norm": 9.719646453857422,
      "learning_rate": 1.902669377503756e-07,
      "logits/chosen": -1.835792064666748,
      "logits/rejected": -1.7160398960113525,
      "logps/chosen": -239.24859619140625,
      "logps/ref_chosen": -78.44993591308594,
      "logps/ref_rejected": -92.04652404785156,
      "logps/rejected": -348.8692626953125,
      "loss": 1.0926,
      "margin_dpo/margin_mean": 96.02409362792969,
      "margin_dpo/margin_std": 130.7830352783203,
      "step": 423
    },
    {
      "KL/chosen_KL_mean": -180.33721923828125,
      "KL/mean": -226.78573608398438,
      "KL/rejected_KL_mean": -273.2342529296875,
      "KL/std": 102.81611633300781,
      "epoch": 0.6226138032305433,
      "fcm_dpo/beta": 0.004220573231577873,
      "fcm_dpo/delta": 0.008978116326034069,
      "fcm_dpo/margin": 92.89704895019531,
      "fcm_dpo/q_t": 0.40758711099624634,
      "grad_norm": 8.20677661895752,
      "learning_rate": 1.890215699729057e-07,
      "logits/chosen": -1.8386406898498535,
      "logits/rejected": -1.2513036727905273,
      "logps/chosen": -267.97955322265625,
      "logps/ref_chosen": -87.64236450195312,
      "logps/ref_rejected": -72.36566162109375,
      "logps/rejected": -345.59991455078125,
      "loss": 1.0852,
      "margin_dpo/margin_mean": 92.89704895019531,
      "margin_dpo/margin_std": 111.38961029052734,
      "step": 424
    },
    {
      "KL/chosen_KL_mean": -187.97508239746094,
      "KL/mean": -219.7484130859375,
      "KL/rejected_KL_mean": -251.521728515625,
      "KL/std": 106.50869750976562,
      "epoch": 0.6240822320117474,
      "fcm_dpo/beta": 0.004459555726498365,
      "fcm_dpo/delta": 0.1278761327266693,
      "fcm_dpo/margin": 63.54664993286133,
      "fcm_dpo/q_t": 0.43854033946990967,
      "grad_norm": 10.282400131225586,
      "learning_rate": 1.8777780903377732e-07,
      "logits/chosen": -1.5902190208435059,
      "logits/rejected": -1.5254371166229248,
      "logps/chosen": -266.494873046875,
      "logps/ref_chosen": -78.51979064941406,
      "logps/ref_rejected": -102.74864196777344,
      "logps/rejected": -354.2703857421875,
      "loss": 1.2062,
      "margin_dpo/margin_mean": 63.54664993286133,
      "margin_dpo/margin_std": 133.04296875,
      "step": 425
    },
    {
      "KL/chosen_KL_mean": -160.654052734375,
      "KL/mean": -206.70767211914062,
      "KL/rejected_KL_mean": -252.7612762451172,
      "KL/std": 97.38905334472656,
      "epoch": 0.6255506607929515,
      "fcm_dpo/beta": 0.004557386506348848,
      "fcm_dpo/delta": -0.0228448286652565,
      "fcm_dpo/margin": 92.10722351074219,
      "fcm_dpo/q_t": 0.4017777144908905,
      "grad_norm": 10.697813987731934,
      "learning_rate": 1.8653568770724803e-07,
      "logits/chosen": -1.6788634061813354,
      "logits/rejected": -0.9344384670257568,
      "logps/chosen": -269.1598815917969,
      "logps/ref_chosen": -108.50582885742188,
      "logps/ref_rejected": -88.300048828125,
      "logps/rejected": -341.06134033203125,
      "loss": 1.0744,
      "margin_dpo/margin_mean": 92.10722351074219,
      "margin_dpo/margin_std": 113.7043685913086,
      "step": 426
    },
    {
      "KL/chosen_KL_mean": -132.01043701171875,
      "KL/mean": -174.29811096191406,
      "KL/rejected_KL_mean": -216.5857696533203,
      "KL/std": 92.57152557373047,
      "epoch": 0.6270190895741556,
      "fcm_dpo/beta": 0.004553237464278936,
      "fcm_dpo/delta": 0.01674714870750904,
      "fcm_dpo/margin": 84.57534790039062,
      "fcm_dpo/q_t": 0.40978169441223145,
      "grad_norm": 9.079821586608887,
      "learning_rate": 1.8529523872436977e-07,
      "logits/chosen": -2.437335729598999,
      "logits/rejected": -1.807018756866455,
      "logps/chosen": -231.13088989257812,
      "logps/ref_chosen": -99.12046813964844,
      "logps/ref_rejected": -85.724609375,
      "logps/rejected": -302.31036376953125,
      "loss": 1.0869,
      "margin_dpo/margin_mean": 84.57534790039062,
      "margin_dpo/margin_std": 99.69166564941406,
      "step": 427
    },
    {
      "KL/chosen_KL_mean": -188.3460235595703,
      "KL/mean": -224.98880004882812,
      "KL/rejected_KL_mean": -261.631591796875,
      "KL/std": 104.60513305664062,
      "epoch": 0.6284875183553598,
      "fcm_dpo/beta": 0.004630422219634056,
      "fcm_dpo/delta": 0.06722044199705124,
      "fcm_dpo/margin": 73.28558349609375,
      "fcm_dpo/q_t": 0.4228600263595581,
      "grad_norm": 10.577934265136719,
      "learning_rate": 1.8405649477212697e-07,
      "logits/chosen": -1.5618274211883545,
      "logits/rejected": -1.295703411102295,
      "logps/chosen": -294.3152770996094,
      "logps/ref_chosen": -105.96925354003906,
      "logps/ref_rejected": -109.10218048095703,
      "logps/rejected": -370.7337646484375,
      "loss": 1.1472,
      "margin_dpo/margin_mean": 73.28557586669922,
      "margin_dpo/margin_std": 116.00077819824219,
      "step": 428
    },
    {
      "KL/chosen_KL_mean": -172.6846923828125,
      "KL/mean": -219.82415771484375,
      "KL/rejected_KL_mean": -266.963623046875,
      "KL/std": 109.58072662353516,
      "epoch": 0.6299559471365639,
      "fcm_dpo/beta": 0.004730083979666233,
      "fcm_dpo/delta": -0.05524929612874985,
      "fcm_dpo/margin": 94.2789306640625,
      "fcm_dpo/q_t": 0.39745980501174927,
      "grad_norm": 8.733163833618164,
      "learning_rate": 1.828194884925749e-07,
      "logits/chosen": -1.8972601890563965,
      "logits/rejected": -1.17545485496521,
      "logps/chosen": -286.22955322265625,
      "logps/ref_chosen": -113.54487609863281,
      "logps/ref_rejected": -98.24201965332031,
      "logps/rejected": -365.20562744140625,
      "loss": 1.0763,
      "margin_dpo/margin_mean": 94.27892303466797,
      "margin_dpo/margin_std": 124.11846923828125,
      "step": 429
    },
    {
      "KL/chosen_KL_mean": -163.82456970214844,
      "KL/mean": -200.0740509033203,
      "KL/rejected_KL_mean": -236.32354736328125,
      "KL/std": 101.28489685058594,
      "epoch": 0.631424375917768,
      "fcm_dpo/beta": 0.0047556450590491295,
      "fcm_dpo/delta": 0.06030133739113808,
      "fcm_dpo/margin": 72.49897766113281,
      "fcm_dpo/q_t": 0.4218756854534149,
      "grad_norm": 9.376683235168457,
      "learning_rate": 1.8158425248197928e-07,
      "logits/chosen": -1.2355812788009644,
      "logits/rejected": -1.1989240646362305,
      "logps/chosen": -255.14393615722656,
      "logps/ref_chosen": -91.31936645507812,
      "logps/ref_rejected": -110.1096420288086,
      "logps/rejected": -346.43316650390625,
      "loss": 1.1454,
      "margin_dpo/margin_mean": 72.49897003173828,
      "margin_dpo/margin_std": 117.64134216308594,
      "step": 430
    },
    {
      "KL/chosen_KL_mean": -164.9702606201172,
      "KL/mean": -208.40426635742188,
      "KL/rejected_KL_mean": -251.83824157714844,
      "KL/std": 104.21295166015625,
      "epoch": 0.6328928046989721,
      "fcm_dpo/beta": 0.0047088805586099625,
      "fcm_dpo/delta": -0.010538321919739246,
      "fcm_dpo/margin": 86.86798095703125,
      "fcm_dpo/q_t": 0.40513521432876587,
      "grad_norm": 10.713359832763672,
      "learning_rate": 1.8035081928995788e-07,
      "logits/chosen": -2.0422730445861816,
      "logits/rejected": -1.6641854047775269,
      "logps/chosen": -258.1514892578125,
      "logps/ref_chosen": -93.18122100830078,
      "logps/ref_rejected": -98.13226318359375,
      "logps/rejected": -349.97052001953125,
      "loss": 1.0918,
      "margin_dpo/margin_mean": 86.86798858642578,
      "margin_dpo/margin_std": 118.10989379882812,
      "step": 431
    },
    {
      "KL/chosen_KL_mean": -144.2672119140625,
      "KL/mean": -196.8187713623047,
      "KL/rejected_KL_mean": -249.370361328125,
      "KL/std": 108.03378295898438,
      "epoch": 0.6343612334801763,
      "fcm_dpo/beta": 0.004574599210172892,
      "fcm_dpo/delta": -0.09415516257286072,
      "fcm_dpo/margin": 105.10311889648438,
      "fcm_dpo/q_t": 0.385455846786499,
      "grad_norm": 8.79653263092041,
      "learning_rate": 1.791192214186223e-07,
      "logits/chosen": -1.4437077045440674,
      "logits/rejected": -0.8842568397521973,
      "logps/chosen": -248.70199584960938,
      "logps/ref_chosen": -104.43478393554688,
      "logps/ref_rejected": -105.08955383300781,
      "logps/rejected": -354.45989990234375,
      "loss": 1.019,
      "margin_dpo/margin_mean": 105.10311889648438,
      "margin_dpo/margin_std": 104.12774658203125,
      "step": 432
    },
    {
      "KL/chosen_KL_mean": -172.90261840820312,
      "KL/mean": -213.42002868652344,
      "KL/rejected_KL_mean": -253.9374237060547,
      "KL/std": 101.65010070800781,
      "epoch": 0.6358296622613803,
      "fcm_dpo/beta": 0.004587068688124418,
      "fcm_dpo/delta": 0.030077029019594193,
      "fcm_dpo/margin": 81.03480529785156,
      "fcm_dpo/q_t": 0.4143836498260498,
      "grad_norm": 9.124029159545898,
      "learning_rate": 1.7788949132172193e-07,
      "logits/chosen": -1.6968985795974731,
      "logits/rejected": -1.378476619720459,
      "logps/chosen": -262.745849609375,
      "logps/ref_chosen": -89.84322357177734,
      "logps/ref_rejected": -101.73345947265625,
      "logps/rejected": -355.6708984375,
      "loss": 1.1171,
      "margin_dpo/margin_mean": 81.03479766845703,
      "margin_dpo/margin_std": 115.22895050048828,
      "step": 433
    },
    {
      "KL/chosen_KL_mean": -149.13882446289062,
      "KL/mean": -195.56573486328125,
      "KL/rejected_KL_mean": -241.99267578125,
      "KL/std": 113.62652587890625,
      "epoch": 0.6372980910425844,
      "fcm_dpo/beta": 0.004514506086707115,
      "fcm_dpo/delta": -0.021943101659417152,
      "fcm_dpo/margin": 92.85387420654297,
      "fcm_dpo/q_t": 0.40593937039375305,
      "grad_norm": 12.128223419189453,
      "learning_rate": 1.7666166140378853e-07,
      "logits/chosen": -2.62402606010437,
      "logits/rejected": -1.9750876426696777,
      "logps/chosen": -246.8313446044922,
      "logps/ref_chosen": -97.6925277709961,
      "logps/ref_rejected": -84.09129333496094,
      "logps/rejected": -326.083984375,
      "loss": 1.0907,
      "margin_dpo/margin_mean": 92.85387420654297,
      "margin_dpo/margin_std": 132.69508361816406,
      "step": 434
    },
    {
      "KL/chosen_KL_mean": -147.19473266601562,
      "KL/mean": -194.84548950195312,
      "KL/rejected_KL_mean": -242.49627685546875,
      "KL/std": 105.22879791259766,
      "epoch": 0.6387665198237885,
      "fcm_dpo/beta": 0.004449313506484032,
      "fcm_dpo/delta": -0.027526747435331345,
      "fcm_dpo/margin": 95.30155944824219,
      "fcm_dpo/q_t": 0.4018566310405731,
      "grad_norm": 10.277752876281738,
      "learning_rate": 1.7543576401928218e-07,
      "logits/chosen": -2.336972713470459,
      "logits/rejected": -1.8863847255706787,
      "logps/chosen": -233.3666534423828,
      "logps/ref_chosen": -86.17192077636719,
      "logps/ref_rejected": -93.751708984375,
      "logps/rejected": -336.24798583984375,
      "loss": 1.0699,
      "margin_dpo/margin_mean": 95.30156707763672,
      "margin_dpo/margin_std": 117.22315216064453,
      "step": 435
    },
    {
      "KL/chosen_KL_mean": -122.23216247558594,
      "KL/mean": -171.57887268066406,
      "KL/rejected_KL_mean": -220.92556762695312,
      "KL/std": 88.68540954589844,
      "epoch": 0.6402349486049926,
      "fcm_dpo/beta": 0.0043383194133639336,
      "fcm_dpo/delta": -0.03245806321501732,
      "fcm_dpo/margin": 98.69340515136719,
      "fcm_dpo/q_t": 0.39757591485977173,
      "grad_norm": 9.920540809631348,
      "learning_rate": 1.742118314717391e-07,
      "logits/chosen": -1.7444126605987549,
      "logits/rejected": -0.8790333867073059,
      "logps/chosen": -228.01925659179688,
      "logps/ref_chosen": -105.78710174560547,
      "logps/ref_rejected": -88.62471008300781,
      "logps/rejected": -309.55029296875,
      "loss": 1.0408,
      "margin_dpo/margin_mean": 98.69340515136719,
      "margin_dpo/margin_std": 91.44416046142578,
      "step": 436
    },
    {
      "KL/chosen_KL_mean": -143.0594482421875,
      "KL/mean": -183.3048095703125,
      "KL/rejected_KL_mean": -223.55015563964844,
      "KL/std": 95.158447265625,
      "epoch": 0.6417033773861968,
      "fcm_dpo/beta": 0.004439051263034344,
      "fcm_dpo/delta": 0.04747384414076805,
      "fcm_dpo/margin": 80.49072265625,
      "fcm_dpo/q_t": 0.4154191315174103,
      "grad_norm": 10.372462272644043,
      "learning_rate": 1.7298989601292036e-07,
      "logits/chosen": -1.8373775482177734,
      "logits/rejected": -1.1662853956222534,
      "logps/chosen": -239.12149047851562,
      "logps/ref_chosen": -96.06204223632812,
      "logps/ref_rejected": -89.01220703125,
      "logps/rejected": -312.5623779296875,
      "loss": 1.1029,
      "margin_dpo/margin_mean": 80.49072265625,
      "margin_dpo/margin_std": 94.30258178710938,
      "step": 437
    },
    {
      "KL/chosen_KL_mean": -147.1654052734375,
      "KL/mean": -188.3097686767578,
      "KL/rejected_KL_mean": -229.45411682128906,
      "KL/std": 96.0766372680664,
      "epoch": 0.6431718061674009,
      "fcm_dpo/beta": 0.0044701602309942245,
      "fcm_dpo/delta": 0.03555918484926224,
      "fcm_dpo/margin": 82.28868865966797,
      "fcm_dpo/q_t": 0.41408777236938477,
      "grad_norm": 9.660764694213867,
      "learning_rate": 1.7176998984196144e-07,
      "logits/chosen": -2.1728670597076416,
      "logits/rejected": -1.3671059608459473,
      "logps/chosen": -249.0207977294922,
      "logps/ref_chosen": -101.85537719726562,
      "logps/ref_rejected": -89.44766235351562,
      "logps/rejected": -318.90179443359375,
      "loss": 1.1053,
      "margin_dpo/margin_mean": 82.2886962890625,
      "margin_dpo/margin_std": 103.2186508178711,
      "step": 438
    },
    {
      "KL/chosen_KL_mean": -158.0364227294922,
      "KL/mean": -199.07489013671875,
      "KL/rejected_KL_mean": -240.11334228515625,
      "KL/std": 104.54864501953125,
      "epoch": 0.644640234948605,
      "fcm_dpo/beta": 0.004521198570728302,
      "fcm_dpo/delta": 0.030363779515028,
      "fcm_dpo/margin": 82.0769271850586,
      "fcm_dpo/q_t": 0.41379404067993164,
      "grad_norm": 9.90652084350586,
      "learning_rate": 1.7055214510452458e-07,
      "logits/chosen": -1.7813705205917358,
      "logits/rejected": -1.5516353845596313,
      "logps/chosen": -239.79205322265625,
      "logps/ref_chosen": -81.75563049316406,
      "logps/ref_rejected": -90.58636474609375,
      "logps/rejected": -330.69970703125,
      "loss": 1.1058,
      "margin_dpo/margin_mean": 82.0769271850586,
      "margin_dpo/margin_std": 102.00228881835938,
      "step": 439
    },
    {
      "KL/chosen_KL_mean": -141.97576904296875,
      "KL/mean": -192.90823364257812,
      "KL/rejected_KL_mean": -243.84071350097656,
      "KL/std": 105.65991973876953,
      "epoch": 0.6461086637298091,
      "fcm_dpo/beta": 0.004432066343724728,
      "fcm_dpo/delta": -0.06259665638208389,
      "fcm_dpo/margin": 101.86494445800781,
      "fcm_dpo/q_t": 0.3940266966819763,
      "grad_norm": 7.6525678634643555,
      "learning_rate": 1.6933639389195134e-07,
      "logits/chosen": -2.4826197624206543,
      "logits/rejected": -1.9908812046051025,
      "logps/chosen": -247.61685180664062,
      "logps/ref_chosen": -105.64108276367188,
      "logps/ref_rejected": -103.4010009765625,
      "logps/rejected": -347.2417297363281,
      "loss": 1.0496,
      "margin_dpo/margin_mean": 101.86494445800781,
      "margin_dpo/margin_std": 116.87601470947266,
      "step": 440
    },
    {
      "KL/chosen_KL_mean": -183.71011352539062,
      "KL/mean": -226.33160400390625,
      "KL/rejected_KL_mean": -268.953125,
      "KL/std": 100.30284118652344,
      "epoch": 0.6475770925110133,
      "fcm_dpo/beta": 0.004483656957745552,
      "fcm_dpo/delta": 0.020073164254426956,
      "fcm_dpo/margin": 85.24299621582031,
      "fcm_dpo/q_t": 0.4125629663467407,
      "grad_norm": 8.66860294342041,
      "learning_rate": 1.681227682404166e-07,
      "logits/chosen": -1.8231854438781738,
      "logits/rejected": -1.4671202898025513,
      "logps/chosen": -275.2396545410156,
      "logps/ref_chosen": -91.529541015625,
      "logps/ref_rejected": -103.619384765625,
      "logps/rejected": -372.572509765625,
      "loss": 1.0963,
      "margin_dpo/margin_mean": 85.24299621582031,
      "margin_dpo/margin_std": 110.10692596435547,
      "step": 441
    },
    {
      "KL/chosen_KL_mean": -179.138427734375,
      "KL/mean": -233.36590576171875,
      "KL/rejected_KL_mean": -287.5933837890625,
      "KL/std": 106.05074310302734,
      "epoch": 0.6490455212922174,
      "fcm_dpo/beta": 0.00435262406244874,
      "fcm_dpo/delta": -0.08395692706108093,
      "fcm_dpo/margin": 108.45498657226562,
      "fcm_dpo/q_t": 0.3873000741004944,
      "grad_norm": 10.513046264648438,
      "learning_rate": 1.669113001300851e-07,
      "logits/chosen": -1.5017528533935547,
      "logits/rejected": -1.0180792808532715,
      "logps/chosen": -263.9159851074219,
      "logps/ref_chosen": -84.77755737304688,
      "logps/ref_rejected": -83.82415771484375,
      "logps/rejected": -371.41754150390625,
      "loss": 1.02,
      "margin_dpo/margin_mean": 108.45498657226562,
      "margin_dpo/margin_std": 107.21076202392578,
      "step": 442
    },
    {
      "KL/chosen_KL_mean": -174.88504028320312,
      "KL/mean": -208.43109130859375,
      "KL/rejected_KL_mean": -241.9771270751953,
      "KL/std": 103.67234802246094,
      "epoch": 0.6505139500734214,
      "fcm_dpo/beta": 0.0044685304164886475,
      "fcm_dpo/delta": 0.11051709949970245,
      "fcm_dpo/margin": 67.09209442138672,
      "fcm_dpo/q_t": 0.43155020475387573,
      "grad_norm": 10.2890625,
      "learning_rate": 1.6570202148426815e-07,
      "logits/chosen": -1.8728140592575073,
      "logits/rejected": -1.3166320323944092,
      "logps/chosen": -277.5343017578125,
      "logps/ref_chosen": -102.64927673339844,
      "logps/ref_rejected": -93.03807067871094,
      "logps/rejected": -335.01519775390625,
      "loss": 1.1611,
      "margin_dpo/margin_mean": 67.09209442138672,
      "margin_dpo/margin_std": 100.7808837890625,
      "step": 443
    },
    {
      "KL/chosen_KL_mean": -180.04351806640625,
      "KL/mean": -219.99757385253906,
      "KL/rejected_KL_mean": -259.95166015625,
      "KL/std": 100.77328491210938,
      "epoch": 0.6519823788546255,
      "fcm_dpo/beta": 0.004663398489356041,
      "fcm_dpo/delta": 0.03066403418779373,
      "fcm_dpo/margin": 79.90814208984375,
      "fcm_dpo/q_t": 0.41475749015808105,
      "grad_norm": 11.357672691345215,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": -1.694549798965454,
      "logits/rejected": -1.5304756164550781,
      "logps/chosen": -267.9632263183594,
      "logps/ref_chosen": -87.91971588134766,
      "logps/ref_rejected": -103.32345581054688,
      "logps/rejected": -363.2751159667969,
      "loss": 1.1186,
      "margin_dpo/margin_mean": 79.90814208984375,
      "margin_dpo/margin_std": 117.36084747314453,
      "step": 444
    },
    {
      "KL/chosen_KL_mean": -157.38235473632812,
      "KL/mean": -203.77230834960938,
      "KL/rejected_KL_mean": -250.1622314453125,
      "KL/std": 98.03376770019531,
      "epoch": 0.6534508076358296,
      "fcm_dpo/beta": 0.004563162103295326,
      "fcm_dpo/delta": -0.02805071324110031,
      "fcm_dpo/margin": 92.77989196777344,
      "fcm_dpo/q_t": 0.4000878632068634,
      "grad_norm": 8.790597915649414,
      "learning_rate": 1.6329015999011182e-07,
      "logits/chosen": -1.849029302597046,
      "logits/rejected": -1.4269061088562012,
      "logps/chosen": -258.7832336425781,
      "logps/ref_chosen": -101.40086364746094,
      "logps/ref_rejected": -99.03790283203125,
      "logps/rejected": -349.20013427734375,
      "loss": 1.061,
      "margin_dpo/margin_mean": 92.77989959716797,
      "margin_dpo/margin_std": 103.64060974121094,
      "step": 445
    },
    {
      "KL/chosen_KL_mean": -151.82879638671875,
      "KL/mean": -193.26800537109375,
      "KL/rejected_KL_mean": -234.7071990966797,
      "KL/std": 103.36875915527344,
      "epoch": 0.6549192364170338,
      "fcm_dpo/beta": 0.004653014708310366,
      "fcm_dpo/delta": 0.015950417146086693,
      "fcm_dpo/margin": 82.87838745117188,
      "fcm_dpo/q_t": 0.41009950637817383,
      "grad_norm": 13.165481567382812,
      "learning_rate": 1.6208764069656578e-07,
      "logits/chosen": -1.8748112916946411,
      "logits/rejected": -1.7350090742111206,
      "logps/chosen": -239.2511444091797,
      "logps/ref_chosen": -87.4223403930664,
      "logps/ref_rejected": -106.70075988769531,
      "logps/rejected": -341.407958984375,
      "loss": 1.0891,
      "margin_dpo/margin_mean": 82.8783950805664,
      "margin_dpo/margin_std": 99.96321868896484,
      "step": 446
    },
    {
      "KL/chosen_KL_mean": -150.55990600585938,
      "KL/mean": -205.0340576171875,
      "KL/rejected_KL_mean": -259.50823974609375,
      "KL/std": 107.98631286621094,
      "epoch": 0.6563876651982379,
      "fcm_dpo/beta": 0.004435893148183823,
      "fcm_dpo/delta": -0.09765411913394928,
      "fcm_dpo/margin": 108.94831848144531,
      "fcm_dpo/q_t": 0.38633695244789124,
      "grad_norm": 8.529854774475098,
      "learning_rate": 1.608874379754465e-07,
      "logits/chosen": -2.0928096771240234,
      "logits/rejected": -2.049389600753784,
      "logps/chosen": -234.17515563964844,
      "logps/ref_chosen": -83.6152572631836,
      "logps/ref_rejected": -104.91239929199219,
      "logps/rejected": -364.420654296875,
      "loss": 1.032,
      "margin_dpo/margin_mean": 108.94833374023438,
      "margin_dpo/margin_std": 123.88082885742188,
      "step": 447
    },
    {
      "KL/chosen_KL_mean": -161.83956909179688,
      "KL/mean": -214.49819946289062,
      "KL/rejected_KL_mean": -267.15679931640625,
      "KL/std": 112.66374206542969,
      "epoch": 0.657856093979442,
      "fcm_dpo/beta": 0.0043128253892064095,
      "fcm_dpo/delta": -0.06316135078668594,
      "fcm_dpo/margin": 105.31723022460938,
      "fcm_dpo/q_t": 0.3942852020263672,
      "grad_norm": 9.834023475646973,
      "learning_rate": 1.5968958345321177e-07,
      "logits/chosen": -1.8544933795928955,
      "logits/rejected": -1.5904440879821777,
      "logps/chosen": -254.41534423828125,
      "logps/ref_chosen": -92.5757827758789,
      "logps/ref_rejected": -107.68977355957031,
      "logps/rejected": -374.8465881347656,
      "loss": 1.0625,
      "margin_dpo/margin_mean": 105.31722259521484,
      "margin_dpo/margin_std": 131.88259887695312,
      "step": 448
    },
    {
      "KL/chosen_KL_mean": -159.3519287109375,
      "KL/mean": -214.3835906982422,
      "KL/rejected_KL_mean": -269.41522216796875,
      "KL/std": 105.94546508789062,
      "epoch": 0.6593245227606461,
      "fcm_dpo/beta": 0.004099005367606878,
      "fcm_dpo/delta": -0.05925465375185013,
      "fcm_dpo/margin": 110.06328582763672,
      "fcm_dpo/q_t": 0.3932005763053894,
      "grad_norm": 8.413712501525879,
      "learning_rate": 1.584941086944423e-07,
      "logits/chosen": -1.3945093154907227,
      "logits/rejected": -0.9447757601737976,
      "logps/chosen": -261.7508544921875,
      "logps/ref_chosen": -102.39893341064453,
      "logps/ref_rejected": -95.14886474609375,
      "logps/rejected": -364.5640869140625,
      "loss": 1.0439,
      "margin_dpo/margin_mean": 110.06328582763672,
      "margin_dpo/margin_std": 121.78463745117188,
      "step": 449
    },
    {
      "KL/chosen_KL_mean": -142.34555053710938,
      "KL/mean": -193.530029296875,
      "KL/rejected_KL_mean": -244.7144775390625,
      "KL/std": 107.21269226074219,
      "epoch": 0.6607929515418502,
      "fcm_dpo/beta": 0.004027402028441429,
      "fcm_dpo/delta": -0.014135586097836494,
      "fcm_dpo/margin": 102.3689193725586,
      "fcm_dpo/q_t": 0.4025944173336029,
      "grad_norm": 8.689202308654785,
      "learning_rate": 1.573010452010098e-07,
      "logits/chosen": -2.3539681434631348,
      "logits/rejected": -2.1625888347625732,
      "logps/chosen": -229.33840942382812,
      "logps/ref_chosen": -86.99285888671875,
      "logps/ref_rejected": -108.53203582763672,
      "logps/rejected": -353.24652099609375,
      "loss": 1.0684,
      "margin_dpo/margin_mean": 102.36891174316406,
      "margin_dpo/margin_std": 116.56834411621094,
      "step": 450
    },
    {
      "KL/chosen_KL_mean": -161.6459503173828,
      "KL/mean": -202.09527587890625,
      "KL/rejected_KL_mean": -242.54457092285156,
      "KL/std": 106.30159759521484,
      "epoch": 0.6622613803230544,
      "fcm_dpo/beta": 0.004178112838417292,
      "fcm_dpo/delta": 0.0684027448296547,
      "fcm_dpo/margin": 80.89863586425781,
      "fcm_dpo/q_t": 0.42326316237449646,
      "grad_norm": 12.71623420715332,
      "learning_rate": 1.5611042441124687e-07,
      "logits/chosen": -2.4614641666412354,
      "logits/rejected": -1.962374210357666,
      "logps/chosen": -248.45724487304688,
      "logps/ref_chosen": -86.811279296875,
      "logps/ref_rejected": -79.8555908203125,
      "logps/rejected": -322.400146484375,
      "loss": 1.1495,
      "margin_dpo/margin_mean": 80.89862823486328,
      "margin_dpo/margin_std": 132.44105529785156,
      "step": 451
    },
    {
      "KL/chosen_KL_mean": -140.71702575683594,
      "KL/mean": -186.70399475097656,
      "KL/rejected_KL_mean": -232.6909637451172,
      "KL/std": 101.13810729980469,
      "epoch": 0.6637298091042585,
      "fcm_dpo/beta": 0.004149815067648888,
      "fcm_dpo/delta": 0.018123138695955276,
      "fcm_dpo/margin": 91.97393798828125,
      "fcm_dpo/q_t": 0.4097292721271515,
      "grad_norm": 8.2000150680542,
      "learning_rate": 1.549222776991186e-07,
      "logits/chosen": -1.8395628929138184,
      "logits/rejected": -1.7836477756500244,
      "logps/chosen": -220.09666442871094,
      "logps/ref_chosen": -79.379638671875,
      "logps/ref_rejected": -103.71539306640625,
      "logps/rejected": -336.4063720703125,
      "loss": 1.0834,
      "margin_dpo/margin_mean": 91.97393798828125,
      "margin_dpo/margin_std": 94.09562683105469,
      "step": 452
    },
    {
      "KL/chosen_KL_mean": -170.69662475585938,
      "KL/mean": -226.325439453125,
      "KL/rejected_KL_mean": -281.9542541503906,
      "KL/std": 103.5924072265625,
      "epoch": 0.6651982378854625,
      "fcm_dpo/beta": 0.004129510372877121,
      "fcm_dpo/delta": -0.06882303208112717,
      "fcm_dpo/margin": 111.25761413574219,
      "fcm_dpo/q_t": 0.3913623094558716,
      "grad_norm": 11.710333824157715,
      "learning_rate": 1.5373663637339584e-07,
      "logits/chosen": -1.754602074623108,
      "logits/rejected": -1.2722032070159912,
      "logps/chosen": -258.3918151855469,
      "logps/ref_chosen": -87.6951904296875,
      "logps/ref_rejected": -90.0582275390625,
      "logps/rejected": -372.0124816894531,
      "loss": 1.0295,
      "margin_dpo/margin_mean": 111.25761413574219,
      "margin_dpo/margin_std": 111.42896270751953,
      "step": 453
    },
    {
      "KL/chosen_KL_mean": -178.69833374023438,
      "KL/mean": -223.53146362304688,
      "KL/rejected_KL_mean": -268.3646240234375,
      "KL/std": 106.26595306396484,
      "epoch": 0.6666666666666666,
      "fcm_dpo/beta": 0.004182643257081509,
      "fcm_dpo/delta": 0.026231005787849426,
      "fcm_dpo/margin": 89.66627502441406,
      "fcm_dpo/q_t": 0.41411131620407104,
      "grad_norm": 9.209973335266113,
      "learning_rate": 1.5255353167683017e-07,
      "logits/chosen": -2.101715087890625,
      "logits/rejected": -1.6619293689727783,
      "logps/chosen": -268.26458740234375,
      "logps/ref_chosen": -89.56623840332031,
      "logps/ref_rejected": -92.92105102539062,
      "logps/rejected": -361.28564453125,
      "loss": 1.1268,
      "margin_dpo/margin_mean": 89.66627502441406,
      "margin_dpo/margin_std": 137.81520080566406,
      "step": 454
    },
    {
      "KL/chosen_KL_mean": -168.0302276611328,
      "KL/mean": -210.7840576171875,
      "KL/rejected_KL_mean": -253.53787231445312,
      "KL/std": 104.66192626953125,
      "epoch": 0.6681350954478708,
      "fcm_dpo/beta": 0.004253532271832228,
      "fcm_dpo/delta": 0.039673611521720886,
      "fcm_dpo/margin": 85.50763702392578,
      "fcm_dpo/q_t": 0.41585594415664673,
      "grad_norm": 9.767982482910156,
      "learning_rate": 1.5137299478533064e-07,
      "logits/chosen": -1.7660424709320068,
      "logits/rejected": -1.7275199890136719,
      "logps/chosen": -245.66021728515625,
      "logps/ref_chosen": -77.62998962402344,
      "logps/ref_rejected": -118.97795104980469,
      "logps/rejected": -372.51580810546875,
      "loss": 1.1177,
      "margin_dpo/margin_mean": 85.50764465332031,
      "margin_dpo/margin_std": 119.07386016845703,
      "step": 455
    },
    {
      "KL/chosen_KL_mean": -170.60720825195312,
      "KL/mean": -213.54885864257812,
      "KL/rejected_KL_mean": -256.49053955078125,
      "KL/std": 106.12969207763672,
      "epoch": 0.6696035242290749,
      "fcm_dpo/beta": 0.004322017543017864,
      "fcm_dpo/delta": 0.03205161914229393,
      "fcm_dpo/margin": 85.88331604003906,
      "fcm_dpo/q_t": 0.41360485553741455,
      "grad_norm": 11.281591415405273,
      "learning_rate": 1.5019505680714232e-07,
      "logits/chosen": -2.0219459533691406,
      "logits/rejected": -1.890674352645874,
      "logps/chosen": -260.22406005859375,
      "logps/ref_chosen": -89.61686706542969,
      "logps/ref_rejected": -109.5597152709961,
      "logps/rejected": -366.05023193359375,
      "loss": 1.1004,
      "margin_dpo/margin_mean": 85.88330841064453,
      "margin_dpo/margin_std": 107.34074401855469,
      "step": 456
    },
    {
      "KL/chosen_KL_mean": -138.10423278808594,
      "KL/mean": -194.77252197265625,
      "KL/rejected_KL_mean": -251.44082641601562,
      "KL/std": 105.86021423339844,
      "epoch": 0.671071953010279,
      "fcm_dpo/beta": 0.004078992176800966,
      "fcm_dpo/delta": -0.07811039686203003,
      "fcm_dpo/margin": 113.33657836914062,
      "fcm_dpo/q_t": 0.38749006390571594,
      "grad_norm": 9.32401180267334,
      "learning_rate": 1.4901974878202627e-07,
      "logits/chosen": -2.111004590988159,
      "logits/rejected": -1.7944074869155884,
      "logps/chosen": -225.42591857910156,
      "logps/ref_chosen": -87.32168579101562,
      "logps/ref_rejected": -90.76660919189453,
      "logps/rejected": -342.2074279785156,
      "loss": 1.0163,
      "margin_dpo/margin_mean": 113.33657836914062,
      "margin_dpo/margin_std": 94.3172378540039,
      "step": 457
    },
    {
      "KL/chosen_KL_mean": -161.30873107910156,
      "KL/mean": -202.31646728515625,
      "KL/rejected_KL_mean": -243.32418823242188,
      "KL/std": 104.93587493896484,
      "epoch": 0.6725403817914831,
      "fcm_dpo/beta": 0.00425717793405056,
      "fcm_dpo/delta": 0.05602229759097099,
      "fcm_dpo/margin": 82.01546478271484,
      "fcm_dpo/q_t": 0.420804500579834,
      "grad_norm": 9.552821159362793,
      "learning_rate": 1.4784710168044212e-07,
      "logits/chosen": -1.6335763931274414,
      "logits/rejected": -1.3973894119262695,
      "logps/chosen": -254.8291778564453,
      "logps/ref_chosen": -93.52044677734375,
      "logps/ref_rejected": -103.36898803710938,
      "logps/rejected": -346.69317626953125,
      "loss": 1.1306,
      "margin_dpo/margin_mean": 82.01545715332031,
      "margin_dpo/margin_std": 122.19290161132812,
      "step": 458
    },
    {
      "KL/chosen_KL_mean": -154.06982421875,
      "KL/mean": -205.28054809570312,
      "KL/rejected_KL_mean": -256.49127197265625,
      "KL/std": 110.05068969726562,
      "epoch": 0.6740088105726872,
      "fcm_dpo/beta": 0.004148690029978752,
      "fcm_dpo/delta": -0.03041214309632778,
      "fcm_dpo/margin": 102.42147064208984,
      "fcm_dpo/q_t": 0.40067756175994873,
      "grad_norm": 7.182371616363525,
      "learning_rate": 1.466771464027316e-07,
      "logits/chosen": -1.5687854290008545,
      "logits/rejected": -1.3169442415237427,
      "logps/chosen": -229.75802612304688,
      "logps/ref_chosen": -75.68820190429688,
      "logps/ref_rejected": -92.17048645019531,
      "logps/rejected": -348.6617431640625,
      "loss": 1.064,
      "margin_dpo/margin_mean": 102.42147064208984,
      "margin_dpo/margin_std": 118.15473175048828,
      "step": 459
    },
    {
      "KL/chosen_KL_mean": -157.5782928466797,
      "KL/mean": -204.96127319335938,
      "KL/rejected_KL_mean": -252.34426879882812,
      "KL/std": 107.24034118652344,
      "epoch": 0.6754772393538914,
      "fcm_dpo/beta": 0.004154226742684841,
      "fcm_dpo/delta": 0.0064100101590156555,
      "fcm_dpo/margin": 94.76599884033203,
      "fcm_dpo/q_t": 0.4084334373474121,
      "grad_norm": 10.360857009887695,
      "learning_rate": 1.4550991377830423e-07,
      "logits/chosen": -2.112626552581787,
      "logits/rejected": -2.139860153198242,
      "logps/chosen": -238.6961669921875,
      "logps/ref_chosen": -81.11788940429688,
      "logps/ref_rejected": -110.31238555908203,
      "logps/rejected": -362.65667724609375,
      "loss": 1.0939,
      "margin_dpo/margin_mean": 94.76599884033203,
      "margin_dpo/margin_std": 122.21580505371094,
      "step": 460
    },
    {
      "KL/chosen_KL_mean": -193.7060546875,
      "KL/mean": -232.63333129882812,
      "KL/rejected_KL_mean": -271.56060791015625,
      "KL/std": 93.86126708984375,
      "epoch": 0.6769456681350955,
      "fcm_dpo/beta": 0.0043110474944114685,
      "fcm_dpo/delta": 0.07173923403024673,
      "fcm_dpo/margin": 77.85453796386719,
      "fcm_dpo/q_t": 0.42324110865592957,
      "grad_norm": 8.506885528564453,
      "learning_rate": 1.4434543456482518e-07,
      "logits/chosen": -1.6190330982208252,
      "logits/rejected": -1.4288344383239746,
      "logps/chosen": -275.2895812988281,
      "logps/ref_chosen": -81.58352661132812,
      "logps/ref_rejected": -93.87710571289062,
      "logps/rejected": -365.43768310546875,
      "loss": 1.1363,
      "margin_dpo/margin_mean": 77.85453796386719,
      "margin_dpo/margin_std": 112.27577209472656,
      "step": 461
    },
    {
      "KL/chosen_KL_mean": -147.86297607421875,
      "KL/mean": -196.80319213867188,
      "KL/rejected_KL_mean": -245.74343872070312,
      "KL/std": 100.94097137451172,
      "epoch": 0.6784140969162996,
      "fcm_dpo/beta": 0.004349173977971077,
      "fcm_dpo/delta": -0.02945837564766407,
      "fcm_dpo/margin": 97.8804702758789,
      "fcm_dpo/q_t": 0.40008318424224854,
      "grad_norm": 8.70224666595459,
      "learning_rate": 1.4318373944740484e-07,
      "logits/chosen": -2.1651620864868164,
      "logits/rejected": -1.6554949283599854,
      "logps/chosen": -242.0615234375,
      "logps/ref_chosen": -94.19854736328125,
      "logps/ref_rejected": -85.63162994384766,
      "logps/rejected": -331.37506103515625,
      "loss": 1.0583,
      "margin_dpo/margin_mean": 97.8804702758789,
      "margin_dpo/margin_std": 108.88066101074219,
      "step": 462
    },
    {
      "KL/chosen_KL_mean": -143.00173950195312,
      "KL/mean": -202.27598571777344,
      "KL/rejected_KL_mean": -261.5502014160156,
      "KL/std": 108.63600158691406,
      "epoch": 0.6798825256975036,
      "fcm_dpo/beta": 0.004180929623544216,
      "fcm_dpo/delta": -0.11265870928764343,
      "fcm_dpo/margin": 118.5484619140625,
      "fcm_dpo/q_t": 0.38105493783950806,
      "grad_norm": 13.2352294921875,
      "learning_rate": 1.4202485903778976e-07,
      "logits/chosen": -1.8831872940063477,
      "logits/rejected": -1.458146095275879,
      "logps/chosen": -228.92648315429688,
      "logps/ref_chosen": -85.92474365234375,
      "logps/ref_rejected": -96.90184020996094,
      "logps/rejected": -358.4520568847656,
      "loss": 1.0157,
      "margin_dpo/margin_mean": 118.54847717285156,
      "margin_dpo/margin_std": 116.07518005371094,
      "step": 463
    },
    {
      "KL/chosen_KL_mean": -176.09031677246094,
      "KL/mean": -224.56686401367188,
      "KL/rejected_KL_mean": -273.04339599609375,
      "KL/std": 107.88982391357422,
      "epoch": 0.6813509544787077,
      "fcm_dpo/beta": 0.004022590816020966,
      "fcm_dpo/delta": 0.011083902791142464,
      "fcm_dpo/margin": 96.95306396484375,
      "fcm_dpo/q_t": 0.4091755449771881,
      "grad_norm": 11.700246810913086,
      "learning_rate": 1.4086882387355658e-07,
      "logits/chosen": -1.8463342189788818,
      "logits/rejected": -1.926882266998291,
      "logps/chosen": -255.77951049804688,
      "logps/ref_chosen": -79.68919372558594,
      "logps/ref_rejected": -107.29232025146484,
      "logps/rejected": -380.3357238769531,
      "loss": 1.092,
      "margin_dpo/margin_mean": 96.95306396484375,
      "margin_dpo/margin_std": 123.13615417480469,
      "step": 464
    },
    {
      "KL/chosen_KL_mean": -168.25247192382812,
      "KL/mean": -224.71884155273438,
      "KL/rejected_KL_mean": -281.18524169921875,
      "KL/std": 117.23634338378906,
      "epoch": 0.6828193832599119,
      "fcm_dpo/beta": 0.00394630990922451,
      "fcm_dpo/delta": -0.053064748644828796,
      "fcm_dpo/margin": 112.9327621459961,
      "fcm_dpo/q_t": 0.3935995101928711,
      "grad_norm": 8.116555213928223,
      "learning_rate": 1.3971566441730714e-07,
      "logits/chosen": -1.6019678115844727,
      "logits/rejected": -1.2979724407196045,
      "logps/chosen": -260.1127014160156,
      "logps/ref_chosen": -91.8602294921875,
      "logps/ref_rejected": -118.71001434326172,
      "logps/rejected": -399.8952331542969,
      "loss": 1.0352,
      "margin_dpo/margin_mean": 112.9327621459961,
      "margin_dpo/margin_std": 111.34031677246094,
      "step": 465
    },
    {
      "KL/chosen_KL_mean": -149.10772705078125,
      "KL/mean": -206.70587158203125,
      "KL/rejected_KL_mean": -264.30401611328125,
      "KL/std": 111.58323669433594,
      "epoch": 0.684287812041116,
      "fcm_dpo/beta": 0.0038333022966980934,
      "fcm_dpo/delta": -0.04819294810295105,
      "fcm_dpo/margin": 115.19627380371094,
      "fcm_dpo/q_t": 0.3949740529060364,
      "grad_norm": 6.6179327964782715,
      "learning_rate": 1.3856541105586545e-07,
      "logits/chosen": -1.500025749206543,
      "logits/rejected": -1.1787292957305908,
      "logps/chosen": -233.80914306640625,
      "logps/ref_chosen": -84.70140838623047,
      "logps/ref_rejected": -96.05084228515625,
      "logps/rejected": -360.3548583984375,
      "loss": 1.0404,
      "margin_dpo/margin_mean": 115.19627380371094,
      "margin_dpo/margin_std": 117.2901611328125,
      "step": 466
    },
    {
      "KL/chosen_KL_mean": -193.5010528564453,
      "KL/mean": -240.939208984375,
      "KL/rejected_KL_mean": -288.37738037109375,
      "KL/std": 115.8831787109375,
      "epoch": 0.6857562408223201,
      "fcm_dpo/beta": 0.0038376045413315296,
      "fcm_dpo/delta": 0.03971167653799057,
      "fcm_dpo/margin": 94.87631225585938,
      "fcm_dpo/q_t": 0.4154435396194458,
      "grad_norm": 11.842679023742676,
      "learning_rate": 1.3741809409947729e-07,
      "logits/chosen": -2.413191318511963,
      "logits/rejected": -1.8436540365219116,
      "logps/chosen": -302.79937744140625,
      "logps/ref_chosen": -109.29832458496094,
      "logps/ref_rejected": -108.8436508178711,
      "logps/rejected": -397.22100830078125,
      "loss": 1.1179,
      "margin_dpo/margin_mean": 94.87630462646484,
      "margin_dpo/margin_std": 130.9418182373047,
      "step": 467
    },
    {
      "KL/chosen_KL_mean": -149.3004913330078,
      "KL/mean": -193.76126098632812,
      "KL/rejected_KL_mean": -238.2220001220703,
      "KL/std": 114.36317443847656,
      "epoch": 0.6872246696035242,
      "fcm_dpo/beta": 0.003988948650658131,
      "fcm_dpo/delta": 0.05076995864510536,
      "fcm_dpo/margin": 88.92147827148438,
      "fcm_dpo/q_t": 0.4194767475128174,
      "grad_norm": 9.085844039916992,
      "learning_rate": 1.362737437810114e-07,
      "logits/chosen": -1.9358673095703125,
      "logits/rejected": -1.7348198890686035,
      "logps/chosen": -247.62213134765625,
      "logps/ref_chosen": -98.32164764404297,
      "logps/ref_rejected": -106.68048095703125,
      "logps/rejected": -344.9024658203125,
      "loss": 1.1277,
      "margin_dpo/margin_mean": 88.9214859008789,
      "margin_dpo/margin_std": 131.6277313232422,
      "step": 468
    },
    {
      "KL/chosen_KL_mean": -183.00335693359375,
      "KL/mean": -237.8058624267578,
      "KL/rejected_KL_mean": -292.6083984375,
      "KL/std": 114.1338882446289,
      "epoch": 0.6886930983847284,
      "fcm_dpo/beta": 0.003907148726284504,
      "fcm_dpo/delta": -0.03473624959588051,
      "fcm_dpo/margin": 109.60501098632812,
      "fcm_dpo/q_t": 0.3989104926586151,
      "grad_norm": 9.854275703430176,
      "learning_rate": 1.351323902551631e-07,
      "logits/chosen": -2.301421642303467,
      "logits/rejected": -1.8870830535888672,
      "logps/chosen": -279.767578125,
      "logps/ref_chosen": -96.76420593261719,
      "logps/ref_rejected": -109.59500885009766,
      "logps/rejected": -402.203369140625,
      "loss": 1.0598,
      "margin_dpo/margin_mean": 109.60501098632812,
      "margin_dpo/margin_std": 122.76976776123047,
      "step": 469
    },
    {
      "KL/chosen_KL_mean": -168.78964233398438,
      "KL/mean": -218.98104858398438,
      "KL/rejected_KL_mean": -269.17242431640625,
      "KL/std": 106.06327819824219,
      "epoch": 0.6901615271659325,
      "fcm_dpo/beta": 0.003949745092540979,
      "fcm_dpo/delta": 0.003983854781836271,
      "fcm_dpo/margin": 100.3828125,
      "fcm_dpo/q_t": 0.4064105749130249,
      "grad_norm": 8.96664047241211,
      "learning_rate": 1.339940635976592e-07,
      "logits/chosen": -1.6086562871932983,
      "logits/rejected": -1.186509609222412,
      "logps/chosen": -252.28628540039062,
      "logps/ref_chosen": -83.49665832519531,
      "logps/ref_rejected": -88.48578643798828,
      "logps/rejected": -357.6582336425781,
      "loss": 1.0712,
      "margin_dpo/margin_mean": 100.3828125,
      "margin_dpo/margin_std": 106.93407440185547,
      "step": 470
    },
    {
      "KL/chosen_KL_mean": -192.95614624023438,
      "KL/mean": -240.3473358154297,
      "KL/rejected_KL_mean": -287.738525390625,
      "KL/std": 104.89599609375,
      "epoch": 0.6916299559471366,
      "fcm_dpo/beta": 0.003996941260993481,
      "fcm_dpo/delta": 0.023881491273641586,
      "fcm_dpo/margin": 94.78238677978516,
      "fcm_dpo/q_t": 0.41103821992874146,
      "grad_norm": 8.530881881713867,
      "learning_rate": 1.3285879380446563e-07,
      "logits/chosen": -1.416385531425476,
      "logits/rejected": -0.9859724044799805,
      "logps/chosen": -281.4304504394531,
      "logps/ref_chosen": -88.47430419921875,
      "logps/ref_rejected": -90.48171997070312,
      "logps/rejected": -378.2202453613281,
      "loss": 1.0854,
      "margin_dpo/margin_mean": 94.78238677978516,
      "margin_dpo/margin_std": 105.94229888916016,
      "step": 471
    },
    {
      "KL/chosen_KL_mean": -181.6434326171875,
      "KL/mean": -237.92559814453125,
      "KL/rejected_KL_mean": -294.207763671875,
      "KL/std": 106.37899017333984,
      "epoch": 0.6930983847283406,
      "fcm_dpo/beta": 0.0039006415754556656,
      "fcm_dpo/delta": -0.045889031141996384,
      "fcm_dpo/margin": 112.56432342529297,
      "fcm_dpo/q_t": 0.3962559998035431,
      "grad_norm": 7.7508039474487305,
      "learning_rate": 1.317266107909975e-07,
      "logits/chosen": -2.0804696083068848,
      "logits/rejected": -1.3716495037078857,
      "logps/chosen": -306.87713623046875,
      "logps/ref_chosen": -125.23370361328125,
      "logps/ref_rejected": -121.05349731445312,
      "logps/rejected": -415.2612609863281,
      "loss": 1.0413,
      "margin_dpo/margin_mean": 112.56431579589844,
      "margin_dpo/margin_std": 113.22309875488281,
      "step": 472
    },
    {
      "KL/chosen_KL_mean": -206.3040313720703,
      "KL/mean": -241.03343200683594,
      "KL/rejected_KL_mean": -275.7628173828125,
      "KL/std": 107.08447265625,
      "epoch": 0.6945668135095447,
      "fcm_dpo/beta": 0.004143530502915382,
      "fcm_dpo/delta": 0.12196655571460724,
      "fcm_dpo/margin": 69.45880126953125,
      "fcm_dpo/q_t": 0.43569839000701904,
      "grad_norm": 10.00335693359375,
      "learning_rate": 1.3059754439133002e-07,
      "logits/chosen": -1.9376657009124756,
      "logits/rejected": -1.457770586013794,
      "logps/chosen": -301.9154052734375,
      "logps/ref_chosen": -95.61137390136719,
      "logps/ref_rejected": -88.15115356445312,
      "logps/rejected": -363.91400146484375,
      "loss": 1.1819,
      "margin_dpo/margin_mean": 69.45880889892578,
      "margin_dpo/margin_std": 125.18821716308594,
      "step": 473
    },
    {
      "KL/chosen_KL_mean": -191.80516052246094,
      "KL/mean": -228.06393432617188,
      "KL/rejected_KL_mean": -264.32269287109375,
      "KL/std": 101.44027709960938,
      "epoch": 0.6960352422907489,
      "fcm_dpo/beta": 0.004180447664111853,
      "fcm_dpo/delta": -0.016434278339147568,
      "fcm_dpo/margin": 72.51751708984375,
      "fcm_dpo/q_t": 0.4293387532234192,
      "grad_norm": 14.038281440734863,
      "learning_rate": 1.2947162435741277e-07,
      "logits/chosen": -1.2898651361465454,
      "logits/rejected": -1.094130516052246,
      "logps/chosen": -273.284912109375,
      "logps/ref_chosen": -81.47975158691406,
      "logps/ref_rejected": -96.46562957763672,
      "logps/rejected": -360.788330078125,
      "loss": 1.1758,
      "margin_dpo/margin_mean": 72.51751708984375,
      "margin_dpo/margin_std": 123.5782470703125,
      "step": 474
    },
    {
      "KL/chosen_KL_mean": -145.2252197265625,
      "KL/mean": -200.84214782714844,
      "KL/rejected_KL_mean": -256.4590759277344,
      "KL/std": 108.71817016601562,
      "epoch": 0.697503671071953,
      "fcm_dpo/beta": 0.004056777339428663,
      "fcm_dpo/delta": -0.059229180216789246,
      "fcm_dpo/margin": 111.23384857177734,
      "fcm_dpo/q_t": 0.3942123055458069,
      "grad_norm": 8.188705444335938,
      "learning_rate": 1.2834888035828596e-07,
      "logits/chosen": -2.222337245941162,
      "logits/rejected": -2.1707024574279785,
      "logps/chosen": -219.42120361328125,
      "logps/ref_chosen": -74.19598388671875,
      "logps/ref_rejected": -94.69242095947266,
      "logps/rejected": -351.1514892578125,
      "loss": 1.0449,
      "margin_dpo/margin_mean": 111.23385620117188,
      "margin_dpo/margin_std": 124.65406799316406,
      "step": 475
    },
    {
      "KL/chosen_KL_mean": -169.00270080566406,
      "KL/mean": -218.57568359375,
      "KL/rejected_KL_mean": -268.1486511230469,
      "KL/std": 110.83822631835938,
      "epoch": 0.6989720998531571,
      "fcm_dpo/beta": 0.003990978933870792,
      "fcm_dpo/delta": 0.004480969160795212,
      "fcm_dpo/margin": 99.1459732055664,
      "fcm_dpo/q_t": 0.4083649516105652,
      "grad_norm": 9.474737167358398,
      "learning_rate": 1.2722934197929802e-07,
      "logits/chosen": -1.9126064777374268,
      "logits/rejected": -1.5320851802825928,
      "logps/chosen": -240.9738006591797,
      "logps/ref_chosen": -71.97109985351562,
      "logps/ref_rejected": -80.26224517822266,
      "logps/rejected": -348.410888671875,
      "loss": 1.0845,
      "margin_dpo/margin_mean": 99.1459732055664,
      "margin_dpo/margin_std": 120.8576431274414,
      "step": 476
    },
    {
      "KL/chosen_KL_mean": -169.4651336669922,
      "KL/mean": -218.65560913085938,
      "KL/rejected_KL_mean": -267.8460998535156,
      "KL/std": 97.45889282226562,
      "epoch": 0.7004405286343612,
      "fcm_dpo/beta": 0.004040460102260113,
      "fcm_dpo/delta": 0.0027820132672786713,
      "fcm_dpo/margin": 98.38095092773438,
      "fcm_dpo/q_t": 0.4065762460231781,
      "grad_norm": 8.953621864318848,
      "learning_rate": 1.2611303872132631e-07,
      "logits/chosen": -2.045901298522949,
      "logits/rejected": -1.105233907699585,
      "logps/chosen": -274.470703125,
      "logps/ref_chosen": -105.00555419921875,
      "logps/ref_rejected": -81.87843322753906,
      "logps/rejected": -349.7245178222656,
      "loss": 1.0849,
      "margin_dpo/margin_mean": 98.38095092773438,
      "margin_dpo/margin_std": 121.16651916503906,
      "step": 477
    },
    {
      "KL/chosen_KL_mean": -164.43063354492188,
      "KL/mean": -215.2110595703125,
      "KL/rejected_KL_mean": -265.99151611328125,
      "KL/std": 105.384033203125,
      "epoch": 0.7019089574155654,
      "fcm_dpo/beta": 0.004013546742498875,
      "fcm_dpo/delta": -0.00869231391698122,
      "fcm_dpo/margin": 101.56088256835938,
      "fcm_dpo/q_t": 0.404168963432312,
      "grad_norm": 8.760366439819336,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -1.4831123352050781,
      "logits/rejected": -1.3039507865905762,
      "logps/chosen": -241.2188262939453,
      "logps/ref_chosen": -76.7882080078125,
      "logps/ref_rejected": -90.43994140625,
      "logps/rejected": -356.43145751953125,
      "loss": 1.072,
      "margin_dpo/margin_mean": 101.56088256835938,
      "margin_dpo/margin_std": 116.87362670898438,
      "step": 478
    },
    {
      "KL/chosen_KL_mean": -167.02737426757812,
      "KL/mean": -218.26727294921875,
      "KL/rejected_KL_mean": -269.5071716308594,
      "KL/std": 107.28239440917969,
      "epoch": 0.7033773861967695,
      "fcm_dpo/beta": 0.0040304576978087425,
      "fcm_dpo/delta": -0.015662338584661484,
      "fcm_dpo/margin": 102.47981262207031,
      "fcm_dpo/q_t": 0.40213337540626526,
      "grad_norm": 12.357301712036133,
      "learning_rate": 1.2389025514492456e-07,
      "logits/chosen": -1.3637654781341553,
      "logits/rejected": -1.220349907875061,
      "logps/chosen": -248.38967895507812,
      "logps/ref_chosen": -81.3623046875,
      "logps/ref_rejected": -101.09114074707031,
      "logps/rejected": -370.59832763671875,
      "loss": 1.0651,
      "margin_dpo/margin_mean": 102.47980499267578,
      "margin_dpo/margin_std": 111.10867309570312,
      "step": 479
    },
    {
      "KL/chosen_KL_mean": -194.60247802734375,
      "KL/mean": -234.75927734375,
      "KL/rejected_KL_mean": -274.916015625,
      "KL/std": 102.36738586425781,
      "epoch": 0.7048458149779736,
      "fcm_dpo/beta": 0.004118351265788078,
      "fcm_dpo/delta": 0.07670682668685913,
      "fcm_dpo/margin": 80.31356048583984,
      "fcm_dpo/q_t": 0.4237343668937683,
      "grad_norm": 9.772912979125977,
      "learning_rate": 1.227838333989088e-07,
      "logits/chosen": -1.7868592739105225,
      "logits/rejected": -1.1106810569763184,
      "logps/chosen": -291.37640380859375,
      "logps/ref_chosen": -96.7739028930664,
      "logps/ref_rejected": -86.40473937988281,
      "logps/rejected": -361.3207702636719,
      "loss": 1.1342,
      "margin_dpo/margin_mean": 80.31355285644531,
      "margin_dpo/margin_std": 112.59440612792969,
      "step": 480
    },
    {
      "KL/chosen_KL_mean": -189.77908325195312,
      "KL/mean": -237.0020751953125,
      "KL/rejected_KL_mean": -284.22503662109375,
      "KL/std": 106.51249694824219,
      "epoch": 0.7063142437591777,
      "fcm_dpo/beta": 0.004167920909821987,
      "fcm_dpo/delta": 0.007203071843832731,
      "fcm_dpo/margin": 94.44598388671875,
      "fcm_dpo/q_t": 0.4079453945159912,
      "grad_norm": 12.417893409729004,
      "learning_rate": 1.2168076391719489e-07,
      "logits/chosen": -1.8917310237884521,
      "logits/rejected": -1.453464388847351,
      "logps/chosen": -281.4492492675781,
      "logps/ref_chosen": -91.670166015625,
      "logps/ref_rejected": -98.69490051269531,
      "logps/rejected": -382.9199523925781,
      "loss": 1.0878,
      "margin_dpo/margin_mean": 94.44598388671875,
      "margin_dpo/margin_std": 117.65678405761719,
      "step": 481
    },
    {
      "KL/chosen_KL_mean": -183.85081481933594,
      "KL/mean": -229.92916870117188,
      "KL/rejected_KL_mean": -276.00750732421875,
      "KL/std": 105.1105728149414,
      "epoch": 0.7077826725403817,
      "fcm_dpo/beta": 0.004176510497927666,
      "fcm_dpo/delta": 0.01675822213292122,
      "fcm_dpo/margin": 92.1567153930664,
      "fcm_dpo/q_t": 0.4096459746360779,
      "grad_norm": 12.101914405822754,
      "learning_rate": 1.2058107576668938e-07,
      "logits/chosen": -1.4066884517669678,
      "logits/rejected": -1.0418941974639893,
      "logps/chosen": -282.37091064453125,
      "logps/ref_chosen": -98.52011108398438,
      "logps/ref_rejected": -94.82945251464844,
      "logps/rejected": -370.83697509765625,
      "loss": 1.0964,
      "margin_dpo/margin_mean": 92.1567153930664,
      "margin_dpo/margin_std": 117.62144470214844,
      "step": 482
    },
    {
      "KL/chosen_KL_mean": -189.35488891601562,
      "KL/mean": -238.31362915039062,
      "KL/rejected_KL_mean": -287.2723388671875,
      "KL/std": 111.03453063964844,
      "epoch": 0.7092511013215859,
      "fcm_dpo/beta": 0.004258490167558193,
      "fcm_dpo/delta": -0.021335046738386154,
      "fcm_dpo/margin": 97.91744995117188,
      "fcm_dpo/q_t": 0.4013490676879883,
      "grad_norm": 7.687160491943359,
      "learning_rate": 1.194847979251979e-07,
      "logits/chosen": -2.1149864196777344,
      "logits/rejected": -1.480512261390686,
      "logps/chosen": -296.4735107421875,
      "logps/ref_chosen": -107.11860656738281,
      "logps/ref_rejected": -101.11499786376953,
      "logps/rejected": -388.3873291015625,
      "loss": 1.0688,
      "margin_dpo/margin_mean": 97.9174575805664,
      "margin_dpo/margin_std": 110.48838806152344,
      "step": 483
    },
    {
      "KL/chosen_KL_mean": -172.55519104003906,
      "KL/mean": -227.68484497070312,
      "KL/rejected_KL_mean": -282.8144836425781,
      "KL/std": 112.48652648925781,
      "epoch": 0.71071953010279,
      "fcm_dpo/beta": 0.004095167387276888,
      "fcm_dpo/delta": -0.059618186205625534,
      "fcm_dpo/margin": 110.25928497314453,
      "fcm_dpo/q_t": 0.393044114112854,
      "grad_norm": 8.42590045928955,
      "learning_rate": 1.1839195928066101e-07,
      "logits/chosen": -2.295564889907837,
      "logits/rejected": -1.8077372312545776,
      "logps/chosen": -259.53509521484375,
      "logps/ref_chosen": -86.97991943359375,
      "logps/ref_rejected": -90.72367095947266,
      "logps/rejected": -373.53814697265625,
      "loss": 1.0459,
      "margin_dpo/margin_mean": 110.25928497314453,
      "margin_dpo/margin_std": 122.91812133789062,
      "step": 484
    },
    {
      "KL/chosen_KL_mean": -164.4088897705078,
      "KL/mean": -223.42672729492188,
      "KL/rejected_KL_mean": -282.4445495605469,
      "KL/std": 117.02507781982422,
      "epoch": 0.7121879588839941,
      "fcm_dpo/beta": 0.0038996217772364616,
      "fcm_dpo/delta": -0.06995357573032379,
      "fcm_dpo/margin": 118.03565979003906,
      "fcm_dpo/q_t": 0.39068442583084106,
      "grad_norm": 8.556241989135742,
      "learning_rate": 1.1730258863039347e-07,
      "logits/chosen": -1.6264275312423706,
      "logits/rejected": -1.2020370960235596,
      "logps/chosen": -258.4676513671875,
      "logps/ref_chosen": -94.05874633789062,
      "logps/ref_rejected": -108.56298828125,
      "logps/rejected": -391.0075378417969,
      "loss": 1.0288,
      "margin_dpo/margin_mean": 118.0356674194336,
      "margin_dpo/margin_std": 119.15333557128906,
      "step": 485
    },
    {
      "KL/chosen_KL_mean": -187.94143676757812,
      "KL/mean": -239.74790954589844,
      "KL/rejected_KL_mean": -291.5543518066406,
      "KL/std": 113.54922485351562,
      "epoch": 0.7136563876651982,
      "fcm_dpo/beta": 0.003881358541548252,
      "fcm_dpo/delta": -0.0028743166476488113,
      "fcm_dpo/margin": 103.6129150390625,
      "fcm_dpo/q_t": 0.40533795952796936,
      "grad_norm": 9.63407039642334,
      "learning_rate": 1.1621671468032493e-07,
      "logits/chosen": -2.030885696411133,
      "logits/rejected": -1.3269312381744385,
      "logps/chosen": -281.68731689453125,
      "logps/ref_chosen": -93.74588012695312,
      "logps/ref_rejected": -98.07064819335938,
      "logps/rejected": -389.625,
      "loss": 1.0779,
      "margin_dpo/margin_mean": 103.6129150390625,
      "margin_dpo/margin_std": 121.29579162597656,
      "step": 486
    },
    {
      "KL/chosen_KL_mean": -181.73214721679688,
      "KL/mean": -228.88397216796875,
      "KL/rejected_KL_mean": -276.0357666015625,
      "KL/std": 107.39261627197266,
      "epoch": 0.7151248164464024,
      "fcm_dpo/beta": 0.003907639998942614,
      "fcm_dpo/delta": 0.03542683646082878,
      "fcm_dpo/margin": 94.30361938476562,
      "fcm_dpo/q_t": 0.4139290452003479,
      "grad_norm": 9.943181037902832,
      "learning_rate": 1.1513436604424378e-07,
      "logits/chosen": -2.1441264152526855,
      "logits/rejected": -1.834014654159546,
      "logps/chosen": -269.7657165527344,
      "logps/ref_chosen": -88.0335693359375,
      "logps/ref_rejected": -98.47209930419922,
      "logps/rejected": -374.50787353515625,
      "loss": 1.0984,
      "margin_dpo/margin_mean": 94.30361938476562,
      "margin_dpo/margin_std": 113.72567749023438,
      "step": 487
    },
    {
      "KL/chosen_KL_mean": -163.90289306640625,
      "KL/mean": -216.395751953125,
      "KL/rejected_KL_mean": -268.88861083984375,
      "KL/std": 113.51566314697266,
      "epoch": 0.7165932452276065,
      "fcm_dpo/beta": 0.0039230533875525,
      "fcm_dpo/delta": -0.013587091118097305,
      "fcm_dpo/margin": 104.98572540283203,
      "fcm_dpo/q_t": 0.40178993344306946,
      "grad_norm": 8.780184745788574,
      "learning_rate": 1.1405557124304335e-07,
      "logits/chosen": -1.4068915843963623,
      "logits/rejected": -1.1362580060958862,
      "logps/chosen": -248.69253540039062,
      "logps/ref_chosen": -84.78964233398438,
      "logps/ref_rejected": -90.2734603881836,
      "logps/rejected": -359.1620788574219,
      "loss": 1.0556,
      "margin_dpo/margin_mean": 104.98572540283203,
      "margin_dpo/margin_std": 100.84841918945312,
      "step": 488
    },
    {
      "KL/chosen_KL_mean": -185.52249145507812,
      "KL/mean": -233.20367431640625,
      "KL/rejected_KL_mean": -280.8848876953125,
      "KL/std": 104.05052185058594,
      "epoch": 0.7180616740088106,
      "fcm_dpo/beta": 0.0039506712928414345,
      "fcm_dpo/delta": 0.026166200637817383,
      "fcm_dpo/margin": 95.36238098144531,
      "fcm_dpo/q_t": 0.4121713638305664,
      "grad_norm": 9.202054023742676,
      "learning_rate": 1.1298035870396985e-07,
      "logits/chosen": -1.6997641324996948,
      "logits/rejected": -1.2144057750701904,
      "logps/chosen": -275.9917907714844,
      "logps/ref_chosen": -90.46929931640625,
      "logps/ref_rejected": -86.39761352539062,
      "logps/rejected": -367.2825012207031,
      "loss": 1.0938,
      "margin_dpo/margin_mean": 95.36238098144531,
      "margin_dpo/margin_std": 115.34426879882812,
      "step": 489
    },
    {
      "KL/chosen_KL_mean": -185.74050903320312,
      "KL/mean": -230.7015380859375,
      "KL/rejected_KL_mean": -275.66253662109375,
      "KL/std": 112.8437271118164,
      "epoch": 0.7195301027900147,
      "fcm_dpo/beta": 0.004104304127395153,
      "fcm_dpo/delta": 0.030782140791416168,
      "fcm_dpo/margin": 89.92202758789062,
      "fcm_dpo/q_t": 0.4161868393421173,
      "grad_norm": 8.340727806091309,
      "learning_rate": 1.1190875675987355e-07,
      "logits/chosen": -2.097835063934326,
      "logits/rejected": -2.179457187652588,
      "logps/chosen": -271.0606384277344,
      "logps/ref_chosen": -85.32012939453125,
      "logps/ref_rejected": -115.99385070800781,
      "logps/rejected": -391.6564025878906,
      "loss": 1.1443,
      "margin_dpo/margin_mean": 89.92202758789062,
      "margin_dpo/margin_std": 148.9972381591797,
      "step": 490
    },
    {
      "KL/chosen_KL_mean": -189.65643310546875,
      "KL/mean": -231.71966552734375,
      "KL/rejected_KL_mean": -273.78289794921875,
      "KL/std": 114.92924499511719,
      "epoch": 0.7209985315712188,
      "fcm_dpo/beta": 0.004135506227612495,
      "fcm_dpo/delta": 0.05822698399424553,
      "fcm_dpo/margin": 84.12648010253906,
      "fcm_dpo/q_t": 0.4194020628929138,
      "grad_norm": 10.846050262451172,
      "learning_rate": 1.1084079364846241e-07,
      "logits/chosen": -1.7809171676635742,
      "logits/rejected": -1.2839205265045166,
      "logps/chosen": -275.7999572753906,
      "logps/ref_chosen": -86.14352416992188,
      "logps/ref_rejected": -80.67945861816406,
      "logps/rejected": -354.4623718261719,
      "loss": 1.1242,
      "margin_dpo/margin_mean": 84.12648010253906,
      "margin_dpo/margin_std": 116.64431762695312,
      "step": 491
    },
    {
      "KL/chosen_KL_mean": -176.97998046875,
      "KL/mean": -210.51626586914062,
      "KL/rejected_KL_mean": -244.05255126953125,
      "KL/std": 112.82335662841797,
      "epoch": 0.7224669603524229,
      "fcm_dpo/beta": 0.004387136548757553,
      "fcm_dpo/delta": 0.11627358198165894,
      "fcm_dpo/margin": 67.07254028320312,
      "fcm_dpo/q_t": 0.43462103605270386,
      "grad_norm": 14.320441246032715,
      "learning_rate": 1.097764975115576e-07,
      "logits/chosen": -2.0363385677337646,
      "logits/rejected": -1.6502289772033691,
      "logps/chosen": -258.08758544921875,
      "logps/ref_chosen": -81.10757446289062,
      "logps/ref_rejected": -80.75199890136719,
      "logps/rejected": -324.8045349121094,
      "loss": 1.204,
      "margin_dpo/margin_mean": 67.07254791259766,
      "margin_dpo/margin_std": 140.54183959960938,
      "step": 492
    },
    {
      "KL/chosen_KL_mean": -175.73397827148438,
      "KL/mean": -220.67312622070312,
      "KL/rejected_KL_mean": -265.61224365234375,
      "KL/std": 101.95720672607422,
      "epoch": 0.723935389133627,
      "fcm_dpo/beta": 0.004419373348355293,
      "fcm_dpo/delta": 0.0016717202961444855,
      "fcm_dpo/margin": 89.87828063964844,
      "fcm_dpo/q_t": 0.4066775143146515,
      "grad_norm": 16.911212921142578,
      "learning_rate": 1.0871589639435203e-07,
      "logits/chosen": -2.4268736839294434,
      "logits/rejected": -1.697644591331482,
      "logps/chosen": -287.9413146972656,
      "logps/ref_chosen": -112.20733642578125,
      "logps/ref_rejected": -93.60719299316406,
      "logps/rejected": -359.2194519042969,
      "loss": 1.086,
      "margin_dpo/margin_mean": 89.87828063964844,
      "margin_dpo/margin_std": 108.97923278808594,
      "step": 493
    },
    {
      "KL/chosen_KL_mean": -170.6858367919922,
      "KL/mean": -228.9185791015625,
      "KL/rejected_KL_mean": -287.1513366699219,
      "KL/std": 112.67691040039062,
      "epoch": 0.7254038179148311,
      "fcm_dpo/beta": 0.004193182103335857,
      "fcm_dpo/delta": -0.10850289463996887,
      "fcm_dpo/margin": 116.46548461914062,
      "fcm_dpo/q_t": 0.38332119584083557,
      "grad_norm": 10.341791152954102,
      "learning_rate": 1.0765901824467166e-07,
      "logits/chosen": -1.5063891410827637,
      "logits/rejected": -1.43217134475708,
      "logps/chosen": -243.80072021484375,
      "logps/ref_chosen": -73.11489868164062,
      "logps/ref_rejected": -92.16300201416016,
      "logps/rejected": -379.3143310546875,
      "loss": 1.0204,
      "margin_dpo/margin_mean": 116.46549224853516,
      "margin_dpo/margin_std": 122.39671325683594,
      "step": 494
    },
    {
      "KL/chosen_KL_mean": -174.98202514648438,
      "KL/mean": -225.7245635986328,
      "KL/rejected_KL_mean": -276.46710205078125,
      "KL/std": 107.17288970947266,
      "epoch": 0.7268722466960352,
      "fcm_dpo/beta": 0.004147041589021683,
      "fcm_dpo/delta": -0.02399694360792637,
      "fcm_dpo/margin": 101.4850845336914,
      "fcm_dpo/q_t": 0.4019627273082733,
      "grad_norm": 11.289752960205078,
      "learning_rate": 1.0660589091223854e-07,
      "logits/chosen": -2.3117270469665527,
      "logits/rejected": -1.852834701538086,
      "logps/chosen": -274.5023498535156,
      "logps/ref_chosen": -99.52031707763672,
      "logps/ref_rejected": -97.93089294433594,
      "logps/rejected": -374.39801025390625,
      "loss": 1.0754,
      "margin_dpo/margin_mean": 101.48507690429688,
      "margin_dpo/margin_std": 128.7162322998047,
      "step": 495
    },
    {
      "KL/chosen_KL_mean": -195.72642517089844,
      "KL/mean": -238.96051025390625,
      "KL/rejected_KL_mean": -282.194580078125,
      "KL/std": 107.98262786865234,
      "epoch": 0.7283406754772394,
      "fcm_dpo/beta": 0.004245154559612274,
      "fcm_dpo/delta": 0.03618233650922775,
      "fcm_dpo/margin": 86.4681625366211,
      "fcm_dpo/q_t": 0.413979172706604,
      "grad_norm": 9.434197425842285,
      "learning_rate": 1.0555654214793722e-07,
      "logits/chosen": -2.123263359069824,
      "logits/rejected": -1.4741357564926147,
      "logps/chosen": -303.58319091796875,
      "logps/ref_chosen": -107.85675048828125,
      "logps/ref_rejected": -92.77056121826172,
      "logps/rejected": -374.96514892578125,
      "loss": 1.1051,
      "margin_dpo/margin_mean": 86.46817016601562,
      "margin_dpo/margin_std": 108.55066680908203,
      "step": 496
    },
    {
      "KL/chosen_KL_mean": -207.91024780273438,
      "KL/mean": -235.85293579101562,
      "KL/rejected_KL_mean": -263.7956237792969,
      "KL/std": 109.07431030273438,
      "epoch": 0.7298091042584435,
      "fcm_dpo/beta": 0.004374855197966099,
      "fcm_dpo/delta": 0.05478700250387192,
      "fcm_dpo/margin": 55.885353088378906,
      "fcm_dpo/q_t": 0.4441932439804077,
      "grad_norm": 9.232996940612793,
      "learning_rate": 1.0451099960308374e-07,
      "logits/chosen": -1.845696210861206,
      "logits/rejected": -1.4055943489074707,
      "logps/chosen": -299.99346923828125,
      "logps/ref_chosen": -92.08322143554688,
      "logps/ref_rejected": -81.79503631591797,
      "logps/rejected": -345.59063720703125,
      "loss": 1.2213,
      "margin_dpo/margin_mean": 55.885353088378906,
      "margin_dpo/margin_std": 117.14604949951172,
      "step": 497
    },
    {
      "KL/chosen_KL_mean": -176.36544799804688,
      "KL/mean": -222.79989624023438,
      "KL/rejected_KL_mean": -269.234375,
      "KL/std": 107.23832702636719,
      "epoch": 0.7312775330396476,
      "fcm_dpo/beta": 0.004355347715318203,
      "fcm_dpo/delta": -0.005140620283782482,
      "fcm_dpo/margin": 92.86890411376953,
      "fcm_dpo/q_t": 0.4052576422691345,
      "grad_norm": 12.30578899383545,
      "learning_rate": 1.0346929082869641e-07,
      "logits/chosen": -1.7426387071609497,
      "logits/rejected": -1.2391643524169922,
      "logps/chosen": -274.559814453125,
      "logps/ref_chosen": -98.19437408447266,
      "logps/ref_rejected": -90.68745422363281,
      "logps/rejected": -359.92181396484375,
      "loss": 1.085,
      "margin_dpo/margin_mean": 92.86890411376953,
      "margin_dpo/margin_std": 118.38478088378906,
      "step": 498
    },
    {
      "KL/chosen_KL_mean": -185.40194702148438,
      "KL/mean": -234.6136474609375,
      "KL/rejected_KL_mean": -283.8253173828125,
      "KL/std": 116.11276245117188,
      "epoch": 0.7327459618208517,
      "fcm_dpo/beta": 0.004318951163440943,
      "fcm_dpo/delta": -0.02872832491993904,
      "fcm_dpo/margin": 98.42337036132812,
      "fcm_dpo/q_t": 0.4011166989803314,
      "grad_norm": 9.665292739868164,
      "learning_rate": 1.0243144327477013e-07,
      "logits/chosen": -1.88462233543396,
      "logits/rejected": -1.8346773386001587,
      "logps/chosen": -266.4418640136719,
      "logps/ref_chosen": -81.0399169921875,
      "logps/ref_rejected": -106.92170715332031,
      "logps/rejected": -390.74700927734375,
      "loss": 1.0723,
      "margin_dpo/margin_mean": 98.42337036132812,
      "margin_dpo/margin_std": 123.51121520996094,
      "step": 499
    },
    {
      "KL/chosen_KL_mean": -206.201416015625,
      "KL/mean": -247.8850555419922,
      "KL/rejected_KL_mean": -289.5687255859375,
      "KL/std": 111.6468276977539,
      "epoch": 0.7342143906020558,
      "fcm_dpo/beta": 0.004351096227765083,
      "fcm_dpo/delta": 0.041869163513183594,
      "fcm_dpo/margin": 83.36729431152344,
      "fcm_dpo/q_t": 0.4181423783302307,
      "grad_norm": 11.676789283752441,
      "learning_rate": 1.0139748428955333e-07,
      "logits/chosen": -1.6080248355865479,
      "logits/rejected": -1.436173439025879,
      "logps/chosen": -295.449462890625,
      "logps/ref_chosen": -89.248046875,
      "logps/ref_rejected": -100.41021728515625,
      "logps/rejected": -389.97894287109375,
      "loss": 1.1279,
      "margin_dpo/margin_mean": 83.36729431152344,
      "margin_dpo/margin_std": 126.88575744628906,
      "step": 500
    },
    {
      "KL/chosen_KL_mean": -180.46432495117188,
      "KL/mean": -226.80780029296875,
      "KL/rejected_KL_mean": -273.1512756347656,
      "KL/std": 120.66539764404297,
      "epoch": 0.73568281938326,
      "fcm_dpo/beta": 0.0044645629823207855,
      "fcm_dpo/delta": -0.019292451441287994,
      "fcm_dpo/margin": 92.68696594238281,
      "fcm_dpo/q_t": 0.4053162932395935,
      "grad_norm": 9.838613510131836,
      "learning_rate": 1.0036744111882672e-07,
      "logits/chosen": -1.4612936973571777,
      "logits/rejected": -1.0161983966827393,
      "logps/chosen": -271.0612487792969,
      "logps/ref_chosen": -90.596923828125,
      "logps/ref_rejected": -85.2972640991211,
      "logps/rejected": -358.44854736328125,
      "loss": 1.1087,
      "margin_dpo/margin_mean": 92.68696594238281,
      "margin_dpo/margin_std": 136.26300048828125,
      "step": 501
    },
    {
      "KL/chosen_KL_mean": -180.6841278076172,
      "KL/mean": -222.086669921875,
      "KL/rejected_KL_mean": -263.48919677734375,
      "KL/std": 100.54071044921875,
      "epoch": 0.737151248164464,
      "fcm_dpo/beta": 0.004467714577913284,
      "fcm_dpo/delta": 0.032055605202913284,
      "fcm_dpo/margin": 82.80506134033203,
      "fcm_dpo/q_t": 0.4143716096878052,
      "grad_norm": 10.912914276123047,
      "learning_rate": 9.934134090518592e-08,
      "logits/chosen": -1.8379855155944824,
      "logits/rejected": -1.0940444469451904,
      "logps/chosen": -284.15484619140625,
      "logps/ref_chosen": -103.47073364257812,
      "logps/ref_rejected": -89.04415893554688,
      "logps/rejected": -352.5333251953125,
      "loss": 1.1263,
      "margin_dpo/margin_mean": 82.80506134033203,
      "margin_dpo/margin_std": 124.6265869140625,
      "step": 502
    },
    {
      "KL/chosen_KL_mean": -180.292724609375,
      "KL/mean": -226.88925170898438,
      "KL/rejected_KL_mean": -273.48577880859375,
      "KL/std": 107.92990112304688,
      "epoch": 0.7386196769456681,
      "fcm_dpo/beta": 0.004453142639249563,
      "fcm_dpo/delta": -0.01812731847167015,
      "fcm_dpo/margin": 93.19305419921875,
      "fcm_dpo/q_t": 0.4034411311149597,
      "grad_norm": 10.069133758544922,
      "learning_rate": 9.831921068732571e-08,
      "logits/chosen": -1.6783623695373535,
      "logits/rejected": -1.1730220317840576,
      "logps/chosen": -274.022705078125,
      "logps/ref_chosen": -93.72998809814453,
      "logps/ref_rejected": -87.60896301269531,
      "logps/rejected": -361.0947265625,
      "loss": 1.0829,
      "margin_dpo/margin_mean": 93.19305419921875,
      "margin_dpo/margin_std": 119.51693725585938,
      "step": 503
    },
    {
      "KL/chosen_KL_mean": -182.89537048339844,
      "KL/mean": -243.07260131835938,
      "KL/rejected_KL_mean": -303.24981689453125,
      "KL/std": 112.14630126953125,
      "epoch": 0.7400881057268722,
      "fcm_dpo/beta": 0.00416524289175868,
      "fcm_dpo/delta": -0.11909794807434082,
      "fcm_dpo/margin": 120.35443878173828,
      "fcm_dpo/q_t": 0.37938666343688965,
      "grad_norm": 9.975341796875,
      "learning_rate": 9.730107739932805e-08,
      "logits/chosen": -2.1692469120025635,
      "logits/rejected": -1.841339111328125,
      "logps/chosen": -275.187255859375,
      "logps/ref_chosen": -92.2918701171875,
      "logps/ref_rejected": -109.03765106201172,
      "logps/rejected": -412.2874755859375,
      "loss": 1.0,
      "margin_dpo/margin_mean": 120.35443878173828,
      "margin_dpo/margin_std": 114.09609985351562,
      "step": 504
    },
    {
      "KL/chosen_KL_mean": -200.64779663085938,
      "KL/mean": -244.1832275390625,
      "KL/rejected_KL_mean": -287.7186279296875,
      "KL/std": 111.23428344726562,
      "epoch": 0.7415565345080763,
      "fcm_dpo/beta": 0.004152392037212849,
      "fcm_dpo/delta": 0.04317181557416916,
      "fcm_dpo/margin": 87.07086181640625,
      "fcm_dpo/q_t": 0.4158153831958771,
      "grad_norm": 8.597580909729004,
      "learning_rate": 9.628696786995188e-08,
      "logits/chosen": -2.3641650676727295,
      "logits/rejected": -1.8044345378875732,
      "logps/chosen": -306.92474365234375,
      "logps/ref_chosen": -106.27693176269531,
      "logps/ref_rejected": -94.20611572265625,
      "logps/rejected": -381.92474365234375,
      "loss": 1.1057,
      "margin_dpo/margin_mean": 87.07086181640625,
      "margin_dpo/margin_std": 107.66104125976562,
      "step": 505
    },
    {
      "KL/chosen_KL_mean": -194.68734741210938,
      "KL/mean": -243.30615234375,
      "KL/rejected_KL_mean": -291.92498779296875,
      "KL/std": 104.80792999267578,
      "epoch": 0.7430249632892805,
      "fcm_dpo/beta": 0.004134034737944603,
      "fcm_dpo/delta": -0.003200523555278778,
      "fcm_dpo/margin": 97.23762512207031,
      "fcm_dpo/q_t": 0.4043969511985779,
      "grad_norm": 9.155471801757812,
      "learning_rate": 9.527690882192635e-08,
      "logits/chosen": -1.8887653350830078,
      "logits/rejected": -1.5247442722320557,
      "logps/chosen": -276.1282043457031,
      "logps/ref_chosen": -81.44086456298828,
      "logps/ref_rejected": -85.66439819335938,
      "logps/rejected": -377.58935546875,
      "loss": 1.0759,
      "margin_dpo/margin_mean": 97.23762512207031,
      "margin_dpo/margin_std": 110.78254699707031,
      "step": 506
    },
    {
      "KL/chosen_KL_mean": -201.9251708984375,
      "KL/mean": -251.26966857910156,
      "KL/rejected_KL_mean": -300.61419677734375,
      "KL/std": 110.33625793457031,
      "epoch": 0.7444933920704846,
      "fcm_dpo/beta": 0.004159481264650822,
      "fcm_dpo/delta": -0.01196424663066864,
      "fcm_dpo/margin": 98.68901824951172,
      "fcm_dpo/q_t": 0.40490034222602844,
      "grad_norm": 8.995759963989258,
      "learning_rate": 9.427092687124691e-08,
      "logits/chosen": -2.1724071502685547,
      "logits/rejected": -1.7310421466827393,
      "logps/chosen": -301.35211181640625,
      "logps/ref_chosen": -99.42694854736328,
      "logps/ref_rejected": -102.25296020507812,
      "logps/rejected": -402.86712646484375,
      "loss": 1.088,
      "margin_dpo/margin_mean": 98.68901824951172,
      "margin_dpo/margin_std": 129.8388671875,
      "step": 507
    },
    {
      "KL/chosen_KL_mean": -195.6327362060547,
      "KL/mean": -237.36141967773438,
      "KL/rejected_KL_mean": -279.090087890625,
      "KL/std": 118.32302856445312,
      "epoch": 0.7459618208516887,
      "fcm_dpo/beta": 0.004246965050697327,
      "fcm_dpo/delta": 0.050929464399814606,
      "fcm_dpo/margin": 83.45734405517578,
      "fcm_dpo/q_t": 0.42007291316986084,
      "grad_norm": 15.97614860534668,
      "learning_rate": 9.326904852647344e-08,
      "logits/chosen": -2.0033986568450928,
      "logits/rejected": -1.5179505348205566,
      "logps/chosen": -293.54840087890625,
      "logps/ref_chosen": -97.9156494140625,
      "logps/ref_rejected": -103.79782104492188,
      "logps/rejected": -382.8879089355469,
      "loss": 1.1465,
      "margin_dpo/margin_mean": 83.45735168457031,
      "margin_dpo/margin_std": 138.01278686523438,
      "step": 508
    },
    {
      "KL/chosen_KL_mean": -182.44894409179688,
      "KL/mean": -220.63461303710938,
      "KL/rejected_KL_mean": -258.82025146484375,
      "KL/std": 92.4969482421875,
      "epoch": 0.7474302496328928,
      "fcm_dpo/beta": 0.004431824199855328,
      "fcm_dpo/delta": 0.06791459023952484,
      "fcm_dpo/margin": 76.37129211425781,
      "fcm_dpo/q_t": 0.4211653769016266,
      "grad_norm": 15.37939739227295,
      "learning_rate": 9.227130018803195e-08,
      "logits/chosen": -1.9323134422302246,
      "logits/rejected": -1.2580387592315674,
      "logps/chosen": -283.1788024902344,
      "logps/ref_chosen": -100.72984313964844,
      "logps/ref_rejected": -86.943359375,
      "logps/rejected": -345.76361083984375,
      "loss": 1.1479,
      "margin_dpo/margin_mean": 76.37129211425781,
      "margin_dpo/margin_std": 122.81903076171875,
      "step": 509
    },
    {
      "KL/chosen_KL_mean": -216.40817260742188,
      "KL/mean": -262.9380187988281,
      "KL/rejected_KL_mean": -309.46783447265625,
      "KL/std": 111.3251724243164,
      "epoch": 0.748898678414097,
      "fcm_dpo/beta": 0.00444612093269825,
      "fcm_dpo/delta": -0.015794921666383743,
      "fcm_dpo/margin": 93.05966186523438,
      "fcm_dpo/q_t": 0.4032546579837799,
      "grad_norm": 12.400285720825195,
      "learning_rate": 9.127770814751932e-08,
      "logits/chosen": -1.6825957298278809,
      "logits/rejected": -1.5014233589172363,
      "logps/chosen": -305.57757568359375,
      "logps/ref_chosen": -89.16938781738281,
      "logps/ref_rejected": -107.91939544677734,
      "logps/rejected": -417.3872375488281,
      "loss": 1.0752,
      "margin_dpo/margin_mean": 93.05966186523438,
      "margin_dpo/margin_std": 111.50645446777344,
      "step": 510
    },
    {
      "KL/chosen_KL_mean": -197.39901733398438,
      "KL/mean": -247.62960815429688,
      "KL/rejected_KL_mean": -297.86016845703125,
      "KL/std": 109.91946411132812,
      "epoch": 0.750367107195301,
      "fcm_dpo/beta": 0.00435635494068265,
      "fcm_dpo/delta": -0.04333028569817543,
      "fcm_dpo/margin": 100.461181640625,
      "fcm_dpo/q_t": 0.39838695526123047,
      "grad_norm": 11.415409088134766,
      "learning_rate": 9.028829858700973e-08,
      "logits/chosen": -2.0648157596588135,
      "logits/rejected": -1.4905649423599243,
      "logps/chosen": -287.1488037109375,
      "logps/ref_chosen": -89.74979400634766,
      "logps/ref_rejected": -97.73127746582031,
      "logps/rejected": -395.59149169921875,
      "loss": 1.0628,
      "margin_dpo/margin_mean": 100.461181640625,
      "margin_dpo/margin_std": 122.69171142578125,
      "step": 511
    },
    {
      "KL/chosen_KL_mean": -186.61770629882812,
      "KL/mean": -235.6119384765625,
      "KL/rejected_KL_mean": -284.606201171875,
      "KL/std": 107.2967529296875,
      "epoch": 0.7518355359765051,
      "fcm_dpo/beta": 0.004315720405429602,
      "fcm_dpo/delta": -0.027514342218637466,
      "fcm_dpo/margin": 97.98849487304688,
      "fcm_dpo/q_t": 0.4010455310344696,
      "grad_norm": 13.794711112976074,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": -1.859352946281433,
      "logits/rejected": -1.4198825359344482,
      "logps/chosen": -272.35284423828125,
      "logps/ref_chosen": -85.73515319824219,
      "logps/ref_rejected": -87.6248779296875,
      "logps/rejected": -372.2310791015625,
      "loss": 1.0827,
      "margin_dpo/margin_mean": 97.98848724365234,
      "margin_dpo/margin_std": 127.53663635253906,
      "step": 512
    },
    {
      "KL/chosen_KL_mean": -199.0638427734375,
      "KL/mean": -239.73486328125,
      "KL/rejected_KL_mean": -280.4058837890625,
      "KL/std": 102.68348693847656,
      "epoch": 0.7533039647577092,
      "fcm_dpo/beta": 0.004285678267478943,
      "fcm_dpo/delta": 0.057033706456422806,
      "fcm_dpo/margin": 81.34205627441406,
      "fcm_dpo/q_t": 0.4186411499977112,
      "grad_norm": 11.278692245483398,
      "learning_rate": 8.832213108254863e-08,
      "logits/chosen": -2.2976670265197754,
      "logits/rejected": -1.5994980335235596,
      "logps/chosen": -295.25433349609375,
      "logps/ref_chosen": -96.19050598144531,
      "logps/ref_rejected": -82.91773986816406,
      "logps/rejected": -363.3236389160156,
      "loss": 1.1313,
      "margin_dpo/margin_mean": 81.34205627441406,
      "margin_dpo/margin_std": 116.39095306396484,
      "step": 513
    },
    {
      "KL/chosen_KL_mean": -192.87234497070312,
      "KL/mean": -237.02496337890625,
      "KL/rejected_KL_mean": -281.1775817871094,
      "KL/std": 103.3597412109375,
      "epoch": 0.7547723935389133,
      "fcm_dpo/beta": 0.004405788145959377,
      "fcm_dpo/delta": 0.012369304895401001,
      "fcm_dpo/margin": 88.30525207519531,
      "fcm_dpo/q_t": 0.41094091534614563,
      "grad_norm": 11.946328163146973,
      "learning_rate": 8.734542494893954e-08,
      "logits/chosen": -1.8340444564819336,
      "logits/rejected": -1.273947834968567,
      "logps/chosen": -290.69598388671875,
      "logps/ref_chosen": -97.82363891601562,
      "logps/ref_rejected": -85.36982727050781,
      "logps/rejected": -366.54742431640625,
      "loss": 1.1024,
      "margin_dpo/margin_mean": 88.30523681640625,
      "margin_dpo/margin_std": 122.05049133300781,
      "step": 514
    },
    {
      "KL/chosen_KL_mean": -184.59890747070312,
      "KL/mean": -228.1392822265625,
      "KL/rejected_KL_mean": -271.6796875,
      "KL/std": 107.04838562011719,
      "epoch": 0.7562408223201175,
      "fcm_dpo/beta": 0.004437828902155161,
      "fcm_dpo/delta": 0.015332169830799103,
      "fcm_dpo/margin": 87.08074188232422,
      "fcm_dpo/q_t": 0.4105073809623718,
      "grad_norm": 11.34633731842041,
      "learning_rate": 8.637300491465272e-08,
      "logits/chosen": -1.6633715629577637,
      "logits/rejected": -1.4290482997894287,
      "logps/chosen": -266.532470703125,
      "logps/ref_chosen": -81.93355560302734,
      "logps/ref_rejected": -93.57656860351562,
      "logps/rejected": -365.2562255859375,
      "loss": 1.1029,
      "margin_dpo/margin_mean": 87.08074188232422,
      "margin_dpo/margin_std": 117.69140625,
      "step": 515
    },
    {
      "KL/chosen_KL_mean": -189.98904418945312,
      "KL/mean": -231.99630737304688,
      "KL/rejected_KL_mean": -274.0035400390625,
      "KL/std": 123.15557861328125,
      "epoch": 0.7577092511013216,
      "fcm_dpo/beta": 0.004517971538007259,
      "fcm_dpo/delta": 0.022752288728952408,
      "fcm_dpo/margin": 84.01449584960938,
      "fcm_dpo/q_t": 0.41542524099349976,
      "grad_norm": 14.295713424682617,
      "learning_rate": 8.540489660386064e-08,
      "logits/chosen": -2.328911304473877,
      "logits/rejected": -2.1572179794311523,
      "logps/chosen": -287.0807189941406,
      "logps/ref_chosen": -97.09166717529297,
      "logps/ref_rejected": -118.55667114257812,
      "logps/rejected": -392.5602111816406,
      "loss": 1.1425,
      "margin_dpo/margin_mean": 84.01449584960938,
      "margin_dpo/margin_std": 145.2054443359375,
      "step": 516
    },
    {
      "KL/chosen_KL_mean": -201.66485595703125,
      "KL/mean": -253.37411499023438,
      "KL/rejected_KL_mean": -305.0833740234375,
      "KL/std": 110.84556579589844,
      "epoch": 0.7591776798825257,
      "fcm_dpo/beta": 0.004304789938032627,
      "fcm_dpo/delta": -0.056877367198467255,
      "fcm_dpo/margin": 103.41851806640625,
      "fcm_dpo/q_t": 0.39553573727607727,
      "grad_norm": 9.448169708251953,
      "learning_rate": 8.444112552711752e-08,
      "logits/chosen": -1.9091458320617676,
      "logits/rejected": -1.4851069450378418,
      "logps/chosen": -294.8514404296875,
      "logps/ref_chosen": -93.18656921386719,
      "logps/ref_rejected": -95.5927963256836,
      "logps/rejected": -400.6761779785156,
      "loss": 1.0653,
      "margin_dpo/margin_mean": 103.41851806640625,
      "margin_dpo/margin_std": 126.35054016113281,
      "step": 517
    },
    {
      "KL/chosen_KL_mean": -210.82733154296875,
      "KL/mean": -249.6283721923828,
      "KL/rejected_KL_mean": -288.42938232421875,
      "KL/std": 103.99676513671875,
      "epoch": 0.7606461086637298,
      "fcm_dpo/beta": 0.004469710402190685,
      "fcm_dpo/delta": 0.05938464403152466,
      "fcm_dpo/margin": 77.6020736694336,
      "fcm_dpo/q_t": 0.4217384457588196,
      "grad_norm": 10.333759307861328,
      "learning_rate": 8.348171708068747e-08,
      "logits/chosen": -1.9001715183258057,
      "logits/rejected": -1.5711462497711182,
      "logps/chosen": -297.84869384765625,
      "logps/ref_chosen": -87.0213623046875,
      "logps/ref_rejected": -91.24456024169922,
      "logps/rejected": -379.6739501953125,
      "loss": 1.1479,
      "margin_dpo/margin_mean": 77.6020736694336,
      "margin_dpo/margin_std": 129.43360900878906,
      "step": 518
    },
    {
      "KL/chosen_KL_mean": -195.07455444335938,
      "KL/mean": -236.94871520996094,
      "KL/rejected_KL_mean": -278.8228759765625,
      "KL/std": 97.37399291992188,
      "epoch": 0.762114537444934,
      "fcm_dpo/beta": 0.00453560147434473,
      "fcm_dpo/delta": 0.022611137479543686,
      "fcm_dpo/margin": 83.74832916259766,
      "fcm_dpo/q_t": 0.4120112657546997,
      "grad_norm": 10.589884757995605,
      "learning_rate": 8.25266965458755e-08,
      "logits/chosen": -2.1106581687927246,
      "logits/rejected": -1.749329924583435,
      "logps/chosen": -295.7620849609375,
      "logps/ref_chosen": -100.68755340576172,
      "logps/ref_rejected": -109.03042602539062,
      "logps/rejected": -387.8533020019531,
      "loss": 1.0933,
      "margin_dpo/margin_mean": 83.74832916259766,
      "margin_dpo/margin_std": 102.3615951538086,
      "step": 519
    },
    {
      "KL/chosen_KL_mean": -181.1324462890625,
      "KL/mean": -230.76329040527344,
      "KL/rejected_KL_mean": -280.3941345214844,
      "KL/std": 109.32801818847656,
      "epoch": 0.7635829662261381,
      "fcm_dpo/beta": 0.0044534169137477875,
      "fcm_dpo/delta": -0.04909754544496536,
      "fcm_dpo/margin": 99.26168823242188,
      "fcm_dpo/q_t": 0.395121306180954,
      "grad_norm": 9.547185897827148,
      "learning_rate": 8.15760890883607e-08,
      "logits/chosen": -1.7385896444320679,
      "logits/rejected": -1.3117560148239136,
      "logps/chosen": -279.91009521484375,
      "logps/ref_chosen": -98.77762603759766,
      "logps/ref_rejected": -106.9457015991211,
      "logps/rejected": -387.33984375,
      "loss": 1.0459,
      "margin_dpo/margin_mean": 99.26168823242188,
      "margin_dpo/margin_std": 106.48385620117188,
      "step": 520
    },
    {
      "KL/chosen_KL_mean": -182.2061309814453,
      "KL/mean": -233.32696533203125,
      "KL/rejected_KL_mean": -284.4477844238281,
      "KL/std": 102.20556640625,
      "epoch": 0.7650513950073421,
      "fcm_dpo/beta": 0.004394838586449623,
      "fcm_dpo/delta": -0.0575142502784729,
      "fcm_dpo/margin": 102.24165344238281,
      "fcm_dpo/q_t": 0.39347124099731445,
      "grad_norm": 10.807952880859375,
      "learning_rate": 8.062991975753378e-08,
      "logits/chosen": -1.9632799625396729,
      "logits/rejected": -1.625704288482666,
      "logps/chosen": -268.5572204589844,
      "logps/ref_chosen": -86.35108947753906,
      "logps/ref_rejected": -90.39317321777344,
      "logps/rejected": -374.8409423828125,
      "loss": 1.039,
      "margin_dpo/margin_mean": 102.24165344238281,
      "margin_dpo/margin_std": 104.54931640625,
      "step": 521
    },
    {
      "KL/chosen_KL_mean": -191.02297973632812,
      "KL/mean": -236.50216674804688,
      "KL/rejected_KL_mean": -281.98138427734375,
      "KL/std": 112.11990356445312,
      "epoch": 0.7665198237885462,
      "fcm_dpo/beta": 0.004333316348493099,
      "fcm_dpo/delta": 0.006356468424201012,
      "fcm_dpo/margin": 90.9583969116211,
      "fcm_dpo/q_t": 0.40832364559173584,
      "grad_norm": 10.510030746459961,
      "learning_rate": 7.968821348583643e-08,
      "logits/chosen": -1.7124919891357422,
      "logits/rejected": -1.3453890085220337,
      "logps/chosen": -267.031005859375,
      "logps/ref_chosen": -76.008056640625,
      "logps/ref_rejected": -74.44125366210938,
      "logps/rejected": -356.4226379394531,
      "loss": 1.0925,
      "margin_dpo/margin_mean": 90.95840454101562,
      "margin_dpo/margin_std": 117.66253662109375,
      "step": 522
    },
    {
      "KL/chosen_KL_mean": -211.2274627685547,
      "KL/mean": -244.6627197265625,
      "KL/rejected_KL_mean": -278.09796142578125,
      "KL/std": 110.83004760742188,
      "epoch": 0.7679882525697503,
      "fcm_dpo/beta": 0.004513116087764502,
      "fcm_dpo/delta": 0.10821999609470367,
      "fcm_dpo/margin": 66.87052154541016,
      "fcm_dpo/q_t": 0.4329168498516083,
      "grad_norm": 11.789393424987793,
      "learning_rate": 7.875099508810484e-08,
      "logits/chosen": -1.940371036529541,
      "logits/rejected": -1.2333672046661377,
      "logps/chosen": -307.7237548828125,
      "logps/ref_chosen": -96.49627685546875,
      "logps/ref_rejected": -91.0152816772461,
      "logps/rejected": -369.1132507324219,
      "loss": 1.1883,
      "margin_dpo/margin_mean": 66.87052154541016,
      "margin_dpo/margin_std": 129.43731689453125,
      "step": 523
    },
    {
      "KL/chosen_KL_mean": -193.33154296875,
      "KL/mean": -240.73046875,
      "KL/rejected_KL_mean": -288.12939453125,
      "KL/std": 110.84721374511719,
      "epoch": 0.7694566813509545,
      "fcm_dpo/beta": 0.004519987851381302,
      "fcm_dpo/delta": -0.0328894779086113,
      "fcm_dpo/margin": 94.79788208007812,
      "fcm_dpo/q_t": 0.4008796513080597,
      "grad_norm": 13.451142311096191,
      "learning_rate": 7.781828926091535e-08,
      "logits/chosen": -2.2086641788482666,
      "logits/rejected": -1.5165722370147705,
      "logps/chosen": -296.737060546875,
      "logps/ref_chosen": -103.40550994873047,
      "logps/ref_rejected": -87.91015625,
      "logps/rejected": -376.03955078125,
      "loss": 1.0691,
      "margin_dpo/margin_mean": 94.79788208007812,
      "margin_dpo/margin_std": 117.80243682861328,
      "step": 524
    },
    {
      "KL/chosen_KL_mean": -181.34353637695312,
      "KL/mean": -235.18338012695312,
      "KL/rejected_KL_mean": -289.02325439453125,
      "KL/std": 110.61888122558594,
      "epoch": 0.7709251101321586,
      "fcm_dpo/beta": 0.00423925556242466,
      "fcm_dpo/delta": -0.07411453127861023,
      "fcm_dpo/margin": 107.6796875,
      "fcm_dpo/q_t": 0.39387136697769165,
      "grad_norm": 10.764533996582031,
      "learning_rate": 7.689012058193384e-08,
      "logits/chosen": -1.5993491411209106,
      "logits/rejected": -1.5129973888397217,
      "logps/chosen": -265.550048828125,
      "logps/ref_chosen": -84.20648193359375,
      "logps/ref_rejected": -106.26158905029297,
      "logps/rejected": -395.2848205566406,
      "loss": 1.0575,
      "margin_dpo/margin_mean": 107.6796875,
      "margin_dpo/margin_std": 128.29293823242188,
      "step": 525
    },
    {
      "KL/chosen_KL_mean": -186.61947631835938,
      "KL/mean": -235.61782836914062,
      "KL/rejected_KL_mean": -284.61614990234375,
      "KL/std": 105.01531982421875,
      "epoch": 0.7723935389133627,
      "fcm_dpo/beta": 0.004276934079825878,
      "fcm_dpo/delta": -0.02186579257249832,
      "fcm_dpo/margin": 97.99671936035156,
      "fcm_dpo/q_t": 0.4022785425186157,
      "grad_norm": 8.154433250427246,
      "learning_rate": 7.596651350926836e-08,
      "logits/chosen": -1.663522481918335,
      "logits/rejected": -1.0014985799789429,
      "logps/chosen": -289.47357177734375,
      "logps/ref_chosen": -102.85411071777344,
      "logps/ref_rejected": -93.65145874023438,
      "logps/rejected": -378.26763916015625,
      "loss": 1.077,
      "margin_dpo/margin_mean": 97.99671936035156,
      "margin_dpo/margin_std": 123.90145111083984,
      "step": 526
    },
    {
      "KL/chosen_KL_mean": -189.13092041015625,
      "KL/mean": -239.07363891601562,
      "KL/rejected_KL_mean": -289.016357421875,
      "KL/std": 110.06622314453125,
      "epoch": 0.7738619676945668,
      "fcm_dpo/beta": 0.004209546837955713,
      "fcm_dpo/delta": -0.023477431386709213,
      "fcm_dpo/margin": 99.88546752929688,
      "fcm_dpo/q_t": 0.3996215760707855,
      "grad_norm": 9.525333404541016,
      "learning_rate": 7.504749238082414e-08,
      "logits/chosen": -3.049450397491455,
      "logits/rejected": -2.2254655361175537,
      "logps/chosen": -285.8963623046875,
      "logps/ref_chosen": -96.76543426513672,
      "logps/ref_rejected": -85.39518737792969,
      "logps/rejected": -374.41156005859375,
      "loss": 1.0496,
      "margin_dpo/margin_mean": 99.88546752929688,
      "margin_dpo/margin_std": 97.57967376708984,
      "step": 527
    },
    {
      "KL/chosen_KL_mean": -176.62380981445312,
      "KL/mean": -225.44680786132812,
      "KL/rejected_KL_mean": -274.26983642578125,
      "KL/std": 111.43293762207031,
      "epoch": 0.775330396475771,
      "fcm_dpo/beta": 0.004196016117930412,
      "fcm_dpo/delta": -0.011155126616358757,
      "fcm_dpo/margin": 97.64599609375,
      "fcm_dpo/q_t": 0.4046876132488251,
      "grad_norm": 9.698579788208008,
      "learning_rate": 7.413308141366254e-08,
      "logits/chosen": -2.1452178955078125,
      "logits/rejected": -1.6333657503128052,
      "logps/chosen": -270.7887268066406,
      "logps/ref_chosen": -94.1649169921875,
      "logps/ref_rejected": -100.372314453125,
      "logps/rejected": -374.64215087890625,
      "loss": 1.0843,
      "margin_dpo/margin_mean": 97.64599609375,
      "margin_dpo/margin_std": 126.28627014160156,
      "step": 528
    },
    {
      "KL/chosen_KL_mean": -205.23056030273438,
      "KL/mean": -249.55996704101562,
      "KL/rejected_KL_mean": -293.8893737792969,
      "KL/std": 110.54206848144531,
      "epoch": 0.7767988252569751,
      "fcm_dpo/beta": 0.0042504784651100636,
      "fcm_dpo/delta": 0.02553163468837738,
      "fcm_dpo/margin": 88.65882873535156,
      "fcm_dpo/q_t": 0.4140586853027344,
      "grad_norm": 11.317716598510742,
      "learning_rate": 7.322330470336313e-08,
      "logits/chosen": -1.8090150356292725,
      "logits/rejected": -1.6339879035949707,
      "logps/chosen": -286.0073547363281,
      "logps/ref_chosen": -80.77680206298828,
      "logps/ref_rejected": -97.191162109375,
      "logps/rejected": -391.0805358886719,
      "loss": 1.1179,
      "margin_dpo/margin_mean": 88.65882873535156,
      "margin_dpo/margin_std": 131.17918395996094,
      "step": 529
    },
    {
      "KL/chosen_KL_mean": -187.261474609375,
      "KL/mean": -231.58837890625,
      "KL/rejected_KL_mean": -275.91534423828125,
      "KL/std": 106.54146575927734,
      "epoch": 0.7782672540381792,
      "fcm_dpo/beta": 0.004246166907250881,
      "fcm_dpo/delta": 0.026189567521214485,
      "fcm_dpo/margin": 88.65385437011719,
      "fcm_dpo/q_t": 0.4123299717903137,
      "grad_norm": 10.496817588806152,
      "learning_rate": 7.231818622338822e-08,
      "logits/chosen": -1.704482078552246,
      "logits/rejected": -1.3910592794418335,
      "logps/chosen": -267.0372009277344,
      "logps/ref_chosen": -79.7757339477539,
      "logps/ref_rejected": -92.05390167236328,
      "logps/rejected": -367.96923828125,
      "loss": 1.0902,
      "margin_dpo/margin_mean": 88.65385437011719,
      "margin_dpo/margin_std": 102.09535217285156,
      "step": 530
    },
    {
      "KL/chosen_KL_mean": -181.89878845214844,
      "KL/mean": -237.29168701171875,
      "KL/rejected_KL_mean": -292.6846008300781,
      "KL/std": 112.62481689453125,
      "epoch": 0.7797356828193832,
      "fcm_dpo/beta": 0.004153624176979065,
      "fcm_dpo/delta": -0.07027285546064377,
      "fcm_dpo/margin": 110.78581237792969,
      "fcm_dpo/q_t": 0.3897207975387573,
      "grad_norm": 11.4281644821167,
      "learning_rate": 7.141774982445147e-08,
      "logits/chosen": -1.9349925518035889,
      "logits/rejected": -1.3425521850585938,
      "logps/chosen": -261.2179870605469,
      "logps/ref_chosen": -79.31919860839844,
      "logps/ref_rejected": -78.10832214355469,
      "logps/rejected": -370.7929382324219,
      "loss": 1.0293,
      "margin_dpo/margin_mean": 110.78582000732422,
      "margin_dpo/margin_std": 112.08589935302734,
      "step": 531
    },
    {
      "KL/chosen_KL_mean": -195.58877563476562,
      "KL/mean": -255.87948608398438,
      "KL/rejected_KL_mean": -316.1701965332031,
      "KL/std": 107.40847778320312,
      "epoch": 0.7812041116005873,
      "fcm_dpo/beta": 0.003982946742326021,
      "fcm_dpo/delta": -0.09348351508378983,
      "fcm_dpo/margin": 120.5814208984375,
      "fcm_dpo/q_t": 0.38538652658462524,
      "grad_norm": 10.557217597961426,
      "learning_rate": 7.052201923388953e-08,
      "logits/chosen": -1.5607173442840576,
      "logits/rejected": -0.9219512939453125,
      "logps/chosen": -297.2257080078125,
      "logps/ref_chosen": -101.63691711425781,
      "logps/ref_rejected": -93.34539031982422,
      "logps/rejected": -409.51556396484375,
      "loss": 1.0179,
      "margin_dpo/margin_mean": 120.58142852783203,
      "margin_dpo/margin_std": 118.92073059082031,
      "step": 532
    },
    {
      "KL/chosen_KL_mean": -189.05186462402344,
      "KL/mean": -224.00613403320312,
      "KL/rejected_KL_mean": -258.9604187011719,
      "KL/std": 113.70378112792969,
      "epoch": 0.7826725403817915,
      "fcm_dpo/beta": 0.004135998897254467,
      "fcm_dpo/delta": 0.12060149013996124,
      "fcm_dpo/margin": 69.90855407714844,
      "fcm_dpo/q_t": 0.4355907738208771,
      "grad_norm": 10.824801445007324,
      "learning_rate": 6.963101805503646e-08,
      "logits/chosen": -2.3724262714385986,
      "logits/rejected": -1.76796293258667,
      "logps/chosen": -279.79852294921875,
      "logps/ref_chosen": -90.74664306640625,
      "logps/ref_rejected": -82.88848876953125,
      "logps/rejected": -341.8489074707031,
      "loss": 1.179,
      "margin_dpo/margin_mean": 69.90855407714844,
      "margin_dpo/margin_std": 122.77506256103516,
      "step": 533
    },
    {
      "KL/chosen_KL_mean": -169.17897033691406,
      "KL/mean": -220.79296875,
      "KL/rejected_KL_mean": -272.4069519042969,
      "KL/std": 106.51093292236328,
      "epoch": 0.7841409691629956,
      "fcm_dpo/beta": 0.004149862565100193,
      "fcm_dpo/delta": -0.03254782408475876,
      "fcm_dpo/margin": 103.22799682617188,
      "fcm_dpo/q_t": 0.39809566736221313,
      "grad_norm": 9.98300552368164,
      "learning_rate": 6.874476976660184e-08,
      "logits/chosen": -1.8245972394943237,
      "logits/rejected": -1.4191936254501343,
      "logps/chosen": -255.809814453125,
      "logps/ref_chosen": -86.6308364868164,
      "logps/ref_rejected": -84.87196350097656,
      "logps/rejected": -357.2789306640625,
      "loss": 1.0483,
      "margin_dpo/margin_mean": 103.22799682617188,
      "margin_dpo/margin_std": 104.35403442382812,
      "step": 534
    },
    {
      "KL/chosen_KL_mean": -171.13681030273438,
      "KL/mean": -225.700927734375,
      "KL/rejected_KL_mean": -280.26507568359375,
      "KL/std": 107.44436645507812,
      "epoch": 0.7856093979441997,
      "fcm_dpo/beta": 0.004070422612130642,
      "fcm_dpo/delta": -0.051427628844976425,
      "fcm_dpo/margin": 109.12826538085938,
      "fcm_dpo/q_t": 0.39567461609840393,
      "grad_norm": 7.820010185241699,
      "learning_rate": 6.786329772205246e-08,
      "logits/chosen": -1.8713486194610596,
      "logits/rejected": -1.687537431716919,
      "logps/chosen": -261.482177734375,
      "logps/ref_chosen": -90.34539031982422,
      "logps/ref_rejected": -100.7779769897461,
      "logps/rejected": -381.04302978515625,
      "loss": 1.0511,
      "margin_dpo/margin_mean": 109.12826538085938,
      "margin_dpo/margin_std": 122.62210083007812,
      "step": 535
    },
    {
      "KL/chosen_KL_mean": -169.99746704101562,
      "KL/mean": -223.65606689453125,
      "KL/rejected_KL_mean": -277.314697265625,
      "KL/std": 108.22160339355469,
      "epoch": 0.7870778267254038,
      "fcm_dpo/beta": 0.003922410309314728,
      "fcm_dpo/delta": -0.024340644478797913,
      "fcm_dpo/margin": 107.31721496582031,
      "fcm_dpo/q_t": 0.4000042676925659,
      "grad_norm": 8.489453315734863,
      "learning_rate": 6.698662514899638e-08,
      "logits/chosen": -1.882274866104126,
      "logits/rejected": -1.6320252418518066,
      "logps/chosen": -248.95704650878906,
      "logps/ref_chosen": -78.95956420898438,
      "logps/ref_rejected": -93.08779907226562,
      "logps/rejected": -370.4024658203125,
      "loss": 1.0566,
      "margin_dpo/margin_mean": 107.31721496582031,
      "margin_dpo/margin_std": 113.55787658691406,
      "step": 536
    },
    {
      "KL/chosen_KL_mean": -148.06005859375,
      "KL/mean": -205.0472869873047,
      "KL/rejected_KL_mean": -262.0345153808594,
      "KL/std": 111.74270629882812,
      "epoch": 0.788546255506608,
      "fcm_dpo/beta": 0.0039041107520461082,
      "fcm_dpo/delta": -0.05296957492828369,
      "fcm_dpo/margin": 113.97444152832031,
      "fcm_dpo/q_t": 0.39364349842071533,
      "grad_norm": 9.30515193939209,
      "learning_rate": 6.611477514857114e-08,
      "logits/chosen": -1.5199506282806396,
      "logits/rejected": -0.823725163936615,
      "logps/chosen": -234.76943969726562,
      "logps/ref_chosen": -86.70939636230469,
      "logps/ref_rejected": -78.90766906738281,
      "logps/rejected": -340.94219970703125,
      "loss": 1.0375,
      "margin_dpo/margin_mean": 113.97445678710938,
      "margin_dpo/margin_std": 108.98240661621094,
      "step": 537
    },
    {
      "KL/chosen_KL_mean": -202.70538330078125,
      "KL/mean": -248.8926239013672,
      "KL/rejected_KL_mean": -295.07989501953125,
      "KL/std": 99.61503601074219,
      "epoch": 0.7900146842878121,
      "fcm_dpo/beta": 0.003890986554324627,
      "fcm_dpo/delta": 0.04536451771855354,
      "fcm_dpo/margin": 92.37451171875,
      "fcm_dpo/q_t": 0.4157577455043793,
      "grad_norm": 8.386140823364258,
      "learning_rate": 6.524777069483525e-08,
      "logits/chosen": -1.9386924505233765,
      "logits/rejected": -1.4087029695510864,
      "logps/chosen": -301.14752197265625,
      "logps/ref_chosen": -98.44214630126953,
      "logps/ref_rejected": -91.09465026855469,
      "logps/rejected": -386.1745300292969,
      "loss": 1.0998,
      "margin_dpo/margin_mean": 92.37451171875,
      "margin_dpo/margin_std": 106.43948364257812,
      "step": 538
    },
    {
      "KL/chosen_KL_mean": -176.45428466796875,
      "KL/mean": -225.40818786621094,
      "KL/rejected_KL_mean": -274.36212158203125,
      "KL/std": 109.82302856445312,
      "epoch": 0.7914831130690162,
      "fcm_dpo/beta": 0.003942383453249931,
      "fcm_dpo/delta": 0.015825822949409485,
      "fcm_dpo/margin": 97.90782165527344,
      "fcm_dpo/q_t": 0.40988391637802124,
      "grad_norm": 8.388779640197754,
      "learning_rate": 6.438563463416221e-08,
      "logits/chosen": -2.4419641494750977,
      "logits/rejected": -1.9623963832855225,
      "logps/chosen": -267.38641357421875,
      "logps/ref_chosen": -90.93211364746094,
      "logps/ref_rejected": -97.44546508789062,
      "logps/rejected": -371.80755615234375,
      "loss": 1.0947,
      "margin_dpo/margin_mean": 97.90782165527344,
      "margin_dpo/margin_std": 125.05751037597656,
      "step": 539
    },
    {
      "KL/chosen_KL_mean": -188.11778259277344,
      "KL/mean": -247.31631469726562,
      "KL/rejected_KL_mean": -306.51483154296875,
      "KL/std": 119.82254028320312,
      "epoch": 0.7929515418502202,
      "fcm_dpo/beta": 0.0038778926245868206,
      "fcm_dpo/delta": -0.06876187771558762,
      "fcm_dpo/margin": 118.39701843261719,
      "fcm_dpo/q_t": 0.39333227276802063,
      "grad_norm": 10.159441947937012,
      "learning_rate": 6.352838968463919e-08,
      "logits/chosen": -2.1140952110290527,
      "logits/rejected": -1.8822753429412842,
      "logps/chosen": -295.9876708984375,
      "logps/ref_chosen": -107.86990356445312,
      "logps/ref_rejected": -121.39350128173828,
      "logps/rejected": -427.9083251953125,
      "loss": 1.0495,
      "margin_dpo/margin_mean": 118.39702606201172,
      "margin_dpo/margin_std": 140.55667114257812,
      "step": 540
    },
    {
      "KL/chosen_KL_mean": -208.94760131835938,
      "KL/mean": -246.2579345703125,
      "KL/rejected_KL_mean": -283.5682678222656,
      "KL/std": 99.24880981445312,
      "epoch": 0.7944199706314243,
      "fcm_dpo/beta": 0.0037974445149302483,
      "fcm_dpo/delta": 0.003563603851944208,
      "fcm_dpo/margin": 74.62067413330078,
      "fcm_dpo/q_t": 0.43320125341415405,
      "grad_norm": 8.794666290283203,
      "learning_rate": 6.267605843546767e-08,
      "logits/chosen": -2.41945743560791,
      "logits/rejected": -1.9616498947143555,
      "logps/chosen": -315.51507568359375,
      "logps/ref_chosen": -106.56748962402344,
      "logps/ref_rejected": -109.06633758544922,
      "logps/rejected": -392.6346130371094,
      "loss": 1.1672,
      "margin_dpo/margin_mean": 74.62067413330078,
      "margin_dpo/margin_std": 107.01324462890625,
      "step": 541
    },
    {
      "KL/chosen_KL_mean": -178.8602752685547,
      "KL/mean": -235.1538848876953,
      "KL/rejected_KL_mean": -291.447509765625,
      "KL/std": 111.1739730834961,
      "epoch": 0.7958883994126285,
      "fcm_dpo/beta": 0.003750790376216173,
      "fcm_dpo/delta": -0.02556237205862999,
      "fcm_dpo/margin": 112.58721923828125,
      "fcm_dpo/q_t": 0.39956605434417725,
      "grad_norm": 9.33528995513916,
      "learning_rate": 6.182866334636888e-08,
      "logits/chosen": -2.3600540161132812,
      "logits/rejected": -2.2404234409332275,
      "logps/chosen": -266.22955322265625,
      "logps/ref_chosen": -87.36929321289062,
      "logps/ref_rejected": -102.82034301757812,
      "logps/rejected": -394.267822265625,
      "loss": 1.0567,
      "margin_dpo/margin_mean": 112.58721923828125,
      "margin_dpo/margin_std": 120.4144287109375,
      "step": 542
    },
    {
      "KL/chosen_KL_mean": -203.34115600585938,
      "KL/mean": -233.813232421875,
      "KL/rejected_KL_mean": -264.2852783203125,
      "KL/std": 118.22736358642578,
      "epoch": 0.7973568281938326,
      "fcm_dpo/beta": 0.003860362572595477,
      "fcm_dpo/delta": 0.052950333803892136,
      "fcm_dpo/margin": 60.94415283203125,
      "fcm_dpo/q_t": 0.4464949071407318,
      "grad_norm": 12.585554122924805,
      "learning_rate": 6.098622674699147e-08,
      "logits/chosen": -1.8398051261901855,
      "logits/rejected": -1.886562705039978,
      "logps/chosen": -290.4937438964844,
      "logps/ref_chosen": -87.152587890625,
      "logps/ref_rejected": -111.53172302246094,
      "logps/rejected": -375.8170166015625,
      "loss": 1.2394,
      "margin_dpo/margin_mean": 60.944156646728516,
      "margin_dpo/margin_std": 143.9779052734375,
      "step": 543
    },
    {
      "KL/chosen_KL_mean": -213.6553955078125,
      "KL/mean": -252.4432373046875,
      "KL/rejected_KL_mean": -291.2310791015625,
      "KL/std": 111.27857971191406,
      "epoch": 0.7988252569750367,
      "fcm_dpo/beta": 0.004025270231068134,
      "fcm_dpo/delta": 0.09691111743450165,
      "fcm_dpo/margin": 77.57566833496094,
      "fcm_dpo/q_t": 0.4288977384567261,
      "grad_norm": 8.990296363830566,
      "learning_rate": 6.01487708363232e-08,
      "logits/chosen": -1.5993623733520508,
      "logits/rejected": -1.5061476230621338,
      "logps/chosen": -299.80230712890625,
      "logps/ref_chosen": -86.14691162109375,
      "logps/ref_rejected": -106.3939208984375,
      "logps/rejected": -397.625,
      "loss": 1.1499,
      "margin_dpo/margin_mean": 77.57566833496094,
      "margin_dpo/margin_std": 116.20040893554688,
      "step": 544
    },
    {
      "KL/chosen_KL_mean": -189.72689819335938,
      "KL/mean": -236.7539825439453,
      "KL/rejected_KL_mean": -283.78106689453125,
      "KL/std": 110.43536376953125,
      "epoch": 0.8002936857562408,
      "fcm_dpo/beta": 0.0041359784081578255,
      "fcm_dpo/delta": 0.012169305235147476,
      "fcm_dpo/margin": 94.05418395996094,
      "fcm_dpo/q_t": 0.4101467430591583,
      "grad_norm": 9.021712303161621,
      "learning_rate": 5.9316317682106294e-08,
      "logits/chosen": -1.6832289695739746,
      "logits/rejected": -1.26765775680542,
      "logps/chosen": -286.151123046875,
      "logps/ref_chosen": -96.42424011230469,
      "logps/ref_rejected": -100.73405456542969,
      "logps/rejected": -384.51513671875,
      "loss": 1.1106,
      "margin_dpo/margin_mean": 94.05418395996094,
      "margin_dpo/margin_std": 136.57858276367188,
      "step": 545
    },
    {
      "KL/chosen_KL_mean": -195.30642700195312,
      "KL/mean": -236.9781494140625,
      "KL/rejected_KL_mean": -278.64990234375,
      "KL/std": 111.65972137451172,
      "epoch": 0.801762114537445,
      "fcm_dpo/beta": 0.004204741679131985,
      "fcm_dpo/delta": 0.05549057945609093,
      "fcm_dpo/margin": 83.34346008300781,
      "fcm_dpo/q_t": 0.418612539768219,
      "grad_norm": 9.415722846984863,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": -1.5976366996765137,
      "logits/rejected": -1.2471153736114502,
      "logps/chosen": -281.3275451660156,
      "logps/ref_chosen": -86.02112579345703,
      "logps/ref_rejected": -87.37263488769531,
      "logps/rejected": -366.02252197265625,
      "loss": 1.1307,
      "margin_dpo/margin_mean": 83.34346008300781,
      "margin_dpo/margin_std": 123.3160400390625,
      "step": 546
    },
    {
      "KL/chosen_KL_mean": -187.77781677246094,
      "KL/mean": -238.57717895507812,
      "KL/rejected_KL_mean": -289.3765563964844,
      "KL/std": 108.89897155761719,
      "epoch": 0.8032305433186491,
      "fcm_dpo/beta": 0.004230954684317112,
      "fcm_dpo/delta": -0.03446131944656372,
      "fcm_dpo/margin": 101.59874725341797,
      "fcm_dpo/q_t": 0.39737388491630554,
      "grad_norm": 9.553152084350586,
      "learning_rate": 5.7666507254280265e-08,
      "logits/chosen": -1.8938827514648438,
      "logits/rejected": -1.433687448501587,
      "logps/chosen": -288.4288330078125,
      "logps/ref_chosen": -100.6510009765625,
      "logps/ref_rejected": -97.9117431640625,
      "logps/rejected": -387.2882995605469,
      "loss": 1.0436,
      "margin_dpo/margin_mean": 101.59874725341797,
      "margin_dpo/margin_std": 97.37659454345703,
      "step": 547
    },
    {
      "KL/chosen_KL_mean": -184.2064208984375,
      "KL/mean": -226.0633087158203,
      "KL/rejected_KL_mean": -267.9201965332031,
      "KL/std": 111.45756530761719,
      "epoch": 0.8046989720998532,
      "fcm_dpo/beta": 0.0042680734768509865,
      "fcm_dpo/delta": 0.04776104539632797,
      "fcm_dpo/margin": 83.7137451171875,
      "fcm_dpo/q_t": 0.4184664487838745,
      "grad_norm": 10.770520210266113,
      "learning_rate": 5.684919345471029e-08,
      "logits/chosen": -2.3766584396362305,
      "logits/rejected": -2.0264182090759277,
      "logps/chosen": -280.2365417480469,
      "logps/ref_chosen": -96.03010559082031,
      "logps/ref_rejected": -98.41322326660156,
      "logps/rejected": -366.33343505859375,
      "loss": 1.1468,
      "margin_dpo/margin_mean": 83.7137451171875,
      "margin_dpo/margin_std": 140.7174835205078,
      "step": 548
    },
    {
      "KL/chosen_KL_mean": -174.92916870117188,
      "KL/mean": -217.70770263671875,
      "KL/rejected_KL_mean": -260.48626708984375,
      "KL/std": 105.59712219238281,
      "epoch": 0.8061674008810573,
      "fcm_dpo/beta": 0.004407069180160761,
      "fcm_dpo/delta": 0.023885484784841537,
      "fcm_dpo/margin": 85.55708312988281,
      "fcm_dpo/q_t": 0.41243159770965576,
      "grad_norm": 11.569587707519531,
      "learning_rate": 5.603696935852426e-08,
      "logits/chosen": -1.8492355346679688,
      "logits/rejected": -1.4330787658691406,
      "logps/chosen": -258.380126953125,
      "logps/ref_chosen": -83.45095825195312,
      "logps/ref_rejected": -81.38023376464844,
      "logps/rejected": -341.8664855957031,
      "loss": 1.1274,
      "margin_dpo/margin_mean": 85.55708312988281,
      "margin_dpo/margin_std": 130.06541442871094,
      "step": 549
    },
    {
      "KL/chosen_KL_mean": -194.205078125,
      "KL/mean": -236.12789916992188,
      "KL/rejected_KL_mean": -278.0506896972656,
      "KL/std": 100.21720123291016,
      "epoch": 0.8076358296622613,
      "fcm_dpo/beta": 0.004443068988621235,
      "fcm_dpo/delta": 0.03047604113817215,
      "fcm_dpo/margin": 83.84562683105469,
      "fcm_dpo/q_t": 0.41390302777290344,
      "grad_norm": 9.870973587036133,
      "learning_rate": 5.5229856368582376e-08,
      "logits/chosen": -1.379887580871582,
      "logits/rejected": -1.1600041389465332,
      "logps/chosen": -283.7994384765625,
      "logps/ref_chosen": -89.59434509277344,
      "logps/ref_rejected": -101.31549835205078,
      "logps/rejected": -379.3662109375,
      "loss": 1.106,
      "margin_dpo/margin_mean": 83.84562683105469,
      "margin_dpo/margin_std": 109.32859802246094,
      "step": 550
    },
    {
      "KL/chosen_KL_mean": -172.8737030029297,
      "KL/mean": -224.17605590820312,
      "KL/rejected_KL_mean": -275.4783935546875,
      "KL/std": 114.53459167480469,
      "epoch": 0.8091042584434655,
      "fcm_dpo/beta": 0.004280459135770798,
      "fcm_dpo/delta": -0.047517456114292145,
      "fcm_dpo/margin": 102.60468292236328,
      "fcm_dpo/q_t": 0.397729754447937,
      "grad_norm": 10.708085060119629,
      "learning_rate": 5.4427875753062734e-08,
      "logits/chosen": -1.7007884979248047,
      "logits/rejected": -1.5609946250915527,
      "logps/chosen": -267.95587158203125,
      "logps/ref_chosen": -95.08216094970703,
      "logps/ref_rejected": -117.7192611694336,
      "logps/rejected": -393.1976318359375,
      "loss": 1.0537,
      "margin_dpo/margin_mean": 102.60468292236328,
      "margin_dpo/margin_std": 116.69213104248047,
      "step": 551
    },
    {
      "KL/chosen_KL_mean": -165.56558227539062,
      "KL/mean": -238.70684814453125,
      "KL/rejected_KL_mean": -311.8481140136719,
      "KL/std": 123.98436737060547,
      "epoch": 0.8105726872246696,
      "fcm_dpo/beta": 0.003835270181298256,
      "fcm_dpo/delta": -0.2050240933895111,
      "fcm_dpo/margin": 146.28253173828125,
      "fcm_dpo/q_t": 0.36199963092803955,
      "grad_norm": 10.958539962768555,
      "learning_rate": 5.363104864490034e-08,
      "logits/chosen": -2.1056160926818848,
      "logits/rejected": -1.7475719451904297,
      "logps/chosen": -266.9747619628906,
      "logps/ref_chosen": -101.40918731689453,
      "logps/ref_rejected": -110.83843994140625,
      "logps/rejected": -422.6865539550781,
      "loss": 0.9622,
      "margin_dpo/margin_mean": 146.28253173828125,
      "margin_dpo/margin_std": 129.02548217773438,
      "step": 552
    },
    {
      "KL/chosen_KL_mean": -177.8858642578125,
      "KL/mean": -221.59130859375,
      "KL/rejected_KL_mean": -265.2967529296875,
      "KL/std": 114.97552490234375,
      "epoch": 0.8120411160058737,
      "fcm_dpo/beta": 0.003941618371754885,
      "fcm_dpo/delta": 0.0612381249666214,
      "fcm_dpo/margin": 87.41089630126953,
      "fcm_dpo/q_t": 0.4197506904602051,
      "grad_norm": 9.161893844604492,
      "learning_rate": 5.2839396041230415e-08,
      "logits/chosen": -1.5530469417572021,
      "logits/rejected": -1.2235181331634521,
      "logps/chosen": -275.8497314453125,
      "logps/ref_chosen": -97.96388244628906,
      "logps/ref_rejected": -103.57719421386719,
      "logps/rejected": -368.87396240234375,
      "loss": 1.1278,
      "margin_dpo/margin_mean": 87.41090393066406,
      "margin_dpo/margin_std": 122.52503967285156,
      "step": 553
    },
    {
      "KL/chosen_KL_mean": -187.28656005859375,
      "KL/mean": -235.57254028320312,
      "KL/rejected_KL_mean": -283.85858154296875,
      "KL/std": 115.59172821044922,
      "epoch": 0.8135095447870778,
      "fcm_dpo/beta": 0.004022675100713968,
      "fcm_dpo/delta": 0.011999618262052536,
      "fcm_dpo/margin": 96.57200622558594,
      "fcm_dpo/q_t": 0.41062504053115845,
      "grad_norm": 8.901398658752441,
      "learning_rate": 5.205293880283551e-08,
      "logits/chosen": -1.9454165697097778,
      "logits/rejected": -1.178072214126587,
      "logps/chosen": -290.9733581542969,
      "logps/ref_chosen": -103.68680572509766,
      "logps/ref_rejected": -92.99210357666016,
      "logps/rejected": -376.8506774902344,
      "loss": 1.1212,
      "margin_dpo/margin_mean": 96.57200622558594,
      "margin_dpo/margin_std": 149.0669708251953,
      "step": 554
    },
    {
      "KL/chosen_KL_mean": -195.4247589111328,
      "KL/mean": -239.95315551757812,
      "KL/rejected_KL_mean": -284.48150634765625,
      "KL/std": 118.93658447265625,
      "epoch": 0.8149779735682819,
      "fcm_dpo/beta": 0.004061169922351837,
      "fcm_dpo/delta": 0.042991891503334045,
      "fcm_dpo/margin": 89.05674743652344,
      "fcm_dpo/q_t": 0.41784989833831787,
      "grad_norm": 9.341143608093262,
      "learning_rate": 5.127169765359515e-08,
      "logits/chosen": -1.7601932287216187,
      "logits/rejected": -1.6699495315551758,
      "logps/chosen": -281.6283874511719,
      "logps/ref_chosen": -86.20362854003906,
      "logps/ref_rejected": -114.54719543457031,
      "logps/rejected": -399.0287170410156,
      "loss": 1.1307,
      "margin_dpo/margin_mean": 89.0567398071289,
      "margin_dpo/margin_std": 137.84878540039062,
      "step": 555
    },
    {
      "KL/chosen_KL_mean": -176.10076904296875,
      "KL/mean": -224.221923828125,
      "KL/rejected_KL_mean": -272.34307861328125,
      "KL/std": 105.5389404296875,
      "epoch": 0.8164464023494861,
      "fcm_dpo/beta": 0.004068740177899599,
      "fcm_dpo/delta": 0.00903857871890068,
      "fcm_dpo/margin": 96.2422866821289,
      "fcm_dpo/q_t": 0.4085083603858948,
      "grad_norm": 10.64559268951416,
      "learning_rate": 5.049569317994012e-08,
      "logits/chosen": -1.6232037544250488,
      "logits/rejected": -1.3248238563537598,
      "logps/chosen": -264.5042724609375,
      "logps/ref_chosen": -88.40348815917969,
      "logps/ref_rejected": -106.63911437988281,
      "logps/rejected": -378.982177734375,
      "loss": 1.0895,
      "margin_dpo/margin_mean": 96.2422866821289,
      "margin_dpo/margin_std": 119.52124786376953,
      "step": 556
    },
    {
      "KL/chosen_KL_mean": -189.38868713378906,
      "KL/mean": -244.8987579345703,
      "KL/rejected_KL_mean": -300.4088134765625,
      "KL/std": 122.444580078125,
      "epoch": 0.8179148311306902,
      "fcm_dpo/beta": 0.003968755714595318,
      "fcm_dpo/delta": -0.04884597659111023,
      "fcm_dpo/margin": 111.02012634277344,
      "fcm_dpo/q_t": 0.39723390340805054,
      "grad_norm": 7.967887878417969,
      "learning_rate": 4.9724945830310144e-08,
      "logits/chosen": -2.104808807373047,
      "logits/rejected": -1.9345542192459106,
      "logps/chosen": -288.22393798828125,
      "logps/ref_chosen": -98.83523559570312,
      "logps/ref_rejected": -115.64888763427734,
      "logps/rejected": -416.05767822265625,
      "loss": 1.0615,
      "margin_dpo/margin_mean": 111.02011108398438,
      "margin_dpo/margin_std": 134.34664916992188,
      "step": 557
    },
    {
      "KL/chosen_KL_mean": -187.15078735351562,
      "KL/mean": -246.51300048828125,
      "KL/rejected_KL_mean": -305.87518310546875,
      "KL/std": 105.57743835449219,
      "epoch": 0.8193832599118943,
      "fcm_dpo/beta": 0.0038486546836793423,
      "fcm_dpo/delta": -0.06685430556535721,
      "fcm_dpo/margin": 118.72438049316406,
      "fcm_dpo/q_t": 0.3903621733188629,
      "grad_norm": 10.927623748779297,
      "learning_rate": 4.8959475914614554e-08,
      "logits/chosen": -2.22155499458313,
      "logits/rejected": -1.5982849597930908,
      "logps/chosen": -293.1626281738281,
      "logps/ref_chosen": -106.01183319091797,
      "logps/ref_rejected": -108.39131927490234,
      "logps/rejected": -414.2664794921875,
      "loss": 1.0218,
      "margin_dpo/margin_mean": 118.72438049316406,
      "margin_dpo/margin_std": 106.77873229980469,
      "step": 558
    },
    {
      "KL/chosen_KL_mean": -202.4082489013672,
      "KL/mean": -249.16998291015625,
      "KL/rejected_KL_mean": -295.93170166015625,
      "KL/std": 108.4201889038086,
      "epoch": 0.8208516886930984,
      "fcm_dpo/beta": 0.003912343643605709,
      "fcm_dpo/delta": 0.038132064044475555,
      "fcm_dpo/margin": 93.5234375,
      "fcm_dpo/q_t": 0.4142046272754669,
      "grad_norm": 9.9929838180542,
      "learning_rate": 4.8199303603697614e-08,
      "logits/chosen": -2.3181519508361816,
      "logits/rejected": -1.839745283126831,
      "logps/chosen": -302.8330078125,
      "logps/ref_chosen": -100.42474365234375,
      "logps/ref_rejected": -100.41270446777344,
      "logps/rejected": -396.34442138671875,
      "loss": 1.1088,
      "margin_dpo/margin_mean": 93.52342224121094,
      "margin_dpo/margin_std": 122.3238754272461,
      "step": 559
    },
    {
      "KL/chosen_KL_mean": -192.26071166992188,
      "KL/mean": -238.5875244140625,
      "KL/rejected_KL_mean": -284.91436767578125,
      "KL/std": 106.11720275878906,
      "epoch": 0.8223201174743024,
      "fcm_dpo/beta": 0.003977730870246887,
      "fcm_dpo/delta": 0.03540133312344551,
      "fcm_dpo/margin": 92.65364837646484,
      "fcm_dpo/q_t": 0.4144192337989807,
      "grad_norm": 11.127015113830566,
      "learning_rate": 4.7444448928806615e-08,
      "logits/chosen": -1.5218555927276611,
      "logits/rejected": -1.086158037185669,
      "logps/chosen": -279.896240234375,
      "logps/ref_chosen": -87.63551330566406,
      "logps/ref_rejected": -87.37985229492188,
      "logps/rejected": -372.294189453125,
      "loss": 1.1078,
      "margin_dpo/margin_mean": 92.65364837646484,
      "margin_dpo/margin_std": 122.2921142578125,
      "step": 560
    },
    {
      "KL/chosen_KL_mean": -200.4535369873047,
      "KL/mean": -243.33480834960938,
      "KL/rejected_KL_mean": -286.2160949707031,
      "KL/std": 104.30130004882812,
      "epoch": 0.8237885462555066,
      "fcm_dpo/beta": 0.004126845858991146,
      "fcm_dpo/delta": 0.050900254398584366,
      "fcm_dpo/margin": 85.76255798339844,
      "fcm_dpo/q_t": 0.4173031747341156,
      "grad_norm": 8.778356552124023,
      "learning_rate": 4.669493178106432e-08,
      "logits/chosen": -1.893129587173462,
      "logits/rejected": -1.8658044338226318,
      "logps/chosen": -275.3349304199219,
      "logps/ref_chosen": -74.88140106201172,
      "logps/ref_rejected": -106.04796600341797,
      "logps/rejected": -392.2640380859375,
      "loss": 1.1042,
      "margin_dpo/margin_mean": 85.76255798339844,
      "margin_dpo/margin_std": 100.20466613769531,
      "step": 561
    },
    {
      "KL/chosen_KL_mean": -183.06106567382812,
      "KL/mean": -226.56558227539062,
      "KL/rejected_KL_mean": -270.07012939453125,
      "KL/std": 105.66160583496094,
      "epoch": 0.8252569750367107,
      "fcm_dpo/beta": 0.004170439206063747,
      "fcm_dpo/delta": 0.04143684729933739,
      "fcm_dpo/margin": 87.0090560913086,
      "fcm_dpo/q_t": 0.4169612228870392,
      "grad_norm": 11.038931846618652,
      "learning_rate": 4.5950771910944596e-08,
      "logits/chosen": -2.0500659942626953,
      "logits/rejected": -1.5897785425186157,
      "logps/chosen": -283.7242736816406,
      "logps/ref_chosen": -100.66322326660156,
      "logps/ref_rejected": -102.24087524414062,
      "logps/rejected": -372.31097412109375,
      "loss": 1.1265,
      "margin_dpo/margin_mean": 87.0090560913086,
      "margin_dpo/margin_std": 130.5972900390625,
      "step": 562
    },
    {
      "KL/chosen_KL_mean": -193.55206298828125,
      "KL/mean": -229.45809936523438,
      "KL/rejected_KL_mean": -265.3641357421875,
      "KL/std": 106.331787109375,
      "epoch": 0.8267254038179148,
      "fcm_dpo/beta": 0.004455032758414745,
      "fcm_dpo/delta": 0.08655049651861191,
      "fcm_dpo/margin": 71.81204986572266,
      "fcm_dpo/q_t": 0.4265323579311371,
      "grad_norm": 10.926514625549316,
      "learning_rate": 4.521198892775202e-08,
      "logits/chosen": -1.801181674003601,
      "logits/rejected": -1.4691920280456543,
      "logps/chosen": -289.811279296875,
      "logps/ref_chosen": -96.25921630859375,
      "logps/ref_rejected": -101.20503997802734,
      "logps/rejected": -366.56915283203125,
      "loss": 1.1539,
      "margin_dpo/margin_mean": 71.81205749511719,
      "margin_dpo/margin_std": 113.65653228759766,
      "step": 563
    },
    {
      "KL/chosen_KL_mean": -189.50576782226562,
      "KL/mean": -229.3646697998047,
      "KL/rejected_KL_mean": -269.22357177734375,
      "KL/std": 106.7225570678711,
      "epoch": 0.8281938325991189,
      "fcm_dpo/beta": 0.004572875797748566,
      "fcm_dpo/delta": 0.03918202966451645,
      "fcm_dpo/margin": 79.71781158447266,
      "fcm_dpo/q_t": 0.4147194027900696,
      "grad_norm": 13.073539733886719,
      "learning_rate": 4.447860229910544e-08,
      "logits/chosen": -2.4111175537109375,
      "logits/rejected": -1.8475167751312256,
      "logps/chosen": -302.4661865234375,
      "logps/ref_chosen": -112.96040344238281,
      "logps/ref_rejected": -99.52803802490234,
      "logps/rejected": -368.7515869140625,
      "loss": 1.1025,
      "margin_dpo/margin_mean": 79.71781158447266,
      "margin_dpo/margin_std": 96.77831268310547,
      "step": 564
    },
    {
      "KL/chosen_KL_mean": -184.76455688476562,
      "KL/mean": -234.74411010742188,
      "KL/rejected_KL_mean": -284.72369384765625,
      "KL/std": 116.3817367553711,
      "epoch": 0.8296622613803231,
      "fcm_dpo/beta": 0.004445655271410942,
      "fcm_dpo/delta": -0.0516103133559227,
      "fcm_dpo/margin": 99.95916748046875,
      "fcm_dpo/q_t": 0.39692285656929016,
      "grad_norm": 9.306631088256836,
      "learning_rate": 4.375063135042445e-08,
      "logits/chosen": -1.7159419059753418,
      "logits/rejected": -1.212406873703003,
      "logps/chosen": -280.130126953125,
      "logps/ref_chosen": -95.36558532714844,
      "logps/ref_rejected": -92.21737670898438,
      "logps/rejected": -376.9410705566406,
      "loss": 1.0658,
      "margin_dpo/margin_mean": 99.95916748046875,
      "margin_dpo/margin_std": 127.66952514648438,
      "step": 565
    },
    {
      "KL/chosen_KL_mean": -194.26251220703125,
      "KL/mean": -241.98165893554688,
      "KL/rejected_KL_mean": -289.7008056640625,
      "KL/std": 118.60604858398438,
      "epoch": 0.8311306901615272,
      "fcm_dpo/beta": 0.0044143348932266235,
      "fcm_dpo/delta": -0.0245785154402256,
      "fcm_dpo/margin": 95.43828582763672,
      "fcm_dpo/q_t": 0.40098801255226135,
      "grad_norm": 8.731414794921875,
      "learning_rate": 4.3028095264420525e-08,
      "logits/chosen": -2.013485908508301,
      "logits/rejected": -1.588104009628296,
      "logps/chosen": -293.2357177734375,
      "logps/ref_chosen": -98.97320556640625,
      "logps/ref_rejected": -110.82090759277344,
      "logps/rejected": -400.5216979980469,
      "loss": 1.0745,
      "margin_dpo/margin_mean": 95.43828582763672,
      "margin_dpo/margin_std": 117.89845275878906,
      "step": 566
    },
    {
      "KL/chosen_KL_mean": -187.82748413085938,
      "KL/mean": -229.95889282226562,
      "KL/rejected_KL_mean": -272.09027099609375,
      "KL/std": 106.5650634765625,
      "epoch": 0.8325991189427313,
      "fcm_dpo/beta": 0.004368951544165611,
      "fcm_dpo/delta": 0.03481277450919151,
      "fcm_dpo/margin": 84.26277160644531,
      "fcm_dpo/q_t": 0.4137539267539978,
      "grad_norm": 10.604565620422363,
      "learning_rate": 4.231101308059165e-08,
      "logits/chosen": -2.297457218170166,
      "logits/rejected": -1.6272627115249634,
      "logps/chosen": -280.53106689453125,
      "logps/ref_chosen": -92.7035903930664,
      "logps/ref_rejected": -91.22431945800781,
      "logps/rejected": -363.3145751953125,
      "loss": 1.1132,
      "margin_dpo/margin_mean": 84.26277160644531,
      "margin_dpo/margin_std": 112.50050354003906,
      "step": 567
    },
    {
      "KL/chosen_KL_mean": -176.75115966796875,
      "KL/mean": -232.93435668945312,
      "KL/rejected_KL_mean": -289.1175537109375,
      "KL/std": 112.77764892578125,
      "epoch": 0.8340675477239354,
      "fcm_dpo/beta": 0.004227595869451761,
      "fcm_dpo/delta": -0.08999736607074738,
      "fcm_dpo/margin": 112.36637878417969,
      "fcm_dpo/q_t": 0.38686317205429077,
      "grad_norm": 8.667065620422363,
      "learning_rate": 4.1599403694720145e-08,
      "logits/chosen": -1.3494253158569336,
      "logits/rejected": -1.2197784185409546,
      "logps/chosen": -249.8834228515625,
      "logps/ref_chosen": -73.13226318359375,
      "logps/ref_rejected": -94.95265197753906,
      "logps/rejected": -384.0701904296875,
      "loss": 1.022,
      "margin_dpo/margin_mean": 112.36637878417969,
      "margin_dpo/margin_std": 114.337646484375,
      "step": 568
    },
    {
      "KL/chosen_KL_mean": -189.0832061767578,
      "KL/mean": -234.06153869628906,
      "KL/rejected_KL_mean": -279.0398864746094,
      "KL/std": 113.72801208496094,
      "epoch": 0.8355359765051396,
      "fcm_dpo/beta": 0.00427539087831974,
      "fcm_dpo/delta": 0.017157766968011856,
      "fcm_dpo/margin": 89.95668029785156,
      "fcm_dpo/q_t": 0.41089075803756714,
      "grad_norm": 14.798583984375,
      "learning_rate": 4.089328585837512e-08,
      "logits/chosen": -2.006195068359375,
      "logits/rejected": -1.6320712566375732,
      "logps/chosen": -278.30029296875,
      "logps/ref_chosen": -89.21708679199219,
      "logps/ref_rejected": -86.41105651855469,
      "logps/rejected": -365.450927734375,
      "loss": 1.1208,
      "margin_dpo/margin_mean": 89.95668029785156,
      "margin_dpo/margin_std": 137.07789611816406,
      "step": 569
    },
    {
      "KL/chosen_KL_mean": -168.63800048828125,
      "KL/mean": -227.6444854736328,
      "KL/rejected_KL_mean": -286.6509704589844,
      "KL/std": 108.15455627441406,
      "epoch": 0.8370044052863436,
      "fcm_dpo/beta": 0.004079293459653854,
      "fcm_dpo/delta": -0.09550732374191284,
      "fcm_dpo/margin": 118.01296997070312,
      "fcm_dpo/q_t": 0.3840605914592743,
      "grad_norm": 9.152946472167969,
      "learning_rate": 4.019267817841834e-08,
      "logits/chosen": -2.423555850982666,
      "logits/rejected": -1.7545349597930908,
      "logps/chosen": -263.4007568359375,
      "logps/ref_chosen": -94.76277160644531,
      "logps/ref_rejected": -88.846923828125,
      "logps/rejected": -375.4978942871094,
      "loss": 1.0116,
      "margin_dpo/margin_mean": 118.01296997070312,
      "margin_dpo/margin_std": 112.75928497314453,
      "step": 570
    },
    {
      "KL/chosen_KL_mean": -176.32920837402344,
      "KL/mean": -227.99671936035156,
      "KL/rejected_KL_mean": -279.66424560546875,
      "KL/std": 106.52212524414062,
      "epoch": 0.8384728340675477,
      "fcm_dpo/beta": 0.004025227855890989,
      "fcm_dpo/delta": -0.018458478152751923,
      "fcm_dpo/margin": 103.33501434326172,
      "fcm_dpo/q_t": 0.4029388725757599,
      "grad_norm": 11.878338813781738,
      "learning_rate": 3.9497599116513705e-08,
      "logits/chosen": -1.527815818786621,
      "logits/rejected": -1.3652896881103516,
      "logps/chosen": -257.1505126953125,
      "logps/ref_chosen": -80.8212890625,
      "logps/ref_rejected": -97.01528930664062,
      "logps/rejected": -376.67950439453125,
      "loss": 1.0769,
      "margin_dpo/margin_mean": 103.33502197265625,
      "margin_dpo/margin_std": 128.2970428466797,
      "step": 571
    },
    {
      "KL/chosen_KL_mean": -180.7730712890625,
      "KL/mean": -233.1273956298828,
      "KL/rejected_KL_mean": -285.4817199707031,
      "KL/std": 113.48738098144531,
      "epoch": 0.8399412628487518,
      "fcm_dpo/beta": 0.003943878225982189,
      "fcm_dpo/delta": -0.014938858337700367,
      "fcm_dpo/margin": 104.70866394042969,
      "fcm_dpo/q_t": 0.40394070744514465,
      "grad_norm": 8.617586135864258,
      "learning_rate": 3.880806698864086e-08,
      "logits/chosen": -1.589660882949829,
      "logits/rejected": -1.4154298305511475,
      "logps/chosen": -258.15301513671875,
      "logps/ref_chosen": -77.37992858886719,
      "logps/ref_rejected": -90.56597900390625,
      "logps/rejected": -376.0476989746094,
      "loss": 1.0695,
      "margin_dpo/margin_mean": 104.70866394042969,
      "margin_dpo/margin_std": 121.6864013671875,
      "step": 572
    },
    {
      "KL/chosen_KL_mean": -175.2093505859375,
      "KL/mean": -224.38743591308594,
      "KL/rejected_KL_mean": -273.56549072265625,
      "KL/std": 112.79222106933594,
      "epoch": 0.8414096916299559,
      "fcm_dpo/beta": 0.004027231130748987,
      "fcm_dpo/delta": 0.0019082464277744293,
      "fcm_dpo/margin": 98.35614776611328,
      "fcm_dpo/q_t": 0.406552791595459,
      "grad_norm": 8.65018081665039,
      "learning_rate": 3.812409996461275e-08,
      "logits/chosen": -2.114858627319336,
      "logits/rejected": -1.8833644390106201,
      "logps/chosen": -258.9154357910156,
      "logps/ref_chosen": -83.7060775756836,
      "logps/ref_rejected": -90.69746398925781,
      "logps/rejected": -364.2629699707031,
      "loss": 1.0884,
      "margin_dpo/margin_mean": 98.35614776611328,
      "margin_dpo/margin_std": 119.68278503417969,
      "step": 573
    },
    {
      "KL/chosen_KL_mean": -184.7670135498047,
      "KL/mean": -237.89739990234375,
      "KL/rejected_KL_mean": -291.02777099609375,
      "KL/std": 111.96734619140625,
      "epoch": 0.8428781204111601,
      "fcm_dpo/beta": 0.00394933857023716,
      "fcm_dpo/delta": -0.022861644625663757,
      "fcm_dpo/margin": 106.26078796386719,
      "fcm_dpo/q_t": 0.40136775374412537,
      "grad_norm": 9.204578399658203,
      "learning_rate": 3.74457160675965e-08,
      "logits/chosen": -2.1955060958862305,
      "logits/rejected": -1.8721802234649658,
      "logps/chosen": -269.896484375,
      "logps/ref_chosen": -85.12948608398438,
      "logps/ref_rejected": -99.05403137207031,
      "logps/rejected": -390.0818176269531,
      "loss": 1.0703,
      "margin_dpo/margin_mean": 106.26078796386719,
      "margin_dpo/margin_std": 126.48181915283203,
      "step": 574
    },
    {
      "KL/chosen_KL_mean": -191.8727264404297,
      "KL/mean": -240.60552978515625,
      "KL/rejected_KL_mean": -289.33831787109375,
      "KL/std": 114.75996398925781,
      "epoch": 0.8443465491923642,
      "fcm_dpo/beta": 0.0039310092106461525,
      "fcm_dpo/delta": 0.019057439640164375,
      "fcm_dpo/margin": 97.46558380126953,
      "fcm_dpo/q_t": 0.4112933576107025,
      "grad_norm": 10.298196792602539,
      "learning_rate": 3.677293317363864e-08,
      "logits/chosen": -1.191744089126587,
      "logits/rejected": -0.709888219833374,
      "logps/chosen": -297.6435546875,
      "logps/ref_chosen": -105.77084350585938,
      "logps/ref_rejected": -100.94944763183594,
      "logps/rejected": -390.2877502441406,
      "loss": 1.1047,
      "margin_dpo/margin_mean": 97.465576171875,
      "margin_dpo/margin_std": 133.80523681640625,
      "step": 575
    },
    {
      "KL/chosen_KL_mean": -189.69064331054688,
      "KL/mean": -236.8587188720703,
      "KL/rejected_KL_mean": -284.02679443359375,
      "KL/std": 106.2891845703125,
      "epoch": 0.8458149779735683,
      "fcm_dpo/beta": 0.00400470569729805,
      "fcm_dpo/delta": 0.02481631189584732,
      "fcm_dpo/margin": 94.33613586425781,
      "fcm_dpo/q_t": 0.411979079246521,
      "grad_norm": 10.575273513793945,
      "learning_rate": 3.6105769011194224e-08,
      "logits/chosen": -1.9179656505584717,
      "logits/rejected": -1.858564853668213,
      "logps/chosen": -270.8951721191406,
      "logps/ref_chosen": -81.20452880859375,
      "logps/ref_rejected": -105.40940856933594,
      "logps/rejected": -389.43621826171875,
      "loss": 1.1062,
      "margin_dpo/margin_mean": 94.33613586425781,
      "margin_dpo/margin_std": 127.31085205078125,
      "step": 576
    },
    {
      "KL/chosen_KL_mean": -170.20147705078125,
      "KL/mean": -229.130615234375,
      "KL/rejected_KL_mean": -288.0597839355469,
      "KL/std": 115.6480712890625,
      "epoch": 0.8472834067547724,
      "fcm_dpo/beta": 0.0038498679641634226,
      "fcm_dpo/delta": -0.06380079686641693,
      "fcm_dpo/margin": 117.8582534790039,
      "fcm_dpo/q_t": 0.3924129605293274,
      "grad_norm": 7.660384654998779,
      "learning_rate": 3.5444241160659304e-08,
      "logits/chosen": -2.076935291290283,
      "logits/rejected": -1.3878109455108643,
      "logps/chosen": -271.6656799316406,
      "logps/ref_chosen": -101.46419525146484,
      "logps/ref_rejected": -89.32637023925781,
      "logps/rejected": -377.3861389160156,
      "loss": 1.0451,
      "margin_dpo/margin_mean": 117.85826110839844,
      "margin_dpo/margin_std": 132.10501098632812,
      "step": 577
    },
    {
      "KL/chosen_KL_mean": -202.87535095214844,
      "KL/mean": -245.7174072265625,
      "KL/rejected_KL_mean": -288.55950927734375,
      "KL/std": 110.0020751953125,
      "epoch": 0.8487518355359766,
      "fcm_dpo/beta": 0.0039657531306147575,
      "fcm_dpo/delta": 0.06708528101444244,
      "fcm_dpo/margin": 85.68412017822266,
      "fcm_dpo/q_t": 0.42172950506210327,
      "grad_norm": 13.724135398864746,
      "learning_rate": 3.478836705390808e-08,
      "logits/chosen": -1.4059717655181885,
      "logits/rejected": -1.1672166585922241,
      "logps/chosen": -280.16778564453125,
      "logps/ref_chosen": -77.29241943359375,
      "logps/ref_rejected": -88.24701690673828,
      "logps/rejected": -376.8065185546875,
      "loss": 1.135,
      "margin_dpo/margin_mean": 85.68412017822266,
      "margin_dpo/margin_std": 126.03876495361328,
      "step": 578
    },
    {
      "KL/chosen_KL_mean": -182.27488708496094,
      "KL/mean": -225.4148712158203,
      "KL/rejected_KL_mean": -268.55487060546875,
      "KL/std": 109.42262268066406,
      "epoch": 0.8502202643171806,
      "fcm_dpo/beta": 0.004063301719725132,
      "fcm_dpo/delta": 0.05491386726498604,
      "fcm_dpo/margin": 86.27996826171875,
      "fcm_dpo/q_t": 0.4192143380641937,
      "grad_norm": 8.481945037841797,
      "learning_rate": 3.41381639738331e-08,
      "logits/chosen": -1.734699010848999,
      "logits/rejected": -1.4410934448242188,
      "logps/chosen": -269.617919921875,
      "logps/ref_chosen": -87.34305572509766,
      "logps/ref_rejected": -102.12100982666016,
      "logps/rejected": -370.67584228515625,
      "loss": 1.1205,
      "margin_dpo/margin_mean": 86.27996826171875,
      "margin_dpo/margin_std": 112.42120361328125,
      "step": 579
    },
    {
      "KL/chosen_KL_mean": -173.71673583984375,
      "KL/mean": -227.40707397460938,
      "KL/rejected_KL_mean": -281.0974426269531,
      "KL/std": 108.97422790527344,
      "epoch": 0.8516886930983847,
      "fcm_dpo/beta": 0.0040215495973825455,
      "fcm_dpo/delta": -0.038409821689128876,
      "fcm_dpo/margin": 107.3807144165039,
      "fcm_dpo/q_t": 0.399572491645813,
      "grad_norm": 9.654060363769531,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -1.412444829940796,
      "logits/rejected": -1.0936086177825928,
      "logps/chosen": -262.6455078125,
      "logps/ref_chosen": -88.92879486083984,
      "logps/ref_rejected": -88.69661712646484,
      "logps/rejected": -369.7940673828125,
      "loss": 1.0668,
      "margin_dpo/margin_mean": 107.38072204589844,
      "margin_dpo/margin_std": 131.157470703125,
      "step": 580
    },
    {
      "KL/chosen_KL_mean": -183.68045043945312,
      "KL/mean": -232.28912353515625,
      "KL/rejected_KL_mean": -280.8978271484375,
      "KL/std": 110.38446807861328,
      "epoch": 0.8531571218795888,
      "fcm_dpo/beta": 0.004071483388543129,
      "fcm_dpo/delta": 0.004670340102165937,
      "fcm_dpo/margin": 97.21736145019531,
      "fcm_dpo/q_t": 0.408061146736145,
      "grad_norm": 11.21069622039795,
      "learning_rate": 3.285483927764726e-08,
      "logits/chosen": -1.8146438598632812,
      "logits/rejected": -1.458327054977417,
      "logps/chosen": -275.9036865234375,
      "logps/ref_chosen": -92.22323608398438,
      "logps/ref_rejected": -99.38943481445312,
      "logps/rejected": -380.2872314453125,
      "loss": 1.104,
      "margin_dpo/margin_mean": 97.21736145019531,
      "margin_dpo/margin_std": 137.60556030273438,
      "step": 581
    },
    {
      "KL/chosen_KL_mean": -163.53561401367188,
      "KL/mean": -207.37521362304688,
      "KL/rejected_KL_mean": -251.2147979736328,
      "KL/std": 114.88983154296875,
      "epoch": 0.8546255506607929,
      "fcm_dpo/beta": 0.003987087868154049,
      "fcm_dpo/delta": -0.06317149102687836,
      "fcm_dpo/margin": 87.67920684814453,
      "fcm_dpo/q_t": 0.41859185695648193,
      "grad_norm": 9.56002140045166,
      "learning_rate": 3.222175147833556e-08,
      "logits/chosen": -1.9726017713546753,
      "logits/rejected": -1.9656271934509277,
      "logps/chosen": -250.87123107910156,
      "logps/ref_chosen": -87.33561706542969,
      "logps/ref_rejected": -115.1544189453125,
      "logps/rejected": -366.36920166015625,
      "loss": 1.1347,
      "margin_dpo/margin_mean": 87.67920684814453,
      "margin_dpo/margin_std": 126.88917541503906,
      "step": 582
    },
    {
      "KL/chosen_KL_mean": -196.4793701171875,
      "KL/mean": -245.21347045898438,
      "KL/rejected_KL_mean": -293.94757080078125,
      "KL/std": 108.7474594116211,
      "epoch": 0.856093979441997,
      "fcm_dpo/beta": 0.0039047710597515106,
      "fcm_dpo/delta": 0.021257011219859123,
      "fcm_dpo/margin": 97.46817016601562,
      "fcm_dpo/q_t": 0.4120529890060425,
      "grad_norm": 10.50943374633789,
      "learning_rate": 3.159440233840763e-08,
      "logits/chosen": -1.3612672090530396,
      "logits/rejected": -1.121570348739624,
      "logps/chosen": -280.1964416503906,
      "logps/ref_chosen": -83.71707153320312,
      "logps/ref_rejected": -94.58465576171875,
      "logps/rejected": -388.5322265625,
      "loss": 1.1029,
      "margin_dpo/margin_mean": 97.46817016601562,
      "margin_dpo/margin_std": 128.89874267578125,
      "step": 583
    },
    {
      "KL/chosen_KL_mean": -174.36782836914062,
      "KL/mean": -225.96583557128906,
      "KL/rejected_KL_mean": -277.5638427734375,
      "KL/std": 112.89099884033203,
      "epoch": 0.8575624082232012,
      "fcm_dpo/beta": 0.004014907404780388,
      "fcm_dpo/delta": -0.01923905313014984,
      "fcm_dpo/margin": 103.19603729248047,
      "fcm_dpo/q_t": 0.402357816696167,
      "grad_norm": 8.047605514526367,
      "learning_rate": 3.0972808389096635e-08,
      "logits/chosen": -1.8914389610290527,
      "logits/rejected": -1.431771993637085,
      "logps/chosen": -268.62738037109375,
      "logps/ref_chosen": -94.25955963134766,
      "logps/ref_rejected": -104.48269653320312,
      "logps/rejected": -382.04656982421875,
      "loss": 1.0833,
      "margin_dpo/margin_mean": 103.196044921875,
      "margin_dpo/margin_std": 128.08058166503906,
      "step": 584
    },
    {
      "KL/chosen_KL_mean": -187.28146362304688,
      "KL/mean": -244.74818420410156,
      "KL/rejected_KL_mean": -302.21490478515625,
      "KL/std": 128.47097778320312,
      "epoch": 0.8590308370044053,
      "fcm_dpo/beta": 0.0038625665474683046,
      "fcm_dpo/delta": -0.05076519399881363,
      "fcm_dpo/margin": 114.93345642089844,
      "fcm_dpo/q_t": 0.3973461091518402,
      "grad_norm": 9.163249015808105,
      "learning_rate": 3.035698600998121e-08,
      "logits/chosen": -2.039374351501465,
      "logits/rejected": -1.7713555097579956,
      "logps/chosen": -272.918212890625,
      "logps/ref_chosen": -85.63676452636719,
      "logps/ref_rejected": -90.28547668457031,
      "logps/rejected": -392.5003662109375,
      "loss": 1.0715,
      "margin_dpo/margin_mean": 114.93345642089844,
      "margin_dpo/margin_std": 151.22824096679688,
      "step": 585
    },
    {
      "KL/chosen_KL_mean": -205.703125,
      "KL/mean": -244.43539428710938,
      "KL/rejected_KL_mean": -283.16766357421875,
      "KL/std": 112.03587341308594,
      "epoch": 0.8604992657856094,
      "fcm_dpo/beta": 0.003974686376750469,
      "fcm_dpo/delta": 0.1014520525932312,
      "fcm_dpo/margin": 77.46456146240234,
      "fcm_dpo/q_t": 0.4288644790649414,
      "grad_norm": 11.312481880187988,
      "learning_rate": 2.974695142855388e-08,
      "logits/chosen": -1.8618745803833008,
      "logits/rejected": -1.5991733074188232,
      "logps/chosen": -289.38427734375,
      "logps/ref_chosen": -83.68114471435547,
      "logps/ref_rejected": -98.35916137695312,
      "logps/rejected": -381.52685546875,
      "loss": 1.1467,
      "margin_dpo/margin_mean": 77.46456146240234,
      "margin_dpo/margin_std": 107.05786895751953,
      "step": 586
    },
    {
      "KL/chosen_KL_mean": -166.3006591796875,
      "KL/mean": -208.59408569335938,
      "KL/rejected_KL_mean": -250.88751220703125,
      "KL/std": 108.68350982666016,
      "epoch": 0.8619676945668135,
      "fcm_dpo/beta": 0.004152378067374229,
      "fcm_dpo/delta": 0.05415453016757965,
      "fcm_dpo/margin": 84.58685302734375,
      "fcm_dpo/q_t": 0.41879117488861084,
      "grad_norm": 10.847317695617676,
      "learning_rate": 2.9142720719793122e-08,
      "logits/chosen": -2.2205851078033447,
      "logits/rejected": -1.9308464527130127,
      "logps/chosen": -243.878173828125,
      "logps/ref_chosen": -77.57750701904297,
      "logps/ref_rejected": -89.32032775878906,
      "logps/rejected": -340.2078552246094,
      "loss": 1.1189,
      "margin_dpo/margin_mean": 84.58685302734375,
      "margin_dpo/margin_std": 114.10688781738281,
      "step": 587
    },
    {
      "KL/chosen_KL_mean": -182.01025390625,
      "KL/mean": -221.15426635742188,
      "KL/rejected_KL_mean": -260.29827880859375,
      "KL/std": 110.57585144042969,
      "epoch": 0.8634361233480177,
      "fcm_dpo/beta": 0.0042855702340602875,
      "fcm_dpo/delta": 0.07189078629016876,
      "fcm_dpo/margin": 78.28800964355469,
      "fcm_dpo/q_t": 0.4248380959033966,
      "grad_norm": 14.562604904174805,
      "learning_rate": 2.8544309805740018e-08,
      "logits/chosen": -1.8698904514312744,
      "logits/rejected": -1.8335736989974976,
      "logps/chosen": -265.81451416015625,
      "logps/ref_chosen": -83.80426025390625,
      "logps/ref_rejected": -113.11943054199219,
      "logps/rejected": -373.4176940917969,
      "loss": 1.1565,
      "margin_dpo/margin_mean": 78.28800964355469,
      "margin_dpo/margin_std": 133.28543090820312,
      "step": 588
    },
    {
      "KL/chosen_KL_mean": -193.998779296875,
      "KL/mean": -246.3238525390625,
      "KL/rejected_KL_mean": -298.64886474609375,
      "KL/std": 109.13973999023438,
      "epoch": 0.8649045521292217,
      "fcm_dpo/beta": 0.004255070351064205,
      "fcm_dpo/delta": -0.0522841215133667,
      "fcm_dpo/margin": 104.65008544921875,
      "fcm_dpo/q_t": 0.3939587473869324,
      "grad_norm": 8.81981372833252,
      "learning_rate": 2.7951734455078786e-08,
      "logits/chosen": -1.5416032075881958,
      "logits/rejected": -1.1429312229156494,
      "logps/chosen": -290.46380615234375,
      "logps/ref_chosen": -96.46501922607422,
      "logps/ref_rejected": -116.03719329833984,
      "logps/rejected": -414.68609619140625,
      "loss": 1.0372,
      "margin_dpo/margin_mean": 104.65008544921875,
      "margin_dpo/margin_std": 104.91950225830078,
      "step": 589
    },
    {
      "KL/chosen_KL_mean": -177.80624389648438,
      "KL/mean": -229.85345458984375,
      "KL/rejected_KL_mean": -281.9006652832031,
      "KL/std": 101.51722717285156,
      "epoch": 0.8663729809104258,
      "fcm_dpo/beta": 0.004192674066871405,
      "fcm_dpo/delta": -0.04244563728570938,
      "fcm_dpo/margin": 104.09440612792969,
      "fcm_dpo/q_t": 0.39698266983032227,
      "grad_norm": 11.557031631469727,
      "learning_rate": 2.736501028272095e-08,
      "logits/chosen": -1.8159635066986084,
      "logits/rejected": -1.6697404384613037,
      "logps/chosen": -266.2341003417969,
      "logps/ref_chosen": -88.4278564453125,
      "logps/ref_rejected": -110.19876861572266,
      "logps/rejected": -392.09942626953125,
      "loss": 1.0448,
      "margin_dpo/margin_mean": 104.09440612792969,
      "margin_dpo/margin_std": 106.209716796875,
      "step": 590
    },
    {
      "KL/chosen_KL_mean": -171.2813720703125,
      "KL/mean": -226.09054565429688,
      "KL/rejected_KL_mean": -280.8997497558594,
      "KL/std": 109.31867980957031,
      "epoch": 0.8678414096916299,
      "fcm_dpo/beta": 0.004002091474831104,
      "fcm_dpo/delta": -0.04532766714692116,
      "fcm_dpo/margin": 109.61836242675781,
      "fcm_dpo/q_t": 0.3954704999923706,
      "grad_norm": 7.1684722900390625,
      "learning_rate": 2.678415274939408e-08,
      "logits/chosen": -1.9345951080322266,
      "logits/rejected": -1.320427656173706,
      "logps/chosen": -259.13104248046875,
      "logps/ref_chosen": -87.84968566894531,
      "logps/ref_rejected": -89.57919311523438,
      "logps/rejected": -370.47894287109375,
      "loss": 1.0387,
      "margin_dpo/margin_mean": 109.61836242675781,
      "margin_dpo/margin_std": 106.60746765136719,
      "step": 591
    },
    {
      "KL/chosen_KL_mean": -201.84735107421875,
      "KL/mean": -244.35409545898438,
      "KL/rejected_KL_mean": -286.86083984375,
      "KL/std": 106.31205749511719,
      "epoch": 0.869309838472834,
      "fcm_dpo/beta": 0.004112501163035631,
      "fcm_dpo/delta": 0.05613921955227852,
      "fcm_dpo/margin": 85.01348114013672,
      "fcm_dpo/q_t": 0.41944199800491333,
      "grad_norm": 11.116849899291992,
      "learning_rate": 2.6209177161234442e-08,
      "logits/chosen": -1.8425564765930176,
      "logits/rejected": -1.5167927742004395,
      "logps/chosen": -276.8230895996094,
      "logps/ref_chosen": -74.97573852539062,
      "logps/ref_rejected": -81.55262756347656,
      "logps/rejected": -368.4134521484375,
      "loss": 1.1389,
      "margin_dpo/margin_mean": 85.01347351074219,
      "margin_dpo/margin_std": 133.73501586914062,
      "step": 592
    },
    {
      "KL/chosen_KL_mean": -184.3921661376953,
      "KL/mean": -220.8485565185547,
      "KL/rejected_KL_mean": -257.3049011230469,
      "KL/std": 111.07266235351562,
      "epoch": 0.8707782672540382,
      "fcm_dpo/beta": 0.004286311566829681,
      "fcm_dpo/delta": 0.09683699160814285,
      "fcm_dpo/margin": 72.91273498535156,
      "fcm_dpo/q_t": 0.4296285808086395,
      "grad_norm": 15.06937026977539,
      "learning_rate": 2.564009866938349e-08,
      "logits/chosen": -0.7559401988983154,
      "logits/rejected": -0.44836944341659546,
      "logps/chosen": -256.6311950683594,
      "logps/ref_chosen": -72.23902893066406,
      "logps/ref_rejected": -70.09652709960938,
      "logps/rejected": -327.40142822265625,
      "loss": 1.1668,
      "margin_dpo/margin_mean": 72.91273498535156,
      "margin_dpo/margin_std": 123.7369384765625,
      "step": 593
    },
    {
      "KL/chosen_KL_mean": -165.35256958007812,
      "KL/mean": -221.48214721679688,
      "KL/rejected_KL_mean": -277.6117248535156,
      "KL/std": 108.01886749267578,
      "epoch": 0.8722466960352423,
      "fcm_dpo/beta": 0.004260594490915537,
      "fcm_dpo/delta": -0.09122254699468613,
      "fcm_dpo/margin": 112.25917053222656,
      "fcm_dpo/q_t": 0.3858344554901123,
      "grad_norm": 9.603059768676758,
      "learning_rate": 2.5076932269588708e-08,
      "logits/chosen": -1.9775947332382202,
      "logits/rejected": -1.4567538499832153,
      "logps/chosen": -255.790283203125,
      "logps/ref_chosen": -90.43771362304688,
      "logps/ref_rejected": -92.267578125,
      "logps/rejected": -369.8793029785156,
      "loss": 1.0134,
      "margin_dpo/margin_mean": 112.2591552734375,
      "margin_dpo/margin_std": 104.70689392089844,
      "step": 594
    },
    {
      "KL/chosen_KL_mean": -178.30355834960938,
      "KL/mean": -224.02267456054688,
      "KL/rejected_KL_mean": -269.7418212890625,
      "KL/std": 108.02765655517578,
      "epoch": 0.8737151248164464,
      "fcm_dpo/beta": 0.004217015113681555,
      "fcm_dpo/delta": 0.015708738937973976,
      "fcm_dpo/margin": 91.43821716308594,
      "fcm_dpo/q_t": 0.41139358282089233,
      "grad_norm": 10.004549026489258,
      "learning_rate": 2.451969280180849e-08,
      "logits/chosen": -1.9869604110717773,
      "logits/rejected": -1.5925554037094116,
      "logps/chosen": -259.818359375,
      "logps/ref_chosen": -81.51480102539062,
      "logps/ref_rejected": -86.52047729492188,
      "logps/rejected": -356.26226806640625,
      "loss": 1.1028,
      "margin_dpo/margin_mean": 91.43821716308594,
      "margin_dpo/margin_std": 124.25677490234375,
      "step": 595
    },
    {
      "KL/chosen_KL_mean": -185.18960571289062,
      "KL/mean": -226.31044006347656,
      "KL/rejected_KL_mean": -267.4312744140625,
      "KL/std": 107.45704650878906,
      "epoch": 0.8751835535976505,
      "fcm_dpo/beta": 0.004303973633795977,
      "fcm_dpo/delta": 0.05125989392399788,
      "fcm_dpo/margin": 82.24169921875,
      "fcm_dpo/q_t": 0.41990286111831665,
      "grad_norm": 13.169529914855957,
      "learning_rate": 2.396839494982103e-08,
      "logits/chosen": -1.4199472665786743,
      "logits/rejected": -0.8174420595169067,
      "logps/chosen": -273.39764404296875,
      "logps/ref_chosen": -88.20805358886719,
      "logps/ref_rejected": -86.09375,
      "logps/rejected": -353.5250244140625,
      "loss": 1.1455,
      "margin_dpo/margin_mean": 82.24169921875,
      "margin_dpo/margin_std": 136.72894287109375,
      "step": 596
    },
    {
      "KL/chosen_KL_mean": -179.56631469726562,
      "KL/mean": -234.31500244140625,
      "KL/rejected_KL_mean": -289.0636901855469,
      "KL/std": 115.5741958618164,
      "epoch": 0.8766519823788547,
      "fcm_dpo/beta": 0.004168538376688957,
      "fcm_dpo/delta": -0.06633562594652176,
      "fcm_dpo/margin": 109.49739074707031,
      "fcm_dpo/q_t": 0.39298683404922485,
      "grad_norm": 9.156472206115723,
      "learning_rate": 2.3423053240837514e-08,
      "logits/chosen": -1.284548044204712,
      "logits/rejected": -1.1680457592010498,
      "logps/chosen": -263.15631103515625,
      "logps/ref_chosen": -83.58998107910156,
      "logps/ref_rejected": -106.94439697265625,
      "logps/rejected": -396.00811767578125,
      "loss": 1.0448,
      "margin_dpo/margin_mean": 109.49739074707031,
      "margin_dpo/margin_std": 125.27772521972656,
      "step": 597
    },
    {
      "KL/chosen_KL_mean": -172.39532470703125,
      "KL/mean": -218.30648803710938,
      "KL/rejected_KL_mean": -264.2176513671875,
      "KL/std": 107.49383544921875,
      "epoch": 0.8781204111600588,
      "fcm_dpo/beta": 0.004148153122514486,
      "fcm_dpo/delta": 0.020910009741783142,
      "fcm_dpo/margin": 91.82232666015625,
      "fcm_dpo/q_t": 0.41076022386550903,
      "grad_norm": 9.49918270111084,
      "learning_rate": 2.2883682045119062e-08,
      "logits/chosen": -1.9958076477050781,
      "logits/rejected": -1.6870863437652588,
      "logps/chosen": -262.57794189453125,
      "logps/ref_chosen": -90.18262481689453,
      "logps/ref_rejected": -94.79856872558594,
      "logps/rejected": -359.0162353515625,
      "loss": 1.1059,
      "margin_dpo/margin_mean": 91.82232666015625,
      "margin_dpo/margin_std": 122.44966125488281,
      "step": 598
    },
    {
      "KL/chosen_KL_mean": -173.83163452148438,
      "KL/mean": -215.24697875976562,
      "KL/rejected_KL_mean": -256.662353515625,
      "KL/std": 112.26593780517578,
      "epoch": 0.8795888399412628,
      "fcm_dpo/beta": 0.004246823489665985,
      "fcm_dpo/delta": 0.0531768798828125,
      "fcm_dpo/margin": 82.8307113647461,
      "fcm_dpo/q_t": 0.41874319314956665,
      "grad_norm": 9.5277738571167,
      "learning_rate": 2.2350295575598367e-08,
      "logits/chosen": -1.650075078010559,
      "logits/rejected": -1.5278490781784058,
      "logps/chosen": -251.67391967773438,
      "logps/ref_chosen": -77.84226989746094,
      "logps/ref_rejected": -89.1976318359375,
      "logps/rejected": -345.8599853515625,
      "loss": 1.1346,
      "margin_dpo/margin_mean": 82.8307113647461,
      "margin_dpo/margin_std": 123.42928314208984,
      "step": 599
    },
    {
      "KL/chosen_KL_mean": -194.17556762695312,
      "KL/mean": -240.72634887695312,
      "KL/rejected_KL_mean": -287.27716064453125,
      "KL/std": 106.1072998046875,
      "epoch": 0.8810572687224669,
      "fcm_dpo/beta": 0.004344576504081488,
      "fcm_dpo/delta": -0.005128633230924606,
      "fcm_dpo/margin": 93.10159301757812,
      "fcm_dpo/q_t": 0.4060867130756378,
      "grad_norm": 9.939950942993164,
      "learning_rate": 2.1822907887504932e-08,
      "logits/chosen": -2.476283311843872,
      "logits/rejected": -2.1362671852111816,
      "logps/chosen": -286.3633728027344,
      "logps/ref_chosen": -92.18781280517578,
      "logps/ref_rejected": -92.95429992675781,
      "logps/rejected": -380.2314453125,
      "loss": 1.082,
      "margin_dpo/margin_mean": 93.1015853881836,
      "margin_dpo/margin_std": 114.73773193359375,
      "step": 600
    },
    {
      "KL/chosen_KL_mean": -195.9962921142578,
      "KL/mean": -245.35531616210938,
      "KL/rejected_KL_mean": -294.7143249511719,
      "KL/std": 107.76451110839844,
      "epoch": 0.882525697503671,
      "fcm_dpo/beta": 0.004245828837156296,
      "fcm_dpo/delta": -0.0228335689753294,
      "fcm_dpo/margin": 98.718017578125,
      "fcm_dpo/q_t": 0.4010237157344818,
      "grad_norm": 9.081266403198242,
      "learning_rate": 2.1301532877994742e-08,
      "logits/chosen": -1.6294050216674805,
      "logits/rejected": -1.2978471517562866,
      "logps/chosen": -285.2124328613281,
      "logps/ref_chosen": -89.21614074707031,
      "logps/ref_rejected": -100.17054748535156,
      "logps/rejected": -394.8848876953125,
      "loss": 1.0569,
      "margin_dpo/margin_mean": 98.718017578125,
      "margin_dpo/margin_std": 103.436279296875,
      "step": 601
    },
    {
      "KL/chosen_KL_mean": -155.35902404785156,
      "KL/mean": -219.5087127685547,
      "KL/rejected_KL_mean": -283.65838623046875,
      "KL/std": 118.00856018066406,
      "epoch": 0.8839941262848752,
      "fcm_dpo/beta": 0.004018221981823444,
      "fcm_dpo/delta": -0.13692793250083923,
      "fcm_dpo/margin": 128.2993621826172,
      "fcm_dpo/q_t": 0.3776431083679199,
      "grad_norm": 8.578645706176758,
      "learning_rate": 2.0786184285784298e-08,
      "logits/chosen": -2.235719680786133,
      "logits/rejected": -1.983658790588379,
      "logps/chosen": -235.4166259765625,
      "logps/ref_chosen": -80.05760192871094,
      "logps/ref_rejected": -93.197509765625,
      "logps/rejected": -376.85589599609375,
      "loss": 1.0034,
      "margin_dpo/margin_mean": 128.2993621826172,
      "margin_dpo/margin_std": 134.89657592773438,
      "step": 602
    },
    {
      "KL/chosen_KL_mean": -165.127685546875,
      "KL/mean": -218.37579345703125,
      "KL/rejected_KL_mean": -271.6239013671875,
      "KL/std": 106.74230194091797,
      "epoch": 0.8854625550660793,
      "fcm_dpo/beta": 0.003912940621376038,
      "fcm_dpo/delta": -0.019099434837698936,
      "fcm_dpo/margin": 106.49618530273438,
      "fcm_dpo/q_t": 0.4022037386894226,
      "grad_norm": 7.809121131896973,
      "learning_rate": 2.0276875690788204e-08,
      "logits/chosen": -2.16447377204895,
      "logits/rejected": -1.5875825881958008,
      "logps/chosen": -267.437255859375,
      "logps/ref_chosen": -102.30958557128906,
      "logps/ref_rejected": -108.06884765625,
      "logps/rejected": -379.6927490234375,
      "loss": 1.0664,
      "margin_dpo/margin_mean": 106.49618530273438,
      "margin_dpo/margin_std": 122.96781158447266,
      "step": 603
    },
    {
      "KL/chosen_KL_mean": -173.65676879882812,
      "KL/mean": -228.09283447265625,
      "KL/rejected_KL_mean": -282.5289306640625,
      "KL/std": 111.92857360839844,
      "epoch": 0.8869309838472834,
      "fcm_dpo/beta": 0.003822739701718092,
      "fcm_dpo/delta": -0.01928185485303402,
      "fcm_dpo/margin": 108.87216186523438,
      "fcm_dpo/q_t": 0.40042081475257874,
      "grad_norm": 8.151638984680176,
      "learning_rate": 1.977362051376158e-08,
      "logits/chosen": -1.6713311672210693,
      "logits/rejected": -1.518505334854126,
      "logps/chosen": -251.83084106445312,
      "logps/ref_chosen": -78.17408752441406,
      "logps/ref_rejected": -99.49610900878906,
      "logps/rejected": -382.0250244140625,
      "loss": 1.0601,
      "margin_dpo/margin_mean": 108.87216186523438,
      "margin_dpo/margin_std": 115.10411071777344,
      "step": 604
    },
    {
      "KL/chosen_KL_mean": -189.547607421875,
      "KL/mean": -233.97152709960938,
      "KL/rejected_KL_mean": -278.39544677734375,
      "KL/std": 99.37776184082031,
      "epoch": 0.8883994126284875,
      "fcm_dpo/beta": 0.003940519876778126,
      "fcm_dpo/delta": 0.05577125400304794,
      "fcm_dpo/margin": 88.84783935546875,
      "fcm_dpo/q_t": 0.41883206367492676,
      "grad_norm": 9.278206825256348,
      "learning_rate": 1.9276432015946446e-08,
      "logits/chosen": -1.8954675197601318,
      "logits/rejected": -1.4069048166275024,
      "logps/chosen": -284.3209228515625,
      "logps/ref_chosen": -94.77332305908203,
      "logps/ref_rejected": -107.30490112304688,
      "logps/rejected": -385.7003173828125,
      "loss": 1.1136,
      "margin_dpo/margin_mean": 88.84783935546875,
      "margin_dpo/margin_std": 112.6802978515625,
      "step": 605
    },
    {
      "KL/chosen_KL_mean": -182.50619506835938,
      "KL/mean": -229.21676635742188,
      "KL/rejected_KL_mean": -275.92730712890625,
      "KL/std": 109.27296447753906,
      "epoch": 0.8898678414096917,
      "fcm_dpo/beta": 0.004077109508216381,
      "fcm_dpo/delta": 0.01939527317881584,
      "fcm_dpo/margin": 93.42111206054688,
      "fcm_dpo/q_t": 0.4103202819824219,
      "grad_norm": 8.657791137695312,
      "learning_rate": 1.8785323298722093e-08,
      "logits/chosen": -1.7141348123550415,
      "logits/rejected": -1.3218364715576172,
      "logps/chosen": -270.25958251953125,
      "logps/ref_chosen": -87.7533950805664,
      "logps/ref_rejected": -104.2422103881836,
      "logps/rejected": -380.1695251464844,
      "loss": 1.0983,
      "margin_dpo/margin_mean": 93.42111206054688,
      "margin_dpo/margin_std": 116.87020111083984,
      "step": 606
    },
    {
      "KL/chosen_KL_mean": -176.13632202148438,
      "KL/mean": -228.93984985351562,
      "KL/rejected_KL_mean": -281.743408203125,
      "KL/std": 116.71160888671875,
      "epoch": 0.8913362701908958,
      "fcm_dpo/beta": 0.003988754935562611,
      "fcm_dpo/delta": -0.024293113499879837,
      "fcm_dpo/margin": 105.60706329345703,
      "fcm_dpo/q_t": 0.4021585285663605,
      "grad_norm": 11.808521270751953,
      "learning_rate": 1.8300307303259904e-08,
      "logits/chosen": -1.6713643074035645,
      "logits/rejected": -1.135755181312561,
      "logps/chosen": -264.4653625488281,
      "logps/ref_chosen": -88.32904052734375,
      "logps/ref_rejected": -86.76810455322266,
      "logps/rejected": -368.511474609375,
      "loss": 1.086,
      "margin_dpo/margin_mean": 105.60707092285156,
      "margin_dpo/margin_std": 143.4172821044922,
      "step": 607
    },
    {
      "KL/chosen_KL_mean": -161.14195251464844,
      "KL/mean": -216.47586059570312,
      "KL/rejected_KL_mean": -271.80975341796875,
      "KL/std": 115.39044952392578,
      "epoch": 0.8928046989720999,
      "fcm_dpo/beta": 0.0038361474871635437,
      "fcm_dpo/delta": -0.031420789659023285,
      "fcm_dpo/margin": 110.66780090332031,
      "fcm_dpo/q_t": 0.39868104457855225,
      "grad_norm": 8.820941925048828,
      "learning_rate": 1.7821396810182437e-08,
      "logits/chosen": -1.84756600856781,
      "logits/rejected": -1.5501123666763306,
      "logps/chosen": -246.9113311767578,
      "logps/ref_chosen": -85.76937866210938,
      "logps/ref_rejected": -100.23281860351562,
      "logps/rejected": -372.0425720214844,
      "loss": 1.066,
      "margin_dpo/margin_mean": 110.66780090332031,
      "margin_dpo/margin_std": 125.43177795410156,
      "step": 608
    },
    {
      "KL/chosen_KL_mean": -180.3653564453125,
      "KL/mean": -231.83041381835938,
      "KL/rejected_KL_mean": -283.29547119140625,
      "KL/std": 116.36293029785156,
      "epoch": 0.8942731277533039,
      "fcm_dpo/beta": 0.003847625106573105,
      "fcm_dpo/delta": 0.0033652260899543762,
      "fcm_dpo/margin": 102.93013000488281,
      "fcm_dpo/q_t": 0.40762239694595337,
      "grad_norm": 10.073235511779785,
      "learning_rate": 1.7348604439226617e-08,
      "logits/chosen": -2.1374337673187256,
      "logits/rejected": -1.7081608772277832,
      "logps/chosen": -273.3319396972656,
      "logps/ref_chosen": -92.9665756225586,
      "logps/ref_rejected": -95.91818237304688,
      "logps/rejected": -379.21368408203125,
      "loss": 1.0886,
      "margin_dpo/margin_mean": 102.93013000488281,
      "margin_dpo/margin_std": 123.33586120605469,
      "step": 609
    },
    {
      "KL/chosen_KL_mean": -165.817626953125,
      "KL/mean": -220.8531494140625,
      "KL/rejected_KL_mean": -275.888671875,
      "KL/std": 110.08412170410156,
      "epoch": 0.895741556534508,
      "fcm_dpo/beta": 0.0038311737589538097,
      "fcm_dpo/delta": -0.025305751711130142,
      "fcm_dpo/margin": 110.07103729248047,
      "fcm_dpo/q_t": 0.3993644118309021,
      "grad_norm": 8.227020263671875,
      "learning_rate": 1.6881942648911074e-08,
      "logits/chosen": -1.8285706043243408,
      "logits/rejected": -1.2927263975143433,
      "logps/chosen": -260.5179138183594,
      "logps/ref_chosen": -94.70028686523438,
      "logps/ref_rejected": -89.68739318847656,
      "logps/rejected": -365.5760498046875,
      "loss": 1.0561,
      "margin_dpo/margin_mean": 110.07102966308594,
      "margin_dpo/margin_std": 116.29500579833984,
      "step": 610
    },
    {
      "KL/chosen_KL_mean": -174.34642028808594,
      "KL/mean": -228.42263793945312,
      "KL/rejected_KL_mean": -282.49884033203125,
      "KL/std": 114.41613006591797,
      "epoch": 0.8972099853157122,
      "fcm_dpo/beta": 0.003858196549117565,
      "fcm_dpo/delta": -0.020787972956895828,
      "fcm_dpo/margin": 108.15242767333984,
      "fcm_dpo/q_t": 0.4018462300300598,
      "grad_norm": 10.385506629943848,
      "learning_rate": 1.6421423736208e-08,
      "logits/chosen": -2.1318676471710205,
      "logits/rejected": -1.7906298637390137,
      "logps/chosen": -261.1297607421875,
      "logps/ref_chosen": -86.78334045410156,
      "logps/ref_rejected": -89.84307861328125,
      "logps/rejected": -372.3419189453125,
      "loss": 1.0619,
      "margin_dpo/margin_mean": 108.15242767333984,
      "margin_dpo/margin_std": 116.29814147949219,
      "step": 611
    },
    {
      "KL/chosen_KL_mean": -191.07974243164062,
      "KL/mean": -246.47535705566406,
      "KL/rejected_KL_mean": -301.8709716796875,
      "KL/std": 108.44122314453125,
      "epoch": 0.8986784140969163,
      "fcm_dpo/beta": 0.0037478278391063213,
      "fcm_dpo/delta": -0.017591699957847595,
      "fcm_dpo/margin": 110.79122924804688,
      "fcm_dpo/q_t": 0.40154868364334106,
      "grad_norm": 8.865962982177734,
      "learning_rate": 1.5967059836219042e-08,
      "logits/chosen": -2.072700262069702,
      "logits/rejected": -1.2294178009033203,
      "logps/chosen": -292.09991455078125,
      "logps/ref_chosen": -101.02015686035156,
      "logps/ref_rejected": -93.78302764892578,
      "logps/rejected": -395.65399169921875,
      "loss": 1.0563,
      "margin_dpo/margin_mean": 110.79122924804688,
      "margin_dpo/margin_std": 113.46830749511719,
      "step": 612
    },
    {
      "KL/chosen_KL_mean": -153.84869384765625,
      "KL/mean": -216.03414916992188,
      "KL/rejected_KL_mean": -278.2196044921875,
      "KL/std": 110.82473754882812,
      "epoch": 0.9001468428781204,
      "fcm_dpo/beta": 0.0036348921712487936,
      "fcm_dpo/delta": -0.06076829880475998,
      "fcm_dpo/margin": 124.37092590332031,
      "fcm_dpo/q_t": 0.3911229372024536,
      "grad_norm": 7.848884582519531,
      "learning_rate": 1.551886292185553e-08,
      "logits/chosen": -1.9390192031860352,
      "logits/rejected": -1.7436946630477905,
      "logps/chosen": -242.83731079101562,
      "logps/ref_chosen": -88.98861694335938,
      "logps/ref_rejected": -109.99551391601562,
      "logps/rejected": -388.21514892578125,
      "loss": 1.0215,
      "margin_dpo/margin_mean": 124.37092590332031,
      "margin_dpo/margin_std": 108.96775817871094,
      "step": 613
    },
    {
      "KL/chosen_KL_mean": -192.30221557617188,
      "KL/mean": -242.63790893554688,
      "KL/rejected_KL_mean": -292.97357177734375,
      "KL/std": 104.78083801269531,
      "epoch": 0.9016152716593245,
      "fcm_dpo/beta": 0.0036650048568844795,
      "fcm_dpo/delta": 0.034944821149110794,
      "fcm_dpo/margin": 100.67137145996094,
      "fcm_dpo/q_t": 0.4135802388191223,
      "grad_norm": 10.017874717712402,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": -1.33028244972229,
      "logits/rejected": -1.3223223686218262,
      "logps/chosen": -272.50225830078125,
      "logps/ref_chosen": -80.20005798339844,
      "logps/ref_rejected": -109.86239624023438,
      "logps/rejected": -402.83599853515625,
      "loss": 1.1057,
      "margin_dpo/margin_mean": 100.67137145996094,
      "margin_dpo/margin_std": 128.9800262451172,
      "step": 614
    },
    {
      "KL/chosen_KL_mean": -180.2340087890625,
      "KL/mean": -221.71786499023438,
      "KL/rejected_KL_mean": -263.20166015625,
      "KL/std": 106.22239685058594,
      "epoch": 0.9030837004405287,
      "fcm_dpo/beta": 0.0036848513409495354,
      "fcm_dpo/delta": -0.009419679641723633,
      "fcm_dpo/margin": 82.9676513671875,
      "fcm_dpo/q_t": 0.42649781703948975,
      "grad_norm": 8.315711975097656,
      "learning_rate": 1.4641017128809801e-08,
      "logits/chosen": -1.6824076175689697,
      "logits/rejected": -1.280740737915039,
      "logps/chosen": -280.6692810058594,
      "logps/ref_chosen": -100.43525695800781,
      "logps/ref_rejected": -101.1800537109375,
      "logps/rejected": -364.3817138671875,
      "loss": 1.1411,
      "margin_dpo/margin_mean": 82.9676513671875,
      "margin_dpo/margin_std": 101.78868865966797,
      "step": 615
    },
    {
      "KL/chosen_KL_mean": -183.19662475585938,
      "KL/mean": -239.76309204101562,
      "KL/rejected_KL_mean": -296.32952880859375,
      "KL/std": 106.27557373046875,
      "epoch": 0.9045521292217328,
      "fcm_dpo/beta": 0.003684100229293108,
      "fcm_dpo/delta": -0.01987517438828945,
      "fcm_dpo/margin": 113.13290405273438,
      "fcm_dpo/q_t": 0.40143367648124695,
      "grad_norm": 8.17743968963623,
      "learning_rate": 1.4211391382180637e-08,
      "logits/chosen": -1.7779088020324707,
      "logits/rejected": -1.277719259262085,
      "logps/chosen": -275.6895751953125,
      "logps/ref_chosen": -92.49292755126953,
      "logps/ref_rejected": -82.06065368652344,
      "logps/rejected": -378.39019775390625,
      "loss": 1.0636,
      "margin_dpo/margin_mean": 113.13290405273438,
      "margin_dpo/margin_std": 124.62448120117188,
      "step": 616
    },
    {
      "KL/chosen_KL_mean": -189.6644287109375,
      "KL/mean": -234.6483154296875,
      "KL/rejected_KL_mean": -279.6322021484375,
      "KL/std": 103.3016357421875,
      "epoch": 0.9060205580029369,
      "fcm_dpo/beta": 0.0037447253707796335,
      "fcm_dpo/delta": 0.07020445168018341,
      "fcm_dpo/margin": 89.96778869628906,
      "fcm_dpo/q_t": 0.4213234484195709,
      "grad_norm": 9.407095909118652,
      "learning_rate": 1.378797888467345e-08,
      "logits/chosen": -1.6116180419921875,
      "logits/rejected": -0.9741203784942627,
      "logps/chosen": -280.76141357421875,
      "logps/ref_chosen": -91.09699249267578,
      "logps/ref_rejected": -70.41004943847656,
      "logps/rejected": -350.042236328125,
      "loss": 1.1236,
      "margin_dpo/margin_mean": 89.96778869628906,
      "margin_dpo/margin_std": 116.06523895263672,
      "step": 617
    },
    {
      "KL/chosen_KL_mean": -192.35955810546875,
      "KL/mean": -242.781982421875,
      "KL/rejected_KL_mean": -293.20440673828125,
      "KL/std": 114.80499267578125,
      "epoch": 0.9074889867841409,
      "fcm_dpo/beta": 0.0038045356050133705,
      "fcm_dpo/delta": 0.018432492390275,
      "fcm_dpo/margin": 100.84483337402344,
      "fcm_dpo/q_t": 0.40946248173713684,
      "grad_norm": 8.008707046508789,
      "learning_rate": 1.3370790793601371e-08,
      "logits/chosen": -2.0133304595947266,
      "logits/rejected": -1.650322675704956,
      "logps/chosen": -294.3801574707031,
      "logps/ref_chosen": -102.02059936523438,
      "logps/ref_rejected": -99.80119323730469,
      "logps/rejected": -393.0055847167969,
      "loss": 1.096,
      "margin_dpo/margin_mean": 100.84483337402344,
      "margin_dpo/margin_std": 127.48600769042969,
      "step": 618
    },
    {
      "KL/chosen_KL_mean": -185.9556884765625,
      "KL/mean": -230.57415771484375,
      "KL/rejected_KL_mean": -275.192626953125,
      "KL/std": 112.90042877197266,
      "epoch": 0.908957415565345,
      "fcm_dpo/beta": 0.003921686206012964,
      "fcm_dpo/delta": 0.055955126881599426,
      "fcm_dpo/margin": 89.23692321777344,
      "fcm_dpo/q_t": 0.42064523696899414,
      "grad_norm": 8.78959846496582,
      "learning_rate": 1.2959838102258535e-08,
      "logits/chosen": -1.5636637210845947,
      "logits/rejected": -1.2033238410949707,
      "logps/chosen": -275.69708251953125,
      "logps/ref_chosen": -89.74136352539062,
      "logps/ref_rejected": -99.90138244628906,
      "logps/rejected": -375.093994140625,
      "loss": 1.132,
      "margin_dpo/margin_mean": 89.23692321777344,
      "margin_dpo/margin_std": 134.693603515625,
      "step": 619
    },
    {
      "KL/chosen_KL_mean": -177.39907836914062,
      "KL/mean": -226.64346313476562,
      "KL/rejected_KL_mean": -275.8878479003906,
      "KL/std": 118.87477111816406,
      "epoch": 0.9104258443465492,
      "fcm_dpo/beta": 0.003763884771615267,
      "fcm_dpo/delta": -0.08463147282600403,
      "fcm_dpo/margin": 98.48875427246094,
      "fcm_dpo/q_t": 0.4104871153831482,
      "grad_norm": 8.828258514404297,
      "learning_rate": 1.2555131639630567e-08,
      "logits/chosen": -2.402806282043457,
      "logits/rejected": -1.8947151899337769,
      "logps/chosen": -262.52337646484375,
      "logps/ref_chosen": -85.12431335449219,
      "logps/ref_rejected": -85.41253662109375,
      "logps/rejected": -361.3003845214844,
      "loss": 1.1119,
      "margin_dpo/margin_mean": 98.48876190185547,
      "margin_dpo/margin_std": 127.98451232910156,
      "step": 620
    },
    {
      "KL/chosen_KL_mean": -182.19570922851562,
      "KL/mean": -229.78408813476562,
      "KL/rejected_KL_mean": -277.3724670410156,
      "KL/std": 114.10252380371094,
      "epoch": 0.9118942731277533,
      "fcm_dpo/beta": 0.003697324078530073,
      "fcm_dpo/delta": -0.060014449059963226,
      "fcm_dpo/margin": 95.1767578125,
      "fcm_dpo/q_t": 0.4163517355918884,
      "grad_norm": 8.189714431762695,
      "learning_rate": 1.2156682070109086e-08,
      "logits/chosen": -2.039632558822632,
      "logits/rejected": -1.7806200981140137,
      "logps/chosen": -271.4441223144531,
      "logps/ref_chosen": -89.24842071533203,
      "logps/ref_rejected": -95.46463775634766,
      "logps/rejected": -372.83709716796875,
      "loss": 1.1316,
      "margin_dpo/margin_mean": 95.1767578125,
      "margin_dpo/margin_std": 133.5888671875,
      "step": 621
    },
    {
      "KL/chosen_KL_mean": -178.44033813476562,
      "KL/mean": -230.14962768554688,
      "KL/rejected_KL_mean": -281.85894775390625,
      "KL/std": 102.61433410644531,
      "epoch": 0.9133627019089574,
      "fcm_dpo/beta": 0.003651430830359459,
      "fcm_dpo/delta": 0.02515832893550396,
      "fcm_dpo/margin": 103.41859436035156,
      "fcm_dpo/q_t": 0.4108109176158905,
      "grad_norm": 9.445230484008789,
      "learning_rate": 1.1764499893210878e-08,
      "logits/chosen": -1.4331793785095215,
      "logits/rejected": -0.771018385887146,
      "logps/chosen": -278.2344665527344,
      "logps/ref_chosen": -99.79412841796875,
      "logps/ref_rejected": -90.82821655273438,
      "logps/rejected": -372.6871643066406,
      "loss": 1.0842,
      "margin_dpo/margin_mean": 103.4186019897461,
      "margin_dpo/margin_std": 111.04049682617188,
      "step": 622
    },
    {
      "KL/chosen_KL_mean": -186.33175659179688,
      "KL/mean": -222.98974609375,
      "KL/rejected_KL_mean": -259.647705078125,
      "KL/std": 110.78649139404297,
      "epoch": 0.9148311306901615,
      "fcm_dpo/beta": 0.0038737263530492783,
      "fcm_dpo/delta": 0.12734903395175934,
      "fcm_dpo/margin": 73.31593322753906,
      "fcm_dpo/q_t": 0.4363425076007843,
      "grad_norm": 9.467825889587402,
      "learning_rate": 1.1378595443300998e-08,
      "logits/chosen": -2.3177402019500732,
      "logits/rejected": -1.886216402053833,
      "logps/chosen": -276.7873229980469,
      "logps/ref_chosen": -90.45555114746094,
      "logps/ref_rejected": -91.32276153564453,
      "logps/rejected": -350.970458984375,
      "loss": 1.1909,
      "margin_dpo/margin_mean": 73.31593322753906,
      "margin_dpo/margin_std": 136.51876831054688,
      "step": 623
    },
    {
      "KL/chosen_KL_mean": -192.45436096191406,
      "KL/mean": -250.53494262695312,
      "KL/rejected_KL_mean": -308.6155090332031,
      "KL/std": 109.55197143554688,
      "epoch": 0.9162995594713657,
      "fcm_dpo/beta": 0.0038736960850656033,
      "fcm_dpo/delta": -0.05786335840821266,
      "fcm_dpo/margin": 116.1611328125,
      "fcm_dpo/q_t": 0.3936132788658142,
      "grad_norm": 10.294363975524902,
      "learning_rate": 1.0998978889320582e-08,
      "logits/chosen": -2.5094587802886963,
      "logits/rejected": -1.592193603515625,
      "logps/chosen": -302.32958984375,
      "logps/ref_chosen": -109.87522888183594,
      "logps/ref_rejected": -104.77320861816406,
      "logps/rejected": -413.38873291015625,
      "loss": 1.0425,
      "margin_dpo/margin_mean": 116.16114044189453,
      "margin_dpo/margin_std": 126.40806579589844,
      "step": 624
    },
    {
      "KL/chosen_KL_mean": -200.47938537597656,
      "KL/mean": -241.61041259765625,
      "KL/rejected_KL_mean": -282.7414245605469,
      "KL/std": 108.52574157714844,
      "epoch": 0.9177679882525698,
      "fcm_dpo/beta": 0.004003247246146202,
      "fcm_dpo/delta": 0.07762551307678223,
      "fcm_dpo/margin": 82.26203918457031,
      "fcm_dpo/q_t": 0.4241800308227539,
      "grad_norm": 8.993022918701172,
      "learning_rate": 1.0625660234518913e-08,
      "logits/chosen": -1.6689283847808838,
      "logits/rejected": -1.2931513786315918,
      "logps/chosen": -287.6475524902344,
      "logps/ref_chosen": -87.16815948486328,
      "logps/ref_rejected": -91.86148071289062,
      "logps/rejected": -374.6029052734375,
      "loss": 1.1357,
      "margin_dpo/margin_mean": 82.26203918457031,
      "margin_dpo/margin_std": 116.55612182617188,
      "step": 625
    },
    {
      "KL/chosen_KL_mean": -188.78207397460938,
      "KL/mean": -230.69921875,
      "KL/rejected_KL_mean": -272.6163330078125,
      "KL/std": 120.68630981445312,
      "epoch": 0.9192364170337739,
      "fcm_dpo/beta": 0.004159306176006794,
      "fcm_dpo/delta": 0.05646456778049469,
      "fcm_dpo/margin": 83.83427429199219,
      "fcm_dpo/q_t": 0.4208095669746399,
      "grad_norm": 12.154741287231445,
      "learning_rate": 1.0258649316189721e-08,
      "logits/chosen": -1.4332330226898193,
      "logits/rejected": -1.001382827758789,
      "logps/chosen": -293.00628662109375,
      "logps/ref_chosen": -104.22421264648438,
      "logps/ref_rejected": -104.1774673461914,
      "logps/rejected": -376.7938232421875,
      "loss": 1.1383,
      "margin_dpo/margin_mean": 83.83425903320312,
      "margin_dpo/margin_std": 131.4295654296875,
      "step": 626
    },
    {
      "KL/chosen_KL_mean": -167.7424774169922,
      "KL/mean": -220.47506713867188,
      "KL/rejected_KL_mean": -273.2076721191406,
      "KL/std": 123.44642639160156,
      "epoch": 0.920704845814978,
      "fcm_dpo/beta": 0.00410048570483923,
      "fcm_dpo/delta": -0.0372723713517189,
      "fcm_dpo/margin": 105.46516418457031,
      "fcm_dpo/q_t": 0.4005392789840698,
      "grad_norm": 7.906949520111084,
      "learning_rate": 9.897955805412e-09,
      "logits/chosen": -1.5610864162445068,
      "logits/rejected": -1.7094752788543701,
      "logps/chosen": -242.67709350585938,
      "logps/ref_chosen": -74.93461608886719,
      "logps/ref_rejected": -112.57289123535156,
      "logps/rejected": -385.7805480957031,
      "loss": 1.0719,
      "margin_dpo/margin_mean": 105.46516418457031,
      "margin_dpo/margin_std": 136.65206909179688,
      "step": 627
    },
    {
      "KL/chosen_KL_mean": -178.42202758789062,
      "KL/mean": -232.58169555664062,
      "KL/rejected_KL_mean": -286.7413635253906,
      "KL/std": 111.38035583496094,
      "epoch": 0.922173274596182,
      "fcm_dpo/beta": 0.004004572983831167,
      "fcm_dpo/delta": -0.03881898522377014,
      "fcm_dpo/margin": 108.3193359375,
      "fcm_dpo/q_t": 0.3981897532939911,
      "grad_norm": 9.837762832641602,
      "learning_rate": 9.543589206795238e-09,
      "logits/chosen": -1.7508156299591064,
      "logits/rejected": -1.4829498529434204,
      "logps/chosen": -272.11309814453125,
      "logps/ref_chosen": -93.69107818603516,
      "logps/ref_rejected": -107.34395599365234,
      "logps/rejected": -394.0853271484375,
      "loss": 1.0586,
      "margin_dpo/margin_mean": 108.3193359375,
      "margin_dpo/margin_std": 124.96466064453125,
      "step": 628
    },
    {
      "KL/chosen_KL_mean": -191.2006378173828,
      "KL/mean": -240.4420166015625,
      "KL/rejected_KL_mean": -289.6833801269531,
      "KL/std": 112.03657531738281,
      "epoch": 0.9236417033773862,
      "fcm_dpo/beta": 0.003959874622523785,
      "fcm_dpo/delta": 0.010792192071676254,
      "fcm_dpo/margin": 98.48272705078125,
      "fcm_dpo/q_t": 0.40805143117904663,
      "grad_norm": 9.49244213104248,
      "learning_rate": 9.19555885822887e-09,
      "logits/chosen": -2.1008148193359375,
      "logits/rejected": -1.6697466373443604,
      "logps/chosen": -294.4310302734375,
      "logps/ref_chosen": -103.23037719726562,
      "logps/ref_rejected": -97.16841888427734,
      "logps/rejected": -386.851806640625,
      "loss": 1.0891,
      "margin_dpo/margin_mean": 98.48272705078125,
      "margin_dpo/margin_std": 119.67965698242188,
      "step": 629
    },
    {
      "KL/chosen_KL_mean": -173.60821533203125,
      "KL/mean": -205.44793701171875,
      "KL/rejected_KL_mean": -237.2876434326172,
      "KL/std": 104.0506591796875,
      "epoch": 0.9251101321585903,
      "fcm_dpo/beta": 0.0040569365955889225,
      "fcm_dpo/delta": 0.04280588775873184,
      "fcm_dpo/margin": 63.67940902709961,
      "fcm_dpo/q_t": 0.43956220149993896,
      "grad_norm": 11.427755355834961,
      "learning_rate": 8.85387393063622e-09,
      "logits/chosen": -1.9688040018081665,
      "logits/rejected": -1.4407211542129517,
      "logps/chosen": -267.5057678222656,
      "logps/ref_chosen": -93.89755249023438,
      "logps/ref_rejected": -89.3743896484375,
      "logps/rejected": -326.66204833984375,
      "loss": 1.2005,
      "margin_dpo/margin_mean": 63.679412841796875,
      "margin_dpo/margin_std": 117.69474792480469,
      "step": 630
    },
    {
      "KL/chosen_KL_mean": -171.47262573242188,
      "KL/mean": -218.9988250732422,
      "KL/rejected_KL_mean": -266.5250244140625,
      "KL/std": 103.52610778808594,
      "epoch": 0.9265785609397944,
      "fcm_dpo/beta": 0.004116623662412167,
      "fcm_dpo/delta": 0.009842852130532265,
      "fcm_dpo/margin": 95.05242156982422,
      "fcm_dpo/q_t": 0.40826502442359924,
      "grad_norm": 8.386008262634277,
      "learning_rate": 8.518543427732949e-09,
      "logits/chosen": -2.2490625381469727,
      "logits/rejected": -1.802931308746338,
      "logps/chosen": -259.2434387207031,
      "logps/ref_chosen": -87.77082061767578,
      "logps/ref_rejected": -88.68241882324219,
      "logps/rejected": -355.2074279785156,
      "loss": 1.0902,
      "margin_dpo/margin_mean": 95.05241394042969,
      "margin_dpo/margin_std": 119.06468200683594,
      "step": 631
    },
    {
      "KL/chosen_KL_mean": -167.69561767578125,
      "KL/mean": -220.05978393554688,
      "KL/rejected_KL_mean": -272.4239807128906,
      "KL/std": 108.30876922607422,
      "epoch": 0.9280469897209985,
      "fcm_dpo/beta": 0.004024748224765062,
      "fcm_dpo/delta": -0.02661992982029915,
      "fcm_dpo/margin": 104.72835540771484,
      "fcm_dpo/q_t": 0.40156620740890503,
      "grad_norm": 11.140682220458984,
      "learning_rate": 8.189576185789637e-09,
      "logits/chosen": -2.078778028488159,
      "logits/rejected": -1.6558153629302979,
      "logps/chosen": -256.3221435546875,
      "logps/ref_chosen": -88.62652587890625,
      "logps/ref_rejected": -91.45091247558594,
      "logps/rejected": -363.8748779296875,
      "loss": 1.0783,
      "margin_dpo/margin_mean": 104.72834777832031,
      "margin_dpo/margin_std": 133.15834045410156,
      "step": 632
    },
    {
      "KL/chosen_KL_mean": -208.97193908691406,
      "KL/mean": -239.7500457763672,
      "KL/rejected_KL_mean": -270.5281982421875,
      "KL/std": 110.12379455566406,
      "epoch": 0.9295154185022027,
      "fcm_dpo/beta": 0.004099705722182989,
      "fcm_dpo/delta": 0.025879479944705963,
      "fcm_dpo/margin": 61.55625534057617,
      "fcm_dpo/q_t": 0.4432462155818939,
      "grad_norm": 11.145735740661621,
      "learning_rate": 7.866980873399015e-09,
      "logits/chosen": -2.105466842651367,
      "logits/rejected": -1.9132448434829712,
      "logps/chosen": -290.34637451171875,
      "logps/ref_chosen": -81.37442016601562,
      "logps/ref_rejected": -98.62571716308594,
      "logps/rejected": -369.1539001464844,
      "loss": 1.2375,
      "margin_dpo/margin_mean": 61.55625534057617,
      "margin_dpo/margin_std": 147.75909423828125,
      "step": 633
    },
    {
      "KL/chosen_KL_mean": -193.0508270263672,
      "KL/mean": -232.63101196289062,
      "KL/rejected_KL_mean": -272.211181640625,
      "KL/std": 99.96620178222656,
      "epoch": 0.9309838472834068,
      "fcm_dpo/beta": 0.004088785964995623,
      "fcm_dpo/delta": -0.03485163673758507,
      "fcm_dpo/margin": 79.16036987304688,
      "fcm_dpo/q_t": 0.4229212701320648,
      "grad_norm": 10.306833267211914,
      "learning_rate": 7.550765991247654e-09,
      "logits/chosen": -1.3886170387268066,
      "logits/rejected": -1.216518759727478,
      "logps/chosen": -289.1736755371094,
      "logps/ref_chosen": -96.12284851074219,
      "logps/ref_rejected": -112.84780883789062,
      "logps/rejected": -385.0589904785156,
      "loss": 1.1365,
      "margin_dpo/margin_mean": 79.16036987304688,
      "margin_dpo/margin_std": 102.98652648925781,
      "step": 634
    },
    {
      "KL/chosen_KL_mean": -201.38473510742188,
      "KL/mean": -232.27484130859375,
      "KL/rejected_KL_mean": -263.16497802734375,
      "KL/std": 109.38069152832031,
      "epoch": 0.9324522760646109,
      "fcm_dpo/beta": 0.004085876047611237,
      "fcm_dpo/delta": 0.032502274960279465,
      "fcm_dpo/margin": 61.780250549316406,
      "fcm_dpo/q_t": 0.4420062303543091,
      "grad_norm": 12.405533790588379,
      "learning_rate": 7.240939871891699e-09,
      "logits/chosen": -1.7879489660263062,
      "logits/rejected": -1.2496676445007324,
      "logps/chosen": -300.06884765625,
      "logps/ref_chosen": -98.68411254882812,
      "logps/ref_rejected": -89.89920043945312,
      "logps/rejected": -353.0641784667969,
      "loss": 1.2198,
      "margin_dpo/margin_mean": 61.780250549316406,
      "margin_dpo/margin_std": 131.87481689453125,
      "step": 635
    },
    {
      "KL/chosen_KL_mean": -178.98391723632812,
      "KL/mean": -231.87326049804688,
      "KL/rejected_KL_mean": -284.76263427734375,
      "KL/std": 112.23828125,
      "epoch": 0.933920704845815,
      "fcm_dpo/beta": 0.004075915087014437,
      "fcm_dpo/delta": -0.035734739154577255,
      "fcm_dpo/margin": 105.77870178222656,
      "fcm_dpo/q_t": 0.3982580602169037,
      "grad_norm": 8.8456392288208,
      "learning_rate": 6.937510679537628e-09,
      "logits/chosen": -1.467139482498169,
      "logits/rejected": -0.9828199148178101,
      "logps/chosen": -269.40185546875,
      "logps/ref_chosen": -90.41796112060547,
      "logps/ref_rejected": -87.70687866210938,
      "logps/rejected": -372.4695129394531,
      "loss": 1.051,
      "margin_dpo/margin_mean": 105.77870178222656,
      "margin_dpo/margin_std": 112.53289031982422,
      "step": 636
    },
    {
      "KL/chosen_KL_mean": -180.4730224609375,
      "KL/mean": -236.8293914794922,
      "KL/rejected_KL_mean": -293.1857604980469,
      "KL/std": 124.46146392822266,
      "epoch": 0.9353891336270191,
      "fcm_dpo/beta": 0.003960088826715946,
      "fcm_dpo/delta": -0.05341949686408043,
      "fcm_dpo/margin": 112.71275329589844,
      "fcm_dpo/q_t": 0.3939940333366394,
      "grad_norm": 7.860083103179932,
      "learning_rate": 6.640486409826785e-09,
      "logits/chosen": -1.8184857368469238,
      "logits/rejected": -1.714880108833313,
      "logps/chosen": -262.9227294921875,
      "logps/ref_chosen": -82.44971466064453,
      "logps/ref_rejected": -104.02860260009766,
      "logps/rejected": -397.21435546875,
      "loss": 1.0425,
      "margin_dpo/margin_mean": 112.71275329589844,
      "margin_dpo/margin_std": 118.40499114990234,
      "step": 637
    },
    {
      "KL/chosen_KL_mean": -168.74307250976562,
      "KL/mean": -223.7769775390625,
      "KL/rejected_KL_mean": -278.81085205078125,
      "KL/std": 110.57144165039062,
      "epoch": 0.9368575624082232,
      "fcm_dpo/beta": 0.00390727398917079,
      "fcm_dpo/delta": -0.03550812602043152,
      "fcm_dpo/margin": 110.06779479980469,
      "fcm_dpo/q_t": 0.39716166257858276,
      "grad_norm": 8.251981735229492,
      "learning_rate": 6.349874889624962e-09,
      "logits/chosen": -1.6402854919433594,
      "logits/rejected": -0.967639684677124,
      "logps/chosen": -260.6680603027344,
      "logps/ref_chosen": -91.92498779296875,
      "logps/ref_rejected": -86.28703308105469,
      "logps/rejected": -365.097900390625,
      "loss": 1.0437,
      "margin_dpo/margin_mean": 110.06779479980469,
      "margin_dpo/margin_std": 102.91815948486328,
      "step": 638
    },
    {
      "KL/chosen_KL_mean": -224.83413696289062,
      "KL/mean": -250.46652221679688,
      "KL/rejected_KL_mean": -276.0989074707031,
      "KL/std": 102.34616088867188,
      "epoch": 0.9383259911894273,
      "fcm_dpo/beta": 0.0039174361154437065,
      "fcm_dpo/delta": 0.07579009234905243,
      "fcm_dpo/margin": 51.26478576660156,
      "fcm_dpo/q_t": 0.45317190885543823,
      "grad_norm": 9.303876876831055,
      "learning_rate": 6.065683776815933e-09,
      "logits/chosen": -1.3510522842407227,
      "logits/rejected": -0.5846160054206848,
      "logps/chosen": -329.3616943359375,
      "logps/ref_chosen": -104.52755737304688,
      "logps/ref_rejected": -81.4803466796875,
      "logps/rejected": -357.5792541503906,
      "loss": 1.2556,
      "margin_dpo/margin_mean": 51.26478576660156,
      "margin_dpo/margin_std": 127.74617767333984,
      "step": 639
    },
    {
      "KL/chosen_KL_mean": -190.80662536621094,
      "KL/mean": -240.49502563476562,
      "KL/rejected_KL_mean": -290.18341064453125,
      "KL/std": 110.78170013427734,
      "epoch": 0.9397944199706314,
      "fcm_dpo/beta": 0.004091430455446243,
      "fcm_dpo/delta": -0.0119662806391716,
      "fcm_dpo/margin": 99.37680053710938,
      "fcm_dpo/q_t": 0.40346354246139526,
      "grad_norm": 12.729485511779785,
      "learning_rate": 5.7879205600998296e-09,
      "logits/chosen": -1.7147471904754639,
      "logits/rejected": -1.1900854110717773,
      "logps/chosen": -288.69189453125,
      "logps/ref_chosen": -97.88526916503906,
      "logps/ref_rejected": -112.70502471923828,
      "logps/rejected": -402.8884582519531,
      "loss": 1.0791,
      "margin_dpo/margin_mean": 99.37680053710938,
      "margin_dpo/margin_std": 110.28523254394531,
      "step": 640
    },
    {
      "KL/chosen_KL_mean": -174.5283203125,
      "KL/mean": -226.17282104492188,
      "KL/rejected_KL_mean": -277.8173522949219,
      "KL/std": 129.3271942138672,
      "epoch": 0.9412628487518355,
      "fcm_dpo/beta": 0.00396689772605896,
      "fcm_dpo/delta": -0.011110333725810051,
      "fcm_dpo/margin": 103.28903198242188,
      "fcm_dpo/q_t": 0.4074035882949829,
      "grad_norm": 8.47859001159668,
      "learning_rate": 5.516592558795746e-09,
      "logits/chosen": -2.287022829055786,
      "logits/rejected": -1.7447799444198608,
      "logps/chosen": -270.9739990234375,
      "logps/ref_chosen": -96.4456787109375,
      "logps/ref_rejected": -95.13568878173828,
      "logps/rejected": -372.95306396484375,
      "loss": 1.0907,
      "margin_dpo/margin_mean": 103.28902435302734,
      "margin_dpo/margin_std": 143.61830139160156,
      "step": 641
    },
    {
      "KL/chosen_KL_mean": -193.540283203125,
      "KL/mean": -241.69232177734375,
      "KL/rejected_KL_mean": -289.8443603515625,
      "KL/std": 111.28330993652344,
      "epoch": 0.9427312775330396,
      "fcm_dpo/beta": 0.004009862430393696,
      "fcm_dpo/delta": 0.015067409723997116,
      "fcm_dpo/margin": 96.30409240722656,
      "fcm_dpo/q_t": 0.4123472571372986,
      "grad_norm": 10.641100883483887,
      "learning_rate": 5.251706922648868e-09,
      "logits/chosen": -1.605882167816162,
      "logits/rejected": -1.2775386571884155,
      "logps/chosen": -294.30010986328125,
      "logps/ref_chosen": -100.75984954833984,
      "logps/ref_rejected": -114.7076416015625,
      "logps/rejected": -404.552001953125,
      "loss": 1.1082,
      "margin_dpo/margin_mean": 96.30408477783203,
      "margin_dpo/margin_std": 138.79312133789062,
      "step": 642
    },
    {
      "KL/chosen_KL_mean": -180.88125610351562,
      "KL/mean": -226.73399353027344,
      "KL/rejected_KL_mean": -272.58673095703125,
      "KL/std": 102.54206848144531,
      "epoch": 0.9441997063142438,
      "fcm_dpo/beta": 0.003977928310632706,
      "fcm_dpo/delta": 0.03762334957718849,
      "fcm_dpo/margin": 91.7054672241211,
      "fcm_dpo/q_t": 0.4145483374595642,
      "grad_norm": 11.654149055480957,
      "learning_rate": 4.993270631642038e-09,
      "logits/chosen": -2.205990791320801,
      "logits/rejected": -1.835115909576416,
      "logps/chosen": -265.6249084472656,
      "logps/ref_chosen": -84.74365997314453,
      "logps/ref_rejected": -94.31842041015625,
      "logps/rejected": -366.9051513671875,
      "loss": 1.1142,
      "margin_dpo/margin_mean": 91.7054672241211,
      "margin_dpo/margin_std": 119.0499038696289,
      "step": 643
    },
    {
      "KL/chosen_KL_mean": -189.25759887695312,
      "KL/mean": -226.84396362304688,
      "KL/rejected_KL_mean": -264.4302978515625,
      "KL/std": 103.91043090820312,
      "epoch": 0.9456681350954479,
      "fcm_dpo/beta": 0.004247503355145454,
      "fcm_dpo/delta": 0.08914369344711304,
      "fcm_dpo/margin": 75.1727294921875,
      "fcm_dpo/q_t": 0.4266725778579712,
      "grad_norm": 11.14410400390625,
      "learning_rate": 4.741290495811873e-09,
      "logits/chosen": -1.840881109237671,
      "logits/rejected": -1.5747301578521729,
      "logps/chosen": -274.5803527832031,
      "logps/ref_chosen": -85.32274627685547,
      "logps/ref_rejected": -94.60861206054688,
      "logps/rejected": -359.0389404296875,
      "loss": 1.1475,
      "margin_dpo/margin_mean": 75.1727294921875,
      "margin_dpo/margin_std": 113.23483276367188,
      "step": 644
    },
    {
      "KL/chosen_KL_mean": -200.5806884765625,
      "KL/mean": -225.034423828125,
      "KL/rejected_KL_mean": -249.48817443847656,
      "KL/std": 113.1276626586914,
      "epoch": 0.947136563876652,
      "fcm_dpo/beta": 0.004399304278194904,
      "fcm_dpo/delta": 0.07344308495521545,
      "fcm_dpo/margin": 48.90745544433594,
      "fcm_dpo/q_t": 0.45111578702926636,
      "grad_norm": 11.00709056854248,
      "learning_rate": 4.495773155069299e-09,
      "logits/chosen": -1.6048392057418823,
      "logits/rejected": -1.4421747922897339,
      "logps/chosen": -283.17095947265625,
      "logps/ref_chosen": -82.59024047851562,
      "logps/ref_rejected": -103.00375366210938,
      "logps/rejected": -352.491943359375,
      "loss": 1.2587,
      "margin_dpo/margin_mean": 48.90745544433594,
      "margin_dpo/margin_std": 127.94683074951172,
      "step": 645
    },
    {
      "KL/chosen_KL_mean": -176.19619750976562,
      "KL/mean": -225.47738647460938,
      "KL/rejected_KL_mean": -274.7585754394531,
      "KL/std": 98.56561279296875,
      "epoch": 0.9486049926578561,
      "fcm_dpo/beta": 0.004363452084362507,
      "fcm_dpo/delta": -0.0364360511302948,
      "fcm_dpo/margin": 98.56236267089844,
      "fcm_dpo/q_t": 0.3980404734611511,
      "grad_norm": 9.435596466064453,
      "learning_rate": 4.256725079024553e-09,
      "logits/chosen": -1.8110512495040894,
      "logits/rejected": -1.2063713073730469,
      "logps/chosen": -267.38824462890625,
      "logps/ref_chosen": -91.1920394897461,
      "logps/ref_rejected": -83.77833557128906,
      "logps/rejected": -358.5368957519531,
      "loss": 1.0556,
      "margin_dpo/margin_mean": 98.56236267089844,
      "margin_dpo/margin_std": 107.29556274414062,
      "step": 646
    },
    {
      "KL/chosen_KL_mean": -190.65725708007812,
      "KL/mean": -234.0261688232422,
      "KL/rejected_KL_mean": -277.39508056640625,
      "KL/std": 118.84233856201172,
      "epoch": 0.9500734214390602,
      "fcm_dpo/beta": 0.004433132242411375,
      "fcm_dpo/delta": 0.01749793253839016,
      "fcm_dpo/margin": 86.73784637451172,
      "fcm_dpo/q_t": 0.4127277135848999,
      "grad_norm": 11.159293174743652,
      "learning_rate": 4.024152566816791e-09,
      "logits/chosen": -1.5164682865142822,
      "logits/rejected": -1.3807823657989502,
      "logps/chosen": -279.501708984375,
      "logps/ref_chosen": -88.8444595336914,
      "logps/ref_rejected": -99.49833679199219,
      "logps/rejected": -376.8934326171875,
      "loss": 1.1176,
      "margin_dpo/margin_mean": 86.73783874511719,
      "margin_dpo/margin_std": 131.2561798095703,
      "step": 647
    },
    {
      "KL/chosen_KL_mean": -178.75689697265625,
      "KL/mean": -237.659912109375,
      "KL/rejected_KL_mean": -296.56292724609375,
      "KL/std": 122.16462707519531,
      "epoch": 0.9515418502202643,
      "fcm_dpo/beta": 0.004247863776981831,
      "fcm_dpo/delta": -0.11772247403860092,
      "fcm_dpo/margin": 117.80604553222656,
      "fcm_dpo/q_t": 0.38164299726486206,
      "grad_norm": 9.88953685760498,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": -1.9277691841125488,
      "logits/rejected": -1.8568875789642334,
      "logps/chosen": -266.60498046875,
      "logps/ref_chosen": -87.84810638427734,
      "logps/ref_rejected": -104.67005920410156,
      "logps/rejected": -401.2330017089844,
      "loss": 1.0197,
      "margin_dpo/margin_mean": 117.80604553222656,
      "margin_dpo/margin_std": 129.9287109375,
      "step": 648
    },
    {
      "KL/chosen_KL_mean": -163.22027587890625,
      "KL/mean": -213.9481201171875,
      "KL/rejected_KL_mean": -264.6759948730469,
      "KL/std": 106.234619140625,
      "epoch": 0.9530102790014684,
      "fcm_dpo/beta": 0.004056960344314575,
      "fcm_dpo/delta": -0.01491134986281395,
      "fcm_dpo/margin": 101.4557113647461,
      "fcm_dpo/q_t": 0.40221071243286133,
      "grad_norm": 10.65735149383545,
      "learning_rate": 3.5784585771215235e-09,
      "logits/chosen": -2.4481735229492188,
      "logits/rejected": -2.048150062561035,
      "logps/chosen": -252.91278076171875,
      "logps/ref_chosen": -89.6925048828125,
      "logps/ref_rejected": -88.70658111572266,
      "logps/rejected": -353.382568359375,
      "loss": 1.0608,
      "margin_dpo/margin_mean": 101.45571899414062,
      "margin_dpo/margin_std": 102.66673278808594,
      "step": 649
    },
    {
      "KL/chosen_KL_mean": -193.930908203125,
      "KL/mean": -240.65000915527344,
      "KL/rejected_KL_mean": -287.369140625,
      "KL/std": 114.84461975097656,
      "epoch": 0.9544787077826725,
      "fcm_dpo/beta": 0.0041417321190238,
      "fcm_dpo/delta": 0.01463753916323185,
      "fcm_dpo/margin": 93.43819427490234,
      "fcm_dpo/q_t": 0.4107799232006073,
      "grad_norm": 9.450539588928223,
      "learning_rate": 3.3653488440851253e-09,
      "logits/chosen": -1.2442814111709595,
      "logits/rejected": -1.0440998077392578,
      "logps/chosen": -283.86151123046875,
      "logps/ref_chosen": -89.93060302734375,
      "logps/ref_rejected": -102.61282348632812,
      "logps/rejected": -389.98193359375,
      "loss": 1.1096,
      "margin_dpo/margin_mean": 93.43820190429688,
      "margin_dpo/margin_std": 135.64544677734375,
      "step": 650
    },
    {
      "KL/chosen_KL_mean": -160.2197265625,
      "KL/mean": -217.8218994140625,
      "KL/rejected_KL_mean": -275.424072265625,
      "KL/std": 111.20906066894531,
      "epoch": 0.9559471365638766,
      "fcm_dpo/beta": 0.0040015289559960365,
      "fcm_dpo/delta": -0.07108767330646515,
      "fcm_dpo/margin": 115.2043685913086,
      "fcm_dpo/q_t": 0.3892034888267517,
      "grad_norm": 9.030444145202637,
      "learning_rate": 3.158738163478475e-09,
      "logits/chosen": -2.018219232559204,
      "logits/rejected": -1.8415499925613403,
      "logps/chosen": -239.40704345703125,
      "logps/ref_chosen": -79.18731689453125,
      "logps/ref_rejected": -105.93333435058594,
      "logps/rejected": -381.3573913574219,
      "loss": 1.0234,
      "margin_dpo/margin_mean": 115.2043685913086,
      "margin_dpo/margin_std": 109.60478973388672,
      "step": 651
    },
    {
      "KL/chosen_KL_mean": -168.90875244140625,
      "KL/mean": -217.0749053955078,
      "KL/rejected_KL_mean": -265.2410583496094,
      "KL/std": 103.22172546386719,
      "epoch": 0.9574155653450808,
      "fcm_dpo/beta": 0.004022186156362295,
      "fcm_dpo/delta": 0.013697346672415733,
      "fcm_dpo/margin": 96.33229064941406,
      "fcm_dpo/q_t": 0.4084562063217163,
      "grad_norm": 9.211309432983398,
      "learning_rate": 2.9586319796851555e-09,
      "logits/chosen": -2.432347297668457,
      "logits/rejected": -2.1363515853881836,
      "logps/chosen": -270.698974609375,
      "logps/ref_chosen": -101.79022216796875,
      "logps/ref_rejected": -116.3245849609375,
      "logps/rejected": -381.5656433105469,
      "loss": 1.0737,
      "margin_dpo/margin_mean": 96.33229064941406,
      "margin_dpo/margin_std": 98.68157958984375,
      "step": 652
    },
    {
      "KL/chosen_KL_mean": -191.82275390625,
      "KL/mean": -238.53665161132812,
      "KL/rejected_KL_mean": -285.25054931640625,
      "KL/std": 113.37013244628906,
      "epoch": 0.9588839941262849,
      "fcm_dpo/beta": 0.004061164800077677,
      "fcm_dpo/delta": 0.02295197919011116,
      "fcm_dpo/margin": 93.42774963378906,
      "fcm_dpo/q_t": 0.4100278317928314,
      "grad_norm": 9.342035293579102,
      "learning_rate": 2.7650355656892166e-09,
      "logits/chosen": -2.1997170448303223,
      "logits/rejected": -1.947645902633667,
      "logps/chosen": -285.17633056640625,
      "logps/ref_chosen": -93.35359191894531,
      "logps/ref_rejected": -109.12324523925781,
      "logps/rejected": -394.373779296875,
      "loss": 1.0919,
      "margin_dpo/margin_mean": 93.4277572631836,
      "margin_dpo/margin_std": 110.88983154296875,
      "step": 653
    },
    {
      "KL/chosen_KL_mean": -192.51150512695312,
      "KL/mean": -232.98602294921875,
      "KL/rejected_KL_mean": -273.46051025390625,
      "KL/std": 108.46007537841797,
      "epoch": 0.960352422907489,
      "fcm_dpo/beta": 0.004182050004601479,
      "fcm_dpo/delta": 0.06840844452381134,
      "fcm_dpo/margin": 80.94902038574219,
      "fcm_dpo/q_t": 0.42445051670074463,
      "grad_norm": 12.556596755981445,
      "learning_rate": 2.577954022936174e-09,
      "logits/chosen": -2.1144731044769287,
      "logits/rejected": -1.9274542331695557,
      "logps/chosen": -281.6270446777344,
      "logps/ref_chosen": -89.11553955078125,
      "logps/ref_rejected": -104.91995239257812,
      "logps/rejected": -378.3804626464844,
      "loss": 1.153,
      "margin_dpo/margin_mean": 80.94902038574219,
      "margin_dpo/margin_std": 137.09866333007812,
      "step": 654
    },
    {
      "KL/chosen_KL_mean": -176.19192504882812,
      "KL/mean": -224.8978729248047,
      "KL/rejected_KL_mean": -273.60382080078125,
      "KL/std": 117.61151123046875,
      "epoch": 0.9618208516886931,
      "fcm_dpo/beta": 0.004216545261442661,
      "fcm_dpo/delta": -0.012277199886739254,
      "fcm_dpo/margin": 97.41191101074219,
      "fcm_dpo/q_t": 0.40380024909973145,
      "grad_norm": 9.40658950805664,
      "learning_rate": 2.397392281198729e-09,
      "logits/chosen": -2.075349807739258,
      "logits/rejected": -2.078049659729004,
      "logps/chosen": -257.22802734375,
      "logps/ref_chosen": -81.03610229492188,
      "logps/ref_rejected": -102.80233764648438,
      "logps/rejected": -376.4061584472656,
      "loss": 1.0854,
      "margin_dpo/margin_mean": 97.41191101074219,
      "margin_dpo/margin_std": 126.36045837402344,
      "step": 655
    },
    {
      "KL/chosen_KL_mean": -177.41543579101562,
      "KL/mean": -239.0623779296875,
      "KL/rejected_KL_mean": -300.7093200683594,
      "KL/std": 118.25332641601562,
      "epoch": 0.9632892804698973,
      "fcm_dpo/beta": 0.003984308801591396,
      "fcm_dpo/delta": -0.10766580700874329,
      "fcm_dpo/margin": 123.29388427734375,
      "fcm_dpo/q_t": 0.3832453191280365,
      "grad_norm": 9.735573768615723,
      "learning_rate": 2.223355098446622e-09,
      "logits/chosen": -1.3222362995147705,
      "logits/rejected": -1.3949482440948486,
      "logps/chosen": -262.74078369140625,
      "logps/ref_chosen": -85.32534790039062,
      "logps/ref_rejected": -118.33866119384766,
      "logps/rejected": -419.0479736328125,
      "loss": 1.0089,
      "margin_dpo/margin_mean": 123.29388427734375,
      "margin_dpo/margin_std": 122.13935852050781,
      "step": 656
    },
    {
      "KL/chosen_KL_mean": -192.8689727783203,
      "KL/mean": -238.75621032714844,
      "KL/rejected_KL_mean": -284.6434631347656,
      "KL/std": 109.27452087402344,
      "epoch": 0.9647577092511013,
      "fcm_dpo/beta": 0.003953919745981693,
      "fcm_dpo/delta": 0.04099327698349953,
      "fcm_dpo/margin": 91.77452850341797,
      "fcm_dpo/q_t": 0.4168602228164673,
      "grad_norm": 10.440885543823242,
      "learning_rate": 2.055847060721566e-09,
      "logits/chosen": -1.7998554706573486,
      "logits/rejected": -1.6285185813903809,
      "logps/chosen": -273.06671142578125,
      "logps/ref_chosen": -80.19772338867188,
      "logps/ref_rejected": -102.581298828125,
      "logps/rejected": -387.22479248046875,
      "loss": 1.1231,
      "margin_dpo/margin_mean": 91.77452850341797,
      "margin_dpo/margin_std": 131.9253387451172,
      "step": 657
    },
    {
      "KL/chosen_KL_mean": -189.9351348876953,
      "KL/mean": -232.38685607910156,
      "KL/rejected_KL_mean": -274.83856201171875,
      "KL/std": 102.71928405761719,
      "epoch": 0.9662261380323054,
      "fcm_dpo/beta": 0.0041618868708610535,
      "fcm_dpo/delta": 0.05108172446489334,
      "fcm_dpo/margin": 84.9034652709961,
      "fcm_dpo/q_t": 0.4177822470664978,
      "grad_norm": 9.109355926513672,
      "learning_rate": 1.8948725820160662e-09,
      "logits/chosen": -1.9578932523727417,
      "logits/rejected": -1.4846608638763428,
      "logps/chosen": -284.56964111328125,
      "logps/ref_chosen": -94.634521484375,
      "logps/ref_rejected": -101.6316146850586,
      "logps/rejected": -376.47021484375,
      "loss": 1.1153,
      "margin_dpo/margin_mean": 84.90345764160156,
      "margin_dpo/margin_std": 110.26458740234375,
      "step": 658
    },
    {
      "KL/chosen_KL_mean": -172.4923095703125,
      "KL/mean": -223.6247100830078,
      "KL/rejected_KL_mean": -274.7571105957031,
      "KL/std": 110.60137939453125,
      "epoch": 0.9676945668135095,
      "fcm_dpo/beta": 0.00407042633742094,
      "fcm_dpo/delta": -0.019901590421795845,
      "fcm_dpo/margin": 102.26481628417969,
      "fcm_dpo/q_t": 0.4038000702857971,
      "grad_norm": 8.847050666809082,
      "learning_rate": 1.7404359041573723e-09,
      "logits/chosen": -2.1049180030822754,
      "logits/rejected": -1.135005235671997,
      "logps/chosen": -285.04815673828125,
      "logps/ref_chosen": -112.55587005615234,
      "logps/ref_rejected": -93.9216079711914,
      "logps/rejected": -368.6787109375,
      "loss": 1.0967,
      "margin_dpo/margin_mean": 102.26481628417969,
      "margin_dpo/margin_std": 143.35914611816406,
      "step": 659
    },
    {
      "KL/chosen_KL_mean": -181.3097686767578,
      "KL/mean": -234.0750732421875,
      "KL/rejected_KL_mean": -286.84039306640625,
      "KL/std": 107.89698028564453,
      "epoch": 0.9691629955947136,
      "fcm_dpo/beta": 0.004048524424433708,
      "fcm_dpo/delta": -0.031274110078811646,
      "fcm_dpo/margin": 105.53065490722656,
      "fcm_dpo/q_t": 0.39950621128082275,
      "grad_norm": 11.322731971740723,
      "learning_rate": 1.592541096695571e-09,
      "logits/chosen": -1.8453764915466309,
      "logits/rejected": -1.2196061611175537,
      "logps/chosen": -274.68719482421875,
      "logps/ref_chosen": -93.37742614746094,
      "logps/ref_rejected": -81.39482116699219,
      "logps/rejected": -368.2352294921875,
      "loss": 1.0581,
      "margin_dpo/margin_mean": 105.53065490722656,
      "margin_dpo/margin_std": 118.83258056640625,
      "step": 660
    },
    {
      "KL/chosen_KL_mean": -179.54922485351562,
      "KL/mean": -238.81178283691406,
      "KL/rejected_KL_mean": -298.0743408203125,
      "KL/std": 121.1986312866211,
      "epoch": 0.9706314243759178,
      "fcm_dpo/beta": 0.003921539057046175,
      "fcm_dpo/delta": -0.07510429620742798,
      "fcm_dpo/margin": 118.52513885498047,
      "fcm_dpo/q_t": 0.39108771085739136,
      "grad_norm": 9.02489948272705,
      "learning_rate": 1.4511920567963908e-09,
      "logits/chosen": -1.7943828105926514,
      "logits/rejected": -1.2804639339447021,
      "logps/chosen": -267.4043884277344,
      "logps/ref_chosen": -87.85516357421875,
      "logps/ref_rejected": -92.40330505371094,
      "logps/rejected": -390.4776611328125,
      "loss": 1.041,
      "margin_dpo/margin_mean": 118.525146484375,
      "margin_dpo/margin_std": 134.84886169433594,
      "step": 661
    },
    {
      "KL/chosen_KL_mean": -175.84609985351562,
      "KL/mean": -229.76480102539062,
      "KL/rejected_KL_mean": -283.68353271484375,
      "KL/std": 105.36199188232422,
      "epoch": 0.9720998531571219,
      "fcm_dpo/beta": 0.0038440036587417126,
      "fcm_dpo/delta": -0.01667799800634384,
      "fcm_dpo/margin": 107.83743286132812,
      "fcm_dpo/q_t": 0.401845246553421,
      "grad_norm": 10.03638744354248,
      "learning_rate": 1.3163925091384532e-09,
      "logits/chosen": -1.7027244567871094,
      "logits/rejected": -1.1287614107131958,
      "logps/chosen": -278.6258850097656,
      "logps/ref_chosen": -102.77980041503906,
      "logps/ref_rejected": -95.22531127929688,
      "logps/rejected": -378.9088134765625,
      "loss": 1.0592,
      "margin_dpo/margin_mean": 107.83743286132812,
      "margin_dpo/margin_std": 111.76766967773438,
      "step": 662
    },
    {
      "KL/chosen_KL_mean": -176.41421508789062,
      "KL/mean": -225.64804077148438,
      "KL/rejected_KL_mean": -274.8818664550781,
      "KL/std": 109.10700988769531,
      "epoch": 0.973568281938326,
      "fcm_dpo/beta": 0.0038744539488106966,
      "fcm_dpo/delta": 0.020457852631807327,
      "fcm_dpo/margin": 98.4676513671875,
      "fcm_dpo/q_t": 0.41110944747924805,
      "grad_norm": 10.528714179992676,
      "learning_rate": 1.1881460058152382e-09,
      "logits/chosen": -2.193429946899414,
      "logits/rejected": -1.9768908023834229,
      "logps/chosen": -272.76080322265625,
      "logps/ref_chosen": -96.34658813476562,
      "logps/ref_rejected": -120.52645874023438,
      "logps/rejected": -395.4083251953125,
      "loss": 1.0987,
      "margin_dpo/margin_mean": 98.4676513671875,
      "margin_dpo/margin_std": 126.62174987792969,
      "step": 663
    },
    {
      "KL/chosen_KL_mean": -196.37579345703125,
      "KL/mean": -240.9063262939453,
      "KL/rejected_KL_mean": -285.4368591308594,
      "KL/std": 112.23881530761719,
      "epoch": 0.9750367107195301,
      "fcm_dpo/beta": 0.003976534120738506,
      "fcm_dpo/delta": 0.050621699541807175,
      "fcm_dpo/margin": 89.06106567382812,
      "fcm_dpo/q_t": 0.41742372512817383,
      "grad_norm": 8.374518394470215,
      "learning_rate": 1.066455926241383e-09,
      "logits/chosen": -1.638286828994751,
      "logits/rejected": -1.465651512145996,
      "logps/chosen": -288.21820068359375,
      "logps/ref_chosen": -91.84242248535156,
      "logps/ref_rejected": -111.8366928100586,
      "logps/rejected": -397.2735595703125,
      "loss": 1.1112,
      "margin_dpo/margin_mean": 89.06106567382812,
      "margin_dpo/margin_std": 110.68853759765625,
      "step": 664
    },
    {
      "KL/chosen_KL_mean": -169.04135131835938,
      "KL/mean": -217.60885620117188,
      "KL/rejected_KL_mean": -266.1763610839844,
      "KL/std": 107.85867309570312,
      "epoch": 0.9765051395007343,
      "fcm_dpo/beta": 0.0040243263356387615,
      "fcm_dpo/delta": 0.0099556315690279,
      "fcm_dpo/margin": 97.13502502441406,
      "fcm_dpo/q_t": 0.40765365958213806,
      "grad_norm": 8.033467292785645,
      "learning_rate": 9.513254770636137e-10,
      "logits/chosen": -2.4374639987945557,
      "logits/rejected": -2.011871337890625,
      "logps/chosen": -257.2275390625,
      "logps/ref_chosen": -88.18618774414062,
      "logps/ref_rejected": -91.9120101928711,
      "logps/rejected": -358.08837890625,
      "loss": 1.0817,
      "margin_dpo/margin_mean": 97.13502502441406,
      "margin_dpo/margin_std": 111.77503967285156,
      "step": 665
    },
    {
      "KL/chosen_KL_mean": -184.5423583984375,
      "KL/mean": -234.1253662109375,
      "KL/rejected_KL_mean": -283.7083740234375,
      "KL/std": 105.77641296386719,
      "epoch": 0.9779735682819384,
      "fcm_dpo/beta": 0.003985341638326645,
      "fcm_dpo/delta": 0.005097856745123863,
      "fcm_dpo/margin": 99.16600799560547,
      "fcm_dpo/q_t": 0.4082358479499817,
      "grad_norm": 9.614357948303223,
      "learning_rate": 8.427576920763956e-10,
      "logits/chosen": -1.461584448814392,
      "logits/rejected": -1.0725932121276855,
      "logps/chosen": -285.5169677734375,
      "logps/ref_chosen": -100.97459411621094,
      "logps/ref_rejected": -101.24992370605469,
      "logps/rejected": -384.9582824707031,
      "loss": 1.0911,
      "margin_dpo/margin_mean": 99.16600036621094,
      "margin_dpo/margin_std": 125.64874267578125,
      "step": 666
    },
    {
      "KL/chosen_KL_mean": -202.74740600585938,
      "KL/mean": -253.18911743164062,
      "KL/rejected_KL_mean": -303.630859375,
      "KL/std": 110.25039672851562,
      "epoch": 0.9794419970631424,
      "fcm_dpo/beta": 0.003996386658400297,
      "fcm_dpo/delta": -0.0037121502682566643,
      "fcm_dpo/margin": 100.88343811035156,
      "fcm_dpo/q_t": 0.4044594168663025,
      "grad_norm": 9.661558151245117,
      "learning_rate": 7.407554321417764e-10,
      "logits/chosen": -1.557248592376709,
      "logits/rejected": -0.9094034433364868,
      "logps/chosen": -300.3185729980469,
      "logps/ref_chosen": -97.5711669921875,
      "logps/ref_rejected": -93.58476257324219,
      "logps/rejected": -397.2156066894531,
      "loss": 1.0672,
      "margin_dpo/margin_mean": 100.88343811035156,
      "margin_dpo/margin_std": 107.23255920410156,
      "step": 667
    },
    {
      "KL/chosen_KL_mean": -178.87619018554688,
      "KL/mean": -237.47244262695312,
      "KL/rejected_KL_mean": -296.0687255859375,
      "KL/std": 117.48635864257812,
      "epoch": 0.9809104258443465,
      "fcm_dpo/beta": 0.0039010439068078995,
      "fcm_dpo/delta": -0.06616520881652832,
      "fcm_dpo/margin": 117.19256591796875,
      "fcm_dpo/q_t": 0.3930302858352661,
      "grad_norm": 9.57116985321045,
      "learning_rate": 6.453213851142225e-10,
      "logits/chosen": -2.1760995388031006,
      "logits/rejected": -1.6460621356964111,
      "logps/chosen": -281.45123291015625,
      "logps/ref_chosen": -102.5750503540039,
      "logps/ref_rejected": -108.81768798828125,
      "logps/rejected": -404.88641357421875,
      "loss": 1.0388,
      "margin_dpo/margin_mean": 117.19256591796875,
      "margin_dpo/margin_std": 128.64874267578125,
      "step": 668
    },
    {
      "KL/chosen_KL_mean": -162.84051513671875,
      "KL/mean": -219.26083374023438,
      "KL/rejected_KL_mean": -275.6811218261719,
      "KL/std": 114.61653900146484,
      "epoch": 0.9823788546255506,
      "fcm_dpo/beta": 0.0038002287037670612,
      "fcm_dpo/delta": -0.033051859587430954,
      "fcm_dpo/margin": 112.84059143066406,
      "fcm_dpo/q_t": 0.39865195751190186,
      "grad_norm": 8.925080299377441,
      "learning_rate": 5.564580657695939e-10,
      "logits/chosen": -1.6794602870941162,
      "logits/rejected": -1.2831504344940186,
      "logps/chosen": -252.33529663085938,
      "logps/ref_chosen": -89.49478149414062,
      "logps/ref_rejected": -82.51950073242188,
      "logps/rejected": -358.20062255859375,
      "loss": 1.0539,
      "margin_dpo/margin_mean": 112.84059143066406,
      "margin_dpo/margin_std": 122.3788833618164,
      "step": 669
    },
    {
      "KL/chosen_KL_mean": -163.03060913085938,
      "KL/mean": -225.31527709960938,
      "KL/rejected_KL_mean": -287.5999755859375,
      "KL/std": 117.09683227539062,
      "epoch": 0.9838472834067548,
      "fcm_dpo/beta": 0.0036794953048229218,
      "fcm_dpo/delta": -0.06750808656215668,
      "fcm_dpo/margin": 124.56935119628906,
      "fcm_dpo/q_t": 0.38974037766456604,
      "grad_norm": 7.269100189208984,
      "learning_rate": 4.741678157389739e-10,
      "logits/chosen": -1.8112390041351318,
      "logits/rejected": -1.2800588607788086,
      "logps/chosen": -258.4851989746094,
      "logps/ref_chosen": -95.45459747314453,
      "logps/ref_rejected": -101.53292846679688,
      "logps/rejected": -389.13287353515625,
      "loss": 1.029,
      "margin_dpo/margin_mean": 124.56935119628906,
      "margin_dpo/margin_std": 122.92074584960938,
      "step": 670
    },
    {
      "KL/chosen_KL_mean": -199.5172119140625,
      "KL/mean": -241.38645935058594,
      "KL/rejected_KL_mean": -283.2557373046875,
      "KL/std": 108.14494323730469,
      "epoch": 0.9853157121879589,
      "fcm_dpo/beta": 0.003827909706160426,
      "fcm_dpo/delta": 0.08459925651550293,
      "fcm_dpo/margin": 83.738525390625,
      "fcm_dpo/q_t": 0.426110178232193,
      "grad_norm": 9.172981262207031,
      "learning_rate": 3.9845280344705245e-10,
      "logits/chosen": -1.682816982269287,
      "logits/rejected": -1.459043264389038,
      "logps/chosen": -281.64031982421875,
      "logps/ref_chosen": -82.12312316894531,
      "logps/ref_rejected": -90.21969604492188,
      "logps/rejected": -373.47540283203125,
      "loss": 1.1449,
      "margin_dpo/margin_mean": 83.738525390625,
      "margin_dpo/margin_std": 120.976806640625,
      "step": 671
    },
    {
      "KL/chosen_KL_mean": -198.61087036132812,
      "KL/mean": -235.1762237548828,
      "KL/rejected_KL_mean": -271.7415771484375,
      "KL/std": 104.26085662841797,
      "epoch": 0.986784140969163,
      "fcm_dpo/beta": 0.0040086861699819565,
      "fcm_dpo/delta": 0.11742755770683289,
      "fcm_dpo/margin": 73.1307373046875,
      "fcm_dpo/q_t": 0.433044970035553,
      "grad_norm": 12.416485786437988,
      "learning_rate": 3.293150240547549e-10,
      "logits/chosen": -2.0012292861938477,
      "logits/rejected": -1.6718947887420654,
      "logps/chosen": -288.67279052734375,
      "logps/ref_chosen": -90.0619125366211,
      "logps/ref_rejected": -100.45323181152344,
      "logps/rejected": -372.19482421875,
      "loss": 1.1659,
      "margin_dpo/margin_mean": 73.1307373046875,
      "margin_dpo/margin_std": 110.65998077392578,
      "step": 672
    },
    {
      "KL/chosen_KL_mean": -178.1288604736328,
      "KL/mean": -219.59744262695312,
      "KL/rejected_KL_mean": -261.0660400390625,
      "KL/std": 107.0048599243164,
      "epoch": 0.9882525697503671,
      "fcm_dpo/beta": 0.004226578865200281,
      "fcm_dpo/delta": 0.05442439019680023,
      "fcm_dpo/margin": 82.93717956542969,
      "fcm_dpo/q_t": 0.41882503032684326,
      "grad_norm": 11.982220649719238,
      "learning_rate": 2.6675629940689504e-10,
      "logits/chosen": -1.7503137588500977,
      "logits/rejected": -1.374958872795105,
      "logps/chosen": -257.3919982910156,
      "logps/ref_chosen": -79.26315307617188,
      "logps/ref_rejected": -91.34925079345703,
      "logps/rejected": -352.415283203125,
      "loss": 1.1172,
      "margin_dpo/margin_mean": 82.93717956542969,
      "margin_dpo/margin_std": 109.45301055908203,
      "step": 673
    },
    {
      "KL/chosen_KL_mean": -167.10311889648438,
      "KL/mean": -226.53466796875,
      "KL/rejected_KL_mean": -285.9662170410156,
      "KL/std": 107.55072021484375,
      "epoch": 0.9897209985315712,
      "fcm_dpo/beta": 0.004042995162308216,
      "fcm_dpo/delta": -0.09460575878620148,
      "fcm_dpo/margin": 118.86309051513672,
      "fcm_dpo/q_t": 0.3860313296318054,
      "grad_norm": 7.587854862213135,
      "learning_rate": 2.1077827798404725e-10,
      "logits/chosen": -1.3744256496429443,
      "logits/rejected": -0.9566362500190735,
      "logps/chosen": -242.56143188476562,
      "logps/ref_chosen": -75.45831298828125,
      "logps/ref_rejected": -76.20362854003906,
      "logps/rejected": -362.16986083984375,
      "loss": 1.0145,
      "margin_dpo/margin_mean": 118.86308288574219,
      "margin_dpo/margin_std": 117.67463684082031,
      "step": 674
    },
    {
      "KL/chosen_KL_mean": -182.51014709472656,
      "KL/mean": -235.1077880859375,
      "KL/rejected_KL_mean": -287.70538330078125,
      "KL/std": 116.31755065917969,
      "epoch": 0.9911894273127754,
      "fcm_dpo/beta": 0.0038986974395811558,
      "fcm_dpo/delta": -0.0143016017973423,
      "fcm_dpo/margin": 105.19525909423828,
      "fcm_dpo/q_t": 0.40380582213401794,
      "grad_norm": 11.767951965332031,
      "learning_rate": 1.6138243485910863e-10,
      "logits/chosen": -1.3053442239761353,
      "logits/rejected": -0.9535959959030151,
      "logps/chosen": -262.419677734375,
      "logps/ref_chosen": -79.90953063964844,
      "logps/ref_rejected": -81.21824645996094,
      "logps/rejected": -368.92364501953125,
      "loss": 1.0946,
      "margin_dpo/margin_mean": 105.19525146484375,
      "margin_dpo/margin_std": 142.34506225585938,
      "step": 675
    },
    {
      "KL/chosen_KL_mean": -196.711181640625,
      "KL/mean": -248.2302703857422,
      "KL/rejected_KL_mean": -299.74932861328125,
      "KL/std": 111.70800018310547,
      "epoch": 0.9926578560939795,
      "fcm_dpo/beta": 0.003926532343029976,
      "fcm_dpo/delta": -0.005574008449912071,
      "fcm_dpo/margin": 103.03812408447266,
      "fcm_dpo/q_t": 0.40483275055885315,
      "grad_norm": 8.410001754760742,
      "learning_rate": 1.1857007165852472e-10,
      "logits/chosen": -1.8184659481048584,
      "logits/rejected": -1.2175357341766357,
      "logps/chosen": -294.88232421875,
      "logps/ref_chosen": -98.17111206054688,
      "logps/ref_rejected": -95.024658203125,
      "logps/rejected": -394.77398681640625,
      "loss": 1.0817,
      "margin_dpo/margin_mean": 103.03813171386719,
      "margin_dpo/margin_std": 125.82829284667969,
      "step": 676
    },
    {
      "KL/chosen_KL_mean": -177.29995727539062,
      "KL/mean": -228.74456787109375,
      "KL/rejected_KL_mean": -280.1891784667969,
      "KL/std": 110.22747802734375,
      "epoch": 0.9941262848751835,
      "fcm_dpo/beta": 0.003914497792720795,
      "fcm_dpo/delta": -0.00354817695915699,
      "fcm_dpo/margin": 102.88920593261719,
      "fcm_dpo/q_t": 0.4048548936843872,
      "grad_norm": 8.262255668640137,
      "learning_rate": 8.23423165278725e-11,
      "logits/chosen": -1.5762054920196533,
      "logits/rejected": -0.8877253532409668,
      "logps/chosen": -268.67926025390625,
      "logps/ref_chosen": -91.37928771972656,
      "logps/ref_rejected": -82.87776947021484,
      "logps/rejected": -363.06695556640625,
      "loss": 1.0678,
      "margin_dpo/margin_mean": 102.88920593261719,
      "margin_dpo/margin_std": 108.11406707763672,
      "step": 677
    },
    {
      "KL/chosen_KL_mean": -167.33026123046875,
      "KL/mean": -228.53323364257812,
      "KL/rejected_KL_mean": -289.7362365722656,
      "KL/std": 117.3753662109375,
      "epoch": 0.9955947136563876,
      "fcm_dpo/beta": 0.003846466541290283,
      "fcm_dpo/delta": -0.0824912041425705,
      "fcm_dpo/margin": 122.40594482421875,
      "fcm_dpo/q_t": 0.3875318169593811,
      "grad_norm": 9.975910186767578,
      "learning_rate": 5.270012410216185e-11,
      "logits/chosen": -1.5534570217132568,
      "logits/rejected": -1.2218315601348877,
      "logps/chosen": -242.97613525390625,
      "logps/ref_chosen": -75.6458740234375,
      "logps/ref_rejected": -86.96611022949219,
      "logps/rejected": -376.70233154296875,
      "loss": 1.0225,
      "margin_dpo/margin_mean": 122.40593719482422,
      "margin_dpo/margin_std": 119.95831298828125,
      "step": 678
    },
    {
      "KL/chosen_KL_mean": -193.5703125,
      "KL/mean": -244.12229919433594,
      "KL/rejected_KL_mean": -294.67425537109375,
      "KL/std": 98.23150634765625,
      "epoch": 0.9970631424375918,
      "fcm_dpo/beta": 0.003776824800297618,
      "fcm_dpo/delta": 0.020507557317614555,
      "fcm_dpo/margin": 101.10397338867188,
      "fcm_dpo/q_t": 0.41029322147369385,
      "grad_norm": 8.542757987976074,
      "learning_rate": 2.9644275480772416e-11,
      "logits/chosen": -2.0917084217071533,
      "logits/rejected": -1.5641454458236694,
      "logps/chosen": -274.34375,
      "logps/ref_chosen": -80.77344512939453,
      "logps/ref_rejected": -82.87850189208984,
      "logps/rejected": -377.55279541015625,
      "loss": 1.0775,
      "margin_dpo/margin_mean": 101.10397338867188,
      "margin_dpo/margin_std": 105.12115478515625,
      "step": 679
    },
    {
      "KL/chosen_KL_mean": -193.62445068359375,
      "KL/mean": -248.60015869140625,
      "KL/rejected_KL_mean": -303.57586669921875,
      "KL/std": 116.4595947265625,
      "epoch": 0.9985315712187959,
      "fcm_dpo/beta": 0.0038294773548841476,
      "fcm_dpo/delta": -0.026379596441984177,
      "fcm_dpo/margin": 109.951416015625,
      "fcm_dpo/q_t": 0.40134620666503906,
      "grad_norm": 7.018282413482666,
      "learning_rate": 1.31753782067201e-11,
      "logits/chosen": -1.7872827053070068,
      "logits/rejected": -1.4453976154327393,
      "logps/chosen": -301.307373046875,
      "logps/ref_chosen": -107.68292999267578,
      "logps/ref_rejected": -116.0948715209961,
      "logps/rejected": -419.6707763671875,
      "loss": 1.0695,
      "margin_dpo/margin_mean": 109.951416015625,
      "margin_dpo/margin_std": 126.60879516601562,
      "step": 680
    },
    {
      "KL/chosen_KL_mean": -176.53546142578125,
      "KL/mean": -225.2855987548828,
      "KL/rejected_KL_mean": -274.0357360839844,
      "KL/std": 110.3056640625,
      "epoch": 1.0,
      "fcm_dpo/beta": 0.0038111673202365637,
      "fcm_dpo/delta": 0.03157641738653183,
      "fcm_dpo/margin": 97.50028991699219,
      "fcm_dpo/q_t": 0.413496732711792,
      "grad_norm": 8.830599784851074,
      "learning_rate": 3.2938662507808745e-12,
      "logits/chosen": -2.1820719242095947,
      "logits/rejected": -1.8646330833435059,
      "logps/chosen": -269.5465087890625,
      "logps/ref_chosen": -93.01106262207031,
      "logps/ref_rejected": -94.8221664428711,
      "logps/rejected": -368.85791015625,
      "loss": 1.0996,
      "margin_dpo/margin_mean": 97.50028228759766,
      "margin_dpo/margin_std": 120.5243911743164,
      "step": 681
    },
    {
      "epoch": 1.0,
      "step": 681,
      "total_flos": 0.0,
      "train_loss": 1.1119352786551489,
      "train_runtime": 1971.8986,
      "train_samples_per_second": 22.11,
      "train_steps_per_second": 0.345
    }
  ],
  "logging_steps": 1,
  "max_steps": 681,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}