{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.008681427945217196, "eval_steps": 100, "global_step": 203, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 0.0, "completion_length": 5461.5, "epoch": 4.276565490254776e-05, "grad_norm": 0.16267029941082, "kl": 0.0, "learning_rate": 1.0000000000000002e-06, "loss": 0.0819, "reward": 0.0625, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.0625, "step": 1 }, { "clip_ratio": 0.0, "completion_length": 4270.875, "epoch": 8.553130980509552e-05, "grad_norm": 0.11525221914052963, "kl": 0.0, "learning_rate": 2.0000000000000003e-06, "loss": 0.0462, "reward": 0.1875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 2 }, { "clip_ratio": 0.0, "completion_length": 1373.25, "epoch": 0.0001282969647076433, "grad_norm": 0.4355790913105011, "kl": 8.341670036315918e-05, "learning_rate": 3e-06, "loss": -0.0442, "reward": 0.53125, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 3 }, { "clip_ratio": 0.0, "completion_length": 4077.75, "epoch": 0.00017106261961019105, "grad_norm": 0.19235004484653473, "kl": 9.560585021972656e-05, "learning_rate": 4.000000000000001e-06, "loss": 0.075, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 4 }, { "clip_ratio": 0.0, "completion_length": 3586.125, "epoch": 0.00021382827451273882, "grad_norm": 0.2733210325241089, "kl": 0.00010514259338378906, "learning_rate": 5e-06, "loss": 0.0966, "reward": 0.3125, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 5 }, { "clip_ratio": 0.0, "completion_length": 2410.625, "epoch": 0.0002565939294152866, "grad_norm": 0.001054590567946434, "kl": 9.357929229736328e-05, "learning_rate": 6e-06, "loss": 0.0, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 6 }, { "clip_ratio": 0.0, "completion_length": 4083.0, "epoch": 0.00029935958431783437, "grad_norm": 0.26410973072052, "kl": 6.258487701416016e-05, "learning_rate": 7e-06, "loss": 0.2727, "reward": 0.1875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 7 }, { "clip_ratio": 0.0, "completion_length": 4407.5, "epoch": 0.0003421252392203821, "grad_norm": 0.1048470214009285, "kl": 5.3938478231430054e-05, "learning_rate": 8.000000000000001e-06, "loss": 0.1556, "reward": 0.125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 8 }, { "clip_ratio": 0.0, "completion_length": 3154.25, "epoch": 0.00038489089412292987, "grad_norm": 0.002674901857972145, "kl": 0.00015789270401000977, "learning_rate": 9e-06, "loss": 0.0976, "reward": 0.21875, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 9 }, { "clip_ratio": 0.0, "completion_length": 1865.75, "epoch": 0.00042765654902547764, "grad_norm": 0.4874477982521057, "kl": 0.0008847713470458984, "learning_rate": 1e-05, "loss": -0.0731, "reward": 0.3125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 10 }, { "clip_ratio": 0.0, "completion_length": 2669.5, "epoch": 0.0004704222039280254, "grad_norm": 0.002774802502244711, "kl": 0.000431060791015625, "learning_rate": 1.1000000000000001e-05, "loss": 0.1372, "reward": 0.625, "reward_std": 0.2651650421321392, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 11 }, { "clip_ratio": 0.0, "completion_length": 4684.875, "epoch": 0.0005131878588305732, "grad_norm": 0.1472734808921814, "kl": 0.00113677978515625, "learning_rate": 1.2e-05, "loss": 0.0161, "reward": 0.4375, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 12 }, { "clip_ratio": 0.0, "completion_length": 3399.875, "epoch": 0.0005559535137331209, "grad_norm": 0.004856433253735304, "kl": 0.001171112060546875, "learning_rate": 1.3000000000000001e-05, "loss": 0.0, "reward": 0.4375, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 13 }, { "clip_ratio": 0.0, "completion_length": 2858.5, "epoch": 0.0005987191686356687, "grad_norm": 0.005653587635606527, "kl": 0.007244110107421875, "learning_rate": 1.4e-05, "loss": 0.0001, "reward": 0.125, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 14 }, { "clip_ratio": 0.0, "completion_length": 2483.125, "epoch": 0.0006414848235382165, "grad_norm": 0.142883762717247, "kl": 0.0027256011962890625, "learning_rate": 1.5000000000000002e-05, "loss": 0.0606, "reward": 0.15625, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 15 }, { "clip_ratio": 0.0, "completion_length": 3462.625, "epoch": 0.0006842504784407642, "grad_norm": 0.20137140154838562, "kl": 0.003688812255859375, "learning_rate": 1.6000000000000003e-05, "loss": 0.1641, "reward": 0.34375, "reward_std": 0.30935920402407646, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 16 }, { "clip_ratio": 0.0, "completion_length": 3050.5, "epoch": 0.000727016133343312, "grad_norm": 0.39956989884376526, "kl": 0.0066070556640625, "learning_rate": 1.7e-05, "loss": 0.0287, "reward": 0.71875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 17 }, { "clip_ratio": 0.0, "completion_length": 2627.75, "epoch": 0.0007697817882458597, "grad_norm": 0.2573762834072113, "kl": 0.005321502685546875, "learning_rate": 1.8e-05, "loss": -0.105, "reward": 0.34375, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 18 }, { "clip_ratio": 0.0, "completion_length": 3989.375, "epoch": 0.0008125474431484076, "grad_norm": 0.00889465119689703, "kl": 0.00554656982421875, "learning_rate": 1.9e-05, "loss": 0.009, "reward": 0.28125, "reward_std": 0.22097086906433105, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 19 }, { "clip_ratio": 0.0, "completion_length": 2171.5, "epoch": 0.0008553130980509553, "grad_norm": 0.02196628600358963, "kl": 0.0075836181640625, "learning_rate": 2e-05, "loss": 0.0003, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 20 }, { "clip_ratio": 0.0, "completion_length": 3278.625, "epoch": 0.000898078752953503, "grad_norm": 0.20426563918590546, "kl": 0.0113067626953125, "learning_rate": 1.9998476951563914e-05, "loss": 0.0314, "reward": 0.59375, "reward_std": 0.5745242461562157, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 21 }, { "clip_ratio": 0.0, "completion_length": 2568.5, "epoch": 0.0009408444078560508, "grad_norm": 0.01476654689759016, "kl": 0.00714874267578125, "learning_rate": 1.999390827019096e-05, "loss": 0.0468, "reward": 0.21875, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 22 }, { "clip_ratio": 0.0, "completion_length": 3102.75, "epoch": 0.0009836100627585987, "grad_norm": 0.008334783837199211, "kl": 0.0075531005859375, "learning_rate": 1.9986295347545738e-05, "loss": 0.0003, "reward": 0.4375, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 23 }, { "clip_ratio": 0.0, "completion_length": 2844.625, "epoch": 0.0010263757176611464, "grad_norm": 0.008713163435459137, "kl": 0.007537841796875, "learning_rate": 1.9975640502598243e-05, "loss": -0.0284, "reward": 0.375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 24 }, { "clip_ratio": 0.0, "completion_length": 2529.625, "epoch": 0.001069141372563694, "grad_norm": 0.35095760226249695, "kl": 0.0089111328125, "learning_rate": 1.9961946980917457e-05, "loss": 0.0603, "reward": 0.65625, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 25 }, { "clip_ratio": 0.0, "completion_length": 3729.25, "epoch": 0.0011119070274662418, "grad_norm": 0.11682198941707611, "kl": 0.0083770751953125, "learning_rate": 1.9945218953682736e-05, "loss": -0.0589, "reward": 0.5, "reward_std": 0.3535533845424652, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 26 }, { "clip_ratio": 0.0, "completion_length": 2405.625, "epoch": 0.0011546726823687895, "grad_norm": 0.15771378576755524, "kl": 0.00885009765625, "learning_rate": 1.9925461516413224e-05, "loss": -0.0339, "reward": 0.9375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.625, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 27 }, { "clip_ratio": 0.0, "completion_length": 3080.0, "epoch": 0.0011974383372713375, "grad_norm": 0.42599111795425415, "kl": 0.0114593505859375, "learning_rate": 1.9902680687415704e-05, "loss": -0.1403, "reward": 0.5, "reward_std": 0.3535533770918846, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 28 }, { "clip_ratio": 0.0, "completion_length": 1790.125, "epoch": 0.0012402039921738852, "grad_norm": 0.27475643157958984, "kl": 0.01348876953125, "learning_rate": 1.9876883405951378e-05, "loss": -0.0633, "reward": 0.625, "reward_std": 0.3535533845424652, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 29 }, { "clip_ratio": 0.0, "completion_length": 1943.25, "epoch": 0.001282969647076433, "grad_norm": 0.19931074976921082, "kl": 0.016143798828125, "learning_rate": 1.9848077530122083e-05, "loss": 0.0565, "reward": 0.4375, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 30 }, { "clip_ratio": 0.0, "completion_length": 2680.0, "epoch": 0.0013257353019789806, "grad_norm": 0.218612402677536, "kl": 0.0192718505859375, "learning_rate": 1.9816271834476642e-05, "loss": -0.0318, "reward": 0.59375, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 31 }, { "clip_ratio": 0.0, "completion_length": 2215.75, "epoch": 0.0013685009568815284, "grad_norm": 0.1676703840494156, "kl": 0.01806640625, "learning_rate": 1.9781476007338058e-05, "loss": 0.0563, "reward": 0.4375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 32 }, { "clip_ratio": 0.0, "completion_length": 1401.375, "epoch": 0.0014112666117840763, "grad_norm": 0.2553260028362274, "kl": 0.03045654296875, "learning_rate": 1.9743700647852356e-05, "loss": 0.0216, "reward": 0.71875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 33 }, { "clip_ratio": 0.0, "completion_length": 1934.375, "epoch": 0.001454032266686624, "grad_norm": 0.1874990314245224, "kl": 0.03179931640625, "learning_rate": 1.9702957262759964e-05, "loss": -0.04, "reward": 0.78125, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 34 }, { "clip_ratio": 0.0, "completion_length": 1011.5, "epoch": 0.0014967979215891717, "grad_norm": 0.027488980442285538, "kl": 0.03265380859375, "learning_rate": 1.9659258262890683e-05, "loss": 0.0013, "reward": 0.625, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 35 }, { "clip_ratio": 0.0, "completion_length": 3376.5, "epoch": 0.0015395635764917195, "grad_norm": 0.13317273557186127, "kl": 0.024627685546875, "learning_rate": 1.961261695938319e-05, "loss": 0.0843, "reward": 0.375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 36 }, { "clip_ratio": 0.0, "completion_length": 2231.75, "epoch": 0.0015823292313942672, "grad_norm": 0.19944815337657928, "kl": 0.02044677734375, "learning_rate": 1.9563047559630356e-05, "loss": 0.1045, "reward": 0.5, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 37 }, { "clip_ratio": 0.0, "completion_length": 1510.125, "epoch": 0.0016250948862968151, "grad_norm": 0.21087314188480377, "kl": 0.02874755859375, "learning_rate": 1.9510565162951538e-05, "loss": 0.0719, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 38 }, { "clip_ratio": 0.0, "completion_length": 1850.125, "epoch": 0.0016678605411993628, "grad_norm": 0.5604463815689087, "kl": 0.032257080078125, "learning_rate": 1.945518575599317e-05, "loss": 0.0618, "reward": 0.78125, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 39 }, { "clip_ratio": 0.0, "completion_length": 2188.0, "epoch": 0.0017106261961019106, "grad_norm": 0.19170281291007996, "kl": 0.028106689453125, "learning_rate": 1.9396926207859085e-05, "loss": 0.0225, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 40 }, { "clip_ratio": 0.0, "completion_length": 3787.0, "epoch": 0.0017533918510044583, "grad_norm": 0.1880658119916916, "kl": 0.02386474609375, "learning_rate": 1.9335804264972018e-05, "loss": 0.0619, "reward": 1.0, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 41 }, { "clip_ratio": 0.0, "completion_length": 3914.75, "epoch": 0.001796157505907006, "grad_norm": 0.2238849401473999, "kl": 0.020660400390625, "learning_rate": 1.9271838545667876e-05, "loss": 0.1911, "reward": 0.40625, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 42 }, { "clip_ratio": 0.0, "completion_length": 3120.0, "epoch": 0.001838923160809554, "grad_norm": 0.4404824674129486, "kl": 0.063812255859375, "learning_rate": 1.9205048534524405e-05, "loss": 0.1613, "reward": 0.625, "reward_std": 0.3535533845424652, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 43 }, { "clip_ratio": 0.0, "completion_length": 4166.875, "epoch": 0.0018816888157121017, "grad_norm": 0.2383565753698349, "kl": 0.04571533203125, "learning_rate": 1.913545457642601e-05, "loss": 0.1937, "reward": 0.375, "reward_std": 0.3535533770918846, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 44 }, { "clip_ratio": 0.0, "completion_length": 2751.625, "epoch": 0.0019244544706146494, "grad_norm": 0.29551273584365845, "kl": 0.05401611328125, "learning_rate": 1.9063077870366504e-05, "loss": 0.1973, "reward": 0.4375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 45 }, { "clip_ratio": 0.0, "completion_length": 2760.5, "epoch": 0.0019672201255171973, "grad_norm": 0.18511998653411865, "kl": 0.0732421875, "learning_rate": 1.8987940462991673e-05, "loss": 0.0234, "reward": 0.59375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 46 }, { "clip_ratio": 0.0, "completion_length": 3005.875, "epoch": 0.002009985780419745, "grad_norm": 0.23445166647434235, "kl": 0.073974609375, "learning_rate": 1.891006524188368e-05, "loss": 0.024, "reward": 0.75, "reward_std": 0.3535533770918846, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 47 }, { "clip_ratio": 0.0, "completion_length": 3649.0, "epoch": 0.0020527514353222928, "grad_norm": 0.3580359220504761, "kl": 0.0859375, "learning_rate": 1.8829475928589272e-05, "loss": 0.0065, "reward": 0.4375, "reward_std": 0.3535533882677555, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 48 }, { "clip_ratio": 0.0, "completion_length": 2737.375, "epoch": 0.0020955170902248405, "grad_norm": 0.5151046514511108, "kl": 0.07086181640625, "learning_rate": 1.874619707139396e-05, "loss": -0.0511, "reward": 0.8125, "reward_std": 0.4419417232275009, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 49 }, { "clip_ratio": 0.0, "completion_length": 2900.625, "epoch": 0.002138282745127388, "grad_norm": 0.31255099177360535, "kl": 0.0771484375, "learning_rate": 1.866025403784439e-05, "loss": 0.0061, "reward": 0.4375, "reward_std": 0.4419417232275009, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 50 }, { "clip_ratio": 0.0, "completion_length": 3572.25, "epoch": 0.002181048400029936, "grad_norm": 0.1769428700208664, "kl": 0.078125, "learning_rate": 1.8571673007021124e-05, "loss": 0.016, "reward": 0.34375, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 51 }, { "clip_ratio": 0.0, "completion_length": 4179.375, "epoch": 0.0022238140549324836, "grad_norm": 0.28456494212150574, "kl": 0.082275390625, "learning_rate": 1.848048096156426e-05, "loss": 0.0554, "reward": 0.21875, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 52 }, { "clip_ratio": 0.0, "completion_length": 3308.5, "epoch": 0.0022665797098350314, "grad_norm": 0.03375667706131935, "kl": 0.0958251953125, "learning_rate": 1.8386705679454243e-05, "loss": 0.0655, "reward": 0.25, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 53 }, { "clip_ratio": 0.0, "completion_length": 4158.75, "epoch": 0.002309345364737579, "grad_norm": 0.6577324867248535, "kl": 0.116455078125, "learning_rate": 1.8290375725550417e-05, "loss": 0.1795, "reward": 0.46875, "reward_std": 0.30935920774936676, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 54 }, { "clip_ratio": 0.0, "completion_length": 4094.0, "epoch": 0.002352111019640127, "grad_norm": 0.26224833726882935, "kl": 0.1153564453125, "learning_rate": 1.819152044288992e-05, "loss": 0.1842, "reward": 0.46875, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 55 }, { "clip_ratio": 0.0, "completion_length": 5614.25, "epoch": 0.002394876674542675, "grad_norm": 0.20306017994880676, "kl": 0.206787109375, "learning_rate": 1.8090169943749477e-05, "loss": 0.0183, "reward": 0.125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 56 }, { "clip_ratio": 0.0, "completion_length": 4841.375, "epoch": 0.0024376423294452227, "grad_norm": 0.24494034051895142, "kl": 0.24609375, "learning_rate": 1.798635510047293e-05, "loss": 0.0805, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 57 }, { "clip_ratio": 0.0, "completion_length": 2709.5, "epoch": 0.0024804079843477704, "grad_norm": 0.25134870409965515, "kl": 0.205078125, "learning_rate": 1.788010753606722e-05, "loss": 0.0904, "reward": 0.65625, "reward_std": 0.30935921519994736, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 58 }, { "clip_ratio": 0.0, "completion_length": 4798.0, "epoch": 0.002523173639250318, "grad_norm": 0.3875373601913452, "kl": 0.3115234375, "learning_rate": 1.777145961456971e-05, "loss": 0.2274, "reward": 0.46875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 59 }, { "clip_ratio": 0.0, "completion_length": 5811.375, "epoch": 0.002565939294152866, "grad_norm": 0.38781318068504333, "kl": 0.50634765625, "learning_rate": 1.766044443118978e-05, "loss": 0.0203, "reward": 0.03125, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.03125, "step": 60 }, { "clip_ratio": 0.0, "completion_length": 4296.375, "epoch": 0.0026087049490554136, "grad_norm": 0.3625580072402954, "kl": 0.29150390625, "learning_rate": 1.7547095802227723e-05, "loss": 0.0801, "reward": 0.03125, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.03125, "step": 61 }, { "clip_ratio": 0.0, "completion_length": 4872.5, "epoch": 0.0026514706039579613, "grad_norm": 0.3603833317756653, "kl": 0.16796875, "learning_rate": 1.7431448254773943e-05, "loss": 0.0047, "reward": 0.40625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 62 }, { "clip_ratio": 0.0, "completion_length": 3905.875, "epoch": 0.002694236258860509, "grad_norm": 1.132883071899414, "kl": 0.2314453125, "learning_rate": 1.7313537016191706e-05, "loss": 0.0996, "reward": 0.25, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 63 }, { "clip_ratio": 0.0, "completion_length": 4616.875, "epoch": 0.0027370019137630567, "grad_norm": 0.4993176758289337, "kl": 0.302490234375, "learning_rate": 1.7193398003386514e-05, "loss": 0.1914, "reward": 0.0625, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.0625, "step": 64 }, { "clip_ratio": 0.0, "completion_length": 4579.125, "epoch": 0.0027797675686656045, "grad_norm": 0.12003020942211151, "kl": 0.3349609375, "learning_rate": 1.7071067811865477e-05, "loss": 0.0139, "reward": 0.09375, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.09375, "step": 65 }, { "clip_ratio": 0.0, "completion_length": 4744.75, "epoch": 0.0028225332235681526, "grad_norm": 0.3411747217178345, "kl": 0.4326171875, "learning_rate": 1.6946583704589973e-05, "loss": 0.0974, "reward": 0.09375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.09375, "step": 66 }, { "clip_ratio": 0.0, "completion_length": 4106.5, "epoch": 0.0028652988784707003, "grad_norm": 0.40250518918037415, "kl": 0.3525390625, "learning_rate": 1.6819983600624986e-05, "loss": 0.0553, "reward": 0.1875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 67 }, { "clip_ratio": 0.0, "completion_length": 3278.625, "epoch": 0.002908064533373248, "grad_norm": 0.4573908746242523, "kl": 0.290283203125, "learning_rate": 1.6691306063588583e-05, "loss": 0.0124, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 68 }, { "clip_ratio": 0.0, "completion_length": 4635.875, "epoch": 0.0029508301882757958, "grad_norm": 0.20075607299804688, "kl": 0.2783203125, "learning_rate": 1.6560590289905074e-05, "loss": -0.0106, "reward": 0.125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 69 }, { "clip_ratio": 0.0, "completion_length": 4831.625, "epoch": 0.0029935958431783435, "grad_norm": 0.3380205035209656, "kl": 0.35302734375, "learning_rate": 1.6427876096865394e-05, "loss": 0.0636, "reward": 0.125, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 70 }, { "clip_ratio": 0.0, "completion_length": 4100.0, "epoch": 0.003036361498080891, "grad_norm": 0.19683989882469177, "kl": 0.3154296875, "learning_rate": 1.6293203910498375e-05, "loss": 0.1183, "reward": 0.125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.125, "step": 71 }, { "clip_ratio": 0.0, "completion_length": 3273.0, "epoch": 0.003079127152983439, "grad_norm": 0.29972413182258606, "kl": 0.21337890625, "learning_rate": 1.6156614753256583e-05, "loss": 0.1369, "reward": 0.28125, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 72 }, { "clip_ratio": 0.0, "completion_length": 3979.0, "epoch": 0.0031218928078859867, "grad_norm": 0.07617900520563126, "kl": 0.21630859375, "learning_rate": 1.6018150231520486e-05, "loss": 0.0117, "reward": 0.21875, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 73 }, { "clip_ratio": 0.0, "completion_length": 3651.125, "epoch": 0.0031646584627885344, "grad_norm": 0.2953796088695526, "kl": 0.28173828125, "learning_rate": 1.5877852522924733e-05, "loss": -0.104, "reward": 0.15625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 74 }, { "clip_ratio": 0.0, "completion_length": 3373.875, "epoch": 0.003207424117691082, "grad_norm": 0.26317182183265686, "kl": 0.19384765625, "learning_rate": 1.573576436351046e-05, "loss": -0.0421, "reward": 0.1875, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 75 }, { "clip_ratio": 0.0, "completion_length": 3990.375, "epoch": 0.0032501897725936302, "grad_norm": 0.28515392541885376, "kl": 0.242919921875, "learning_rate": 1.5591929034707468e-05, "loss": -0.0168, "reward": 0.15625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 76 }, { "clip_ratio": 0.0, "completion_length": 2018.625, "epoch": 0.003292955427496178, "grad_norm": 0.23012442886829376, "kl": 0.1318359375, "learning_rate": 1.5446390350150272e-05, "loss": -0.039, "reward": 0.4375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 77 }, { "clip_ratio": 0.0, "completion_length": 3627.625, "epoch": 0.0033357210823987257, "grad_norm": 0.4319152235984802, "kl": 0.144775390625, "learning_rate": 1.529919264233205e-05, "loss": 0.0683, "reward": 0.1875, "reward_std": 0.1767766885459423, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 78 }, { "clip_ratio": 0.0, "completion_length": 3797.375, "epoch": 0.0033784867373012734, "grad_norm": 0.17109890282154083, "kl": 0.128662109375, "learning_rate": 1.5150380749100545e-05, "loss": 0.0166, "reward": 0.25, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 79 }, { "clip_ratio": 0.0, "completion_length": 3715.875, "epoch": 0.003421252392203821, "grad_norm": 0.43475019931793213, "kl": 0.1298828125, "learning_rate": 1.5000000000000002e-05, "loss": 0.1031, "reward": 0.34375, "reward_std": 0.39774756133556366, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 80 }, { "clip_ratio": 0.0, "completion_length": 4430.25, "epoch": 0.003464018047106369, "grad_norm": 0.027983365580439568, "kl": 0.1322021484375, "learning_rate": 1.4848096202463373e-05, "loss": 0.0053, "reward": 0.0625, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.0625, "step": 81 }, { "clip_ratio": 0.0, "completion_length": 3278.875, "epoch": 0.0035067837020089166, "grad_norm": 0.181055948138237, "kl": 0.1214599609375, "learning_rate": 1.469471562785891e-05, "loss": 0.0488, "reward": 0.53125, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 82 }, { "clip_ratio": 0.0, "completion_length": 3779.125, "epoch": 0.0035495493569114643, "grad_norm": 0.18667203187942505, "kl": 0.1451416015625, "learning_rate": 1.4539904997395468e-05, "loss": 0.1067, "reward": 0.21875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 83 }, { "clip_ratio": 0.0, "completion_length": 4723.625, "epoch": 0.003592315011814012, "grad_norm": 0.22420822083950043, "kl": 0.1639404296875, "learning_rate": 1.4383711467890776e-05, "loss": 0.0791, "reward": 0.25, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 84 }, { "clip_ratio": 0.0, "completion_length": 4158.5, "epoch": 0.0036350806667165597, "grad_norm": 0.2224516123533249, "kl": 0.206787109375, "learning_rate": 1.4226182617406996e-05, "loss": -0.1824, "reward": 0.28125, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 85 }, { "clip_ratio": 0.0, "completion_length": 4031.375, "epoch": 0.003677846321619108, "grad_norm": 0.029293499886989594, "kl": 0.17236328125, "learning_rate": 1.4067366430758004e-05, "loss": 0.0887, "reward": 0.46875, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 86 }, { "clip_ratio": 0.0, "completion_length": 3825.5, "epoch": 0.0037206119765216556, "grad_norm": 0.30558931827545166, "kl": 0.19287109375, "learning_rate": 1.3907311284892737e-05, "loss": -0.1076, "reward": 0.28125, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.15625, "step": 87 }, { "clip_ratio": 0.0, "completion_length": 4313.375, "epoch": 0.0037633776314242033, "grad_norm": 0.3198181688785553, "kl": 0.17919921875, "learning_rate": 1.3746065934159123e-05, "loss": 0.184, "reward": 0.25, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 88 }, { "clip_ratio": 0.0, "completion_length": 3728.5, "epoch": 0.003806143286326751, "grad_norm": 0.1312406361103058, "kl": 0.113525390625, "learning_rate": 1.3583679495453e-05, "loss": 0.1374, "reward": 0.3125, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 89 }, { "clip_ratio": 0.0, "completion_length": 3167.75, "epoch": 0.0038489089412292988, "grad_norm": 0.39866146445274353, "kl": 0.18115234375, "learning_rate": 1.342020143325669e-05, "loss": 0.0516, "reward": 0.25, "reward_std": 0.1767766885459423, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 90 }, { "clip_ratio": 0.0, "completion_length": 3727.0, "epoch": 0.0038916745961318465, "grad_norm": 0.27305853366851807, "kl": 0.15771484375, "learning_rate": 1.3255681544571568e-05, "loss": 0.0296, "reward": 0.28125, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.28125, "step": 91 }, { "clip_ratio": 0.0, "completion_length": 3970.625, "epoch": 0.003934440251034395, "grad_norm": 0.21286995708942413, "kl": 0.202392578125, "learning_rate": 1.3090169943749475e-05, "loss": 0.0604, "reward": 0.1875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 92 }, { "clip_ratio": 0.0, "completion_length": 3910.0, "epoch": 0.003977205905936942, "grad_norm": 0.258375883102417, "kl": 0.1380615234375, "learning_rate": 1.2923717047227368e-05, "loss": 0.1807, "reward": 0.34375, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 93 }, { "clip_ratio": 0.0, "completion_length": 3416.875, "epoch": 0.00401997156083949, "grad_norm": 0.24168948829174042, "kl": 0.0948486328125, "learning_rate": 1.2756373558169992e-05, "loss": 0.0307, "reward": 0.4375, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 94 }, { "clip_ratio": 0.0, "completion_length": 1900.625, "epoch": 0.004062737215742037, "grad_norm": 0.5117138028144836, "kl": 0.148193359375, "learning_rate": 1.2588190451025209e-05, "loss": -0.0826, "reward": 0.4375, "reward_std": 0.4419417232275009, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 95 }, { "clip_ratio": 0.0, "completion_length": 4070.25, "epoch": 0.0041055028706445855, "grad_norm": 0.3463330566883087, "kl": 0.232421875, "learning_rate": 1.2419218955996677e-05, "loss": 0.1747, "reward": 0.34375, "reward_std": 0.30935920774936676, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 96 }, { "clip_ratio": 0.0, "completion_length": 3395.5, "epoch": 0.004148268525547133, "grad_norm": 0.17179813981056213, "kl": 0.1876220703125, "learning_rate": 1.2249510543438652e-05, "loss": -0.0027, "reward": 0.5625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.1875, "step": 97 }, { "clip_ratio": 0.0, "completion_length": 2927.625, "epoch": 0.004191034180449681, "grad_norm": 0.3246678113937378, "kl": 0.1654052734375, "learning_rate": 1.2079116908177592e-05, "loss": 0.0217, "reward": 0.46875, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 98 }, { "clip_ratio": 0.0, "completion_length": 2730.875, "epoch": 0.004233799835352228, "grad_norm": 0.2528171241283417, "kl": 0.151611328125, "learning_rate": 1.190808995376545e-05, "loss": 0.1882, "reward": 0.71875, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 99 }, { "epoch": 0.004276565490254776, "grad_norm": 0.03388703987002373, "learning_rate": 1.1736481776669307e-05, "loss": -0.0429, "step": 100 }, { "epoch": 0.004276565490254776, "eval_clip_ratio": 0.0, "eval_completion_length": 2586.8975, "eval_kl": 0.1329681396484375, "eval_loss": 0.059333380311727524, "eval_reward": 0.53375, "eval_reward_std": 0.20152542807161808, "eval_rewards/accuracy_reward": 0.0875, "eval_rewards/format_reward": 0.0, "eval_rewards/tag_count_reward": 0.44625, "eval_runtime": 7409.5209, "eval_samples_per_second": 0.027, "eval_steps_per_second": 0.013, "step": 100 }, { "clip_ratio": 0.0, "completion_length": 2146.0625, "epoch": 0.004319331145157325, "grad_norm": 0.4932703375816345, "kl": 0.097381591796875, "learning_rate": 1.156434465040231e-05, "loss": -0.0943, "reward": 0.53125, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0625, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 101 }, { "clip_ratio": 0.0, "completion_length": 3281.25, "epoch": 0.004362096800059872, "grad_norm": 0.2450522631406784, "kl": 0.1661376953125, "learning_rate": 1.1391731009600655e-05, "loss": 0.0702, "reward": 0.4375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 102 }, { "clip_ratio": 0.0, "completion_length": 1487.625, "epoch": 0.00440486245496242, "grad_norm": 0.21485422551631927, "kl": 0.0777587890625, "learning_rate": 1.1218693434051475e-05, "loss": 0.0994, "reward": 0.65625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.65625, "step": 103 }, { "clip_ratio": 0.0, "completion_length": 2665.375, "epoch": 0.004447628109864967, "grad_norm": 0.1788434386253357, "kl": 0.0714111328125, "learning_rate": 1.1045284632676535e-05, "loss": 0.0085, "reward": 0.46875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 104 }, { "clip_ratio": 0.0, "completion_length": 1608.375, "epoch": 0.0044903937647675154, "grad_norm": 0.271541029214859, "kl": 0.103271484375, "learning_rate": 1.0871557427476585e-05, "loss": 0.0177, "reward": 0.875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 105 }, { "clip_ratio": 0.0, "completion_length": 2189.375, "epoch": 0.004533159419670063, "grad_norm": 0.1872943937778473, "kl": 0.1141357421875, "learning_rate": 1.0697564737441254e-05, "loss": -0.0148, "reward": 0.84375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 106 }, { "clip_ratio": 0.0, "completion_length": 2014.125, "epoch": 0.004575925074572611, "grad_norm": 0.021891843527555466, "kl": 0.0697021484375, "learning_rate": 1.0523359562429441e-05, "loss": 0.0475, "reward": 0.84375, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 107 }, { "clip_ratio": 0.0, "completion_length": 1231.75, "epoch": 0.004618690729475158, "grad_norm": 0.017184779047966003, "kl": 0.0758056640625, "learning_rate": 1.0348994967025012e-05, "loss": -0.0081, "reward": 0.65625, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.65625, "step": 108 }, { "clip_ratio": 0.0, "completion_length": 2696.0, "epoch": 0.004661456384377706, "grad_norm": 0.4246922433376312, "kl": 0.088134765625, "learning_rate": 1.0174524064372837e-05, "loss": -0.1442, "reward": 0.5, "reward_std": 0.3535533808171749, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 109 }, { "clip_ratio": 0.0, "completion_length": 1631.625, "epoch": 0.004704222039280254, "grad_norm": 0.2600347697734833, "kl": 0.0823974609375, "learning_rate": 1e-05, "loss": 0.0767, "reward": 0.59375, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 110 }, { "clip_ratio": 0.0, "completion_length": 1957.625, "epoch": 0.004746987694182802, "grad_norm": 0.2815832197666168, "kl": 0.069091796875, "learning_rate": 9.825475935627165e-06, "loss": -0.0326, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 111 }, { "clip_ratio": 0.0, "completion_length": 3396.0, "epoch": 0.00478975334908535, "grad_norm": 0.1351148933172226, "kl": 0.0657958984375, "learning_rate": 9.651005032974994e-06, "loss": -0.0247, "reward": 0.5, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 112 }, { "clip_ratio": 0.0, "completion_length": 3885.875, "epoch": 0.004832519003987897, "grad_norm": 0.20564581453800201, "kl": 0.0760498046875, "learning_rate": 9.476640437570562e-06, "loss": 0.0884, "reward": 0.5, "reward_std": 0.1767766885459423, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 113 }, { "clip_ratio": 0.0, "completion_length": 3385.25, "epoch": 0.004875284658890445, "grad_norm": 0.017859425395727158, "kl": 0.06280517578125, "learning_rate": 9.302435262558748e-06, "loss": 0.0417, "reward": 0.625, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 114 }, { "clip_ratio": 0.0, "completion_length": 2518.0, "epoch": 0.004918050313792993, "grad_norm": 0.21801644563674927, "kl": 0.064697265625, "learning_rate": 9.128442572523418e-06, "loss": -0.0179, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 115 }, { "clip_ratio": 0.0, "completion_length": 2587.5, "epoch": 0.004960815968695541, "grad_norm": 0.20927733182907104, "kl": 0.07635498046875, "learning_rate": 8.954715367323468e-06, "loss": 0.0768, "reward": 0.84375, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 116 }, { "clip_ratio": 0.0, "completion_length": 4093.375, "epoch": 0.005003581623598088, "grad_norm": 0.12026116997003555, "kl": 0.049072265625, "learning_rate": 8.781306565948528e-06, "loss": 0.0822, "reward": 0.59375, "reward_std": 0.39774755388498306, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 117 }, { "clip_ratio": 0.0, "completion_length": 3249.625, "epoch": 0.005046347278500636, "grad_norm": 0.024114476516842842, "kl": 0.0595703125, "learning_rate": 8.60826899039935e-06, "loss": 0.0455, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 118 }, { "clip_ratio": 0.0, "completion_length": 3661.625, "epoch": 0.0050891129334031835, "grad_norm": 0.2734898626804352, "kl": 0.0621337890625, "learning_rate": 8.43565534959769e-06, "loss": 0.1484, "reward": 0.65625, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 119 }, { "clip_ratio": 0.0, "completion_length": 2190.625, "epoch": 0.005131878588305732, "grad_norm": 0.25671371817588806, "kl": 0.052825927734375, "learning_rate": 8.263518223330698e-06, "loss": -0.0465, "reward": 0.75, "reward_std": 0.3535533845424652, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 120 }, { "clip_ratio": 0.0, "completion_length": 1727.5, "epoch": 0.00517464424320828, "grad_norm": 0.029247041791677475, "kl": 0.05804443359375, "learning_rate": 8.091910046234552e-06, "loss": 0.0543, "reward": 1.1875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.6875, "step": 121 }, { "clip_ratio": 0.0, "completion_length": 2638.875, "epoch": 0.005217409898110827, "grad_norm": 0.3749679923057556, "kl": 0.0845947265625, "learning_rate": 7.92088309182241e-06, "loss": 0.2039, "reward": 0.46875, "reward_std": 0.39774755015969276, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 122 }, { "clip_ratio": 0.0, "completion_length": 2149.625, "epoch": 0.005260175553013375, "grad_norm": 0.2396032065153122, "kl": 0.12109375, "learning_rate": 7.750489456561351e-06, "loss": -0.0977, "reward": 0.625, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 123 }, { "clip_ratio": 0.0, "completion_length": 1615.0, "epoch": 0.005302941207915923, "grad_norm": 0.19491690397262573, "kl": 0.06597900390625, "learning_rate": 7.580781044003324e-06, "loss": -0.0444, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 124 }, { "clip_ratio": 0.0, "completion_length": 1669.875, "epoch": 0.005345706862818471, "grad_norm": 0.02028077282011509, "kl": 0.06610107421875, "learning_rate": 7.411809548974792e-06, "loss": -0.0901, "reward": 0.4375, "reward_std": 0.1767766885459423, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 125 }, { "clip_ratio": 0.0, "completion_length": 1546.0, "epoch": 0.005388472517721018, "grad_norm": 0.3600315749645233, "kl": 0.0616455078125, "learning_rate": 7.243626441830009e-06, "loss": 0.1528, "reward": 0.8125, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.6875, "step": 126 }, { "clip_ratio": 0.0, "completion_length": 3225.0, "epoch": 0.005431238172623566, "grad_norm": 0.249704048037529, "kl": 0.06597900390625, "learning_rate": 7.076282952772634e-06, "loss": 0.0916, "reward": 0.34375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 127 }, { "clip_ratio": 0.0, "completion_length": 3865.75, "epoch": 0.0054740038275261135, "grad_norm": 0.19370591640472412, "kl": 0.06646728515625, "learning_rate": 6.909830056250527e-06, "loss": 0.0723, "reward": 0.71875, "reward_std": 0.6629125773906708, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 128 }, { "clip_ratio": 0.0, "completion_length": 3395.5, "epoch": 0.005516769482428662, "grad_norm": 0.26017823815345764, "kl": 0.07568359375, "learning_rate": 6.744318455428436e-06, "loss": 0.1892, "reward": 0.5, "reward_std": 0.3535533808171749, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 129 }, { "clip_ratio": 0.0, "completion_length": 2424.5, "epoch": 0.005559535137331209, "grad_norm": 0.31396713852882385, "kl": 0.1290283203125, "learning_rate": 6.579798566743314e-06, "loss": -0.0035, "reward": 0.96875, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.625, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 130 }, { "clip_ratio": 0.0, "completion_length": 1908.125, "epoch": 0.005602300792233757, "grad_norm": 0.31851494312286377, "kl": 0.08929443359375, "learning_rate": 6.4163205045469975e-06, "loss": -0.0454, "reward": 0.71875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 131 }, { "clip_ratio": 0.0, "completion_length": 2507.125, "epoch": 0.005645066447136305, "grad_norm": 0.24837914109230042, "kl": 0.1806640625, "learning_rate": 6.25393406584088e-06, "loss": 0.0333, "reward": 0.40625, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 132 }, { "clip_ratio": 0.0, "completion_length": 2167.75, "epoch": 0.0056878321020388525, "grad_norm": 0.22763270139694214, "kl": 0.1214599609375, "learning_rate": 6.092688715107265e-06, "loss": 0.1109, "reward": 0.59375, "reward_std": 0.39774755388498306, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 133 }, { "clip_ratio": 0.0, "completion_length": 2720.0, "epoch": 0.005730597756941401, "grad_norm": 0.188200905919075, "kl": 0.0924072265625, "learning_rate": 5.932633569242e-06, "loss": -0.0248, "reward": 0.4375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 134 }, { "clip_ratio": 0.0, "completion_length": 3277.5, "epoch": 0.005773363411843948, "grad_norm": 0.402265727519989, "kl": 0.140380859375, "learning_rate": 5.773817382593008e-06, "loss": 0.1537, "reward": 0.46875, "reward_std": 0.39774755015969276, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 135 }, { "clip_ratio": 0.0, "completion_length": 1824.625, "epoch": 0.005816129066746496, "grad_norm": 0.5344210267066956, "kl": 0.10400390625, "learning_rate": 5.616288532109225e-06, "loss": 0.0536, "reward": 0.875, "reward_std": 0.4419417269527912, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 136 }, { "clip_ratio": 0.0, "completion_length": 2563.0, "epoch": 0.005858894721649043, "grad_norm": 0.636093020439148, "kl": 0.1328125, "learning_rate": 5.460095002604533e-06, "loss": 0.0735, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 137 }, { "clip_ratio": 0.0, "completion_length": 3144.75, "epoch": 0.0059016603765515915, "grad_norm": 0.13661141693592072, "kl": 0.13507080078125, "learning_rate": 5.305284372141095e-06, "loss": 0.0953, "reward": 0.65625, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 138 }, { "clip_ratio": 0.0, "completion_length": 2669.5, "epoch": 0.005944426031454139, "grad_norm": 0.15402540564537048, "kl": 0.135498046875, "learning_rate": 5.151903797536631e-06, "loss": 0.0917, "reward": 0.59375, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 139 }, { "clip_ratio": 0.0, "completion_length": 2987.5, "epoch": 0.005987191686356687, "grad_norm": 0.19349801540374756, "kl": 0.1014404296875, "learning_rate": 5.000000000000003e-06, "loss": -0.0657, "reward": 0.625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 140 }, { "clip_ratio": 0.0, "completion_length": 3093.875, "epoch": 0.006029957341259235, "grad_norm": 0.024953797459602356, "kl": 0.1361083984375, "learning_rate": 4.849619250899458e-06, "loss": 0.0518, "reward": 0.40625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 141 }, { "clip_ratio": 0.0, "completion_length": 2819.625, "epoch": 0.006072722996161782, "grad_norm": 0.4294950067996979, "kl": 0.1728515625, "learning_rate": 4.700807357667953e-06, "loss": -0.0305, "reward": 0.75, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 142 }, { "clip_ratio": 0.0, "completion_length": 2374.75, "epoch": 0.006115488651064331, "grad_norm": 0.2958958148956299, "kl": 0.1700439453125, "learning_rate": 4.5536096498497295e-06, "loss": 0.0025, "reward": 0.34375, "reward_std": 0.30935921147465706, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 143 }, { "clip_ratio": 0.0, "completion_length": 2263.25, "epoch": 0.006158254305966878, "grad_norm": 0.470819890499115, "kl": 0.10693359375, "learning_rate": 4.408070965292534e-06, "loss": -0.0299, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 144 }, { "clip_ratio": 0.0, "completion_length": 1851.125, "epoch": 0.006201019960869426, "grad_norm": 0.22044295072555542, "kl": 0.145263671875, "learning_rate": 4.264235636489542e-06, "loss": 0.112, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 145 }, { "clip_ratio": 0.0, "completion_length": 2871.375, "epoch": 0.006243785615771973, "grad_norm": 0.16469889879226685, "kl": 0.09478759765625, "learning_rate": 4.12214747707527e-06, "loss": 0.0281, "reward": 0.5, "reward_std": 0.1767766885459423, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 146 }, { "clip_ratio": 0.0, "completion_length": 2444.75, "epoch": 0.0062865512706745215, "grad_norm": 0.20137082040309906, "kl": 0.12255859375, "learning_rate": 3.981849768479516e-06, "loss": 0.234, "reward": 0.65625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 147 }, { "clip_ratio": 0.0, "completion_length": 2147.75, "epoch": 0.006329316925577069, "grad_norm": 0.2639320194721222, "kl": 0.262939453125, "learning_rate": 3.8433852467434175e-06, "loss": 0.0949, "reward": 0.59375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 148 }, { "clip_ratio": 0.0, "completion_length": 3338.5, "epoch": 0.006372082580479617, "grad_norm": 0.4376908242702484, "kl": 0.195068359375, "learning_rate": 3.7067960895016277e-06, "loss": 0.2345, "reward": 0.4375, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 149 }, { "clip_ratio": 0.0, "completion_length": 1790.5, "epoch": 0.006414848235382164, "grad_norm": 0.30710121989250183, "kl": 0.10693359375, "learning_rate": 3.5721239031346067e-06, "loss": -0.0364, "reward": 0.59375, "reward_std": 0.39774756133556366, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 150 }, { "clip_ratio": 0.0, "completion_length": 1792.5, "epoch": 0.006457613890284712, "grad_norm": 0.21631275117397308, "kl": 0.1448974609375, "learning_rate": 3.4394097100949286e-06, "loss": -0.0563, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 151 }, { "clip_ratio": 0.0, "completion_length": 2212.625, "epoch": 0.0065003795451872605, "grad_norm": 0.2937834858894348, "kl": 0.1668701171875, "learning_rate": 3.308693936411421e-06, "loss": 0.1283, "reward": 0.5625, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 152 }, { "clip_ratio": 0.0, "completion_length": 1659.75, "epoch": 0.006543145200089808, "grad_norm": 0.03413737565279007, "kl": 0.0904541015625, "learning_rate": 3.1800163993750166e-06, "loss": -0.0389, "reward": 0.84375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 153 }, { "clip_ratio": 0.0, "completion_length": 1651.0, "epoch": 0.006585910854992356, "grad_norm": 0.4570809304714203, "kl": 0.21484375, "learning_rate": 3.0534162954100264e-06, "loss": 0.3128, "reward": 0.84375, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 154 }, { "clip_ratio": 0.0, "completion_length": 1350.0, "epoch": 0.006628676509894903, "grad_norm": 0.26234403252601624, "kl": 0.16943359375, "learning_rate": 2.9289321881345257e-06, "loss": -0.0651, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 155 }, { "clip_ratio": 0.0, "completion_length": 1446.375, "epoch": 0.006671442164797451, "grad_norm": 0.22783590853214264, "kl": 0.2669677734375, "learning_rate": 2.8066019966134907e-06, "loss": 0.0706, "reward": 0.40625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 156 }, { "clip_ratio": 0.0, "completion_length": 2273.625, "epoch": 0.006714207819699999, "grad_norm": 0.17475581169128418, "kl": 0.1229248046875, "learning_rate": 2.6864629838082957e-06, "loss": 0.1244, "reward": 0.40625, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 157 }, { "clip_ratio": 0.0, "completion_length": 2046.25, "epoch": 0.006756973474602547, "grad_norm": 0.22613146901130676, "kl": 0.20721435546875, "learning_rate": 2.5685517452260566e-06, "loss": -0.0101, "reward": 0.46875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 158 }, { "clip_ratio": 0.0, "completion_length": 2122.125, "epoch": 0.006799739129505094, "grad_norm": 0.2160838544368744, "kl": 0.095855712890625, "learning_rate": 2.45290419777228e-06, "loss": 0.1349, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 159 }, { "clip_ratio": 0.0, "completion_length": 2754.5, "epoch": 0.006842504784407642, "grad_norm": 0.23569229245185852, "kl": 0.2366943359375, "learning_rate": 2.339555568810221e-06, "loss": 0.0055, "reward": 0.71875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 160 }, { "clip_ratio": 0.0, "completion_length": 1442.0, "epoch": 0.00688527043931019, "grad_norm": 0.2198779582977295, "kl": 0.081298828125, "learning_rate": 2.2285403854302912e-06, "loss": 0.1288, "reward": 1.09375, "reward_std": 0.5745242610573769, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.59375, "step": 161 }, { "clip_ratio": 0.0, "completion_length": 1375.625, "epoch": 0.006928036094212738, "grad_norm": 0.365790456533432, "kl": 0.0860595703125, "learning_rate": 2.119892463932781e-06, "loss": 0.1027, "reward": 0.875, "reward_std": 0.5303300879895687, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 162 }, { "clip_ratio": 0.0, "completion_length": 1383.625, "epoch": 0.006970801749115286, "grad_norm": 0.37987950444221497, "kl": 0.260498046875, "learning_rate": 2.013644899527074e-06, "loss": -0.0918, "reward": 0.46875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 163 }, { "clip_ratio": 0.0, "completion_length": 1755.25, "epoch": 0.007013567404017833, "grad_norm": 0.07863836735486984, "kl": 0.17376708984375, "learning_rate": 1.9098300562505266e-06, "loss": -0.034, "reward": 0.375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 164 }, { "clip_ratio": 0.0, "completion_length": 2550.0, "epoch": 0.007056333058920381, "grad_norm": 0.4073450565338135, "kl": 0.25537109375, "learning_rate": 1.808479557110081e-06, "loss": 0.0791, "reward": 0.46875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 165 }, { "clip_ratio": 0.0, "completion_length": 2039.625, "epoch": 0.007099098713822929, "grad_norm": 0.3922102153301239, "kl": 0.2119140625, "learning_rate": 1.709624274449584e-06, "loss": 0.2275, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 166 }, { "clip_ratio": 0.0, "completion_length": 2846.125, "epoch": 0.007141864368725477, "grad_norm": 0.21499007940292358, "kl": 0.36376953125, "learning_rate": 1.6132943205457607e-06, "loss": 0.0935, "reward": 0.875, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.625, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.25, "step": 167 }, { "clip_ratio": 0.0, "completion_length": 1261.625, "epoch": 0.007184630023628024, "grad_norm": 0.3568659722805023, "kl": 0.10186767578125, "learning_rate": 1.5195190384357405e-06, "loss": 0.0013, "reward": 0.6875, "reward_std": 0.4419417306780815, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 168 }, { "clip_ratio": 0.0, "completion_length": 1947.625, "epoch": 0.007227395678530572, "grad_norm": 0.49718061089515686, "kl": 0.1708984375, "learning_rate": 1.4283269929788779e-06, "loss": -0.1155, "reward": 0.59375, "reward_std": 0.39774755761027336, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 169 }, { "clip_ratio": 0.0, "completion_length": 1703.75, "epoch": 0.0072701613334331195, "grad_norm": 0.4116189181804657, "kl": 0.128662109375, "learning_rate": 1.339745962155613e-06, "loss": 0.1535, "reward": 0.53125, "reward_std": 0.22097086161375046, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 170 }, { "clip_ratio": 0.0, "completion_length": 1917.875, "epoch": 0.007312926988335668, "grad_norm": 0.02001749351620674, "kl": 0.1390380859375, "learning_rate": 1.2538029286060428e-06, "loss": -0.0288, "reward": 0.65625, "reward_std": 0.30935920402407646, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 171 }, { "clip_ratio": 0.0, "completion_length": 1852.625, "epoch": 0.007355692643238216, "grad_norm": 0.08485474437475204, "kl": 0.18994140625, "learning_rate": 1.1705240714107301e-06, "loss": 0.0236, "reward": 0.6875, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 172 }, { "clip_ratio": 0.0, "completion_length": 1896.125, "epoch": 0.007398458298140763, "grad_norm": 0.42003291845321655, "kl": 0.30029296875, "learning_rate": 1.0899347581163222e-06, "loss": -0.0369, "reward": 0.40625, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 173 }, { "clip_ratio": 0.0, "completion_length": 1909.875, "epoch": 0.007441223953043311, "grad_norm": 0.08695568889379501, "kl": 0.185302734375, "learning_rate": 1.012059537008332e-06, "loss": 0.1239, "reward": 0.9375, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 174 }, { "clip_ratio": 0.0, "completion_length": 1517.125, "epoch": 0.0074839896079458585, "grad_norm": 0.19628533720970154, "kl": 0.08392333984375, "learning_rate": 9.369221296335007e-07, "loss": 0.0323, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 175 }, { "clip_ratio": 0.0, "completion_length": 1932.0, "epoch": 0.007526755262848407, "grad_norm": 0.591918408870697, "kl": 0.2237548828125, "learning_rate": 8.645454235739903e-07, "loss": -0.1419, "reward": 0.4375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 176 }, { "clip_ratio": 0.0, "completion_length": 2334.875, "epoch": 0.007569520917750954, "grad_norm": 0.043656013906002045, "kl": 0.16259765625, "learning_rate": 7.949514654755963e-07, "loss": 0.0071, "reward": 0.375, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 177 }, { "clip_ratio": 0.0, "completion_length": 2616.25, "epoch": 0.007612286572653502, "grad_norm": 0.386140376329422, "kl": 0.259765625, "learning_rate": 7.281614543321269e-07, "loss": 0.2467, "reward": 0.6875, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 178 }, { "clip_ratio": 0.0, "completion_length": 1977.125, "epoch": 0.007655052227556049, "grad_norm": 0.05279253423213959, "kl": 0.216552734375, "learning_rate": 6.641957350279838e-07, "loss": 0.0435, "reward": 0.53125, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 179 }, { "clip_ratio": 0.0, "completion_length": 3419.375, "epoch": 0.0076978178824585975, "grad_norm": 0.2946254312992096, "kl": 0.1947021484375, "learning_rate": 6.030737921409169e-07, "loss": -0.0765, "reward": 0.375, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 180 }, { "clip_ratio": 0.0, "completion_length": 2742.875, "epoch": 0.007740583537361146, "grad_norm": 0.5300281643867493, "kl": 0.23828125, "learning_rate": 5.448142440068316e-07, "loss": 0.0436, "reward": 0.46875, "reward_std": 0.30935920774936676, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 181 }, { "clip_ratio": 0.0, "completion_length": 1538.75, "epoch": 0.007783349192263693, "grad_norm": 0.3689361810684204, "kl": 0.130126953125, "learning_rate": 4.894348370484648e-07, "loss": 0.1185, "reward": 0.8125, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.6875, "step": 182 }, { "clip_ratio": 0.0, "completion_length": 2989.5, "epoch": 0.00782611484716624, "grad_norm": 0.050531912595033646, "kl": 0.19305419921875, "learning_rate": 4.3695244036964567e-07, "loss": -0.0671, "reward": 1.03125, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.53125, "step": 183 }, { "clip_ratio": 0.0, "completion_length": 2425.375, "epoch": 0.00786888050206879, "grad_norm": 0.1556394398212433, "kl": 0.18603515625, "learning_rate": 3.8738304061681107e-07, "loss": 0.0718, "reward": 0.6875, "reward_std": 0.4419417232275009, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 184 }, { "clip_ratio": 0.0, "completion_length": 1074.375, "epoch": 0.007911646156971337, "grad_norm": 0.570248007774353, "kl": 0.1925048828125, "learning_rate": 3.4074173710931804e-07, "loss": -0.1418, "reward": 0.875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.625, "step": 185 }, { "clip_ratio": 0.0, "completion_length": 1992.625, "epoch": 0.007954411811873884, "grad_norm": 0.10743938386440277, "kl": 0.268310546875, "learning_rate": 2.970427372400353e-07, "loss": 0.1327, "reward": 0.34375, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.34375, "step": 186 }, { "clip_ratio": 0.0, "completion_length": 2096.625, "epoch": 0.007997177466776431, "grad_norm": 0.36785003542900085, "kl": 0.289306640625, "learning_rate": 2.5629935214764866e-07, "loss": 0.0696, "reward": 0.71875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 187 }, { "clip_ratio": 0.0, "completion_length": 1224.375, "epoch": 0.00803994312167898, "grad_norm": 0.36991608142852783, "kl": 0.1092529296875, "learning_rate": 2.1852399266194312e-07, "loss": 0.1875, "reward": 0.875, "reward_std": 0.4419417232275009, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 188 }, { "clip_ratio": 0.0, "completion_length": 2567.5, "epoch": 0.008082708776581527, "grad_norm": 0.07666341215372086, "kl": 0.1605224609375, "learning_rate": 1.8372816552336025e-07, "loss": 0.0068, "reward": 0.5625, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5625, "step": 189 }, { "clip_ratio": 0.0, "completion_length": 1473.5, "epoch": 0.008125474431484075, "grad_norm": 0.42782488465309143, "kl": 0.173828125, "learning_rate": 1.519224698779198e-07, "loss": -0.0216, "reward": 0.375, "reward_std": 0.3535533845424652, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 190 }, { "clip_ratio": 0.0, "completion_length": 3243.75, "epoch": 0.008168240086386622, "grad_norm": 0.23994077742099762, "kl": 0.1695556640625, "learning_rate": 1.231165940486234e-07, "loss": 0.0688, "reward": 0.4375, "reward_std": 0.2651650346815586, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 191 }, { "clip_ratio": 0.0, "completion_length": 1632.75, "epoch": 0.008211005741289171, "grad_norm": 0.3033127188682556, "kl": 0.13330078125, "learning_rate": 9.731931258429638e-08, "loss": 0.0865, "reward": 0.5, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.5, "step": 192 }, { "clip_ratio": 0.0, "completion_length": 1998.375, "epoch": 0.008253771396191718, "grad_norm": 0.05849841609597206, "kl": 0.1898193359375, "learning_rate": 7.453848358678018e-08, "loss": 0.0705, "reward": 0.6875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 193 }, { "clip_ratio": 0.0, "completion_length": 2685.625, "epoch": 0.008296537051094266, "grad_norm": 0.18582133948802948, "kl": 0.258544921875, "learning_rate": 5.4781046317267103e-08, "loss": 0.2169, "reward": 0.53125, "reward_std": 0.30935921519994736, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 194 }, { "clip_ratio": 0.0, "completion_length": 1837.125, "epoch": 0.008339302705996815, "grad_norm": 0.3928259015083313, "kl": 0.1937255859375, "learning_rate": 3.805301908254455e-08, "loss": -0.0365, "reward": 0.53125, "reward_std": 0.48613591492176056, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.40625, "step": 195 }, { "clip_ratio": 0.0, "completion_length": 2399.125, "epoch": 0.008382068360899362, "grad_norm": 0.13430964946746826, "kl": 0.3720703125, "learning_rate": 2.4359497401758026e-08, "loss": -0.1007, "reward": 0.21875, "reward_std": 0.13258251920342445, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 196 }, { "clip_ratio": 0.0, "completion_length": 2257.125, "epoch": 0.00842483401580191, "grad_norm": 0.21241837739944458, "kl": 0.1380615234375, "learning_rate": 1.370465245426167e-08, "loss": 0.1397, "reward": 0.46875, "reward_std": 0.22097086533904076, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 197 }, { "clip_ratio": 0.0, "completion_length": 2047.125, "epoch": 0.008467599670704457, "grad_norm": 0.6166004538536072, "kl": 0.2255859375, "learning_rate": 6.091729809042379e-09, "loss": 0.0647, "reward": 0.375, "reward_std": 0.2651650384068489, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.375, "step": 198 }, { "clip_ratio": 0.0, "completion_length": 1217.5, "epoch": 0.008510365325607006, "grad_norm": 0.37172725796699524, "kl": 0.19580078125, "learning_rate": 1.5230484360873043e-09, "loss": 0.0025, "reward": 0.46875, "reward_std": 0.04419417306780815, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.46875, "step": 199 }, { "epoch": 0.008553130980509553, "grad_norm": 0.46583613753318787, "learning_rate": 0.0, "loss": 0.0334, "step": 200 }, { "epoch": 0.008553130980509553, "eval_clip_ratio": 0.0, "eval_completion_length": 1743.6575, "eval_kl": 0.1995361328125, "eval_loss": 0.05781688541173935, "eval_reward": 0.555625, "eval_reward_std": 0.2324613492190838, "eval_rewards/accuracy_reward": 0.1, "eval_rewards/format_reward": 0.0, "eval_rewards/tag_count_reward": 0.455625, "eval_runtime": 4841.523, "eval_samples_per_second": 0.041, "eval_steps_per_second": 0.021, "step": 200 }, { "clip_ratio": 0.0, "completion_length": 2449.5, "epoch": 0.0085958966354121, "grad_norm": 0.24276599287986755, "kl": 0.104736328125, "learning_rate": 1.5230484360873043e-09, "loss": 0.1196, "reward": 0.4375, "reward_std": 0.3535533808171749, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.3125, "step": 201 }, { "clip_ratio": 0.0, "completion_length": 1351.375, "epoch": 0.00863866229031465, "grad_norm": 0.4390352964401245, "kl": 0.1497802734375, "learning_rate": 6.091729809042379e-09, "loss": 0.0129, "reward": 0.4375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.4375, "step": 202 }, { "clip_ratio": 0.0, "completion_length": 2207.75, "epoch": 0.008681427945217196, "grad_norm": 0.34543412923812866, "kl": 0.314453125, "learning_rate": 1.370465245426167e-08, "loss": 0.0108, "reward": 0.21875, "reward_std": 0.30935920402407646, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "rewards/tag_count_reward": 0.21875, "step": 203 }, { "epoch": 0.008681427945217196, "step": 203, "total_flos": 0.0, "train_loss": 5.297641742405633e-05, "train_runtime": 562.5077, "train_samples_per_second": 2.844, "train_steps_per_second": 0.356 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }