{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0488, "eval_steps": 500, "global_step": 610, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 301.25, "completions/mean_terminated_length": 301.25, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3187313340604305, "epoch": 8e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.11033321917057037, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0007, "num_tokens": 27507.0, "reward": 0.35593751072883606, "reward_std": 0.35650908946990967, "rewards/rollout_reward_func/mean": 0.35593751072883606, "rewards/rollout_reward_func/std": 0.399433970451355, "sampling/importance_sampling_ratio/max": 1.6676214933395386, "sampling/importance_sampling_ratio/mean": 1.0151346921920776, "sampling/importance_sampling_ratio/min": 0.6278224587440491, "sampling/sampling_logp_difference/max": 0.4534909725189209, "sampling/sampling_logp_difference/mean": 0.040589556097984314, "step": 1, "step_time": 9.021057722000478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 300.90625, "completions/mean_terminated_length": 300.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3571012131869793, "epoch": 0.00016, "frac_reward_zero_std": 0.25, "grad_norm": 0.1205478236079216, "kl": 0.0, "learning_rate": 1.2438299999999998e-06, "loss": 0.0002, "num_tokens": 54884.0, "reward": 0.22500000894069672, "reward_std": 0.30521661043167114, "rewards/rollout_reward_func/mean": 0.22500000894069672, "rewards/rollout_reward_func/std": 0.3654427230358124, "sampling/importance_sampling_ratio/max": 1.8328614234924316, "sampling/importance_sampling_ratio/mean": 0.984244704246521, "sampling/importance_sampling_ratio/min": 0.5920599102973938, "sampling/sampling_logp_difference/max": 0.487576961517334, "sampling/sampling_logp_difference/mean": 0.045160628855228424, "step": 2, "step_time": 7.797765201000402 }, { "clip_ratio/high_max": 0.011363636702299118, "clip_ratio/high_mean": 0.005681818351149559, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013494318351149559, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 280.65625, "completions/mean_terminated_length": 280.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3536026570945978, "epoch": 0.00024, "frac_reward_zero_std": 0.0, "grad_norm": 0.12386211007833481, "kl": 0.006677820347249508, "learning_rate": 2.4876599999999997e-06, "loss": 0.0013, "num_tokens": 81060.0, "reward": 0.26218751072883606, "reward_std": 0.35526764392852783, "rewards/rollout_reward_func/mean": 0.26218751072883606, "rewards/rollout_reward_func/std": 0.33889511227607727, "sampling/importance_sampling_ratio/max": 1.7290635108947754, "sampling/importance_sampling_ratio/mean": 0.9902434349060059, "sampling/importance_sampling_ratio/min": 0.3712500035762787, "sampling/sampling_logp_difference/max": 0.6723372936248779, "sampling/sampling_logp_difference/mean": 0.05579252541065216, "step": 3, "step_time": 7.956034166000791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 315.625, "completions/mean_terminated_length": 315.625, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.3639394473284483, "epoch": 0.00032, "frac_reward_zero_std": 0.5, "grad_norm": 0.07511676847934723, "kl": 0.013857253477908671, "learning_rate": 3.73149e-06, "loss": 0.0, "num_tokens": 109483.0, "reward": 0.16874998807907104, "reward_std": 0.233409583568573, "rewards/rollout_reward_func/mean": 0.16874998807907104, "rewards/rollout_reward_func/std": 0.3486725986003876, "sampling/importance_sampling_ratio/max": 2.230165958404541, "sampling/importance_sampling_ratio/mean": 1.059661865234375, "sampling/importance_sampling_ratio/min": 0.6051729917526245, "sampling/sampling_logp_difference/max": 0.5022211074829102, "sampling/sampling_logp_difference/mean": 0.05090917646884918, "step": 4, "step_time": 9.298923759000445 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 305.8125, "completions/mean_terminated_length": 305.8125, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.31169237196445465, "epoch": 0.0004, "frac_reward_zero_std": 0.0, "grad_norm": 0.0898391455411911, "kl": 0.010670528106857091, "learning_rate": 4.975319999999999e-06, "loss": 0.0007, "num_tokens": 137709.0, "reward": 0.3371874988079071, "reward_std": 0.34612250328063965, "rewards/rollout_reward_func/mean": 0.3371874988079071, "rewards/rollout_reward_func/std": 0.37165048718452454, "sampling/importance_sampling_ratio/max": 1.3670653104782104, "sampling/importance_sampling_ratio/mean": 0.9132508039474487, "sampling/importance_sampling_ratio/min": 0.4510490596294403, "sampling/sampling_logp_difference/max": 0.7926580905914307, "sampling/sampling_logp_difference/mean": 0.05983540415763855, "step": 5, "step_time": 7.9873320929996225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.021006944589316845, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021006944589316845, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 270.5, "completions/mean_terminated_length": 270.5, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.3336157724261284, "epoch": 0.00048, "frac_reward_zero_std": 0.125, "grad_norm": 0.10157592594623566, "kl": 0.006093155519920401, "learning_rate": 6.2191499999999996e-06, "loss": -0.0005, "num_tokens": 162822.0, "reward": 0.31812500953674316, "reward_std": 0.34396660327911377, "rewards/rollout_reward_func/mean": 0.31812500953674316, "rewards/rollout_reward_func/std": 0.45726704597473145, "sampling/importance_sampling_ratio/max": 1.6552499532699585, "sampling/importance_sampling_ratio/mean": 1.04984712600708, "sampling/importance_sampling_ratio/min": 0.6131481528282166, "sampling/sampling_logp_difference/max": 0.49155092239379883, "sampling/sampling_logp_difference/mean": 0.045063719153404236, "step": 6, "step_time": 7.835910219000652 }, { "clip_ratio/high_max": 0.060763888992369175, "clip_ratio/high_mean": 0.030381944496184587, "clip_ratio/low_mean": 0.020833333488553762, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.05121527658775449, "completions/clipped_ratio": 0.0, "completions/max_length": 373.0, "completions/max_terminated_length": 373.0, "completions/mean_length": 300.75, "completions/mean_terminated_length": 300.75, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.34452300518751144, "epoch": 0.00056, "frac_reward_zero_std": 0.125, "grad_norm": 0.15673647820949554, "kl": 0.014406466507352889, "learning_rate": 7.46298e-06, "loss": 0.001, "num_tokens": 190669.0, "reward": 0.46812504529953003, "reward_std": 0.45488813519477844, "rewards/rollout_reward_func/mean": 0.46812504529953003, "rewards/rollout_reward_func/std": 0.5222682952880859, "sampling/importance_sampling_ratio/max": 1.5099186897277832, "sampling/importance_sampling_ratio/mean": 0.9870142936706543, "sampling/importance_sampling_ratio/min": 0.6360042691230774, "sampling/sampling_logp_difference/max": 0.49539002776145935, "sampling/sampling_logp_difference/mean": 0.05368976294994354, "step": 7, "step_time": 7.953064536000511 }, { "clip_ratio/high_max": 0.02777777798473835, "clip_ratio/high_mean": 0.013888888992369175, "clip_ratio/low_mean": 0.028819444589316845, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.04270833358168602, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 315.46875, "completions/mean_terminated_length": 315.46875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.3588735293596983, "epoch": 0.00064, "frac_reward_zero_std": 0.0, "grad_norm": 0.11780666559934616, "kl": 0.011717404995579273, "learning_rate": 8.70681e-06, "loss": 0.0022, "num_tokens": 219277.0, "reward": 0.3187500238418579, "reward_std": 0.3812493681907654, "rewards/rollout_reward_func/mean": 0.3187500238418579, "rewards/rollout_reward_func/std": 0.3728162348270416, "sampling/importance_sampling_ratio/max": 2.2589457035064697, "sampling/importance_sampling_ratio/mean": 1.083111047744751, "sampling/importance_sampling_ratio/min": 0.5480707287788391, "sampling/sampling_logp_difference/max": 0.8980450630187988, "sampling/sampling_logp_difference/mean": 0.052481792867183685, "step": 8, "step_time": 9.320614303999264 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 260.21875, "completions/mean_terminated_length": 260.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.30374085530638695, "epoch": 0.00072, "frac_reward_zero_std": 0.0, "grad_norm": 0.0802796483039856, "kl": 0.0036519222194328904, "learning_rate": 9.950639999999999e-06, "loss": -0.0002, "num_tokens": 244816.0, "reward": 0.48750001192092896, "reward_std": 0.4552166163921356, "rewards/rollout_reward_func/mean": 0.48750001192092896, "rewards/rollout_reward_func/std": 0.4681811034679413, "sampling/importance_sampling_ratio/max": 1.2234160900115967, "sampling/importance_sampling_ratio/mean": 0.955081582069397, "sampling/importance_sampling_ratio/min": 0.37016767263412476, "sampling/sampling_logp_difference/max": 0.5537159442901611, "sampling/sampling_logp_difference/mean": 0.0344792939722538, "step": 9, "step_time": 7.954598627999076 }, { "clip_ratio/high_max": 0.05902777798473835, "clip_ratio/high_mean": 0.029513888992369175, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.029513888992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 316.375, "completions/mean_terminated_length": 316.375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3739766962826252, "epoch": 0.0008, "frac_reward_zero_std": 0.25, "grad_norm": 0.07135297358036041, "kl": 0.03596597191062756, "learning_rate": 1.1194469999999999e-05, "loss": 0.001, "num_tokens": 273382.0, "reward": 0.2618750035762787, "reward_std": 0.2825181484222412, "rewards/rollout_reward_func/mean": 0.2618750035762787, "rewards/rollout_reward_func/std": 0.3385780155658722, "sampling/importance_sampling_ratio/max": 1.7461260557174683, "sampling/importance_sampling_ratio/mean": 0.9589649438858032, "sampling/importance_sampling_ratio/min": 0.3435109257698059, "sampling/sampling_logp_difference/max": 0.7510861158370972, "sampling/sampling_logp_difference/mean": 0.053384341299533844, "step": 10, "step_time": 7.973904867999863 }, { "clip_ratio/high_max": 0.02638888917863369, "clip_ratio/high_mean": 0.013194444589316845, "clip_ratio/low_mean": 0.014062500093132257, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027256944682449102, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.84375, "completions/mean_terminated_length": 290.84375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3090610224753618, "epoch": 0.00088, "frac_reward_zero_std": 0.125, "grad_norm": 0.1226002424955368, "kl": 0.022108020319137722, "learning_rate": 1.2438299999999999e-05, "loss": -0.0012, "num_tokens": 300505.0, "reward": 0.39375001192092896, "reward_std": 0.3379480838775635, "rewards/rollout_reward_func/mean": 0.39375001192092896, "rewards/rollout_reward_func/std": 0.4471685290336609, "sampling/importance_sampling_ratio/max": 1.4724481105804443, "sampling/importance_sampling_ratio/mean": 0.9596928358078003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3379878997802734, "sampling/sampling_logp_difference/mean": 0.05969899892807007, "step": 11, "step_time": 9.543189336000069 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.037326388992369175, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.037326388992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 281.21875, "completions/mean_terminated_length": 281.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3723270557820797, "epoch": 0.00096, "frac_reward_zero_std": 0.0, "grad_norm": 0.10062465816736221, "kl": 0.04101349285338074, "learning_rate": 1.368213e-05, "loss": 0.0008, "num_tokens": 326913.0, "reward": 0.35625001788139343, "reward_std": 0.3786410689353943, "rewards/rollout_reward_func/mean": 0.35625001788139343, "rewards/rollout_reward_func/std": 0.39914223551750183, "sampling/importance_sampling_ratio/max": 1.976764440536499, "sampling/importance_sampling_ratio/mean": 1.0276546478271484, "sampling/importance_sampling_ratio/min": 0.6260702013969421, "sampling/sampling_logp_difference/max": 0.8166131973266602, "sampling/sampling_logp_difference/mean": 0.0463852658867836, "step": 12, "step_time": 7.941104143000757 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 270.90625, "completions/mean_terminated_length": 270.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.2762108463793993, "epoch": 0.00104, "frac_reward_zero_std": 0.125, "grad_norm": 0.12515157461166382, "kl": 0.062352439388632774, "learning_rate": 1.492596e-05, "loss": 0.0005, "num_tokens": 352750.0, "reward": 0.5249999761581421, "reward_std": 0.35751211643218994, "rewards/rollout_reward_func/mean": 0.5249999761581421, "rewards/rollout_reward_func/std": 0.4242640733718872, "sampling/importance_sampling_ratio/max": 2.2936055660247803, "sampling/importance_sampling_ratio/mean": 0.949316680431366, "sampling/importance_sampling_ratio/min": 0.4613780081272125, "sampling/sampling_logp_difference/max": 0.9497102499008179, "sampling/sampling_logp_difference/mean": 0.06091103330254555, "step": 13, "step_time": 7.84188028600056 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 296.21875, "completions/mean_terminated_length": 296.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3682078756392002, "epoch": 0.00112, "frac_reward_zero_std": 0.0, "grad_norm": 0.10302872955799103, "kl": 0.11470550671219826, "learning_rate": 1.6169789999999998e-05, "loss": -0.0004, "num_tokens": 379833.0, "reward": 0.43125003576278687, "reward_std": 0.3648782968521118, "rewards/rollout_reward_func/mean": 0.43125003576278687, "rewards/rollout_reward_func/std": 0.40990757942199707, "sampling/importance_sampling_ratio/max": 1.6255182027816772, "sampling/importance_sampling_ratio/mean": 1.0577311515808105, "sampling/importance_sampling_ratio/min": 0.6636798977851868, "sampling/sampling_logp_difference/max": 0.47386693954467773, "sampling/sampling_logp_difference/mean": 0.04928760230541229, "step": 14, "step_time": 8.026380745000097 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.022569444496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 280.03125, "completions/mean_terminated_length": 280.03125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.29517422802746296, "epoch": 0.0012, "frac_reward_zero_std": 0.125, "grad_norm": 0.10121005028486252, "kl": 0.12026547826826572, "learning_rate": 1.741362e-05, "loss": -0.0005, "num_tokens": 405965.0, "reward": 0.5249999761581421, "reward_std": 0.39755725860595703, "rewards/rollout_reward_func/mean": 0.5249999761581421, "rewards/rollout_reward_func/std": 0.4242640733718872, "sampling/importance_sampling_ratio/max": 2.749668836593628, "sampling/importance_sampling_ratio/mean": 1.0825247764587402, "sampling/importance_sampling_ratio/min": 0.3409128487110138, "sampling/sampling_logp_difference/max": 1.171809196472168, "sampling/sampling_logp_difference/mean": 0.08482812345027924, "step": 15, "step_time": 9.515036971999507 }, { "clip_ratio/high_max": 0.029513888992369175, "clip_ratio/high_mean": 0.014756944496184587, "clip_ratio/low_mean": 0.029513888992369175, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.044270833022892475, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 310.6875, "completions/mean_terminated_length": 310.6875, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.37191349267959595, "epoch": 0.00128, "frac_reward_zero_std": 0.0, "grad_norm": 0.0816335529088974, "kl": 0.3604915663599968, "learning_rate": 1.865745e-05, "loss": -0.0028, "num_tokens": 434159.0, "reward": 0.41218751668930054, "reward_std": 0.4416668117046356, "rewards/rollout_reward_func/mean": 0.41218751668930054, "rewards/rollout_reward_func/std": 0.44298645853996277, "sampling/importance_sampling_ratio/max": 1.6169039011001587, "sampling/importance_sampling_ratio/mean": 0.8929818272590637, "sampling/importance_sampling_ratio/min": 0.21065275371074677, "sampling/sampling_logp_difference/max": 2.210373878479004, "sampling/sampling_logp_difference/mean": 0.1168808788061142, "step": 16, "step_time": 7.958728525999959 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 266.21875, "completions/mean_terminated_length": 266.21875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.29791201651096344, "epoch": 0.00136, "frac_reward_zero_std": 0.0, "grad_norm": 0.07244627177715302, "kl": 0.739876018371433, "learning_rate": 1.9901279999999997e-05, "loss": -0.0019, "num_tokens": 459561.0, "reward": 0.5621874928474426, "reward_std": 0.39640867710113525, "rewards/rollout_reward_func/mean": 0.5621874928474426, "rewards/rollout_reward_func/std": 0.429791659116745, "sampling/importance_sampling_ratio/max": 1.9798283576965332, "sampling/importance_sampling_ratio/mean": 0.9799823760986328, "sampling/importance_sampling_ratio/min": 0.13889779150485992, "sampling/sampling_logp_difference/max": 1.9739902019500732, "sampling/sampling_logp_difference/mean": 0.09204481542110443, "step": 17, "step_time": 7.966833985999983 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.04131944477558136, "clip_ratio/low_min": 0.015625, "clip_ratio/region_mean": 0.04826388927176595, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 326.65625, "completions/mean_terminated_length": 326.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.2520571779459715, "epoch": 0.00144, "frac_reward_zero_std": 0.125, "grad_norm": 0.06466308236122131, "kl": 0.2698284504003823, "learning_rate": 2.114511e-05, "loss": 0.0002, "num_tokens": 489362.0, "reward": 0.5437500476837158, "reward_std": 0.3812493681907654, "rewards/rollout_reward_func/mean": 0.5437500476837158, "rewards/rollout_reward_func/std": 0.44062820076942444, "sampling/importance_sampling_ratio/max": 1.9853273630142212, "sampling/importance_sampling_ratio/mean": 0.9817830920219421, "sampling/importance_sampling_ratio/min": 0.2280855029821396, "sampling/sampling_logp_difference/max": 1.4791345596313477, "sampling/sampling_logp_difference/mean": 0.07609017193317413, "step": 18, "step_time": 7.926769493999473 }, { "clip_ratio/high_max": 0.028125000186264515, "clip_ratio/high_mean": 0.014062500093132257, "clip_ratio/low_mean": 0.01145833358168602, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.025520833674818277, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 310.1875, "completions/mean_terminated_length": 310.1875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.2815595418214798, "epoch": 0.00152, "frac_reward_zero_std": 0.125, "grad_norm": 0.328479528427124, "kl": 4.543358171358705, "learning_rate": 2.2388939999999998e-05, "loss": 0.0001, "num_tokens": 517538.0, "reward": 0.5062500238418579, "reward_std": 0.3902435898780823, "rewards/rollout_reward_func/mean": 0.5062500238418579, "rewards/rollout_reward_func/std": 0.4063507914543152, "sampling/importance_sampling_ratio/max": 1.7135324478149414, "sampling/importance_sampling_ratio/mean": 0.8684549331665039, "sampling/importance_sampling_ratio/min": 0.009426064789295197, "sampling/sampling_logp_difference/max": 4.452655792236328, "sampling/sampling_logp_difference/mean": 0.11393614113330841, "step": 19, "step_time": 10.108504762000848 }, { "clip_ratio/high_max": 0.03125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 373.0, "completions/max_terminated_length": 373.0, "completions/mean_length": 275.3125, "completions/mean_terminated_length": 275.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.20874292589724064, "epoch": 0.0016, "frac_reward_zero_std": 0.125, "grad_norm": 0.12303224951028824, "kl": 0.22338835708796978, "learning_rate": 2.3632769999999996e-05, "loss": 0.0007, "num_tokens": 543755.0, "reward": 0.8062499761581421, "reward_std": 0.3331795632839203, "rewards/rollout_reward_func/mean": 0.8062499761581421, "rewards/rollout_reward_func/std": 0.3609463572502136, "sampling/importance_sampling_ratio/max": 2.115467071533203, "sampling/importance_sampling_ratio/mean": 0.8641330003738403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.812043309211731, "sampling/sampling_logp_difference/mean": 0.08788560330867767, "step": 20, "step_time": 7.85372267799994 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 280.65625, "completions/mean_terminated_length": 280.65625, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.2249947115778923, "epoch": 0.00168, "frac_reward_zero_std": 0.125, "grad_norm": 0.048400841653347015, "kl": 0.7632453143596649, "learning_rate": 2.4876599999999998e-05, "loss": -0.002, "num_tokens": 570360.0, "reward": 0.6559374928474426, "reward_std": 0.2926635146141052, "rewards/rollout_reward_func/mean": 0.6559374928474426, "rewards/rollout_reward_func/std": 0.35341522097587585, "sampling/importance_sampling_ratio/max": 1.7896636724472046, "sampling/importance_sampling_ratio/mean": 0.7840760946273804, "sampling/importance_sampling_ratio/min": 0.09646078199148178, "sampling/sampling_logp_difference/max": 2.3575150966644287, "sampling/sampling_logp_difference/mean": 0.13912206888198853, "step": 21, "step_time": 7.949934961000508 }, { "clip_ratio/high_max": 0.015625, "clip_ratio/high_mean": 0.01406249962747097, "clip_ratio/low_mean": 0.03350694477558136, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.047569445334374905, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 286.625, "completions/mean_terminated_length": 286.625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.24710408598184586, "epoch": 0.00176, "frac_reward_zero_std": 0.0, "grad_norm": 0.055163174867630005, "kl": 0.41494268737733364, "learning_rate": 2.6120429999999997e-05, "loss": -0.0022, "num_tokens": 596824.0, "reward": 0.5621874928474426, "reward_std": 0.4664088487625122, "rewards/rollout_reward_func/mean": 0.5621874928474426, "rewards/rollout_reward_func/std": 0.45558711886405945, "sampling/importance_sampling_ratio/max": 1.5321714878082275, "sampling/importance_sampling_ratio/mean": 0.8903067111968994, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3187341690063477, "sampling/sampling_logp_difference/mean": 0.11416112631559372, "step": 22, "step_time": 8.021714273999805 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.028125000186264515, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0350694446824491, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 285.375, "completions/mean_terminated_length": 285.375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.27896914072334766, "epoch": 0.00184, "frac_reward_zero_std": 0.0, "grad_norm": 0.06080702319741249, "kl": 0.7041767947375774, "learning_rate": 2.736426e-05, "loss": -0.0012, "num_tokens": 623034.0, "reward": 0.7121875286102295, "reward_std": 0.4275849461555481, "rewards/rollout_reward_func/mean": 0.7121875286102295, "rewards/rollout_reward_func/std": 0.4427679777145386, "sampling/importance_sampling_ratio/max": 1.942314624786377, "sampling/importance_sampling_ratio/mean": 0.9337605834007263, "sampling/importance_sampling_ratio/min": 0.15043196082115173, "sampling/sampling_logp_difference/max": 1.302234172821045, "sampling/sampling_logp_difference/mean": 0.10249899327754974, "step": 23, "step_time": 9.74794881600019 }, { "clip_ratio/high_max": 0.046875, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 296.1875, "completions/mean_terminated_length": 296.1875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.21369423065334558, "epoch": 0.00192, "frac_reward_zero_std": 0.0, "grad_norm": 0.16228128969669342, "kl": 0.4553202469833195, "learning_rate": 2.8608089999999997e-05, "loss": -0.0003, "num_tokens": 650686.0, "reward": 0.7871875166893005, "reward_std": 0.37607312202453613, "rewards/rollout_reward_func/mean": 0.7871875166893005, "rewards/rollout_reward_func/std": 0.41577163338661194, "sampling/importance_sampling_ratio/max": 2.683306932449341, "sampling/importance_sampling_ratio/mean": 0.9836126565933228, "sampling/importance_sampling_ratio/min": 0.09829449653625488, "sampling/sampling_logp_difference/max": 1.662473440170288, "sampling/sampling_logp_difference/mean": 0.10777679085731506, "step": 24, "step_time": 7.958107905999896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014756944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014756944496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 290.28125, "completions/mean_terminated_length": 290.28125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.18398151453584433, "epoch": 0.002, "frac_reward_zero_std": 0.125, "grad_norm": 0.11078019440174103, "kl": 2.3383586704730988, "learning_rate": 2.985192e-05, "loss": -0.0003, "num_tokens": 677672.0, "reward": 0.768750011920929, "reward_std": 0.33951300382614136, "rewards/rollout_reward_func/mean": 0.768750011920929, "rewards/rollout_reward_func/std": 0.3805238604545593, "sampling/importance_sampling_ratio/max": 1.556268334388733, "sampling/importance_sampling_ratio/mean": 0.938198983669281, "sampling/importance_sampling_ratio/min": 0.12258704751729965, "sampling/sampling_logp_difference/max": 2.061178207397461, "sampling/sampling_logp_difference/mean": 0.09111033380031586, "step": 25, "step_time": 7.751307518999511 }, { "clip_ratio/high_max": 0.02638888917863369, "clip_ratio/high_mean": 0.013194444589316845, "clip_ratio/low_mean": 0.022569444496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03576388908550143, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 296.1875, "completions/mean_terminated_length": 296.1875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.224177073687315, "epoch": 0.00208, "frac_reward_zero_std": 0.0, "grad_norm": 0.06343436241149902, "kl": 0.5240140296518803, "learning_rate": 3.109575e-05, "loss": -0.0008, "num_tokens": 704849.0, "reward": 0.8062499761581421, "reward_std": 0.3353397846221924, "rewards/rollout_reward_func/mean": 0.8062499761581421, "rewards/rollout_reward_func/std": 0.36094632744789124, "sampling/importance_sampling_ratio/max": 1.27411687374115, "sampling/importance_sampling_ratio/mean": 0.9335529804229736, "sampling/importance_sampling_ratio/min": 0.3273059129714966, "sampling/sampling_logp_difference/max": 1.3228144645690918, "sampling/sampling_logp_difference/mean": 0.06958809494972229, "step": 26, "step_time": 7.883683099000791 }, { "clip_ratio/high_max": 0.028125000186264515, "clip_ratio/high_mean": 0.014062500093132257, "clip_ratio/low_mean": 0.01996527798473835, "clip_ratio/low_min": 0.010416666977107525, "clip_ratio/region_mean": 0.03402777807787061, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 281.25, "completions/mean_terminated_length": 281.25, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1963601300958544, "epoch": 0.00216, "frac_reward_zero_std": 0.125, "grad_norm": 0.06295550614595413, "kl": 1.2591982930898666, "learning_rate": 3.2339579999999996e-05, "loss": -0.0012, "num_tokens": 731163.0, "reward": 0.8250000476837158, "reward_std": 0.34590959548950195, "rewards/rollout_reward_func/mean": 0.8250000476837158, "rewards/rollout_reward_func/std": 0.4508057236671448, "sampling/importance_sampling_ratio/max": 2.212244749069214, "sampling/importance_sampling_ratio/mean": 0.957727313041687, "sampling/importance_sampling_ratio/min": 0.025072962045669556, "sampling/sampling_logp_difference/max": 2.8886754512786865, "sampling/sampling_logp_difference/mean": 0.12513552606105804, "step": 27, "step_time": 9.512978467001176 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.012500000186264515, "clip_ratio/low_mean": 0.019176136702299118, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03167613688856363, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 307.34375, "completions/mean_terminated_length": 307.34375, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.23280802555382252, "epoch": 0.00224, "frac_reward_zero_std": 0.25, "grad_norm": 0.06411010026931763, "kl": 0.7862906008958817, "learning_rate": 3.358341e-05, "loss": -0.0003, "num_tokens": 759201.0, "reward": 0.7309374809265137, "reward_std": 0.3519214987754822, "rewards/rollout_reward_func/mean": 0.7309374809265137, "rewards/rollout_reward_func/std": 0.5225086212158203, "sampling/importance_sampling_ratio/max": 2.1141340732574463, "sampling/importance_sampling_ratio/mean": 0.9719194173812866, "sampling/importance_sampling_ratio/min": 0.0668177530169487, "sampling/sampling_logp_difference/max": 2.7348480224609375, "sampling/sampling_logp_difference/mean": 0.09659253060817719, "step": 28, "step_time": 8.000084699000581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 249.78125, "completions/mean_terminated_length": 249.78125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1626758510246873, "epoch": 0.00232, "frac_reward_zero_std": 0.25, "grad_norm": 0.028533341363072395, "kl": 0.7259419336915016, "learning_rate": 3.482724e-05, "loss": -0.0016, "num_tokens": 783120.0, "reward": 0.9375000596046448, "reward_std": 0.30521661043167114, "rewards/rollout_reward_func/mean": 0.9375000596046448, "rewards/rollout_reward_func/std": 0.37135326862335205, "sampling/importance_sampling_ratio/max": 2.606818199157715, "sampling/importance_sampling_ratio/mean": 0.90839022397995, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.4189562797546387, "sampling/sampling_logp_difference/mean": 0.08706843852996826, "step": 29, "step_time": 7.817832963000001 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.030381944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.037326388992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 265.65625, "completions/mean_terminated_length": 265.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1477297474630177, "epoch": 0.0024, "frac_reward_zero_std": 0.125, "grad_norm": 0.09804792702198029, "kl": 2.5584509782493114, "learning_rate": 3.607107e-05, "loss": -0.0024, "num_tokens": 808431.0, "reward": 0.8625000715255737, "reward_std": 0.3031088709831238, "rewards/rollout_reward_func/mean": 0.8625000715255737, "rewards/rollout_reward_func/std": 0.37135326862335205, "sampling/importance_sampling_ratio/max": 1.6572843790054321, "sampling/importance_sampling_ratio/mean": 0.8733506798744202, "sampling/importance_sampling_ratio/min": 0.03966372087597847, "sampling/sampling_logp_difference/max": 3.227372169494629, "sampling/sampling_logp_difference/mean": 0.11239971220493317, "step": 30, "step_time": 7.973769966000873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 276.09375, "completions/mean_terminated_length": 276.09375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.164591652341187, "epoch": 0.00248, "frac_reward_zero_std": 0.25, "grad_norm": 0.0540638193488121, "kl": 0.8059543967247009, "learning_rate": 3.73149e-05, "loss": -0.0016, "num_tokens": 834032.0, "reward": 0.8246874809265137, "reward_std": 0.3055804371833801, "rewards/rollout_reward_func/mean": 0.8246874809265137, "rewards/rollout_reward_func/std": 0.45139917731285095, "sampling/importance_sampling_ratio/max": 1.4981743097305298, "sampling/importance_sampling_ratio/mean": 0.8064203262329102, "sampling/importance_sampling_ratio/min": 0.10818186402320862, "sampling/sampling_logp_difference/max": 2.5553293228149414, "sampling/sampling_logp_difference/mean": 0.11227218806743622, "step": 31, "step_time": 9.292238261999955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 290.28125, "completions/mean_terminated_length": 290.28125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1426389808766544, "epoch": 0.00256, "frac_reward_zero_std": 0.0, "grad_norm": 0.1669316589832306, "kl": 1.312481101602316, "learning_rate": 3.8558729999999996e-05, "loss": -0.0022, "num_tokens": 861303.0, "reward": 0.8812500238418579, "reward_std": 0.426710844039917, "rewards/rollout_reward_func/mean": 0.8812500238418579, "rewards/rollout_reward_func/std": 0.45680344104766846, "sampling/importance_sampling_ratio/max": 1.9040971994400024, "sampling/importance_sampling_ratio/mean": 1.0163731575012207, "sampling/importance_sampling_ratio/min": 0.027382994070649147, "sampling/sampling_logp_difference/max": 3.598799705505371, "sampling/sampling_logp_difference/mean": 0.09276574850082397, "step": 32, "step_time": 7.8575140439997995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 306.53125, "completions/mean_terminated_length": 306.53125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.15779712283983827, "epoch": 0.00264, "frac_reward_zero_std": 0.375, "grad_norm": 0.05883312225341797, "kl": 1.1092037372291088, "learning_rate": 3.9802559999999995e-05, "loss": -0.0014, "num_tokens": 889147.0, "reward": 0.9937500357627869, "reward_std": 0.26510828733444214, "rewards/rollout_reward_func/mean": 0.9937500357627869, "rewards/rollout_reward_func/std": 0.36094632744789124, "sampling/importance_sampling_ratio/max": 2.743377447128296, "sampling/importance_sampling_ratio/mean": 1.1102042198181152, "sampling/importance_sampling_ratio/min": 0.14254987239837646, "sampling/sampling_logp_difference/max": 2.292268991470337, "sampling/sampling_logp_difference/mean": 0.08593671768903732, "step": 33, "step_time": 8.016190576999179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 245.5, "completions/mean_terminated_length": 245.5, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.09760337499028537, "epoch": 0.00272, "frac_reward_zero_std": 0.625, "grad_norm": 0.020595967769622803, "kl": 0.6726802475750446, "learning_rate": 4.104638999999999e-05, "loss": -0.0009, "num_tokens": 913165.0, "reward": 1.0125000476837158, "reward_std": 0.17054426670074463, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.3553735017776489, "sampling/importance_sampling_ratio/max": 1.542810320854187, "sampling/importance_sampling_ratio/mean": 0.9607316255569458, "sampling/importance_sampling_ratio/min": 0.14384639263153076, "sampling/sampling_logp_difference/max": 2.015115261077881, "sampling/sampling_logp_difference/mean": 0.05230295658111572, "step": 34, "step_time": 7.783849855999506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 270.625, "completions/mean_terminated_length": 270.625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.10143160965526477, "epoch": 0.0028, "frac_reward_zero_std": 0.25, "grad_norm": 0.05928665027022362, "kl": 1.0665921121835709, "learning_rate": 4.229022e-05, "loss": -0.0005, "num_tokens": 938898.0, "reward": 1.03125, "reward_std": 0.2308012843132019, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 0.2740820646286011, "sampling/importance_sampling_ratio/max": 2.6422765254974365, "sampling/importance_sampling_ratio/mean": 0.995773196220398, "sampling/importance_sampling_ratio/min": 0.061440229415893555, "sampling/sampling_logp_difference/max": 2.799118995666504, "sampling/sampling_logp_difference/mean": 0.07374110817909241, "step": 35, "step_time": 9.660342224000033 }, { "clip_ratio/high_max": 0.012500000186264515, "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 310.25, "completions/mean_terminated_length": 310.25, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.10469470010139048, "epoch": 0.00288, "frac_reward_zero_std": 0.5, "grad_norm": 0.048865459859371185, "kl": 0.9001794531941414, "learning_rate": 4.353405e-05, "loss": -0.0001, "num_tokens": 967266.0, "reward": 1.0125000476837158, "reward_std": 0.24554428458213806, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.3866731524467468, "sampling/importance_sampling_ratio/max": 2.6484134197235107, "sampling/importance_sampling_ratio/mean": 0.9453653693199158, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.43544602394104, "sampling/sampling_logp_difference/mean": 0.05557909607887268, "step": 36, "step_time": 7.993418754000686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013888888992369175, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013888888992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 271.53125, "completions/mean_terminated_length": 271.53125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.05944679502863437, "epoch": 0.00296, "frac_reward_zero_std": 0.25, "grad_norm": 0.03716575354337692, "kl": 1.2882579416036606, "learning_rate": 4.3534049870739164e-05, "loss": -0.0025, "num_tokens": 993242.0, "reward": 1.0499999523162842, "reward_std": 0.2683012783527374, "rewards/rollout_reward_func/mean": 1.0499999523162842, "rewards/rollout_reward_func/std": 0.3048003017902374, "sampling/importance_sampling_ratio/max": 1.130203127861023, "sampling/importance_sampling_ratio/mean": 0.8906633853912354, "sampling/importance_sampling_ratio/min": 0.05752609297633171, "sampling/sampling_logp_difference/max": 2.7043697834014893, "sampling/sampling_logp_difference/mean": 0.06861257553100586, "step": 37, "step_time": 7.89394630499919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 274.6875, "completions/mean_terminated_length": 274.6875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.055616328900214285, "epoch": 0.00304, "frac_reward_zero_std": 0.5, "grad_norm": 0.08999308198690414, "kl": 0.8978069573640823, "learning_rate": 4.3534049482956666e-05, "loss": -0.0021, "num_tokens": 1018940.0, "reward": 1.068750023841858, "reward_std": 0.19910253584384918, "rewards/rollout_reward_func/mean": 1.068750023841858, "rewards/rollout_reward_func/std": 0.29450616240501404, "sampling/importance_sampling_ratio/max": 2.192216396331787, "sampling/importance_sampling_ratio/mean": 0.9803860187530518, "sampling/importance_sampling_ratio/min": 0.11944349855184555, "sampling/sampling_logp_difference/max": 2.213015079498291, "sampling/sampling_logp_difference/mean": 0.06751498579978943, "step": 38, "step_time": 7.920984497000518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 290.34375, "completions/mean_terminated_length": 290.34375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.03700371854938567, "epoch": 0.00312, "frac_reward_zero_std": 0.625, "grad_norm": 0.02716164104640484, "kl": 0.8606064431369305, "learning_rate": 4.353404883665252e-05, "loss": -0.0009, "num_tokens": 1045833.0, "reward": 1.125, "reward_std": 0.15000000596046448, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.3911504745483398, "sampling/importance_sampling_ratio/mean": 0.9985114336013794, "sampling/importance_sampling_ratio/min": 0.01601959578692913, "sampling/sampling_logp_difference/max": 4.137251853942871, "sampling/sampling_logp_difference/mean": 0.05193096399307251, "step": 39, "step_time": 9.646959263001008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 279.6875, "completions/mean_terminated_length": 279.6875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.014073438418563455, "epoch": 0.0032, "frac_reward_zero_std": 0.625, "grad_norm": 0.7373865842819214, "kl": 11.778141215443611, "learning_rate": 4.353404793182672e-05, "loss": 0.0, "num_tokens": 1072334.0, "reward": 1.1437500715255737, "reward_std": 0.11250000447034836, "rewards/rollout_reward_func/mean": 1.1437500715255737, "rewards/rollout_reward_func/std": 0.17768675088882446, "sampling/importance_sampling_ratio/max": 1.0959906578063965, "sampling/importance_sampling_ratio/mean": 0.9251403212547302, "sampling/importance_sampling_ratio/min": 0.01429319754242897, "sampling/sampling_logp_difference/max": 4.253487586975098, "sampling/sampling_logp_difference/mean": 0.07442593574523926, "step": 40, "step_time": 7.754895531001239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014756944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014756944496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 306.0625, "completions/mean_terminated_length": 306.0625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.036442570431972854, "epoch": 0.00328, "frac_reward_zero_std": 0.5, "grad_norm": 0.021613966673612595, "kl": 0.853540938347578, "learning_rate": 4.3534046768479294e-05, "loss": -0.0009, "num_tokens": 1100709.0, "reward": 1.1062500476837158, "reward_std": 0.15580126643180847, "rewards/rollout_reward_func/mean": 1.1062500476837158, "rewards/rollout_reward_func/std": 0.22134123742580414, "sampling/importance_sampling_ratio/max": 1.5003032684326172, "sampling/importance_sampling_ratio/mean": 0.9760409593582153, "sampling/importance_sampling_ratio/min": 0.29808637499809265, "sampling/sampling_logp_difference/max": 1.2194957733154297, "sampling/sampling_logp_difference/mean": 0.02159947156906128, "step": 41, "step_time": 7.951497732000462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 373.0, "completions/max_terminated_length": 373.0, "completions/mean_length": 284.90625, "completions/mean_terminated_length": 284.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.06934978702338412, "epoch": 0.00336, "frac_reward_zero_std": 0.75, "grad_norm": 0.036122363060712814, "kl": 1.4169140756130219, "learning_rate": 4.353404534661025e-05, "loss": -0.0008, "num_tokens": 1126880.0, "reward": 1.125, "reward_std": 0.11830127239227295, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.3643670082092285, "sampling/importance_sampling_ratio/mean": 0.9154174327850342, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4062061309814453, "sampling/sampling_logp_difference/mean": 0.08993034809827805, "step": 42, "step_time": 7.846576690000347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 266.3125, "completions/mean_terminated_length": 266.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0414350210048724, "epoch": 0.00344, "frac_reward_zero_std": 0.75, "grad_norm": 0.013467486947774887, "kl": 0.816759143024683, "learning_rate": 4.353404366621962e-05, "loss": -0.0002, "num_tokens": 1152426.0, "reward": 1.1437500715255737, "reward_std": 0.08080127090215683, "rewards/rollout_reward_func/mean": 1.1437500715255737, "rewards/rollout_reward_func/std": 0.17768675088882446, "sampling/importance_sampling_ratio/max": 1.943003535270691, "sampling/importance_sampling_ratio/mean": 1.0006059408187866, "sampling/importance_sampling_ratio/min": 0.18744057416915894, "sampling/sampling_logp_difference/max": 1.7138478755950928, "sampling/sampling_logp_difference/mean": 0.03626240789890289, "step": 43, "step_time": 8.73944216100108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 300.03125, "completions/mean_terminated_length": 300.03125, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.03517312743497314, "epoch": 0.00352, "frac_reward_zero_std": 0.5, "grad_norm": 0.0695430189371109, "kl": 0.7897405922412872, "learning_rate": 4.3534041727307414e-05, "loss": -0.0002, "num_tokens": 1180321.0, "reward": 1.125, "reward_std": 0.15000000596046448, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 2.6711955070495605, "sampling/importance_sampling_ratio/mean": 1.0462613105773926, "sampling/importance_sampling_ratio/min": 0.09676270931959152, "sampling/sampling_logp_difference/max": 2.3355016708374023, "sampling/sampling_logp_difference/mean": 0.04019223898649216, "step": 44, "step_time": 7.855489442000817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 336.65625, "completions/mean_terminated_length": 336.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.14152880990877748, "epoch": 0.0036, "frac_reward_zero_std": 0.25, "grad_norm": 0.12354016304016113, "kl": 3.367022179067135, "learning_rate": 4.3534039529873685e-05, "loss": -0.0015, "num_tokens": 1209923.0, "reward": 0.9746875762939453, "reward_std": 0.2828759551048279, "rewards/rollout_reward_func/mean": 0.9746875762939453, "rewards/rollout_reward_func/std": 0.332517147064209, "sampling/importance_sampling_ratio/max": 2.4827404022216797, "sampling/importance_sampling_ratio/mean": 0.9489078521728516, "sampling/importance_sampling_ratio/min": 0.011774348095059395, "sampling/sampling_logp_difference/max": 4.442207336425781, "sampling/sampling_logp_difference/mean": 0.08008399605751038, "step": 45, "step_time": 7.916122444999928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 295.96875, "completions/mean_terminated_length": 295.96875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0650530201382935, "epoch": 0.00368, "frac_reward_zero_std": 0.75, "grad_norm": 0.01695827953517437, "kl": 1.0837746560573578, "learning_rate": 4.3534037073918466e-05, "loss": -0.0008, "num_tokens": 1237093.0, "reward": 1.125, "reward_std": 0.08660253882408142, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 2.1693990230560303, "sampling/importance_sampling_ratio/mean": 1.028033971786499, "sampling/importance_sampling_ratio/min": 0.220290407538414, "sampling/sampling_logp_difference/max": 1.5184905529022217, "sampling/sampling_logp_difference/mean": 0.02910466678440571, "step": 46, "step_time": 7.882395288999305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 275.90625, "completions/mean_terminated_length": 275.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.08647931751329452, "epoch": 0.00376, "frac_reward_zero_std": 0.375, "grad_norm": 0.10817690193653107, "kl": 4.742111161351204, "learning_rate": 4.353403435944177e-05, "loss": 0.0007, "num_tokens": 1263258.0, "reward": 1.0500000715255737, "reward_std": 0.2683012783527374, "rewards/rollout_reward_func/mean": 1.0500000715255737, "rewards/rollout_reward_func/std": 0.34077712893486023, "sampling/importance_sampling_ratio/max": 1.9073151350021362, "sampling/importance_sampling_ratio/mean": 1.0629686117172241, "sampling/importance_sampling_ratio/min": 0.32909950613975525, "sampling/sampling_logp_difference/max": 1.3324027061462402, "sampling/sampling_logp_difference/mean": 0.03618936985731125, "step": 47, "step_time": 9.591066821999448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.84375, "completions/mean_terminated_length": 290.84375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.15111199161037803, "epoch": 0.00384, "frac_reward_zero_std": 0.25, "grad_norm": 0.03760185465216637, "kl": 0.9303759709000587, "learning_rate": 4.353403138644366e-05, "loss": -0.0016, "num_tokens": 1289811.0, "reward": 0.9187500476837158, "reward_std": 0.312582790851593, "rewards/rollout_reward_func/mean": 0.9187500476837158, "rewards/rollout_reward_func/std": 0.40276265144348145, "sampling/importance_sampling_ratio/max": 1.7451053857803345, "sampling/importance_sampling_ratio/mean": 0.9505935311317444, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.5125484466552734, "sampling/sampling_logp_difference/mean": 0.06414590775966644, "step": 48, "step_time": 7.962967010999819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 315.71875, "completions/mean_terminated_length": 315.71875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.13763120886869729, "epoch": 0.00392, "frac_reward_zero_std": 0.25, "grad_norm": 0.045121848583221436, "kl": 1.1776729747653008, "learning_rate": 4.3534028154924186e-05, "loss": -0.0005, "num_tokens": 1318617.0, "reward": 1.0125000476837158, "reward_std": 0.2366025447845459, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.2825574576854706, "sampling/importance_sampling_ratio/max": 1.6848394870758057, "sampling/importance_sampling_ratio/mean": 1.0096924304962158, "sampling/importance_sampling_ratio/min": 0.4476051330566406, "sampling/sampling_logp_difference/max": 0.8038437366485596, "sampling/sampling_logp_difference/mean": 0.04047476127743721, "step": 49, "step_time": 7.942660953000086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 336.46875, "completions/mean_terminated_length": 336.46875, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.18235087627545, "epoch": 0.004, "frac_reward_zero_std": 0.25, "grad_norm": 0.028358319774270058, "kl": 0.7837071008980274, "learning_rate": 4.35340246648834e-05, "loss": -0.0005, "num_tokens": 1348189.0, "reward": 0.9000000357627869, "reward_std": 0.28884556889533997, "rewards/rollout_reward_func/mean": 0.9000000357627869, "rewards/rollout_reward_func/std": 0.40321293473243713, "sampling/importance_sampling_ratio/max": 1.9457248449325562, "sampling/importance_sampling_ratio/mean": 1.0235942602157593, "sampling/importance_sampling_ratio/min": 0.6214995384216309, "sampling/sampling_logp_difference/max": 0.5035011768341064, "sampling/sampling_logp_difference/mean": 0.039148926734924316, "step": 50, "step_time": 8.006501671000024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 270.09375, "completions/mean_terminated_length": 270.09375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.11485931277275085, "epoch": 0.00408, "frac_reward_zero_std": 0.375, "grad_norm": 0.025022653862833977, "kl": 0.788881704211235, "learning_rate": 4.3534020916321335e-05, "loss": 0.0007, "num_tokens": 1373786.0, "reward": 1.068750023841858, "reward_std": 0.2308012843132019, "rewards/rollout_reward_func/mean": 1.068750023841858, "rewards/rollout_reward_func/std": 0.29450616240501404, "sampling/importance_sampling_ratio/max": 2.239361524581909, "sampling/importance_sampling_ratio/mean": 1.0327645540237427, "sampling/importance_sampling_ratio/min": 0.6819923520088196, "sampling/sampling_logp_difference/max": 0.6789684295654297, "sampling/sampling_logp_difference/mean": 0.024581894278526306, "step": 51, "step_time": 9.941127948000485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 275.6875, "completions/mean_terminated_length": 275.6875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.11507323710247874, "epoch": 0.00416, "frac_reward_zero_std": 0.25, "grad_norm": 0.05924519896507263, "kl": 0.8558782301843166, "learning_rate": 4.3534016909238075e-05, "loss": -0.0007, "num_tokens": 1400039.0, "reward": 1.0125000476837158, "reward_std": 0.2709095776081085, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.3210366368293762, "sampling/importance_sampling_ratio/max": 1.4944210052490234, "sampling/importance_sampling_ratio/mean": 0.9199806451797485, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.2766759395599365, "sampling/sampling_logp_difference/mean": 0.04160736873745918, "step": 52, "step_time": 7.940384456000629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 373.0, "completions/max_terminated_length": 373.0, "completions/mean_length": 260.3125, "completions/mean_terminated_length": 260.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1886470913887024, "epoch": 0.00424, "frac_reward_zero_std": 0.25, "grad_norm": 0.0540018193423748, "kl": 0.9725520387291908, "learning_rate": 4.353401264363367e-05, "loss": 0.0004, "num_tokens": 1424821.0, "reward": 1.0125000476837158, "reward_std": 0.30260831117630005, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.3553735017776489, "sampling/importance_sampling_ratio/max": 1.8682571649551392, "sampling/importance_sampling_ratio/mean": 1.0141847133636475, "sampling/importance_sampling_ratio/min": 0.5016090869903564, "sampling/sampling_logp_difference/max": 0.6861862540245056, "sampling/sampling_logp_difference/mean": 0.03720526769757271, "step": 53, "step_time": 7.756263882999519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 255.375, "completions/mean_terminated_length": 255.375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.18109502270817757, "epoch": 0.00432, "frac_reward_zero_std": 0.25, "grad_norm": 0.037197887897491455, "kl": 1.6266912147402763, "learning_rate": 4.35340081195082e-05, "loss": 0.0006, "num_tokens": 1449087.0, "reward": 0.9000000357627869, "reward_std": 0.3321468234062195, "rewards/rollout_reward_func/mean": 0.9000000357627869, "rewards/rollout_reward_func/std": 0.40321293473243713, "sampling/importance_sampling_ratio/max": 2.0104808807373047, "sampling/importance_sampling_ratio/mean": 1.0253084897994995, "sampling/importance_sampling_ratio/min": 0.20926451683044434, "sampling/sampling_logp_difference/max": 1.2017450332641602, "sampling/sampling_logp_difference/mean": 0.04562991484999657, "step": 54, "step_time": 7.661444229000153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 270.0625, "completions/mean_terminated_length": 270.0625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1505243186838925, "epoch": 0.0044, "frac_reward_zero_std": 0.125, "grad_norm": 0.05768263339996338, "kl": 1.240611456334591, "learning_rate": 4.353400333686172e-05, "loss": -0.0005, "num_tokens": 1474873.0, "reward": 0.9937500953674316, "reward_std": 0.3237372636795044, "rewards/rollout_reward_func/mean": 0.9937500953674316, "rewards/rollout_reward_func/std": 0.3609463572502136, "sampling/importance_sampling_ratio/max": 1.6629772186279297, "sampling/importance_sampling_ratio/mean": 0.9884673357009888, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.340965747833252, "sampling/sampling_logp_difference/mean": 0.02907535620033741, "step": 55, "step_time": 9.891223988000093 }, { "clip_ratio/high_max": 0.029513888992369175, "clip_ratio/high_mean": 0.014756944496184587, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.021701388992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 321.15625, "completions/mean_terminated_length": 321.15625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.21555871050804853, "epoch": 0.00448, "frac_reward_zero_std": 0.25, "grad_norm": 0.04618807137012482, "kl": 1.2864653170108795, "learning_rate": 4.353399829569432e-05, "loss": 0.0015, "num_tokens": 1503831.0, "reward": 0.9187500476837158, "reward_std": 0.2424038052558899, "rewards/rollout_reward_func/mean": 0.9187500476837158, "rewards/rollout_reward_func/std": 0.34024423360824585, "sampling/importance_sampling_ratio/max": 2.176079034805298, "sampling/importance_sampling_ratio/mean": 1.0545737743377686, "sampling/importance_sampling_ratio/min": 0.2854979634284973, "sampling/sampling_logp_difference/max": 1.254502773284912, "sampling/sampling_logp_difference/mean": 0.056082580238580704, "step": 56, "step_time": 7.938374097998803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.015625, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 285.1875, "completions/mean_terminated_length": 285.1875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.1718423580750823, "epoch": 0.00456, "frac_reward_zero_std": 0.25, "grad_norm": 0.03429986163973808, "kl": 1.25742681697011, "learning_rate": 4.353399299600607e-05, "loss": 0.0002, "num_tokens": 1530296.0, "reward": 1.0125000476837158, "reward_std": 0.2799038290977478, "rewards/rollout_reward_func/mean": 1.0125000476837158, "rewards/rollout_reward_func/std": 0.3210366368293762, "sampling/importance_sampling_ratio/max": 1.6768337488174438, "sampling/importance_sampling_ratio/mean": 1.0217704772949219, "sampling/importance_sampling_ratio/min": 0.74320387840271, "sampling/sampling_logp_difference/max": 0.5169110298156738, "sampling/sampling_logp_difference/mean": 0.026178814470767975, "step": 57, "step_time": 7.855257410999457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 280.3125, "completions/mean_terminated_length": 280.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.19798619952052832, "epoch": 0.00464, "frac_reward_zero_std": 0.375, "grad_norm": 0.03177686780691147, "kl": 0.9109754916280508, "learning_rate": 4.353398743779707e-05, "loss": 0.0003, "num_tokens": 1556651.0, "reward": 0.956250011920929, "reward_std": 0.25611406564712524, "rewards/rollout_reward_func/mean": 0.956250011920929, "rewards/rollout_reward_func/std": 0.39914223551750183, "sampling/importance_sampling_ratio/max": 1.3695523738861084, "sampling/importance_sampling_ratio/mean": 0.9952094554901123, "sampling/importance_sampling_ratio/min": 0.626275360584259, "sampling/sampling_logp_difference/max": 0.4453871250152588, "sampling/sampling_logp_difference/mean": 0.02335367724299431, "step": 58, "step_time": 7.935053289999814 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 317.0625, "completions/mean_terminated_length": 317.0625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.15008317783940583, "epoch": 0.00472, "frac_reward_zero_std": 0.25, "grad_norm": 0.05143209174275398, "kl": 1.122834250330925, "learning_rate": 4.353398162106738e-05, "loss": -0.0002, "num_tokens": 1585168.0, "reward": 0.9187500476837158, "reward_std": 0.3173513412475586, "rewards/rollout_reward_func/mean": 0.9187500476837158, "rewards/rollout_reward_func/std": 0.4306315779685974, "sampling/importance_sampling_ratio/max": 1.5250457525253296, "sampling/importance_sampling_ratio/mean": 0.993643045425415, "sampling/importance_sampling_ratio/min": 0.39288732409477234, "sampling/sampling_logp_difference/max": 0.9096617698669434, "sampling/sampling_logp_difference/mean": 0.03307843208312988, "step": 59, "step_time": 9.853581047000262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 300.125, "completions/mean_terminated_length": 300.125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.1486674400512129, "epoch": 0.0048, "frac_reward_zero_std": 0.125, "grad_norm": 0.06002556532621384, "kl": 2.8148342967033386, "learning_rate": 4.353397554581712e-05, "loss": 0.0003, "num_tokens": 1612781.0, "reward": 0.9750000238418579, "reward_std": 0.33691534399986267, "rewards/rollout_reward_func/mean": 0.9750000238418579, "rewards/rollout_reward_func/std": 0.3654427230358124, "sampling/importance_sampling_ratio/max": 2.386046886444092, "sampling/importance_sampling_ratio/mean": 1.0082030296325684, "sampling/importance_sampling_ratio/min": 0.312150239944458, "sampling/sampling_logp_difference/max": 1.1684985160827637, "sampling/sampling_logp_difference/mean": 0.02929677441716194, "step": 60, "step_time": 7.82678515499947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 265.8125, "completions/mean_terminated_length": 265.8125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.09197181701892987, "epoch": 0.00488, "frac_reward_zero_std": 0.5, "grad_norm": 0.03861580044031143, "kl": 0.7230320759117603, "learning_rate": 4.3533969212046366e-05, "loss": -0.0002, "num_tokens": 1638097.0, "reward": 1.125, "reward_std": 0.15000000596046448, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 1.6914225816726685, "sampling/importance_sampling_ratio/mean": 0.982423722743988, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.3486332893371582, "sampling/sampling_logp_difference/mean": 0.033385761082172394, "step": 61, "step_time": 7.802646727000592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014756944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014756944496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 321.0, "completions/mean_terminated_length": 321.0, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.15609693014994264, "epoch": 0.00496, "frac_reward_zero_std": 0.125, "grad_norm": 0.03867170214653015, "kl": 1.002743236720562, "learning_rate": 4.3533962619755234e-05, "loss": -0.0026, "num_tokens": 1666575.0, "reward": 0.9937499761581421, "reward_std": 0.34010833501815796, "rewards/rollout_reward_func/mean": 0.9937499761581421, "rewards/rollout_reward_func/std": 0.3609463572502136, "sampling/importance_sampling_ratio/max": 1.8810590505599976, "sampling/importance_sampling_ratio/mean": 0.9881374835968018, "sampling/importance_sampling_ratio/min": 0.34196531772613525, "sampling/sampling_logp_difference/max": 1.1645917892456055, "sampling/sampling_logp_difference/mean": 0.037138473242521286, "step": 62, "step_time": 8.01059179899994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 284.9375, "completions/mean_terminated_length": 284.9375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0790540580637753, "epoch": 0.00504, "frac_reward_zero_std": 0.625, "grad_norm": 0.019840601831674576, "kl": 1.298482820391655, "learning_rate": 4.353395576894381e-05, "loss": -0.0003, "num_tokens": 1692937.0, "reward": 1.1062500476837158, "reward_std": 0.12410254031419754, "rewards/rollout_reward_func/mean": 1.1062500476837158, "rewards/rollout_reward_func/std": 0.22134123742580414, "sampling/importance_sampling_ratio/max": 1.324730396270752, "sampling/importance_sampling_ratio/mean": 0.9982839822769165, "sampling/importance_sampling_ratio/min": 0.21199165284633636, "sampling/sampling_logp_difference/max": 1.6455206871032715, "sampling/sampling_logp_difference/mean": 0.02978626638650894, "step": 63, "step_time": 9.459651282999403 }, { "clip_ratio/high_max": 0.013888888992369175, "clip_ratio/high_mean": 0.0069444444961845875, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.013888888992369175, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 296.78125, "completions/mean_terminated_length": 296.78125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.054197820369154215, "epoch": 0.00512, "frac_reward_zero_std": 0.625, "grad_norm": 0.03027612715959549, "kl": 1.1264933682978153, "learning_rate": 4.353394865961223e-05, "loss": -0.0005, "num_tokens": 1720632.0, "reward": 1.125, "reward_std": 0.11830127239227295, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 1.0344293117523193, "sampling/importance_sampling_ratio/mean": 0.907318115234375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.201116919517517, "sampling/sampling_logp_difference/mean": 0.03517932444810867, "step": 64, "step_time": 7.9200876330005485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.3125, "completions/mean_terminated_length": 290.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.04984044888988137, "epoch": 0.0052, "frac_reward_zero_std": 0.75, "grad_norm": 0.028782792389392853, "kl": 1.0349424369633198, "learning_rate": 4.353394129176058e-05, "loss": -0.0, "num_tokens": 1747524.0, "reward": 1.162500023841858, "reward_std": 0.07500000298023224, "rewards/rollout_reward_func/mean": 1.162500023841858, "rewards/rollout_reward_func/std": 0.14756081998348236, "sampling/importance_sampling_ratio/max": 1.599551796913147, "sampling/importance_sampling_ratio/mean": 1.0256993770599365, "sampling/importance_sampling_ratio/min": 0.5776486992835999, "sampling/sampling_logp_difference/max": 0.6034482717514038, "sampling/sampling_logp_difference/mean": 0.013537315651774406, "step": 65, "step_time": 7.888362634000714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 301.21875, "completions/mean_terminated_length": 301.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.025275442050769925, "epoch": 0.00528, "frac_reward_zero_std": 1.0, "grad_norm": 6.729937740601599e-05, "kl": 0.9362634420394897, "learning_rate": 4.353393366538898e-05, "loss": 0.0002, "num_tokens": 1775315.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.261993169784546, "sampling/importance_sampling_ratio/mean": 1.0223727226257324, "sampling/importance_sampling_ratio/min": 0.9997139573097229, "sampling/sampling_logp_difference/max": 0.15536287426948547, "sampling/sampling_logp_difference/mean": 0.004749409854412079, "step": 66, "step_time": 9.055721840000842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 270.1875, "completions/mean_terminated_length": 270.1875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.05280795122962445, "epoch": 0.00536, "frac_reward_zero_std": 0.5, "grad_norm": 0.3722967207431793, "kl": 6.000564677640796, "learning_rate": 4.353392578049757e-05, "loss": -0.0013, "num_tokens": 1800725.0, "reward": 1.1062500476837158, "reward_std": 0.1875, "rewards/rollout_reward_func/mean": 1.1062500476837158, "rewards/rollout_reward_func/std": 0.26873359084129333, "sampling/importance_sampling_ratio/max": 1.6750870943069458, "sampling/importance_sampling_ratio/mean": 0.9590173959732056, "sampling/importance_sampling_ratio/min": 0.043312977999448776, "sampling/sampling_logp_difference/max": 3.155855178833008, "sampling/sampling_logp_difference/mean": 0.053839217871427536, "step": 67, "step_time": 8.95276670399926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014756944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014756944496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 315.84375, "completions/mean_terminated_length": 315.84375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.05910730984760448, "epoch": 0.00544, "frac_reward_zero_std": 0.375, "grad_norm": 0.045257288962602615, "kl": 1.602789431810379, "learning_rate": 4.353391763708646e-05, "loss": -0.0011, "num_tokens": 1829345.0, "reward": 1.0496875047683716, "reward_std": 0.2689262926578522, "rewards/rollout_reward_func/mean": 1.0496875047683716, "rewards/rollout_reward_func/std": 0.3417741656303406, "sampling/importance_sampling_ratio/max": 2.4444875717163086, "sampling/importance_sampling_ratio/mean": 1.0089073181152344, "sampling/importance_sampling_ratio/min": 0.3840332329273224, "sampling/sampling_logp_difference/max": 1.1453220844268799, "sampling/sampling_logp_difference/mean": 0.03663824126124382, "step": 68, "step_time": 7.980377719999979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 280.6875, "completions/mean_terminated_length": 280.6875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.013986794830998406, "epoch": 0.00552, "frac_reward_zero_std": 0.875, "grad_norm": 0.037016239017248154, "kl": 2.0089162215590477, "learning_rate": 4.353390923515578e-05, "loss": -0.0002, "num_tokens": 1856092.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.0496174097061157, "sampling/importance_sampling_ratio/mean": 0.982447624206543, "sampling/importance_sampling_ratio/min": 0.3039742708206177, "sampling/sampling_logp_difference/max": 1.192826509475708, "sampling/sampling_logp_difference/mean": 0.010450880043208599, "step": 69, "step_time": 7.760221334000107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 280.0, "completions/mean_terminated_length": 280.0, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.052246630541048944, "epoch": 0.0056, "frac_reward_zero_std": 0.875, "grad_norm": 0.012879423797130585, "kl": 0.9396154209971428, "learning_rate": 4.353390057470567e-05, "loss": 0.0003, "num_tokens": 1882603.0, "reward": 1.1437500715255737, "reward_std": 0.07180703431367874, "rewards/rollout_reward_func/mean": 1.1437500715255737, "rewards/rollout_reward_func/std": 0.23409055173397064, "sampling/importance_sampling_ratio/max": 1.5393744707107544, "sampling/importance_sampling_ratio/mean": 1.0376193523406982, "sampling/importance_sampling_ratio/min": 0.9416899085044861, "sampling/sampling_logp_difference/max": 0.41953563690185547, "sampling/sampling_logp_difference/mean": 0.009257299825549126, "step": 70, "step_time": 8.883858626999881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0069444444961845875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0069444444961845875, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 311.09375, "completions/mean_terminated_length": 311.09375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.08476114110089839, "epoch": 0.00568, "frac_reward_zero_std": 0.25, "grad_norm": 0.0340198390185833, "kl": 1.4748088456690311, "learning_rate": 4.353389165573626e-05, "loss": -0.0006, "num_tokens": 1910642.0, "reward": 1.03125, "reward_std": 0.2651083171367645, "rewards/rollout_reward_func/mean": 1.03125, "rewards/rollout_reward_func/std": 0.31360289454460144, "sampling/importance_sampling_ratio/max": 1.9891877174377441, "sampling/importance_sampling_ratio/mean": 0.9928376078605652, "sampling/importance_sampling_ratio/min": 0.41794291138648987, "sampling/sampling_logp_difference/max": 0.8778750896453857, "sampling/sampling_logp_difference/mean": 0.039408549666404724, "step": 71, "step_time": 9.080612050000582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 264.8125, "completions/mean_terminated_length": 264.8125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0572042342973873, "epoch": 0.00576, "frac_reward_zero_std": 0.625, "grad_norm": 0.02946249023079872, "kl": 0.7823966294527054, "learning_rate": 4.353388247824768e-05, "loss": -0.0008, "num_tokens": 1936092.0, "reward": 1.1062500476837158, "reward_std": 0.15580126643180847, "rewards/rollout_reward_func/mean": 1.1062500476837158, "rewards/rollout_reward_func/std": 0.26873359084129333, "sampling/importance_sampling_ratio/max": 1.5228697061538696, "sampling/importance_sampling_ratio/mean": 1.0042927265167236, "sampling/importance_sampling_ratio/min": 0.3932654559612274, "sampling/sampling_logp_difference/max": 1.0526652336120605, "sampling/sampling_logp_difference/mean": 0.021640101447701454, "step": 72, "step_time": 7.777483000999837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 263.9375, "completions/mean_terminated_length": 263.9375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.03995063988259062, "epoch": 0.00584, "frac_reward_zero_std": 0.75, "grad_norm": 0.0132850157096982, "kl": 0.9739180095493793, "learning_rate": 4.3533873042240096e-05, "loss": -0.0005, "num_tokens": 1961205.0, "reward": 1.1621875762939453, "reward_std": 0.07562500238418579, "rewards/rollout_reward_func/mean": 1.1621875762939453, "rewards/rollout_reward_func/std": 0.14879591763019562, "sampling/importance_sampling_ratio/max": 1.1371190547943115, "sampling/importance_sampling_ratio/mean": 0.9878384470939636, "sampling/importance_sampling_ratio/min": 0.48520469665527344, "sampling/sampling_logp_difference/max": 0.7782995700836182, "sampling/sampling_logp_difference/mean": 0.011758463457226753, "step": 73, "step_time": 7.85138319600037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 285.90625, "completions/mean_terminated_length": 285.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.07039971114136279, "epoch": 0.00592, "frac_reward_zero_std": 0.625, "grad_norm": 0.03469613194465637, "kl": 0.8201285600662231, "learning_rate": 4.353386334771366e-05, "loss": -0.0009, "num_tokens": 1987812.0, "reward": 1.125, "reward_std": 0.15000000596046448, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.6029505729675293, "sampling/importance_sampling_ratio/mean": 1.0400995016098022, "sampling/importance_sampling_ratio/min": 0.5984629988670349, "sampling/sampling_logp_difference/max": 0.515189528465271, "sampling/sampling_logp_difference/mean": 0.019253287464380264, "step": 74, "step_time": 9.8747933249997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 255.125, "completions/mean_terminated_length": 255.125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.04503151042445097, "epoch": 0.006, "frac_reward_zero_std": 0.5, "grad_norm": 0.02260386385023594, "kl": 0.9310923218727112, "learning_rate": 4.353385339466851e-05, "loss": -0.0007, "num_tokens": 2012450.0, "reward": 1.0875000953674316, "reward_std": 0.1843070387840271, "rewards/rollout_reward_func/mean": 1.0875000953674316, "rewards/rollout_reward_func/std": 0.2825574576854706, "sampling/importance_sampling_ratio/max": 1.081639051437378, "sampling/importance_sampling_ratio/mean": 0.9646203517913818, "sampling/importance_sampling_ratio/min": 0.49109765887260437, "sampling/sampling_logp_difference/max": 0.7123596668243408, "sampling/sampling_logp_difference/mean": 0.0154666593298316, "step": 75, "step_time": 7.935186095000063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 290.125, "completions/mean_terminated_length": 290.125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.03961527335923165, "epoch": 0.00608, "frac_reward_zero_std": 0.75, "grad_norm": 0.011047380976378918, "kl": 1.1142286211252213, "learning_rate": 4.35338431831048e-05, "loss": -0.0005, "num_tokens": 2039241.0, "reward": 1.162500023841858, "reward_std": 0.07500000298023224, "rewards/rollout_reward_func/mean": 1.162500023841858, "rewards/rollout_reward_func/std": 0.14756081998348236, "sampling/importance_sampling_ratio/max": 1.4051661491394043, "sampling/importance_sampling_ratio/mean": 1.0055490732192993, "sampling/importance_sampling_ratio/min": 0.25845175981521606, "sampling/sampling_logp_difference/max": 1.3545377254486084, "sampling/sampling_logp_difference/mean": 0.01653273031115532, "step": 76, "step_time": 7.943028566999601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 322.3125, "completions/mean_terminated_length": 322.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.06818312918767333, "epoch": 0.00616, "frac_reward_zero_std": 0.75, "grad_norm": 0.013988149352371693, "kl": 0.9275210164487362, "learning_rate": 4.353383271302271e-05, "loss": -0.0001, "num_tokens": 2068379.0, "reward": 1.125, "reward_std": 0.07500000298023224, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 1.2771904468536377, "sampling/importance_sampling_ratio/mean": 1.0031827688217163, "sampling/importance_sampling_ratio/min": 0.41439446806907654, "sampling/sampling_logp_difference/max": 0.9727766513824463, "sampling/sampling_logp_difference/mean": 0.0158405564725399, "step": 77, "step_time": 7.930315806000181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 315.9375, "completions/mean_terminated_length": 315.9375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0645121990237385, "epoch": 0.00624, "frac_reward_zero_std": 0.75, "grad_norm": 0.024502430111169815, "kl": 1.2259608879685402, "learning_rate": 4.3533821984422386e-05, "loss": -0.0002, "num_tokens": 2096741.0, "reward": 1.162500023841858, "reward_std": 0.07500000298023224, "rewards/rollout_reward_func/mean": 1.162500023841858, "rewards/rollout_reward_func/std": 0.14756081998348236, "sampling/importance_sampling_ratio/max": 1.182511329650879, "sampling/importance_sampling_ratio/mean": 1.0009673833847046, "sampling/importance_sampling_ratio/min": 0.753045916557312, "sampling/sampling_logp_difference/max": 0.36170101165771484, "sampling/sampling_logp_difference/mean": 0.010112968273460865, "step": 78, "step_time": 10.007045330999517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 279.75, "completions/mean_terminated_length": 279.75, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.05705379619030282, "epoch": 0.00632, "frac_reward_zero_std": 0.625, "grad_norm": 0.028117062523961067, "kl": 0.8632127866148949, "learning_rate": 4.353381099730402e-05, "loss": -0.0001, "num_tokens": 2122579.0, "reward": 1.1437499523162842, "reward_std": 0.11250000447034836, "rewards/rollout_reward_func/mean": 1.1437499523162842, "rewards/rollout_reward_func/std": 0.17768675088882446, "sampling/importance_sampling_ratio/max": 1.4057514667510986, "sampling/importance_sampling_ratio/mean": 1.0248793363571167, "sampling/importance_sampling_ratio/min": 0.8742473721504211, "sampling/sampling_logp_difference/max": 0.2085188627243042, "sampling/sampling_logp_difference/mean": 0.008420808240771294, "step": 79, "step_time": 7.886672663999889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 316.46875, "completions/mean_terminated_length": 316.46875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.060832820541691035, "epoch": 0.0064, "frac_reward_zero_std": 0.625, "grad_norm": 0.03975361958146095, "kl": 1.6412314306944609, "learning_rate": 4.353379975166777e-05, "loss": -0.0002, "num_tokens": 2151528.0, "reward": 1.1437499523162842, "reward_std": 0.11250000447034836, "rewards/rollout_reward_func/mean": 1.1437499523162842, "rewards/rollout_reward_func/std": 0.17768675088882446, "sampling/importance_sampling_ratio/max": 2.8017160892486572, "sampling/importance_sampling_ratio/mean": 1.0404901504516602, "sampling/importance_sampling_ratio/min": 0.4486793875694275, "sampling/sampling_logp_difference/max": 0.9562540054321289, "sampling/sampling_logp_difference/mean": 0.01940784603357315, "step": 80, "step_time": 7.957194036000146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013194444589316845, "clip_ratio/low_min": 0.012500000186264515, "clip_ratio/region_mean": 0.013194444589316845, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 305.71875, "completions/mean_terminated_length": 305.71875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.04899762541754171, "epoch": 0.00648, "frac_reward_zero_std": 0.75, "grad_norm": 0.02301192656159401, "kl": 1.0116124227643013, "learning_rate": 4.3533788247513815e-05, "loss": -0.0006, "num_tokens": 2179702.0, "reward": 1.125, "reward_std": 0.08660253882408142, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.20160645246505737, "sampling/importance_sampling_ratio/max": 1.875569462776184, "sampling/importance_sampling_ratio/mean": 1.0101540088653564, "sampling/importance_sampling_ratio/min": 0.28503507375717163, "sampling/sampling_logp_difference/max": 1.2559183835983276, "sampling/sampling_logp_difference/mean": 0.01926511526107788, "step": 81, "step_time": 7.978154268000253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 297.03125, "completions/mean_terminated_length": 297.03125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.022272392467129976, "epoch": 0.00656, "frac_reward_zero_std": 0.875, "grad_norm": 0.013292910531163216, "kl": 0.9102301821112633, "learning_rate": 4.3533776484842344e-05, "loss": 0.0001, "num_tokens": 2207120.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.136574149131775, "sampling/importance_sampling_ratio/mean": 1.0120327472686768, "sampling/importance_sampling_ratio/min": 0.9999644160270691, "sampling/sampling_logp_difference/max": 0.11654043197631836, "sampling/sampling_logp_difference/mean": 0.0026099944952875376, "step": 82, "step_time": 8.864897964000193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 326.25, "completions/mean_terminated_length": 326.25, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.019051656941883266, "epoch": 0.00664, "frac_reward_zero_std": 0.875, "grad_norm": 0.004411371424794197, "kl": 0.8523155301809311, "learning_rate": 4.353376446365354e-05, "loss": 0.0006, "num_tokens": 2236599.0, "reward": 1.1968750953674316, "reward_std": 0.006249999161809683, "rewards/rollout_reward_func/mean": 1.1968750953674316, "rewards/rollout_reward_func/std": 0.01767767407000065, "sampling/importance_sampling_ratio/max": 1.1605231761932373, "sampling/importance_sampling_ratio/mean": 1.011759638786316, "sampling/importance_sampling_ratio/min": 0.9993352293968201, "sampling/sampling_logp_difference/max": 0.14883357286453247, "sampling/sampling_logp_difference/mean": 0.0025451905094087124, "step": 83, "step_time": 8.064721024999017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 295.40625, "completions/mean_terminated_length": 295.40625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.028427855169866234, "epoch": 0.00672, "frac_reward_zero_std": 0.875, "grad_norm": 0.017765698954463005, "kl": 0.926544539630413, "learning_rate": 4.353375218394759e-05, "loss": -0.0002, "num_tokens": 2264012.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.2200895547866821, "sampling/importance_sampling_ratio/mean": 1.0012845993041992, "sampling/importance_sampling_ratio/min": 0.5183109641075134, "sampling/sampling_logp_difference/max": 0.6549568176269531, "sampling/sampling_logp_difference/mean": 0.008520926348865032, "step": 84, "step_time": 8.0084450559998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.014756944496184587, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014756944496184587, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 307.21875, "completions/mean_terminated_length": 307.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.03962784991017543, "epoch": 0.0068, "frac_reward_zero_std": 0.75, "grad_norm": 0.04151219502091408, "kl": 0.9067967049777508, "learning_rate": 4.35337396457247e-05, "loss": -0.0004, "num_tokens": 2291712.0, "reward": 1.125, "reward_std": 0.10930703580379486, "rewards/rollout_reward_func/mean": 1.125, "rewards/rollout_reward_func/std": 0.25272706151008606, "sampling/importance_sampling_ratio/max": 1.7602646350860596, "sampling/importance_sampling_ratio/mean": 1.0363727807998657, "sampling/importance_sampling_ratio/min": 0.6363921165466309, "sampling/sampling_logp_difference/max": 0.5586846470832825, "sampling/sampling_logp_difference/mean": 0.013922005891799927, "step": 85, "step_time": 8.0002799949998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0078125, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 265.65625, "completions/mean_terminated_length": 265.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.02815404417924583, "epoch": 0.00688, "frac_reward_zero_std": 0.875, "grad_norm": 0.005533280316740274, "kl": 1.1824943646788597, "learning_rate": 4.353372684898505e-05, "loss": -0.0002, "num_tokens": 2316453.0, "reward": 1.162500023841858, "reward_std": 0.04330126941204071, "rewards/rollout_reward_func/mean": 1.162500023841858, "rewards/rollout_reward_func/std": 0.14756081998348236, "sampling/importance_sampling_ratio/max": 1.162795066833496, "sampling/importance_sampling_ratio/mean": 0.9892362952232361, "sampling/importance_sampling_ratio/min": 0.5705499649047852, "sampling/sampling_logp_difference/max": 0.5614519119262695, "sampling/sampling_logp_difference/mean": 0.010713176801800728, "step": 86, "step_time": 9.556566259999727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 326.03125, "completions/mean_terminated_length": 326.03125, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.01459448179230094, "epoch": 0.00696, "frac_reward_zero_std": 1.0, "grad_norm": 2.0734010831802152e-05, "kl": 0.9236952438950539, "learning_rate": 4.353371379372886e-05, "loss": 0.0002, "num_tokens": 2345735.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0470086336135864, "sampling/importance_sampling_ratio/mean": 1.0028676986694336, "sampling/importance_sampling_ratio/min": 0.9835476279258728, "sampling/sampling_logp_difference/max": 0.029721900820732117, "sampling/sampling_logp_difference/mean": 0.0011603579623624682, "step": 87, "step_time": 7.82755037700008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 260.65625, "completions/mean_terminated_length": 260.65625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.009473193844314665, "epoch": 0.00704, "frac_reward_zero_std": 1.0, "grad_norm": 1.618634269107133e-05, "kl": 0.9003307521343231, "learning_rate": 4.353370047995632e-05, "loss": 0.0001, "num_tokens": 2370623.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0867642164230347, "sampling/importance_sampling_ratio/mean": 1.0046534538269043, "sampling/importance_sampling_ratio/min": 0.9965481758117676, "sampling/sampling_logp_difference/max": 0.08583112061023712, "sampling/sampling_logp_difference/mean": 0.0012172880815342069, "step": 88, "step_time": 7.934258312999646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 284.78125, "completions/mean_terminated_length": 284.78125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.012512716493802145, "epoch": 0.00712, "frac_reward_zero_std": 0.875, "grad_norm": 0.020356306806206703, "kl": 0.9552054032683372, "learning_rate": 4.353368690766766e-05, "loss": 0.0001, "num_tokens": 2396695.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.0297937393188477, "sampling/importance_sampling_ratio/mean": 0.9976717233657837, "sampling/importance_sampling_ratio/min": 0.7878608107566833, "sampling/sampling_logp_difference/max": 0.23846888542175293, "sampling/sampling_logp_difference/mean": 0.0028896075673401356, "step": 89, "step_time": 7.977665292999518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 311.875, "completions/mean_terminated_length": 311.875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.013359704083995894, "epoch": 0.0072, "frac_reward_zero_std": 1.0, "grad_norm": 1.628275276743807e-05, "kl": 0.8991029858589172, "learning_rate": 4.353367307686308e-05, "loss": 0.0002, "num_tokens": 2425068.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.083115577697754, "sampling/importance_sampling_ratio/mean": 1.0064340829849243, "sampling/importance_sampling_ratio/min": 0.9995262026786804, "sampling/sampling_logp_difference/max": 0.04911571741104126, "sampling/sampling_logp_difference/mean": 0.0014375498285517097, "step": 90, "step_time": 9.586448382001436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 281.3125, "completions/mean_terminated_length": 281.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.010835977853275836, "epoch": 0.00728, "frac_reward_zero_std": 0.875, "grad_norm": 0.001290602027438581, "kl": 0.9017965942621231, "learning_rate": 4.35336589875428e-05, "loss": -0.0001, "num_tokens": 2451194.0, "reward": 1.1809375286102295, "reward_std": 0.03812500461935997, "rewards/rollout_reward_func/mean": 1.1809375286102295, "rewards/rollout_reward_func/std": 0.10783379524946213, "sampling/importance_sampling_ratio/max": 1.069079041481018, "sampling/importance_sampling_ratio/mean": 0.992816150188446, "sampling/importance_sampling_ratio/min": 0.6302892565727234, "sampling/sampling_logp_difference/max": 0.46175146102905273, "sampling/sampling_logp_difference/mean": 0.004237859509885311, "step": 91, "step_time": 8.090420361999804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 305.875, "completions/mean_terminated_length": 305.875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.007981971488334239, "epoch": 0.00736, "frac_reward_zero_std": 1.0, "grad_norm": 1.4349090633913875e-05, "kl": 1.0813498087227345, "learning_rate": 4.353364463970706e-05, "loss": 0.0002, "num_tokens": 2479183.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0204559564590454, "sampling/importance_sampling_ratio/mean": 1.0022176504135132, "sampling/importance_sampling_ratio/min": 0.9946155548095703, "sampling/sampling_logp_difference/max": 0.019003435969352722, "sampling/sampling_logp_difference/mean": 0.0006422828882932663, "step": 92, "step_time": 8.067600336999476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 280.5, "completions/mean_terminated_length": 280.5, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.006280163215706125, "epoch": 0.00744, "frac_reward_zero_std": 0.875, "grad_norm": 0.010133400559425354, "kl": 1.1168262362480164, "learning_rate": 4.353363003335607e-05, "loss": -0.0003, "num_tokens": 2505449.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.0139174461364746, "sampling/importance_sampling_ratio/mean": 0.9837093353271484, "sampling/importance_sampling_ratio/min": 0.42087361216545105, "sampling/sampling_logp_difference/max": 0.8669371604919434, "sampling/sampling_logp_difference/mean": 0.006550541613250971, "step": 93, "step_time": 7.835873078999157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 306.90625, "completions/mean_terminated_length": 306.90625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.004679743040469475, "epoch": 0.00752, "frac_reward_zero_std": 1.0, "grad_norm": 4.211693976685638e-06, "kl": 0.886857483536005, "learning_rate": 4.3533615168490066e-05, "loss": 0.0001, "num_tokens": 2533305.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0096253156661987, "sampling/importance_sampling_ratio/mean": 1.001589298248291, "sampling/importance_sampling_ratio/min": 0.9998981356620789, "sampling/sampling_logp_difference/max": 0.009505439549684525, "sampling/sampling_logp_difference/mean": 0.0003720804234035313, "step": 94, "step_time": 8.796367310999813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 352.3125, "completions/mean_terminated_length": 352.3125, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.012984006869373843, "epoch": 0.0076, "frac_reward_zero_std": 1.0, "grad_norm": 2.5323028239654377e-05, "kl": 0.771374050527811, "learning_rate": 4.353360004510928e-05, "loss": 0.0001, "num_tokens": 2564268.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.1055309772491455, "sampling/importance_sampling_ratio/mean": 1.0100034475326538, "sampling/importance_sampling_ratio/min": 0.998989999294281, "sampling/sampling_logp_difference/max": 0.0970330536365509, "sampling/sampling_logp_difference/mean": 0.0020982669666409492, "step": 95, "step_time": 8.040210511000168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 295.09375, "completions/mean_terminated_length": 295.09375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.00713029396138154, "epoch": 0.00768, "frac_reward_zero_std": 0.875, "grad_norm": 0.037605173885822296, "kl": 1.042659742757678, "learning_rate": 4.353358466321396e-05, "loss": 0.0003, "num_tokens": 2591671.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.0504950284957886, "sampling/importance_sampling_ratio/mean": 1.0037702322006226, "sampling/importance_sampling_ratio/min": 0.9998167753219604, "sampling/sampling_logp_difference/max": 0.044113658368587494, "sampling/sampling_logp_difference/mean": 0.0008205314516089857, "step": 96, "step_time": 8.045752045999961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 284.84375, "completions/mean_terminated_length": 284.84375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.004216597953927703, "epoch": 0.00776, "frac_reward_zero_std": 1.0, "grad_norm": 4.133876245759893e-06, "kl": 1.0271097421646118, "learning_rate": 4.353356902280434e-05, "loss": 0.0002, "num_tokens": 2618410.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0063194036483765, "sampling/importance_sampling_ratio/mean": 1.000886082649231, "sampling/importance_sampling_ratio/min": 0.9989850521087646, "sampling/sampling_logp_difference/max": 0.005841049365699291, "sampling/sampling_logp_difference/mean": 0.0002417312643956393, "step": 97, "step_time": 7.76038138100057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 299.84375, "completions/mean_terminated_length": 299.84375, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.003669391735456884, "epoch": 0.00784, "frac_reward_zero_std": 1.0, "grad_norm": 3.876815753756091e-06, "kl": 0.8679007887840271, "learning_rate": 4.3533553123880673e-05, "loss": 0.0001, "num_tokens": 2646109.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.006697177886963, "sampling/importance_sampling_ratio/mean": 1.0007996559143066, "sampling/importance_sampling_ratio/min": 0.9988589286804199, "sampling/sampling_logp_difference/max": 0.006291601806879044, "sampling/sampling_logp_difference/mean": 0.0002218030858784914, "step": 98, "step_time": 8.680122117999872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 285.09375, "completions/mean_terminated_length": 285.09375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.003658386704046279, "epoch": 0.00792, "frac_reward_zero_std": 1.0, "grad_norm": 3.43472038366599e-06, "kl": 0.9053032025694847, "learning_rate": 4.353353696644321e-05, "loss": 0.0002, "num_tokens": 2672666.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0130594968795776, "sampling/importance_sampling_ratio/mean": 1.0010430812835693, "sampling/importance_sampling_ratio/min": 0.9993507862091064, "sampling/sampling_logp_difference/max": 0.011343911290168762, "sampling/sampling_logp_difference/mean": 0.00027817601221613586, "step": 99, "step_time": 7.919323075999273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 305.25, "completions/mean_terminated_length": 305.25, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.005305002472596243, "epoch": 0.008, "frac_reward_zero_std": 1.0, "grad_norm": 9.758991836861242e-06, "kl": 0.8968730792403221, "learning_rate": 4.353352055049221e-05, "loss": 0.0002, "num_tokens": 2700611.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0074819326400757, "sampling/importance_sampling_ratio/mean": 1.0010285377502441, "sampling/importance_sampling_ratio/min": 0.9894261360168457, "sampling/sampling_logp_difference/max": 0.010616688057780266, "sampling/sampling_logp_difference/mean": 0.00037669416633434594, "step": 100, "step_time": 7.928784010999607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 300.4375, "completions/mean_terminated_length": 300.4375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0054996990074869245, "epoch": 0.00808, "frac_reward_zero_std": 1.0, "grad_norm": 6.633994416915812e-06, "kl": 1.2568271979689598, "learning_rate": 4.353350387602792e-05, "loss": 0.0002, "num_tokens": 2728353.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0124917030334473, "sampling/importance_sampling_ratio/mean": 1.0008971691131592, "sampling/importance_sampling_ratio/min": 0.9885252118110657, "sampling/sampling_logp_difference/max": 0.01164676807820797, "sampling/sampling_logp_difference/mean": 0.0003762872365768999, "step": 101, "step_time": 7.90759317400034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 280.5, "completions/mean_terminated_length": 280.5, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.007697945853578858, "epoch": 0.00816, "frac_reward_zero_std": 0.875, "grad_norm": 0.06633217632770538, "kl": 0.8269255049526691, "learning_rate": 4.3533486943050624e-05, "loss": 0.0003, "num_tokens": 2754334.0, "reward": 1.1812500953674316, "reward_std": 0.03750000149011612, "rewards/rollout_reward_func/mean": 1.1812500953674316, "rewards/rollout_reward_func/std": 0.1060660183429718, "sampling/importance_sampling_ratio/max": 1.0106627941131592, "sampling/importance_sampling_ratio/mean": 1.0006964206695557, "sampling/importance_sampling_ratio/min": 0.9870078563690186, "sampling/sampling_logp_difference/max": 0.013049032539129257, "sampling/sampling_logp_difference/mean": 0.00036282118526287377, "step": 102, "step_time": 9.635418801999549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.125, "completions/mean_terminated_length": 290.125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0030942665252950974, "epoch": 0.00824, "frac_reward_zero_std": 1.0, "grad_norm": 3.1073686841409653e-06, "kl": 0.9219110310077667, "learning_rate": 4.353346975156058e-05, "loss": 0.0002, "num_tokens": 2781410.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0142300128936768, "sampling/importance_sampling_ratio/mean": 1.000820279121399, "sampling/importance_sampling_ratio/min": 0.9998006820678711, "sampling/sampling_logp_difference/max": 0.012971777468919754, "sampling/sampling_logp_difference/mean": 0.0001960925292223692, "step": 103, "step_time": 8.004810662999716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 270.8125, "completions/mean_terminated_length": 270.8125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0028748356853611767, "epoch": 0.00832, "frac_reward_zero_std": 1.0, "grad_norm": 2.48886408371618e-06, "kl": 0.9896811014041305, "learning_rate": 4.353345230155805e-05, "loss": 0.0002, "num_tokens": 2807054.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0075089931488037, "sampling/importance_sampling_ratio/mean": 1.0008145570755005, "sampling/importance_sampling_ratio/min": 0.9999532103538513, "sampling/sampling_logp_difference/max": 0.006868699565529823, "sampling/sampling_logp_difference/mean": 0.0001939799403771758, "step": 104, "step_time": 7.939015687999472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 275.53125, "completions/mean_terminated_length": 275.53125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0025608587311580777, "epoch": 0.0084, "frac_reward_zero_std": 1.0, "grad_norm": 1.3131370906194206e-06, "kl": 0.8954698964953423, "learning_rate": 4.353343459304333e-05, "loss": 0.0001, "num_tokens": 2833088.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0074695348739624, "sampling/importance_sampling_ratio/mean": 1.000930905342102, "sampling/importance_sampling_ratio/min": 0.9998204708099365, "sampling/sampling_logp_difference/max": 0.006562128663063049, "sampling/sampling_logp_difference/mean": 0.00022808517678640783, "step": 105, "step_time": 7.898669197999425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 310.21875, "completions/mean_terminated_length": 310.21875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.0032242367160506546, "epoch": 0.00848, "frac_reward_zero_std": 1.0, "grad_norm": 3.351198301970726e-06, "kl": 0.9163210056722164, "learning_rate": 4.3533416626016684e-05, "loss": 0.0002, "num_tokens": 2861360.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0128750801086426, "sampling/importance_sampling_ratio/mean": 1.0012255907058716, "sampling/importance_sampling_ratio/min": 0.9964293241500854, "sampling/sampling_logp_difference/max": 0.012345895171165466, "sampling/sampling_logp_difference/mean": 0.0003278968797530979, "step": 106, "step_time": 9.746941074999995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.03125, "completions/mean_terminated_length": 290.03125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.002230577258160338, "epoch": 0.00856, "frac_reward_zero_std": 1.0, "grad_norm": 1.059197188624239e-06, "kl": 1.0393922738730907, "learning_rate": 4.353339840047841e-05, "loss": 0.0002, "num_tokens": 2888433.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0030412673950195, "sampling/importance_sampling_ratio/mean": 1.00034499168396, "sampling/importance_sampling_ratio/min": 0.9999644160270691, "sampling/sampling_logp_difference/max": 0.00216706283390522, "sampling/sampling_logp_difference/mean": 8.82229651324451e-05, "step": 107, "step_time": 7.97876614200095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 290.21875, "completions/mean_terminated_length": 290.21875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0025749695778358728, "epoch": 0.00864, "frac_reward_zero_std": 1.0, "grad_norm": 5.618621344183339e-06, "kl": 0.7546286396682262, "learning_rate": 4.3533379916428786e-05, "loss": 0.0001, "num_tokens": 2915512.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0056582689285278, "sampling/importance_sampling_ratio/mean": 1.000446081161499, "sampling/importance_sampling_ratio/min": 0.9991776943206787, "sampling/sampling_logp_difference/max": 0.004048171453177929, "sampling/sampling_logp_difference/mean": 0.00011801549408119172, "step": 108, "step_time": 8.033228233000045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 265.59375, "completions/mean_terminated_length": 265.59375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.001603594319021795, "epoch": 0.00872, "frac_reward_zero_std": 1.0, "grad_norm": 6.755781782885606e-07, "kl": 0.767003558576107, "learning_rate": 4.353336117386811e-05, "loss": 0.0001, "num_tokens": 2941106.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0018162727355957, "sampling/importance_sampling_ratio/mean": 1.0002328157424927, "sampling/importance_sampling_ratio/min": 0.9997153878211975, "sampling/sampling_logp_difference/max": 0.0011970610357820988, "sampling/sampling_logp_difference/mean": 6.69076616759412e-05, "step": 109, "step_time": 7.782705512998746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 271.625, "completions/mean_terminated_length": 271.625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.003668708654004149, "epoch": 0.0088, "frac_reward_zero_std": 1.0, "grad_norm": 4.256055035511963e-06, "kl": 1.0576087981462479, "learning_rate": 4.353334217279667e-05, "loss": 0.0002, "num_tokens": 2966610.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0240758657455444, "sampling/importance_sampling_ratio/mean": 1.0015134811401367, "sampling/importance_sampling_ratio/min": 0.9997015595436096, "sampling/sampling_logp_difference/max": 0.017831873148679733, "sampling/sampling_logp_difference/mean": 0.0003493973345030099, "step": 110, "step_time": 9.481656796999232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 336.6875, "completions/mean_terminated_length": 336.6875, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.0033909254852915183, "epoch": 0.00888, "frac_reward_zero_std": 1.0, "grad_norm": 4.280543180357199e-06, "kl": 1.0437197089195251, "learning_rate": 4.353332291321478e-05, "loss": 0.0002, "num_tokens": 2996379.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0058143138885498, "sampling/importance_sampling_ratio/mean": 1.0009870529174805, "sampling/importance_sampling_ratio/min": 1.0000150203704834, "sampling/sampling_logp_difference/max": 0.005332387983798981, "sampling/sampling_logp_difference/mean": 0.00022039904433768243, "step": 111, "step_time": 8.008671350000895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 285.46875, "completions/mean_terminated_length": 285.46875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0032153506472241133, "epoch": 0.00896, "frac_reward_zero_std": 1.0, "grad_norm": 2.782247747745714e-06, "kl": 1.2320617362856865, "learning_rate": 4.353330339512275e-05, "loss": 0.0002, "num_tokens": 3022687.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0098962783813477, "sampling/importance_sampling_ratio/mean": 1.0008645057678223, "sampling/importance_sampling_ratio/min": 0.9998559951782227, "sampling/sampling_logp_difference/max": 0.009321728721261024, "sampling/sampling_logp_difference/mean": 0.00019536318723112345, "step": 112, "step_time": 7.892091949999667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 352.15625, "completions/mean_terminated_length": 352.15625, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.0071789829671615735, "epoch": 0.00904, "frac_reward_zero_std": 1.0, "grad_norm": 5.054842768004164e-05, "kl": 0.905017614364624, "learning_rate": 4.3533283618520865e-05, "loss": 0.0002, "num_tokens": 3053360.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3870736360549927, "sampling/importance_sampling_ratio/mean": 1.0130882263183594, "sampling/importance_sampling_ratio/min": 0.9999829530715942, "sampling/sampling_logp_difference/max": 0.32688939571380615, "sampling/sampling_logp_difference/mean": 0.0023825636599212885, "step": 113, "step_time": 7.901304995999908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 284.21875, "completions/mean_terminated_length": 284.21875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.002386334592301864, "epoch": 0.00912, "frac_reward_zero_std": 1.0, "grad_norm": 2.1994615053699818e-06, "kl": 1.3753302618861198, "learning_rate": 4.3533263583409464e-05, "loss": 0.0002, "num_tokens": 3079865.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0070444345474243, "sampling/importance_sampling_ratio/mean": 1.0007307529449463, "sampling/importance_sampling_ratio/min": 0.9999845623970032, "sampling/sampling_logp_difference/max": 0.005592597648501396, "sampling/sampling_logp_difference/mean": 0.00017743208445608616, "step": 114, "step_time": 10.031195793999814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 241.03125, "completions/mean_terminated_length": 241.03125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0022404507399187423, "epoch": 0.0092, "frac_reward_zero_std": 1.0, "grad_norm": 1.3552677273764857e-06, "kl": 0.5740615949034691, "learning_rate": 4.353324328978884e-05, "loss": 0.0001, "num_tokens": 3103786.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0062806606292725, "sampling/importance_sampling_ratio/mean": 1.0003859996795654, "sampling/importance_sampling_ratio/min": 0.9999957084655762, "sampling/sampling_logp_difference/max": 0.005704356357455254, "sampling/sampling_logp_difference/mean": 9.772858174983412e-05, "step": 115, "step_time": 7.707980992999637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 275.53125, "completions/mean_terminated_length": 275.53125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0017663557518972084, "epoch": 0.00928, "frac_reward_zero_std": 1.0, "grad_norm": 5.630481609841809e-07, "kl": 0.8926740437746048, "learning_rate": 4.3533222737659336e-05, "loss": 0.0001, "num_tokens": 3129915.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0044442415237427, "sampling/importance_sampling_ratio/mean": 1.000339150428772, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 0.0039198449812829494, "sampling/sampling_logp_difference/mean": 8.035312930587679e-05, "step": 116, "step_time": 7.923621838000145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 341.46875, "completions/mean_terminated_length": 341.46875, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.003273058100603521, "epoch": 0.00936, "frac_reward_zero_std": 1.0, "grad_norm": 3.997585281467764e-06, "kl": 0.7832371294498444, "learning_rate": 4.353320192702126e-05, "loss": 0.0001, "num_tokens": 3160290.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0253633260726929, "sampling/importance_sampling_ratio/mean": 1.0013339519500732, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.022482480853796005, "sampling/sampling_logp_difference/mean": 0.000289207004243508, "step": 117, "step_time": 8.041205730999991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 254.40625, "completions/mean_terminated_length": 254.40625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.002076580800348893, "epoch": 0.00944, "frac_reward_zero_std": 1.0, "grad_norm": 2.193146656281897e-06, "kl": 0.9485401250422001, "learning_rate": 4.3533180857874946e-05, "loss": 0.0002, "num_tokens": 3184691.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0412697792053223, "sampling/importance_sampling_ratio/mean": 1.0014978647232056, "sampling/importance_sampling_ratio/min": 1.0000113248825073, "sampling/sampling_logp_difference/max": 0.023405615240335464, "sampling/sampling_logp_difference/mean": 0.00034402255550958216, "step": 118, "step_time": 9.333281152999916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 290.40625, "completions/mean_terminated_length": 290.40625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0016499179546372034, "epoch": 0.00952, "frac_reward_zero_std": 1.0, "grad_norm": 7.621999884577235e-07, "kl": 0.7720878943800926, "learning_rate": 4.353315953022074e-05, "loss": 0.0001, "num_tokens": 3211966.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0031160116195679, "sampling/importance_sampling_ratio/mean": 1.0002719163894653, "sampling/importance_sampling_ratio/min": 0.9997026920318604, "sampling/sampling_logp_difference/max": 0.0027775103226304054, "sampling/sampling_logp_difference/mean": 7.42078700568527e-05, "step": 119, "step_time": 7.7929123470003105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 280.625, "completions/mean_terminated_length": 280.625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0028635957569349557, "epoch": 0.0096, "frac_reward_zero_std": 1.0, "grad_norm": 1.8403419517198927e-06, "kl": 0.8982496745884418, "learning_rate": 4.3533137944058965e-05, "loss": 0.0001, "num_tokens": 3238236.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0031509399414062, "sampling/importance_sampling_ratio/mean": 1.000476360321045, "sampling/importance_sampling_ratio/min": 0.9990882873535156, "sampling/sampling_logp_difference/max": 0.003126264549791813, "sampling/sampling_logp_difference/mean": 0.00013709375343751162, "step": 120, "step_time": 7.969132335000268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 281.1875, "completions/mean_terminated_length": 281.1875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.001712185716314707, "epoch": 0.00968, "frac_reward_zero_std": 1.0, "grad_norm": 1.7779278778107255e-06, "kl": 0.8751526847481728, "learning_rate": 4.3533116099389974e-05, "loss": 0.0001, "num_tokens": 3264548.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0106688737869263, "sampling/importance_sampling_ratio/mean": 1.0009894371032715, "sampling/importance_sampling_ratio/min": 1.0000081062316895, "sampling/sampling_logp_difference/max": 0.009239201433956623, "sampling/sampling_logp_difference/mean": 0.00023393213632516563, "step": 121, "step_time": 7.908907305999037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 230.59375, "completions/mean_terminated_length": 230.59375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0015797761079738848, "epoch": 0.00976, "frac_reward_zero_std": 1.0, "grad_norm": 4.6970549760771974e-07, "kl": 0.8199932724237442, "learning_rate": 4.3533093996214103e-05, "loss": 0.0001, "num_tokens": 3287609.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0020288228988647, "sampling/importance_sampling_ratio/mean": 1.0002607107162476, "sampling/importance_sampling_ratio/min": 0.9999421238899231, "sampling/sampling_logp_difference/max": 0.0016462940257042646, "sampling/sampling_logp_difference/mean": 6.691954331472516e-05, "step": 122, "step_time": 9.306468207999842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 275.3125, "completions/mean_terminated_length": 275.3125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0018206075037596747, "epoch": 0.00984, "frac_reward_zero_std": 1.0, "grad_norm": 5.821110562465037e-07, "kl": 1.0216600224375725, "learning_rate": 4.353307163453171e-05, "loss": 0.0002, "num_tokens": 3313446.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.003731369972229, "sampling/importance_sampling_ratio/mean": 1.0003786087036133, "sampling/importance_sampling_ratio/min": 0.9995545744895935, "sampling/sampling_logp_difference/max": 0.00347282737493515, "sampling/sampling_logp_difference/mean": 0.00010381989704910666, "step": 123, "step_time": 7.876680347000274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 284.9375, "completions/mean_terminated_length": 284.9375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0022123418166302145, "epoch": 0.00992, "frac_reward_zero_std": 1.0, "grad_norm": 1.855613618317875e-06, "kl": 0.9234612509608269, "learning_rate": 4.353304901434314e-05, "loss": 0.0002, "num_tokens": 3339998.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0053061246871948, "sampling/importance_sampling_ratio/mean": 1.00053071975708, "sampling/importance_sampling_ratio/min": 0.9992577433586121, "sampling/sampling_logp_difference/max": 0.005183844827115536, "sampling/sampling_logp_difference/mean": 0.0001446136157028377, "step": 124, "step_time": 7.944355333999738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.0, "completions/max_terminated_length": 332.0, "completions/mean_length": 264.0625, "completions/mean_terminated_length": 264.0625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0018840617922251113, "epoch": 0.01, "frac_reward_zero_std": 1.0, "grad_norm": 1.5587991128995782e-06, "kl": 0.9948739409446716, "learning_rate": 4.3533026135648765e-05, "loss": 0.0002, "num_tokens": 3365115.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0087990760803223, "sampling/importance_sampling_ratio/mean": 1.0005731582641602, "sampling/importance_sampling_ratio/min": 0.9997692704200745, "sampling/sampling_logp_difference/max": 0.008651604875922203, "sampling/sampling_logp_difference/mean": 0.00015504607290495187, "step": 125, "step_time": 7.813175108000905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 294.6875, "completions/mean_terminated_length": 294.6875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.0023748095700284466, "epoch": 0.01008, "frac_reward_zero_std": 1.0, "grad_norm": 1.5568176650049281e-06, "kl": 1.0050797536969185, "learning_rate": 4.3533002998448926e-05, "loss": 0.0002, "num_tokens": 3392101.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0107582807540894, "sampling/importance_sampling_ratio/mean": 1.0007569789886475, "sampling/importance_sampling_ratio/min": 0.9988821744918823, "sampling/sampling_logp_difference/max": 0.010333390906453133, "sampling/sampling_logp_difference/mean": 0.00019616979989223182, "step": 126, "step_time": 9.149904105999667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 375.0, "completions/max_terminated_length": 375.0, "completions/mean_length": 280.5625, "completions/mean_terminated_length": 280.5625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.0026455379629624076, "epoch": 0.01016, "frac_reward_zero_std": 1.0, "grad_norm": 1.6429938114015386e-05, "kl": 0.9262720942497253, "learning_rate": 4.353297960274401e-05, "loss": 0.0002, "num_tokens": 3418369.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0127620697021484, "sampling/importance_sampling_ratio/mean": 0.9973284602165222, "sampling/importance_sampling_ratio/min": 0.8904464840888977, "sampling/sampling_logp_difference/max": 0.12499332427978516, "sampling/sampling_logp_difference/mean": 0.0011542720021679997, "step": 127, "step_time": 7.961577512000531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.0, "completions/max_terminated_length": 374.0, "completions/mean_length": 270.75, "completions/mean_terminated_length": 270.75, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.002954744966700673, "epoch": 0.01024, "frac_reward_zero_std": 1.0, "grad_norm": 9.025873737300572e-07, "kl": 0.9678482711315155, "learning_rate": 4.3532955948534384e-05, "loss": 0.0002, "num_tokens": 3443916.0, "reward": 1.2000000476837158, "reward_std": 0.0, "rewards/rollout_reward_func/mean": 1.2000000476837158, "rewards/rollout_reward_func/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0103626251220703, "sampling/importance_sampling_ratio/mean": 1.0009623765945435, "sampling/importance_sampling_ratio/min": 0.9999430775642395, "sampling/sampling_logp_difference/max": 0.008814064785838127, "sampling/sampling_logp_difference/mean": 0.0002325621317140758, "step": 128, "step_time": 7.812974072999623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 728.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 535.46875, "completions/mean_terminated_length": 535.46875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.011180363624589518, "epoch": 0.01032, "frac_reward_zero_std": 0.5, "grad_norm": 0.032643213868141174, "kl": 0.9731160998344421, "learning_rate": 4.353293203582041e-05, "loss": 0.0004, "num_tokens": 3478102.0, "reward": 1.2455207109451294, "reward_std": 0.08606479316949844, "rewards/rollout_reward_func/mean": 1.2455207109451294, "rewards/rollout_reward_func/std": 0.14549532532691956, "sampling/importance_sampling_ratio/max": 1.242073893547058, "sampling/importance_sampling_ratio/mean": 1.0011532306671143, "sampling/importance_sampling_ratio/min": 0.6836673021316528, "sampling/sampling_logp_difference/max": 0.38035547733306885, "sampling/sampling_logp_difference/mean": 0.0034724250435829163, "step": 129, "step_time": 10.011316147999878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 537.375, "completions/mean_terminated_length": 537.375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.013671449778485112, "epoch": 0.0104, "frac_reward_zero_std": 0.5, "grad_norm": 0.03305273875594139, "kl": 0.8943541534245014, "learning_rate": 4.353290786460249e-05, "loss": -0.0001, "num_tokens": 3512563.0, "reward": 1.2590625286102295, "reward_std": 0.08187499642372131, "rewards/rollout_reward_func/mean": 1.2590625286102295, "rewards/rollout_reward_func/std": 0.12823933362960815, "sampling/importance_sampling_ratio/max": 1.1223937273025513, "sampling/importance_sampling_ratio/mean": 0.9876028895378113, "sampling/importance_sampling_ratio/min": 0.3954014778137207, "sampling/sampling_logp_difference/max": 0.9279760122299194, "sampling/sampling_logp_difference/mean": 0.006061212625354528, "step": 130, "step_time": 11.033380782000677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 526.65625, "completions/mean_terminated_length": 526.65625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.008745982107939199, "epoch": 0.01048, "frac_reward_zero_std": 0.625, "grad_norm": 0.01682690717279911, "kl": 1.124693103134632, "learning_rate": 4.353288343488098e-05, "loss": 0.0001, "num_tokens": 3546038.0, "reward": 1.2590625286102295, "reward_std": 0.08104812353849411, "rewards/rollout_reward_func/mean": 1.2590625286102295, "rewards/rollout_reward_func/std": 0.12823933362960815, "sampling/importance_sampling_ratio/max": 1.7732131481170654, "sampling/importance_sampling_ratio/mean": 1.027938961982727, "sampling/importance_sampling_ratio/min": 0.9991727471351624, "sampling/sampling_logp_difference/max": 0.5727040767669678, "sampling/sampling_logp_difference/mean": 0.0034476923756301403, "step": 131, "step_time": 9.951341013999809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004807692486792803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004807692486792803, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 594.4375, "completions/mean_terminated_length": 594.4375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.003739486273843795, "epoch": 0.01056, "frac_reward_zero_std": 0.625, "grad_norm": 0.0038927544374018908, "kl": 1.0757212564349174, "learning_rate": 4.353285874665628e-05, "loss": -0.0004, "num_tokens": 3583166.0, "reward": 1.2593748569488525, "reward_std": 0.08124999701976776, "rewards/rollout_reward_func/mean": 1.2593748569488525, "rewards/rollout_reward_func/std": 0.1283293068408966, "sampling/importance_sampling_ratio/max": 1.054421305656433, "sampling/importance_sampling_ratio/mean": 0.9791184663772583, "sampling/importance_sampling_ratio/min": 0.24138830602169037, "sampling/sampling_logp_difference/max": 1.4217681884765625, "sampling/sampling_logp_difference/mean": 0.006970229558646679, "step": 132, "step_time": 10.055943976999515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 514.53125, "completions/mean_terminated_length": 514.53125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.015462286304682493, "epoch": 0.01064, "frac_reward_zero_std": 0.625, "grad_norm": 0.04201054573059082, "kl": 1.1028756126761436, "learning_rate": 4.353283379992878e-05, "loss": 0.0001, "num_tokens": 3616726.0, "reward": 1.2458332777023315, "reward_std": 0.08543980121612549, "rewards/rollout_reward_func/mean": 1.2458332777023315, "rewards/rollout_reward_func/std": 0.14560464024543762, "sampling/importance_sampling_ratio/max": 1.4349968433380127, "sampling/importance_sampling_ratio/mean": 0.9945424795150757, "sampling/importance_sampling_ratio/min": 0.6208416819572449, "sampling/sampling_logp_difference/max": 0.47671031951904297, "sampling/sampling_logp_difference/mean": 0.0065397704020142555, "step": 133, "step_time": 9.386514928999986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 522.34375, "completions/mean_terminated_length": 522.34375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.010772602021461353, "epoch": 0.01072, "frac_reward_zero_std": 0.5, "grad_norm": 0.014521813951432705, "kl": 0.8598150089383125, "learning_rate": 4.353280859469887e-05, "loss": 0.0003, "num_tokens": 3650989.0, "reward": 1.231041669845581, "reward_std": 0.10611619055271149, "rewards/rollout_reward_func/mean": 1.231041669845581, "rewards/rollout_reward_func/std": 0.25183117389678955, "sampling/importance_sampling_ratio/max": 1.096602201461792, "sampling/importance_sampling_ratio/mean": 0.9962838888168335, "sampling/importance_sampling_ratio/min": 0.7030482888221741, "sampling/sampling_logp_difference/max": 0.37227439880371094, "sampling/sampling_logp_difference/mean": 0.0026527659501880407, "step": 134, "step_time": 11.787746956000774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 533.8125, "completions/mean_terminated_length": 533.8125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.005057345872046426, "epoch": 0.0108, "frac_reward_zero_std": 0.875, "grad_norm": 0.0052049788646399975, "kl": 0.8191076368093491, "learning_rate": 4.3532783130966965e-05, "loss": 0.0005, "num_tokens": 3685003.0, "reward": 1.299687385559082, "reward_std": 0.0006249994039535522, "rewards/rollout_reward_func/mean": 1.299687385559082, "rewards/rollout_reward_func/std": 0.0017677651485428214, "sampling/importance_sampling_ratio/max": 1.2421886920928955, "sampling/importance_sampling_ratio/mean": 1.012158751487732, "sampling/importance_sampling_ratio/min": 0.9992395043373108, "sampling/sampling_logp_difference/max": 0.2168380171060562, "sampling/sampling_logp_difference/mean": 0.0018088730284944177, "step": 135, "step_time": 9.962781290999374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 730.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 466.0625, "completions/mean_terminated_length": 466.0625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.016410774318501353, "epoch": 0.01088, "frac_reward_zero_std": 0.375, "grad_norm": 0.03194565325975418, "kl": 0.8101879805326462, "learning_rate": 4.353275740873345e-05, "loss": -0.001, "num_tokens": 3716030.0, "reward": 1.245520830154419, "reward_std": 0.10895832628011703, "rewards/rollout_reward_func/mean": 1.245520830154419, "rewards/rollout_reward_func/std": 0.14549532532691956, "sampling/importance_sampling_ratio/max": 1.0557047128677368, "sampling/importance_sampling_ratio/mean": 0.9735875129699707, "sampling/importance_sampling_ratio/min": 0.5073900818824768, "sampling/sampling_logp_difference/max": 0.6857633590698242, "sampling/sampling_logp_difference/mean": 0.006636587902903557, "step": 136, "step_time": 9.789079821000541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 586.34375, "completions/mean_terminated_length": 586.34375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.017932373972143978, "epoch": 0.01096, "frac_reward_zero_std": 0.25, "grad_norm": 0.03019634634256363, "kl": 0.6465921364724636, "learning_rate": 4.3532731427998744e-05, "loss": -0.0008, "num_tokens": 3752660.0, "reward": 1.2184374332427979, "reward_std": 0.16312499344348907, "rewards/rollout_reward_func/mean": 1.2184374332427979, "rewards/rollout_reward_func/std": 0.20394843816757202, "sampling/importance_sampling_ratio/max": 1.1368844509124756, "sampling/importance_sampling_ratio/mean": 0.9817204475402832, "sampling/importance_sampling_ratio/min": 0.6018290519714355, "sampling/sampling_logp_difference/max": 0.5078255534172058, "sampling/sampling_logp_difference/mean": 0.005829981062561274, "step": 137, "step_time": 9.97657412299941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 544.59375, "completions/mean_terminated_length": 544.59375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.014248839084757492, "epoch": 0.01104, "frac_reward_zero_std": 0.25, "grad_norm": 0.03428541123867035, "kl": 0.9007213264703751, "learning_rate": 4.353270518876325e-05, "loss": -0.0001, "num_tokens": 3787187.0, "reward": 1.2452082633972168, "reward_std": 0.10958332568407059, "rewards/rollout_reward_func/mean": 1.2452082633972168, "rewards/rollout_reward_func/std": 0.1453852355480194, "sampling/importance_sampling_ratio/max": 1.339624047279358, "sampling/importance_sampling_ratio/mean": 0.9920567274093628, "sampling/importance_sampling_ratio/min": 0.490577757358551, "sampling/sampling_logp_difference/max": 0.7122424840927124, "sampling/sampling_logp_difference/mean": 0.006347449030727148, "step": 138, "step_time": 10.624866170999212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 606.78125, "completions/mean_terminated_length": 606.78125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.005386328695749398, "epoch": 0.01112, "frac_reward_zero_std": 0.875, "grad_norm": 0.010304288007318974, "kl": 0.9753484353423119, "learning_rate": 4.353267869102739e-05, "loss": 0.0013, "num_tokens": 3824997.0, "reward": 1.299687385559082, "reward_std": 0.0006249994039535522, "rewards/rollout_reward_func/mean": 1.299687385559082, "rewards/rollout_reward_func/std": 0.0017677651485428214, "sampling/importance_sampling_ratio/max": 1.7676399946212769, "sampling/importance_sampling_ratio/mean": 1.024378776550293, "sampling/importance_sampling_ratio/min": 0.9823206663131714, "sampling/sampling_logp_difference/max": 0.5693756341934204, "sampling/sampling_logp_difference/mean": 0.0030128415673971176, "step": 139, "step_time": 10.081947524000043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 556.15625, "completions/mean_terminated_length": 556.15625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.013986769190523773, "epoch": 0.0112, "frac_reward_zero_std": 0.625, "grad_norm": 0.02896770089864731, "kl": 0.8760957419872284, "learning_rate": 4.353265193479158e-05, "loss": -0.0008, "num_tokens": 3860276.0, "reward": 1.2452082633972168, "reward_std": 0.054589785635471344, "rewards/rollout_reward_func/mean": 1.2452082633972168, "rewards/rollout_reward_func/std": 0.1453852355480194, "sampling/importance_sampling_ratio/max": 1.2889293432235718, "sampling/importance_sampling_ratio/mean": 0.9847899675369263, "sampling/importance_sampling_ratio/min": 0.22509172558784485, "sampling/sampling_logp_difference/max": 1.4912656545639038, "sampling/sampling_logp_difference/mean": 0.00937747023999691, "step": 140, "step_time": 10.020960504000868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 544.78125, "completions/mean_terminated_length": 544.78125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.0033063684968510643, "epoch": 0.01128, "frac_reward_zero_std": 0.75, "grad_norm": 0.0006865442846901715, "kl": 0.9232851229608059, "learning_rate": 4.3532624920056237e-05, "loss": 0.0003, "num_tokens": 3895284.0, "reward": 1.2858332395553589, "reward_std": 0.027506457641720772, "rewards/rollout_reward_func/mean": 1.2858332395553589, "rewards/rollout_reward_func/std": 0.07652861624956131, "sampling/importance_sampling_ratio/max": 1.022231936454773, "sampling/importance_sampling_ratio/mean": 1.002477765083313, "sampling/importance_sampling_ratio/min": 0.999949038028717, "sampling/sampling_logp_difference/max": 0.019560128450393677, "sampling/sampling_logp_difference/mean": 0.0003835466632153839, "step": 141, "step_time": 9.952398824001193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 594.75, "completions/mean_terminated_length": 594.75, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.010649462434230372, "epoch": 0.01136, "frac_reward_zero_std": 0.375, "grad_norm": 0.3629268705844879, "kl": 8.501242689788342, "learning_rate": 4.3532597646821805e-05, "loss": 0.0021, "num_tokens": 3932137.0, "reward": 1.2448956966400146, "reward_std": 0.08633317053318024, "rewards/rollout_reward_func/mean": 1.2448956966400146, "rewards/rollout_reward_func/std": 0.14527437090873718, "sampling/importance_sampling_ratio/max": 1.1279447078704834, "sampling/importance_sampling_ratio/mean": 0.9781762361526489, "sampling/importance_sampling_ratio/min": 0.4678066372871399, "sampling/sampling_logp_difference/max": 0.7619372606277466, "sampling/sampling_logp_difference/mean": 0.007314727175980806, "step": 142, "step_time": 11.471841097999459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 546.0, "completions/mean_terminated_length": 546.0, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.021447591483592987, "epoch": 0.01144, "frac_reward_zero_std": 0.625, "grad_norm": 0.014821848832070827, "kl": 0.9655835181474686, "learning_rate": 4.353257011508871e-05, "loss": -0.0001, "num_tokens": 3966638.0, "reward": 1.2719790935516357, "reward_std": 0.05468647554516792, "rewards/rollout_reward_func/mean": 1.2719790935516357, "rewards/rollout_reward_func/std": 0.10636669397354126, "sampling/importance_sampling_ratio/max": 1.337045669555664, "sampling/importance_sampling_ratio/mean": 1.0024406909942627, "sampling/importance_sampling_ratio/min": 0.7598346471786499, "sampling/sampling_logp_difference/max": 0.2880232334136963, "sampling/sampling_logp_difference/mean": 0.004039570689201355, "step": 143, "step_time": 9.944881509000425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 626.625, "completions/mean_terminated_length": 626.625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.011719557514879853, "epoch": 0.01152, "frac_reward_zero_std": 0.75, "grad_norm": 0.025126127526164055, "kl": 0.8558261692523956, "learning_rate": 4.353254232485738e-05, "loss": -0.0005, "num_tokens": 4005205.0, "reward": 1.286145806312561, "reward_std": 0.027708331122994423, "rewards/rollout_reward_func/mean": 1.286145806312561, "rewards/rollout_reward_func/std": 0.07656658440828323, "sampling/importance_sampling_ratio/max": 1.019237995147705, "sampling/importance_sampling_ratio/mean": 0.9867112636566162, "sampling/importance_sampling_ratio/min": 0.5233541131019592, "sampling/sampling_logp_difference/max": 0.6478126049041748, "sampling/sampling_logp_difference/mean": 0.003428862662985921, "step": 144, "step_time": 10.011449411999365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 633.6875, "completions/mean_terminated_length": 633.6875, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 0.03676209389232099, "epoch": 0.0116, "frac_reward_zero_std": 0.625, "grad_norm": 0.043598026037216187, "kl": 1.0050817802548409, "learning_rate": 4.3532514276128264e-05, "loss": -0.0004, "num_tokens": 4043762.0, "reward": 1.259374976158142, "reward_std": 0.08124999701976776, "rewards/rollout_reward_func/mean": 1.259374976158142, "rewards/rollout_reward_func/std": 0.1283293068408966, "sampling/importance_sampling_ratio/max": 1.4694103002548218, "sampling/importance_sampling_ratio/mean": 1.0026533603668213, "sampling/importance_sampling_ratio/min": 0.771098792552948, "sampling/sampling_logp_difference/max": 0.37546873092651367, "sampling/sampling_logp_difference/mean": 0.006022340152412653, "step": 145, "step_time": 10.060867641000641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 728.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 486.125, "completions/mean_terminated_length": 486.125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.033082072623074055, "epoch": 0.01168, "frac_reward_zero_std": 0.375, "grad_norm": 0.04938723146915436, "kl": 1.1520497053861618, "learning_rate": 4.353248596890181e-05, "loss": -0.0002, "num_tokens": 4076052.0, "reward": 1.2581249475479126, "reward_std": 0.08208983391523361, "rewards/rollout_reward_func/mean": 1.2581249475479126, "rewards/rollout_reward_func/std": 0.12796428799629211, "sampling/importance_sampling_ratio/max": 1.5089449882507324, "sampling/importance_sampling_ratio/mean": 0.997917890548706, "sampling/importance_sampling_ratio/min": 0.6395244598388672, "sampling/sampling_logp_difference/max": 0.4487767219543457, "sampling/sampling_logp_difference/mean": 0.009268432855606079, "step": 146, "step_time": 11.893771871000808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 635.75, "completions/mean_terminated_length": 635.75, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.031227883300743997, "epoch": 0.01176, "frac_reward_zero_std": 0.5, "grad_norm": 0.030443115159869194, "kl": 1.1072702631354332, "learning_rate": 4.353245740317844e-05, "loss": -0.0011, "num_tokens": 4115079.0, "reward": 1.2455207109451294, "reward_std": 0.10813145339488983, "rewards/rollout_reward_func/mean": 1.2455207109451294, "rewards/rollout_reward_func/std": 0.14549532532691956, "sampling/importance_sampling_ratio/max": 1.054962158203125, "sampling/importance_sampling_ratio/mean": 0.9734452962875366, "sampling/importance_sampling_ratio/min": 0.6101904511451721, "sampling/sampling_logp_difference/max": 0.4943222999572754, "sampling/sampling_logp_difference/mean": 0.007517709396779537, "step": 147, "step_time": 10.118594939999184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004807692486792803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004807692486792803, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 553.15625, "completions/mean_terminated_length": 553.15625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.0416766325943172, "epoch": 0.01184, "frac_reward_zero_std": 0.375, "grad_norm": 0.039279915392398834, "kl": 1.154223121702671, "learning_rate": 4.353242857895864e-05, "loss": -0.0006, "num_tokens": 4149953.0, "reward": 1.1904165744781494, "reward_std": 0.2166827917098999, "rewards/rollout_reward_func/mean": 1.1904165744781494, "rewards/rollout_reward_func/std": 0.2921870946884155, "sampling/importance_sampling_ratio/max": 1.6439316272735596, "sampling/importance_sampling_ratio/mean": 0.9965168237686157, "sampling/importance_sampling_ratio/min": 0.49807170033454895, "sampling/sampling_logp_difference/max": 0.7143464088439941, "sampling/sampling_logp_difference/mean": 0.014425855129957199, "step": 148, "step_time": 10.024935027999163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004807692486792803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004807692486792803, "completions/clipped_ratio": 0.0, "completions/max_length": 648.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 501.25, "completions/mean_terminated_length": 501.25, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.03798253717832267, "epoch": 0.01192, "frac_reward_zero_std": 0.625, "grad_norm": 0.015526200644671917, "kl": 0.9766840562224388, "learning_rate": 4.353239949624285e-05, "loss": 0.0005, "num_tokens": 4182397.0, "reward": 1.25906240940094, "reward_std": 0.08104812353849411, "rewards/rollout_reward_func/mean": 1.25906240940094, "rewards/rollout_reward_func/std": 0.12823933362960815, "sampling/importance_sampling_ratio/max": 1.2318724393844604, "sampling/importance_sampling_ratio/mean": 0.9984570741653442, "sampling/importance_sampling_ratio/min": 0.8258872032165527, "sampling/sampling_logp_difference/max": 0.20864391326904297, "sampling/sampling_logp_difference/mean": 0.0034210151061415672, "step": 149, "step_time": 9.777033323999603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 730.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 554.90625, "completions/mean_terminated_length": 554.90625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.03949773160275072, "epoch": 0.012, "frac_reward_zero_std": 0.375, "grad_norm": 0.028686366975307465, "kl": 0.9443170726299286, "learning_rate": 4.3532370155031534e-05, "loss": -0.0004, "num_tokens": 4217897.0, "reward": 1.2448958158493042, "reward_std": 0.11020832508802414, "rewards/rollout_reward_func/mean": 1.2448958158493042, "rewards/rollout_reward_func/std": 0.23968464136123657, "sampling/importance_sampling_ratio/max": 1.183545470237732, "sampling/importance_sampling_ratio/mean": 0.9874070882797241, "sampling/importance_sampling_ratio/min": 0.5332264304161072, "sampling/sampling_logp_difference/max": 0.6063394546508789, "sampling/sampling_logp_difference/mean": 0.00850982591509819, "step": 150, "step_time": 10.583405167999445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 543.09375, "completions/mean_terminated_length": 543.09375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.04028109461069107, "epoch": 0.01208, "frac_reward_zero_std": 0.25, "grad_norm": 0.043271876871585846, "kl": 0.8776917830109596, "learning_rate": 4.353234055532515e-05, "loss": 0.0006, "num_tokens": 4252922.0, "reward": 1.2313541173934937, "reward_std": 0.136461541056633, "rewards/rollout_reward_func/mean": 1.2313541173934937, "rewards/rollout_reward_func/std": 0.19377009570598602, "sampling/importance_sampling_ratio/max": 1.4875383377075195, "sampling/importance_sampling_ratio/mean": 1.0117700099945068, "sampling/importance_sampling_ratio/min": 0.9112693667411804, "sampling/sampling_logp_difference/max": 0.39710116386413574, "sampling/sampling_logp_difference/mean": 0.004705607891082764, "step": 151, "step_time": 9.992124632000014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 506.34375, "completions/mean_terminated_length": 506.34375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.04243151342961937, "epoch": 0.01216, "frac_reward_zero_std": 0.5, "grad_norm": 0.026152007281780243, "kl": 0.8471603766083717, "learning_rate": 4.353231069712417e-05, "loss": -0.0002, "num_tokens": 4285411.0, "reward": 1.2322916984558105, "reward_std": 0.1354166567325592, "rewards/rollout_reward_func/mean": 1.2322916984558105, "rewards/rollout_reward_func/std": 0.19408534467220306, "sampling/importance_sampling_ratio/max": 1.2249970436096191, "sampling/importance_sampling_ratio/mean": 0.9809556007385254, "sampling/importance_sampling_ratio/min": 0.5358037352561951, "sampling/sampling_logp_difference/max": 0.6229891777038574, "sampling/sampling_logp_difference/mean": 0.007731186226010323, "step": 152, "step_time": 9.948441414999252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.009615384973585606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009615384973585606, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 575.8125, "completions/mean_terminated_length": 575.8125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.04223974165506661, "epoch": 0.01224, "frac_reward_zero_std": 0.5, "grad_norm": 0.027246542274951935, "kl": 0.7073983624577522, "learning_rate": 4.353228058042907e-05, "loss": 0.0014, "num_tokens": 4321536.0, "reward": 1.2452082633972168, "reward_std": 0.10958331823348999, "rewards/rollout_reward_func/mean": 1.2452082633972168, "rewards/rollout_reward_func/std": 0.18311506509780884, "sampling/importance_sampling_ratio/max": 1.569769263267517, "sampling/importance_sampling_ratio/mean": 1.0485897064208984, "sampling/importance_sampling_ratio/min": 0.9951664805412292, "sampling/sampling_logp_difference/max": 0.4442850351333618, "sampling/sampling_logp_difference/mean": 0.006749277003109455, "step": 153, "step_time": 10.766158639000423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 646.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 542.71875, "completions/mean_terminated_length": 542.71875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.02604771131882444, "epoch": 0.01232, "frac_reward_zero_std": 0.75, "grad_norm": 0.029834095388650894, "kl": 0.7482284530997276, "learning_rate": 4.3532250205240325e-05, "loss": -0.0001, "num_tokens": 4356549.0, "reward": 1.272916555404663, "reward_std": 0.05416666343808174, "rewards/rollout_reward_func/mean": 1.272916555404663, "rewards/rollout_reward_func/std": 0.10657168179750443, "sampling/importance_sampling_ratio/max": 1.2634644508361816, "sampling/importance_sampling_ratio/mean": 0.9966223239898682, "sampling/importance_sampling_ratio/min": 0.48962900042533875, "sampling/sampling_logp_difference/max": 0.7164514064788818, "sampling/sampling_logp_difference/mean": 0.005804511718451977, "step": 154, "step_time": 10.145599140999366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 543.46875, "completions/mean_terminated_length": 543.46875, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.03141283767763525, "epoch": 0.0124, "frac_reward_zero_std": 0.75, "grad_norm": 0.022678229957818985, "kl": 0.9366422109305859, "learning_rate": 4.353221957155843e-05, "loss": -0.0001, "num_tokens": 4391208.0, "reward": 1.2584373950958252, "reward_std": 0.08312499523162842, "rewards/rollout_reward_func/mean": 1.2584373950958252, "rewards/rollout_reward_func/std": 0.23329488933086395, "sampling/importance_sampling_ratio/max": 1.0763630867004395, "sampling/importance_sampling_ratio/mean": 0.9894322156906128, "sampling/importance_sampling_ratio/min": 0.582743227481842, "sampling/sampling_logp_difference/max": 0.5598955154418945, "sampling/sampling_logp_difference/mean": 0.006226009223610163, "step": 155, "step_time": 9.99125198799993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 584.4375, "completions/mean_terminated_length": 584.4375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.025513644912280142, "epoch": 0.01248, "frac_reward_zero_std": 0.875, "grad_norm": 0.016259880736470222, "kl": 0.8116874303668737, "learning_rate": 4.353218867938383e-05, "loss": -0.0004, "num_tokens": 4427848.0, "reward": 1.2864582538604736, "reward_std": 0.02708333171904087, "rewards/rollout_reward_func/mean": 1.2864582538604736, "rewards/rollout_reward_func/std": 0.0766032263636589, "sampling/importance_sampling_ratio/max": 1.0189932584762573, "sampling/importance_sampling_ratio/mean": 0.9890927076339722, "sampling/importance_sampling_ratio/min": 0.6117552518844604, "sampling/sampling_logp_difference/max": 0.4906834363937378, "sampling/sampling_logp_difference/mean": 0.003313068300485611, "step": 156, "step_time": 9.988939616000607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 545.5, "completions/mean_terminated_length": 545.5, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.01934251975035295, "epoch": 0.01256, "frac_reward_zero_std": 0.5, "grad_norm": 0.0012860053684562445, "kl": 0.9941539913415909, "learning_rate": 4.353215752871706e-05, "loss": 0.0004, "num_tokens": 4462713.0, "reward": 1.2987499237060547, "reward_std": 0.002499997615814209, "rewards/rollout_reward_func/mean": 1.2987499237060547, "rewards/rollout_reward_func/std": 0.0033601040486246347, "sampling/importance_sampling_ratio/max": 1.0843442678451538, "sampling/importance_sampling_ratio/mean": 1.0073246955871582, "sampling/importance_sampling_ratio/min": 0.9903218150138855, "sampling/sampling_logp_difference/max": 0.07053746283054352, "sampling/sampling_logp_difference/mean": 0.0014099898980930448, "step": 157, "step_time": 10.744672659000116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0052083334885537624, "completions/clipped_ratio": 0.0, "completions/max_length": 730.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 523.71875, "completions/mean_terminated_length": 523.71875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.02133380400482565, "epoch": 0.01264, "frac_reward_zero_std": 0.625, "grad_norm": 0.025139756500720978, "kl": 0.8151384266093373, "learning_rate": 4.353212611955859e-05, "loss": -0.0011, "num_tokens": 4496521.0, "reward": 1.25906240940094, "reward_std": 0.08104812353849411, "rewards/rollout_reward_func/mean": 1.25906240940094, "rewards/rollout_reward_func/std": 0.12823933362960815, "sampling/importance_sampling_ratio/max": 1.0257768630981445, "sampling/importance_sampling_ratio/mean": 0.969860315322876, "sampling/importance_sampling_ratio/min": 0.5511314868927002, "sampling/sampling_logp_difference/max": 0.5969321727752686, "sampling/sampling_logp_difference/mean": 0.00662821251899004, "step": 158, "step_time": 10.001978536000024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 604.78125, "completions/mean_terminated_length": 604.78125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.02927525871200487, "epoch": 0.01272, "frac_reward_zero_std": 0.75, "grad_norm": 0.021029243245720863, "kl": 0.8227683119475842, "learning_rate": 4.353209445190892e-05, "loss": -0.0001, "num_tokens": 4533958.0, "reward": 1.2858332395553589, "reward_std": 0.027506457641720772, "rewards/rollout_reward_func/mean": 1.2858332395553589, "rewards/rollout_reward_func/std": 0.07652861624956131, "sampling/importance_sampling_ratio/max": 1.2505266666412354, "sampling/importance_sampling_ratio/mean": 0.9992468357086182, "sampling/importance_sampling_ratio/min": 0.7506878972053528, "sampling/sampling_logp_difference/max": 0.29521793127059937, "sampling/sampling_logp_difference/mean": 0.003966997377574444, "step": 159, "step_time": 10.039364564999687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 533.46875, "completions/mean_terminated_length": 533.46875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.011304380663204938, "epoch": 0.0128, "frac_reward_zero_std": 0.625, "grad_norm": 0.011366693302989006, "kl": 0.9064881429076195, "learning_rate": 4.3532062525768555e-05, "loss": -0.0003, "num_tokens": 4568721.0, "reward": 1.2855207920074463, "reward_std": 0.028131457045674324, "rewards/rollout_reward_func/mean": 1.2855207920074463, "rewards/rollout_reward_func/std": 0.07648930698633194, "sampling/importance_sampling_ratio/max": 1.0404860973358154, "sampling/importance_sampling_ratio/mean": 0.9948252439498901, "sampling/importance_sampling_ratio/min": 0.7524406909942627, "sampling/sampling_logp_difference/max": 0.2843436598777771, "sampling/sampling_logp_difference/mean": 0.0018665582174435258, "step": 160, "step_time": 9.98710430499932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 564.875, "completions/mean_terminated_length": 564.875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.029943338769953698, "epoch": 0.01288, "frac_reward_zero_std": 0.375, "grad_norm": 0.03177407756447792, "kl": 1.0525428280234337, "learning_rate": 4.3532030341138e-05, "loss": -0.0011, "num_tokens": 4604518.0, "reward": 1.2313541173934937, "reward_std": 0.1348046213388443, "rewards/rollout_reward_func/mean": 1.2313541173934937, "rewards/rollout_reward_func/std": 0.15947461128234863, "sampling/importance_sampling_ratio/max": 1.0794541835784912, "sampling/importance_sampling_ratio/mean": 0.9691417813301086, "sampling/importance_sampling_ratio/min": 0.432843416929245, "sampling/sampling_logp_difference/max": 0.8386726379394531, "sampling/sampling_logp_difference/mean": 0.009048236533999443, "step": 161, "step_time": 11.919637804000104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 583.53125, "completions/mean_terminated_length": 583.53125, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.016192513925489038, "epoch": 0.01296, "frac_reward_zero_std": 0.875, "grad_norm": 0.004461262375116348, "kl": 0.7707387581467628, "learning_rate": 4.353199789801776e-05, "loss": -0.0001, "num_tokens": 4641105.0, "reward": 1.2864582538604736, "reward_std": 0.02708333171904087, "rewards/rollout_reward_func/mean": 1.2864582538604736, "rewards/rollout_reward_func/std": 0.0766032263636589, "sampling/importance_sampling_ratio/max": 1.1370948553085327, "sampling/importance_sampling_ratio/mean": 1.0060611963272095, "sampling/importance_sampling_ratio/min": 0.8938983082771301, "sampling/sampling_logp_difference/max": 0.12163716554641724, "sampling/sampling_logp_difference/mean": 0.0018839314579963684, "step": 162, "step_time": 9.440906460000406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 574.8125, "completions/mean_terminated_length": 574.8125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.010697373538278043, "epoch": 0.01304, "frac_reward_zero_std": 0.625, "grad_norm": 0.011258811689913273, "kl": 1.2101338878273964, "learning_rate": 4.353196519640835e-05, "loss": -0.0003, "num_tokens": 4677008.0, "reward": 1.2858332395553589, "reward_std": 0.028333330526947975, "rewards/rollout_reward_func/mean": 1.2858332395553589, "rewards/rollout_reward_func/std": 0.07652861624956131, "sampling/importance_sampling_ratio/max": 1.0191643238067627, "sampling/importance_sampling_ratio/mean": 0.9909625053405762, "sampling/importance_sampling_ratio/min": 0.5977583527565002, "sampling/sampling_logp_difference/max": 0.5164883136749268, "sampling/sampling_logp_difference/mean": 0.003011536318808794, "step": 163, "step_time": 9.975566901999628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 514.875, "completions/mean_terminated_length": 514.875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.006927478214493021, "epoch": 0.01312, "frac_reward_zero_std": 0.625, "grad_norm": 3.4697131923167035e-05, "kl": 0.8487632386386395, "learning_rate": 4.3531932236310296e-05, "loss": 0.0002, "num_tokens": 4710389.0, "reward": 1.2990624904632568, "reward_std": 0.0018749982118606567, "rewards/rollout_reward_func/mean": 1.2990624904632568, "rewards/rollout_reward_func/std": 0.002961442805826664, "sampling/importance_sampling_ratio/max": 1.0678205490112305, "sampling/importance_sampling_ratio/mean": 1.0028151273727417, "sampling/importance_sampling_ratio/min": 0.9939342141151428, "sampling/sampling_logp_difference/max": 0.04648207128047943, "sampling/sampling_logp_difference/mean": 0.0006423327140510082, "step": 164, "step_time": 9.91169081399994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 616.0625, "completions/mean_terminated_length": 616.0625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.011562560335732996, "epoch": 0.0132, "frac_reward_zero_std": 0.5, "grad_norm": 0.016466205939650536, "kl": 0.9879420548677444, "learning_rate": 4.353189901772411e-05, "loss": -0.0002, "num_tokens": 4748189.0, "reward": 1.2587499618530273, "reward_std": 0.08249999582767487, "rewards/rollout_reward_func/mean": 1.2587499618530273, "rewards/rollout_reward_func/std": 0.16892819106578827, "sampling/importance_sampling_ratio/max": 1.0204384326934814, "sampling/importance_sampling_ratio/mean": 0.9929096698760986, "sampling/importance_sampling_ratio/min": 0.7229320406913757, "sampling/sampling_logp_difference/max": 0.3667569160461426, "sampling/sampling_logp_difference/mean": 0.0021392167545855045, "step": 165, "step_time": 11.636611924000135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 575.40625, "completions/mean_terminated_length": 575.40625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.010997199249686673, "epoch": 0.01328, "frac_reward_zero_std": 0.75, "grad_norm": 0.003511278424412012, "kl": 0.9452344179153442, "learning_rate": 4.353186554065032e-05, "loss": 0.0, "num_tokens": 4784586.0, "reward": 1.2990624904632568, "reward_std": 0.0013466866221278906, "rewards/rollout_reward_func/mean": 1.2990624904632568, "rewards/rollout_reward_func/std": 0.002961442805826664, "sampling/importance_sampling_ratio/max": 1.0626952648162842, "sampling/importance_sampling_ratio/mean": 1.0035834312438965, "sampling/importance_sampling_ratio/min": 0.999974250793457, "sampling/sampling_logp_difference/max": 0.0606808140873909, "sampling/sampling_logp_difference/mean": 0.0005475750658661127, "step": 166, "step_time": 9.963319588000559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 565.8125, "completions/mean_terminated_length": 565.8125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.017424317193217576, "epoch": 0.01336, "frac_reward_zero_std": 0.375, "grad_norm": 0.029630489647388458, "kl": 1.0189038813114166, "learning_rate": 4.353183180508946e-05, "loss": 0.0003, "num_tokens": 4819938.0, "reward": 1.2716665267944336, "reward_std": 0.05583978444337845, "rewards/rollout_reward_func/mean": 1.2716665267944336, "rewards/rollout_reward_func/std": 0.10629639029502869, "sampling/importance_sampling_ratio/max": 1.0892724990844727, "sampling/importance_sampling_ratio/mean": 0.994946300983429, "sampling/importance_sampling_ratio/min": 0.7306466698646545, "sampling/sampling_logp_difference/max": 0.31448888778686523, "sampling/sampling_logp_difference/mean": 0.0026783260982483625, "step": 167, "step_time": 9.97512074199949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 485.28125, "completions/mean_terminated_length": 485.28125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.012462181664886884, "epoch": 0.01344, "frac_reward_zero_std": 0.375, "grad_norm": 0.014980134554207325, "kl": 0.7726731151342392, "learning_rate": 4.3531797811042064e-05, "loss": 0.0003, "num_tokens": 4851607.0, "reward": 1.2852082252502441, "reward_std": 0.02958332933485508, "rewards/rollout_reward_func/mean": 1.2852082252502441, "rewards/rollout_reward_func/std": 0.07644865661859512, "sampling/importance_sampling_ratio/max": 1.4319759607315063, "sampling/importance_sampling_ratio/mean": 1.0041286945343018, "sampling/importance_sampling_ratio/min": 0.6129228472709656, "sampling/sampling_logp_difference/max": 0.49177223443984985, "sampling/sampling_logp_difference/mean": 0.004754368215799332, "step": 168, "step_time": 9.896625917999245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 593.90625, "completions/mean_terminated_length": 593.90625, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 0.007790367817506194, "epoch": 0.01352, "frac_reward_zero_std": 0.75, "grad_norm": 0.0068872361443936825, "kl": 0.9091915264725685, "learning_rate": 4.3531763558508664e-05, "loss": 0.0006, "num_tokens": 4888409.0, "reward": 1.2993749380111694, "reward_std": 0.0012499988079071045, "rewards/rollout_reward_func/mean": 1.2993749380111694, "rewards/rollout_reward_func/std": 0.0024593444541096687, "sampling/importance_sampling_ratio/max": 1.2029170989990234, "sampling/importance_sampling_ratio/mean": 1.0077660083770752, "sampling/importance_sampling_ratio/min": 0.9987561702728271, "sampling/sampling_logp_difference/max": 0.16436177492141724, "sampling/sampling_logp_difference/mean": 0.00113835115917027, "step": 169, "step_time": 11.338921163000578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 727.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 542.1875, "completions/mean_terminated_length": 542.1875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.004340102357673459, "epoch": 0.0136, "frac_reward_zero_std": 0.625, "grad_norm": 7.956532499520108e-05, "kl": 1.0013542473316193, "learning_rate": 4.3531729047489805e-05, "loss": 0.0002, "num_tokens": 4922835.0, "reward": 1.2990624904632568, "reward_std": 0.0018749982118606567, "rewards/rollout_reward_func/mean": 1.2990624904632568, "rewards/rollout_reward_func/std": 0.002961442805826664, "sampling/importance_sampling_ratio/max": 1.020132064819336, "sampling/importance_sampling_ratio/mean": 1.0019829273223877, "sampling/importance_sampling_ratio/min": 0.9995527863502502, "sampling/sampling_logp_difference/max": 0.014393346384167671, "sampling/sampling_logp_difference/mean": 0.00032068605651147664, "step": 170, "step_time": 9.896850555000128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 650.0, "completions/max_terminated_length": 650.0, "completions/mean_length": 543.65625, "completions/mean_terminated_length": 543.65625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.011625372848357074, "epoch": 0.01368, "frac_reward_zero_std": 0.875, "grad_norm": 0.003127025905996561, "kl": 1.002815157175064, "learning_rate": 4.353169427798605e-05, "loss": -0.0001, "num_tokens": 4957118.0, "reward": 1.286145806312561, "reward_std": 0.026881452649831772, "rewards/rollout_reward_func/mean": 1.286145806312561, "rewards/rollout_reward_func/std": 0.07656658440828323, "sampling/importance_sampling_ratio/max": 1.1243923902511597, "sampling/importance_sampling_ratio/mean": 1.0045157670974731, "sampling/importance_sampling_ratio/min": 0.9840234518051147, "sampling/sampling_logp_difference/max": 0.061580151319503784, "sampling/sampling_logp_difference/mean": 0.0008591283112764359, "step": 171, "step_time": 9.37193212200009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 616.0625, "completions/mean_terminated_length": 616.0625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.004147661180468276, "epoch": 0.01376, "frac_reward_zero_std": 0.75, "grad_norm": 0.002386786276474595, "kl": 0.9730972163379192, "learning_rate": 4.353165924999792e-05, "loss": -0.0, "num_tokens": 4995559.0, "reward": 1.2993749380111694, "reward_std": 0.0012499988079071045, "rewards/rollout_reward_func/mean": 1.2993749380111694, "rewards/rollout_reward_func/std": 0.0024593444541096687, "sampling/importance_sampling_ratio/max": 1.1467556953430176, "sampling/importance_sampling_ratio/mean": 1.0051199197769165, "sampling/importance_sampling_ratio/min": 1.0000191926956177, "sampling/sampling_logp_difference/max": 0.1353173851966858, "sampling/sampling_logp_difference/mean": 0.0007193583878688514, "step": 172, "step_time": 10.0096960879996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 575.0, "completions/mean_terminated_length": 575.0, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.009936308022588491, "epoch": 0.01384, "frac_reward_zero_std": 0.375, "grad_norm": 0.012317944318056107, "kl": 0.9084576740860939, "learning_rate": 4.353162396352599e-05, "loss": 0.0009, "num_tokens": 5031753.0, "reward": 1.2716666460037231, "reward_std": 0.05666665732860565, "rewards/rollout_reward_func/mean": 1.2716666460037231, "rewards/rollout_reward_func/std": 0.10760339349508286, "sampling/importance_sampling_ratio/max": 1.0042599439620972, "sampling/importance_sampling_ratio/mean": 0.9956226944923401, "sampling/importance_sampling_ratio/min": 0.8825201392173767, "sampling/sampling_logp_difference/max": 0.13123559951782227, "sampling/sampling_logp_difference/mean": 0.000915643060579896, "step": 173, "step_time": 11.6172335199999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 596.71875, "completions/mean_terminated_length": 596.71875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.010944611713057384, "epoch": 0.01392, "frac_reward_zero_std": 0.625, "grad_norm": 0.02279139868915081, "kl": 0.8236223645508289, "learning_rate": 4.353158841857081e-05, "loss": -0.0001, "num_tokens": 5068693.0, "reward": 1.272291660308838, "reward_std": 0.055416662245988846, "rewards/rollout_reward_func/mean": 1.272291660308838, "rewards/rollout_reward_func/std": 0.15311211347579956, "sampling/importance_sampling_ratio/max": 1.039664387702942, "sampling/importance_sampling_ratio/mean": 1.0029631853103638, "sampling/importance_sampling_ratio/min": 0.9922633171081543, "sampling/sampling_logp_difference/max": 0.03476575389504433, "sampling/sampling_logp_difference/mean": 0.0005423135589808226, "step": 174, "step_time": 10.011209226000119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 611.34375, "completions/mean_terminated_length": 611.34375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.0057161499571520835, "epoch": 0.014, "frac_reward_zero_std": 0.625, "grad_norm": 0.09799474477767944, "kl": 4.905142150819302, "learning_rate": 4.353155261513295e-05, "loss": 0.001, "num_tokens": 5106698.0, "reward": 1.2587499618530273, "reward_std": 0.08249998837709427, "rewards/rollout_reward_func/mean": 1.2587499618530273, "rewards/rollout_reward_func/std": 0.1705751121044159, "sampling/importance_sampling_ratio/max": 1.108082890510559, "sampling/importance_sampling_ratio/mean": 0.9921061992645264, "sampling/importance_sampling_ratio/min": 0.6078328490257263, "sampling/sampling_logp_difference/max": 0.5001401901245117, "sampling/sampling_logp_difference/mean": 0.002820945344865322, "step": 175, "step_time": 9.997892340999897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 554.75, "completions/mean_terminated_length": 554.75, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.006264741386985406, "epoch": 0.01408, "frac_reward_zero_std": 0.75, "grad_norm": 3.663620736915618e-05, "kl": 0.8191276453435421, "learning_rate": 4.353151655321297e-05, "loss": 0.0002, "num_tokens": 5141623.0, "reward": 1.2993749380111694, "reward_std": 0.0012499988079071045, "rewards/rollout_reward_func/mean": 1.2993749380111694, "rewards/rollout_reward_func/std": 0.0024593444541096687, "sampling/importance_sampling_ratio/max": 1.0189716815948486, "sampling/importance_sampling_ratio/mean": 1.0012321472167969, "sampling/importance_sampling_ratio/min": 0.9934056997299194, "sampling/sampling_logp_difference/max": 0.013934636488556862, "sampling/sampling_logp_difference/mean": 0.000304545828839764, "step": 176, "step_time": 11.050024731999656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 564.90625, "completions/mean_terminated_length": 564.90625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.007204327121144161, "epoch": 0.01416, "frac_reward_zero_std": 0.75, "grad_norm": 0.0009165172814391553, "kl": 0.9098776429891586, "learning_rate": 4.353148023281145e-05, "loss": 0.0003, "num_tokens": 5177231.0, "reward": 1.2993749380111694, "reward_std": 0.0012499988079071045, "rewards/rollout_reward_func/mean": 1.2993749380111694, "rewards/rollout_reward_func/std": 0.0024593444541096687, "sampling/importance_sampling_ratio/max": 1.075183391571045, "sampling/importance_sampling_ratio/mean": 1.0049374103546143, "sampling/importance_sampling_ratio/min": 0.999923825263977, "sampling/sampling_logp_difference/max": 0.07126212120056152, "sampling/sampling_logp_difference/mean": 0.0007679307600483298, "step": 177, "step_time": 10.975029552999786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 617.6875, "completions/mean_terminated_length": 617.6875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.07533728139242157, "epoch": 0.01424, "frac_reward_zero_std": 0.5, "grad_norm": 0.017393946647644043, "kl": 1.0198535099625587, "learning_rate": 4.353144365392895e-05, "loss": -0.0009, "num_tokens": 5215132.0, "reward": 1.282395839691162, "reward_std": 0.03440365940332413, "rewards/rollout_reward_func/mean": 1.282395839691162, "rewards/rollout_reward_func/std": 0.07790827751159668, "sampling/importance_sampling_ratio/max": 1.1691420078277588, "sampling/importance_sampling_ratio/mean": 0.9771996140480042, "sampling/importance_sampling_ratio/min": 8.205423330409248e-19, "sampling/sampling_logp_difference/max": 31.890457153320312, "sampling/sampling_logp_difference/mean": 0.10820688307285309, "step": 178, "step_time": 10.920577911000237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 607.75, "completions/mean_terminated_length": 607.75, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.01950708485674113, "epoch": 0.01432, "frac_reward_zero_std": 0.375, "grad_norm": 0.011749503202736378, "kl": 0.9561147168278694, "learning_rate": 4.353140681656607e-05, "loss": 0.0006, "num_tokens": 5252427.0, "reward": 1.2852082252502441, "reward_std": 0.02958332933485508, "rewards/rollout_reward_func/mean": 1.2852082252502441, "rewards/rollout_reward_func/std": 0.07644865661859512, "sampling/importance_sampling_ratio/max": 1.0295978784561157, "sampling/importance_sampling_ratio/mean": 0.9980581402778625, "sampling/importance_sampling_ratio/min": 0.88387531042099, "sampling/sampling_logp_difference/max": 0.13681524991989136, "sampling/sampling_logp_difference/mean": 0.0012691197916865349, "step": 179, "step_time": 10.0804518369996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 504.46875, "completions/mean_terminated_length": 504.46875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.025836335844360292, "epoch": 0.0144, "frac_reward_zero_std": 0.625, "grad_norm": 0.01983824372291565, "kl": 0.8677699193358421, "learning_rate": 4.353136972072338e-05, "loss": -0.0001, "num_tokens": 5284737.0, "reward": 1.2719790935516357, "reward_std": 0.05468647927045822, "rewards/rollout_reward_func/mean": 1.2719790935516357, "rewards/rollout_reward_func/std": 0.10636669397354126, "sampling/importance_sampling_ratio/max": 1.5236939191818237, "sampling/importance_sampling_ratio/mean": 0.9991315007209778, "sampling/importance_sampling_ratio/min": 0.3289441168308258, "sampling/sampling_logp_difference/max": 1.1190135478973389, "sampling/sampling_logp_difference/mean": 0.00889965333044529, "step": 180, "step_time": 10.52761416499925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 563.125, "completions/mean_terminated_length": 563.125, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.030372591223567724, "epoch": 0.01448, "frac_reward_zero_std": 0.625, "grad_norm": 0.013370750471949577, "kl": 1.1906751468777657, "learning_rate": 4.3531332366401464e-05, "loss": -0.0004, "num_tokens": 5319694.0, "reward": 1.2452082633972168, "reward_std": 0.0796814039349556, "rewards/rollout_reward_func/mean": 1.2452082633972168, "rewards/rollout_reward_func/std": 0.18311506509780884, "sampling/importance_sampling_ratio/max": 1.0893418788909912, "sampling/importance_sampling_ratio/mean": 0.9801419377326965, "sampling/importance_sampling_ratio/min": 0.6132227182388306, "sampling/sampling_logp_difference/max": 0.4896097183227539, "sampling/sampling_logp_difference/mean": 0.005593525245785713, "step": 181, "step_time": 10.985957740999766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 512.96875, "completions/mean_terminated_length": 512.96875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.01676247126306407, "epoch": 0.01456, "frac_reward_zero_std": 0.875, "grad_norm": 0.0010566629935055971, "kl": 0.9723625853657722, "learning_rate": 4.3531294753600915e-05, "loss": 0.0002, "num_tokens": 5352895.0, "reward": 1.299687385559082, "reward_std": 0.0006249994039535522, "rewards/rollout_reward_func/mean": 1.299687385559082, "rewards/rollout_reward_func/std": 0.0017677651485428214, "sampling/importance_sampling_ratio/max": 1.116060733795166, "sampling/importance_sampling_ratio/mean": 1.004180669784546, "sampling/importance_sampling_ratio/min": 0.991771936416626, "sampling/sampling_logp_difference/max": 0.10641026496887207, "sampling/sampling_logp_difference/mean": 0.0008830494480207562, "step": 182, "step_time": 9.851821004001067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004807692486792803, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004807692486792803, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 556.65625, "completions/mean_terminated_length": 556.65625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.03164177911821753, "epoch": 0.01464, "frac_reward_zero_std": 0.5, "grad_norm": 0.03757935389876366, "kl": 0.9158384688198566, "learning_rate": 4.353125688232234e-05, "loss": 0.0007, "num_tokens": 5387905.0, "reward": 1.2316665649414062, "reward_std": 0.1358397901058197, "rewards/rollout_reward_func/mean": 1.2316665649414062, "rewards/rollout_reward_func/std": 0.2229405790567398, "sampling/importance_sampling_ratio/max": 1.4918242692947388, "sampling/importance_sampling_ratio/mean": 1.0210068225860596, "sampling/importance_sampling_ratio/min": 0.7384206652641296, "sampling/sampling_logp_difference/max": 0.3896501064300537, "sampling/sampling_logp_difference/mean": 0.005991713143885136, "step": 183, "step_time": 10.03782695100017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 605.71875, "completions/mean_terminated_length": 605.71875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.03585980128264055, "epoch": 0.01472, "frac_reward_zero_std": 0.625, "grad_norm": 0.024098459631204605, "kl": 0.9632038250565529, "learning_rate": 4.3531218752566334e-05, "loss": 0.0006, "num_tokens": 5424992.0, "reward": 1.272291660308838, "reward_std": 0.05541665852069855, "rewards/rollout_reward_func/mean": 1.272291660308838, "rewards/rollout_reward_func/std": 0.1077413335442543, "sampling/importance_sampling_ratio/max": 1.1494673490524292, "sampling/importance_sampling_ratio/mean": 1.0019748210906982, "sampling/importance_sampling_ratio/min": 0.8459352254867554, "sampling/sampling_logp_difference/max": 0.17202448844909668, "sampling/sampling_logp_difference/mean": 0.003051217645406723, "step": 184, "step_time": 10.84036684199964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 496.15625, "completions/mean_terminated_length": 496.15625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.030629963031969965, "epoch": 0.0148, "frac_reward_zero_std": 0.5, "grad_norm": 0.001491184113547206, "kl": 0.8436816185712814, "learning_rate": 4.3531180364333495e-05, "loss": 0.0002, "num_tokens": 5456892.0, "reward": 1.2987499237060547, "reward_std": 0.002499997615814209, "rewards/rollout_reward_func/mean": 1.2987499237060547, "rewards/rollout_reward_func/std": 0.0033601040486246347, "sampling/importance_sampling_ratio/max": 1.101388692855835, "sampling/importance_sampling_ratio/mean": 0.9931643009185791, "sampling/importance_sampling_ratio/min": 0.7900412082672119, "sampling/sampling_logp_difference/max": 0.23805826902389526, "sampling/sampling_logp_difference/mean": 0.002553326077759266, "step": 185, "step_time": 10.766335931000867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 583.34375, "completions/mean_terminated_length": 583.34375, "completions/min_length": 476.0, "completions/min_terminated_length": 476.0, "entropy": 0.01206105516757816, "epoch": 0.01488, "frac_reward_zero_std": 0.75, "grad_norm": 0.0001422903296770528, "kl": 0.8027200885117054, "learning_rate": 4.353114171762443e-05, "loss": 0.0001, "num_tokens": 5493758.0, "reward": 1.2990624904632568, "reward_std": 0.0013466866221278906, "rewards/rollout_reward_func/mean": 1.2990624904632568, "rewards/rollout_reward_func/std": 0.0029614430386573076, "sampling/importance_sampling_ratio/max": 1.0061222314834595, "sampling/importance_sampling_ratio/mean": 0.9992969632148743, "sampling/importance_sampling_ratio/min": 0.9831962585449219, "sampling/sampling_logp_difference/max": 0.01420692726969719, "sampling/sampling_logp_difference/mean": 0.0003592863504309207, "step": 186, "step_time": 10.007983509000951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 525.1875, "completions/mean_terminated_length": 525.1875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.023783182608895004, "epoch": 0.01496, "frac_reward_zero_std": 0.75, "grad_norm": 0.03161729872226715, "kl": 0.7117518000304699, "learning_rate": 4.353110281243976e-05, "loss": 0.0004, "num_tokens": 5527162.0, "reward": 1.2726041078567505, "reward_std": 0.054791662842035294, "rewards/rollout_reward_func/mean": 1.2726041078567505, "rewards/rollout_reward_func/std": 0.15315963327884674, "sampling/importance_sampling_ratio/max": 1.1010422706604004, "sampling/importance_sampling_ratio/mean": 1.0071568489074707, "sampling/importance_sampling_ratio/min": 0.986549973487854, "sampling/sampling_logp_difference/max": 0.09916400909423828, "sampling/sampling_logp_difference/mean": 0.0015463503077626228, "step": 187, "step_time": 9.74389550899923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 525.0625, "completions/mean_terminated_length": 525.0625, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.013718736357986927, "epoch": 0.01504, "frac_reward_zero_std": 0.75, "grad_norm": 9.582232451066375e-05, "kl": 0.956855520606041, "learning_rate": 4.3531063648780095e-05, "loss": 0.0003, "num_tokens": 5560847.0, "reward": 1.2990624904632568, "reward_std": 0.0013466866221278906, "rewards/rollout_reward_func/mean": 1.2990624904632568, "rewards/rollout_reward_func/std": 0.002961442805826664, "sampling/importance_sampling_ratio/max": 1.0260241031646729, "sampling/importance_sampling_ratio/mean": 1.0013630390167236, "sampling/importance_sampling_ratio/min": 0.9781720042228699, "sampling/sampling_logp_difference/max": 0.017893154174089432, "sampling/sampling_logp_difference/mean": 0.0005979373818263412, "step": 188, "step_time": 10.626862244999757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 544.6875, "completions/mean_terminated_length": 544.6875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.019416438997723162, "epoch": 0.01512, "frac_reward_zero_std": 0.625, "grad_norm": 0.014020721428096294, "kl": 0.9514632374048233, "learning_rate": 4.353102422664606e-05, "loss": -0.0002, "num_tokens": 5594997.0, "reward": 1.2858332395553589, "reward_std": 0.028333330526947975, "rewards/rollout_reward_func/mean": 1.2858332395553589, "rewards/rollout_reward_func/std": 0.07652860879898071, "sampling/importance_sampling_ratio/max": 1.0068893432617188, "sampling/importance_sampling_ratio/mean": 0.9952936172485352, "sampling/importance_sampling_ratio/min": 0.8852721452713013, "sampling/sampling_logp_difference/max": 0.11889767646789551, "sampling/sampling_logp_difference/mean": 0.0010965296532958746, "step": 189, "step_time": 10.943649225998342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 584.09375, "completions/mean_terminated_length": 584.09375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.014031538972631097, "epoch": 0.0152, "frac_reward_zero_std": 0.625, "grad_norm": 0.0026950715109705925, "kl": 0.947668232023716, "learning_rate": 4.353098454603827e-05, "loss": 0.0002, "num_tokens": 5631721.0, "reward": 1.298437476158142, "reward_std": 0.0018749982118606567, "rewards/rollout_reward_func/mean": 1.298437476158142, "rewards/rollout_reward_func/std": 0.003689016681164503, "sampling/importance_sampling_ratio/max": 1.0577576160430908, "sampling/importance_sampling_ratio/mean": 1.0049984455108643, "sampling/importance_sampling_ratio/min": 0.9986441135406494, "sampling/sampling_logp_difference/max": 0.053248122334480286, "sampling/sampling_logp_difference/mean": 0.0008174946415238082, "step": 190, "step_time": 10.047503318000963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 570.71875, "completions/mean_terminated_length": 570.71875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.021503833530005068, "epoch": 0.01528, "frac_reward_zero_std": 0.375, "grad_norm": 0.01589382067322731, "kl": 0.9355600588023663, "learning_rate": 4.353094460695736e-05, "loss": -0.0001, "num_tokens": 5667493.0, "reward": 1.2442708015441895, "reward_std": 0.08191052079200745, "rewards/rollout_reward_func/mean": 1.2442708015441895, "rewards/rollout_reward_func/std": 0.18514594435691833, "sampling/importance_sampling_ratio/max": 1.0161213874816895, "sampling/importance_sampling_ratio/mean": 0.974028468132019, "sampling/importance_sampling_ratio/min": 0.6402817964553833, "sampling/sampling_logp_difference/max": 0.5431919097900391, "sampling/sampling_logp_difference/mean": 0.006446410436183214, "step": 191, "step_time": 10.051030008000453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 614.0, "completions/mean_terminated_length": 614.0, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.01801345427520573, "epoch": 0.01536, "frac_reward_zero_std": 0.75, "grad_norm": 0.0009582291822880507, "kl": 1.0656500235199928, "learning_rate": 4.353090440940396e-05, "loss": -0.0006, "num_tokens": 5705369.0, "reward": 1.2858332395553589, "reward_std": 0.027805019170045853, "rewards/rollout_reward_func/mean": 1.2858332395553589, "rewards/rollout_reward_func/std": 0.07652861624956131, "sampling/importance_sampling_ratio/max": 1.0245709419250488, "sampling/importance_sampling_ratio/mean": 0.9762355089187622, "sampling/importance_sampling_ratio/min": 0.2331194132566452, "sampling/sampling_logp_difference/max": 1.4661226272583008, "sampling/sampling_logp_difference/mean": 0.00718722166493535, "step": 192, "step_time": 11.153203620998283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1073.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 763.34375, "completions/mean_terminated_length": 763.34375, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.006955954362638295, "epoch": 0.01544, "frac_reward_zero_std": 0.125, "grad_norm": 0.0003996436716988683, "kl": 1.1834549978375435, "learning_rate": 4.353086395337871e-05, "loss": -0.0002, "num_tokens": 5747130.0, "reward": 1.395937442779541, "reward_std": 0.00476174708455801, "rewards/rollout_reward_func/mean": 1.395937442779541, "rewards/rollout_reward_func/std": 0.004989904351532459, "sampling/importance_sampling_ratio/max": 1.0339959859848022, "sampling/importance_sampling_ratio/mean": 1.001948356628418, "sampling/importance_sampling_ratio/min": 0.9982563257217407, "sampling/sampling_logp_difference/max": 0.03669443726539612, "sampling/sampling_logp_difference/mean": 0.0003700521483551711, "step": 193, "step_time": 13.352387535998787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 949.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 694.28125, "completions/mean_terminated_length": 694.28125, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.0178364475723356, "epoch": 0.01552, "frac_reward_zero_std": 0.375, "grad_norm": 0.017448442056775093, "kl": 1.0990071222186089, "learning_rate": 4.353082323888225e-05, "loss": 0.0008, "num_tokens": 5785558.0, "reward": 1.3524999618530273, "reward_std": 0.07251521199941635, "rewards/rollout_reward_func/mean": 1.3524999618530273, "rewards/rollout_reward_func/std": 0.12199947237968445, "sampling/importance_sampling_ratio/max": 1.3369085788726807, "sampling/importance_sampling_ratio/mean": 1.0167460441589355, "sampling/importance_sampling_ratio/min": 0.8768891096115112, "sampling/sampling_logp_difference/max": 0.2545987665653229, "sampling/sampling_logp_difference/mean": 0.0028782528825104237, "step": 194, "step_time": 11.507688903000599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 949.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 735.8125, "completions/mean_terminated_length": 735.8125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.013695926463697106, "epoch": 0.0156, "frac_reward_zero_std": 0.125, "grad_norm": 0.00796057004481554, "kl": 1.2297316119074821, "learning_rate": 4.3530782265915224e-05, "loss": -0.0002, "num_tokens": 5825985.0, "reward": 1.3403124809265137, "reward_std": 0.09666693210601807, "rewards/rollout_reward_func/mean": 1.3403124809265137, "rewards/rollout_reward_func/std": 0.13487112522125244, "sampling/importance_sampling_ratio/max": 1.0570554733276367, "sampling/importance_sampling_ratio/mean": 1.0045123100280762, "sampling/importance_sampling_ratio/min": 0.9903717637062073, "sampling/sampling_logp_difference/max": 0.0546388179063797, "sampling/sampling_logp_difference/mean": 0.0007240338018164039, "step": 195, "step_time": 11.49658450700008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0036764706019312143, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036764706019312143, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 849.53125, "completions/mean_terminated_length": 849.53125, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.020247139735147357, "epoch": 0.01568, "frac_reward_zero_std": 0.125, "grad_norm": 0.00807209312915802, "kl": 1.1111843958497047, "learning_rate": 4.353074103447828e-05, "loss": 0.0004, "num_tokens": 5870633.0, "reward": 1.3746875524520874, "reward_std": 0.04761234298348427, "rewards/rollout_reward_func/mean": 1.3746875524520874, "rewards/rollout_reward_func/std": 0.08795102685689926, "sampling/importance_sampling_ratio/max": 1.1836891174316406, "sampling/importance_sampling_ratio/mean": 1.007298231124878, "sampling/importance_sampling_ratio/min": 0.7806801795959473, "sampling/sampling_logp_difference/max": 0.2481861114501953, "sampling/sampling_logp_difference/mean": 0.002712228335440159, "step": 196, "step_time": 13.35118533899913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 860.28125, "completions/mean_terminated_length": 860.28125, "completions/min_length": 695.0, "completions/min_terminated_length": 695.0, "entropy": 0.01277871715137735, "epoch": 0.01576, "frac_reward_zero_std": 0.25, "grad_norm": 0.003799167927354574, "kl": 1.0592977851629257, "learning_rate": 4.353069954457207e-05, "loss": 0.0008, "num_tokens": 5916244.0, "reward": 1.375, "reward_std": 0.048645928502082825, "rewards/rollout_reward_func/mean": 1.375, "rewards/rollout_reward_func/std": 0.09058947116136551, "sampling/importance_sampling_ratio/max": 1.0673081874847412, "sampling/importance_sampling_ratio/mean": 1.0060300827026367, "sampling/importance_sampling_ratio/min": 0.9982129335403442, "sampling/sampling_logp_difference/max": 0.06408122181892395, "sampling/sampling_logp_difference/mean": 0.0008198700961656868, "step": 197, "step_time": 13.484314489998724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 825.8125, "completions/mean_terminated_length": 825.8125, "completions/min_length": 624.0, "completions/min_terminated_length": 624.0, "entropy": 0.009238059108611196, "epoch": 0.01584, "frac_reward_zero_std": 0.125, "grad_norm": 0.003107727039605379, "kl": 1.1252694875001907, "learning_rate": 4.353065779619725e-05, "loss": 0.0011, "num_tokens": 5960869.0, "reward": 1.3068749904632568, "reward_std": 0.1268956959247589, "rewards/rollout_reward_func/mean": 1.3068749904632568, "rewards/rollout_reward_func/std": 0.1617133617401123, "sampling/importance_sampling_ratio/max": 1.2391324043273926, "sampling/importance_sampling_ratio/mean": 1.008238434791565, "sampling/importance_sampling_ratio/min": 0.9860057830810547, "sampling/sampling_logp_difference/max": 0.19947046041488647, "sampling/sampling_logp_difference/mean": 0.0011587428161874413, "step": 198, "step_time": 12.703931843999271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1069.0, "completions/max_terminated_length": 1069.0, "completions/mean_length": 743.0, "completions/mean_terminated_length": 743.0, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.009121247770963237, "epoch": 0.01592, "frac_reward_zero_std": 0.375, "grad_norm": 0.00039089928031899035, "kl": 0.9071030840277672, "learning_rate": 4.353061578935449e-05, "loss": -0.0001, "num_tokens": 6001550.0, "reward": 1.3643748760223389, "reward_std": 0.0698961466550827, "rewards/rollout_reward_func/mean": 1.3643748760223389, "rewards/rollout_reward_func/std": 0.10823920369148254, "sampling/importance_sampling_ratio/max": 1.0110955238342285, "sampling/importance_sampling_ratio/mean": 0.9940100908279419, "sampling/importance_sampling_ratio/min": 0.7571471333503723, "sampling/sampling_logp_difference/max": 0.28101617097854614, "sampling/sampling_logp_difference/mean": 0.001375861349515617, "step": 199, "step_time": 12.635020617999544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 818.375, "completions/mean_terminated_length": 818.375, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.017509670986328274, "epoch": 0.016, "frac_reward_zero_std": 0.25, "grad_norm": 0.01798006147146225, "kl": 1.0504795089364052, "learning_rate": 4.353057352404445e-05, "loss": 0.0011, "num_tokens": 6044984.0, "reward": 1.353124976158142, "reward_std": 0.09209892898797989, "rewards/rollout_reward_func/mean": 1.353124976158142, "rewards/rollout_reward_func/std": 0.12224875390529633, "sampling/importance_sampling_ratio/max": 1.3264251947402954, "sampling/importance_sampling_ratio/mean": 1.0160691738128662, "sampling/importance_sampling_ratio/min": 0.9793035387992859, "sampling/sampling_logp_difference/max": 0.24639105796813965, "sampling/sampling_logp_difference/mean": 0.0019017900340259075, "step": 200, "step_time": 13.25288072299827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00390625, "completions/clipped_ratio": 0.0, "completions/max_length": 1067.0, "completions/max_terminated_length": 1067.0, "completions/mean_length": 759.0, "completions/mean_terminated_length": 759.0, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.01959760277532041, "epoch": 0.01608, "frac_reward_zero_std": 0.0, "grad_norm": 0.03478989005088806, "kl": 1.2558467164635658, "learning_rate": 4.353053100026779e-05, "loss": 0.0015, "num_tokens": 6086299.0, "reward": 1.3174999952316284, "reward_std": 0.12327270209789276, "rewards/rollout_reward_func/mean": 1.3174999952316284, "rewards/rollout_reward_func/std": 0.15385372936725616, "sampling/importance_sampling_ratio/max": 1.6704044342041016, "sampling/importance_sampling_ratio/mean": 1.0220284461975098, "sampling/importance_sampling_ratio/min": 0.7833827137947083, "sampling/sampling_logp_difference/max": 0.5130385160446167, "sampling/sampling_logp_difference/mean": 0.0036636909935623407, "step": 201, "step_time": 13.301217414001258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1079.0, "completions/max_terminated_length": 1079.0, "completions/mean_length": 967.15625, "completions/mean_terminated_length": 967.15625, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.011776998988352716, "epoch": 0.01616, "frac_reward_zero_std": 0.25, "grad_norm": 0.0017125749727711082, "kl": 0.9092152938246727, "learning_rate": 4.353048821802519e-05, "loss": -0.0005, "num_tokens": 6136601.0, "reward": 1.3640625476837158, "reward_std": 0.07104925066232681, "rewards/rollout_reward_func/mean": 1.3640625476837158, "rewards/rollout_reward_func/std": 0.10921595245599747, "sampling/importance_sampling_ratio/max": 1.0726654529571533, "sampling/importance_sampling_ratio/mean": 1.0046420097351074, "sampling/importance_sampling_ratio/min": 0.9932113289833069, "sampling/sampling_logp_difference/max": 0.07011301815509796, "sampling/sampling_logp_difference/mean": 0.000728521088603884, "step": 202, "step_time": 12.82337846000064 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 825.5625, "completions/mean_terminated_length": 825.5625, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "entropy": 0.022307000239379704, "epoch": 0.01624, "frac_reward_zero_std": 0.25, "grad_norm": 0.01964915357530117, "kl": 1.1332311779260635, "learning_rate": 4.353044517731733e-05, "loss": -0.0007, "num_tokens": 6180575.0, "reward": 1.3309375047683716, "reward_std": 0.13481725752353668, "rewards/rollout_reward_func/mean": 1.3309375047683716, "rewards/rollout_reward_func/std": 0.20898152887821198, "sampling/importance_sampling_ratio/max": 1.0395678281784058, "sampling/importance_sampling_ratio/mean": 0.9804682731628418, "sampling/importance_sampling_ratio/min": 0.5053101778030396, "sampling/sampling_logp_difference/max": 0.8161218166351318, "sampling/sampling_logp_difference/mean": 0.005150239914655685, "step": 203, "step_time": 12.551520767998227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 753.71875, "completions/mean_terminated_length": 753.71875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.02391993917990476, "epoch": 0.01632, "frac_reward_zero_std": 0.125, "grad_norm": 0.028941020369529724, "kl": 1.3225923255085945, "learning_rate": 4.3530401878144896e-05, "loss": -0.0021, "num_tokens": 6221482.0, "reward": 1.2768750190734863, "reward_std": 0.17395055294036865, "rewards/rollout_reward_func/mean": 1.2768750190734863, "rewards/rollout_reward_func/std": 0.24707889556884766, "sampling/importance_sampling_ratio/max": 1.0538686513900757, "sampling/importance_sampling_ratio/mean": 0.9353786706924438, "sampling/importance_sampling_ratio/min": 0.3371962308883667, "sampling/sampling_logp_difference/max": 1.0658338069915771, "sampling/sampling_logp_difference/mean": 0.01226288266479969, "step": 204, "step_time": 14.145842533000177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 949.0, "completions/max_terminated_length": 949.0, "completions/mean_length": 713.0625, "completions/mean_terminated_length": 713.0625, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.018618440022692084, "epoch": 0.0164, "frac_reward_zero_std": 0.25, "grad_norm": 0.014042842201888561, "kl": 0.8669618666172028, "learning_rate": 4.3530358320508564e-05, "loss": -0.0012, "num_tokens": 6260679.0, "reward": 1.3606250286102295, "reward_std": 0.07586301863193512, "rewards/rollout_reward_func/mean": 1.3606250286102295, "rewards/rollout_reward_func/std": 0.10841787606477737, "sampling/importance_sampling_ratio/max": 1.06587553024292, "sampling/importance_sampling_ratio/mean": 0.9722824096679688, "sampling/importance_sampling_ratio/min": 1.137015206142402e-15, "sampling/sampling_logp_difference/max": 34.16071701049805, "sampling/sampling_logp_difference/mean": 0.05092347785830498, "step": 205, "step_time": 13.104720227998769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 856.0, "completions/mean_terminated_length": 856.0, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.018581462500151247, "epoch": 0.01648, "frac_reward_zero_std": 0.375, "grad_norm": 0.01393978949636221, "kl": 1.303908795118332, "learning_rate": 4.3530314504409025e-05, "loss": 0.0004, "num_tokens": 6305868.0, "reward": 1.3306249380111694, "reward_std": 0.11686389148235321, "rewards/rollout_reward_func/mean": 1.3306249380111694, "rewards/rollout_reward_func/std": 0.16886600852012634, "sampling/importance_sampling_ratio/max": 1.0740610361099243, "sampling/importance_sampling_ratio/mean": 0.99913489818573, "sampling/importance_sampling_ratio/min": 0.7769865393638611, "sampling/sampling_logp_difference/max": 0.17580747604370117, "sampling/sampling_logp_difference/mean": 0.001751350937411189, "step": 206, "step_time": 12.783024841000042 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 858.3125, "completions/mean_terminated_length": 858.3125, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.012701814412139356, "epoch": 0.01656, "frac_reward_zero_std": 0.125, "grad_norm": 0.02169005014002323, "kl": 1.1269369646906853, "learning_rate": 4.353027042984698e-05, "loss": 0.0003, "num_tokens": 6351274.0, "reward": 1.3521875143051147, "reward_std": 0.09314772486686707, "rewards/rollout_reward_func/mean": 1.3521875143051147, "rewards/rollout_reward_func/std": 0.12283570319414139, "sampling/importance_sampling_ratio/max": 1.028767466545105, "sampling/importance_sampling_ratio/mean": 0.9974038600921631, "sampling/importance_sampling_ratio/min": 0.8480698466300964, "sampling/sampling_logp_difference/max": 0.16614848375320435, "sampling/sampling_logp_difference/mean": 0.00120417564176023, "step": 207, "step_time": 12.755263867001304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1079.0, "completions/max_terminated_length": 1079.0, "completions/mean_length": 858.1875, "completions/mean_terminated_length": 858.1875, "completions/min_length": 697.0, "completions/min_terminated_length": 697.0, "entropy": 0.012995885801501572, "epoch": 0.01664, "frac_reward_zero_std": 0.125, "grad_norm": 0.012265958823263645, "kl": 1.141316644847393, "learning_rate": 4.353022609682312e-05, "loss": 0.0006, "num_tokens": 6396343.0, "reward": 1.3628125190734863, "reward_std": 0.07105694711208344, "rewards/rollout_reward_func/mean": 1.3628125190734863, "rewards/rollout_reward_func/std": 0.10884242504835129, "sampling/importance_sampling_ratio/max": 1.1877846717834473, "sampling/importance_sampling_ratio/mean": 1.0187333822250366, "sampling/importance_sampling_ratio/min": 0.9997079968452454, "sampling/sampling_logp_difference/max": 0.16559074819087982, "sampling/sampling_logp_difference/mean": 0.002028217539191246, "step": 208, "step_time": 13.865726737999466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 810.875, "completions/mean_terminated_length": 810.875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.00929250117042102, "epoch": 0.01672, "frac_reward_zero_std": 0.25, "grad_norm": 0.0195176899433136, "kl": 1.2134939432144165, "learning_rate": 4.3530181505338156e-05, "loss": 0.001, "num_tokens": 6440012.0, "reward": 1.3299999237060547, "reward_std": 0.11827307194471359, "rewards/rollout_reward_func/mean": 1.3299999237060547, "rewards/rollout_reward_func/std": 0.14486925303936005, "sampling/importance_sampling_ratio/max": 1.8637868165969849, "sampling/importance_sampling_ratio/mean": 1.0312659740447998, "sampling/importance_sampling_ratio/min": 0.8888064622879028, "sampling/sampling_logp_difference/max": 0.6224155426025391, "sampling/sampling_logp_difference/mean": 0.0035812484566122293, "step": 209, "step_time": 12.699842445000286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 667.0625, "completions/mean_terminated_length": 667.0625, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.0118083554552868, "epoch": 0.0168, "frac_reward_zero_std": 0.125, "grad_norm": 0.021699773147702217, "kl": 1.3302117586135864, "learning_rate": 4.353013665539278e-05, "loss": -0.0001, "num_tokens": 6477067.0, "reward": 1.3518750667572021, "reward_std": 0.04864614084362984, "rewards/rollout_reward_func/mean": 1.3518750667572021, "rewards/rollout_reward_func/std": 0.1227228045463562, "sampling/importance_sampling_ratio/max": 1.6567604541778564, "sampling/importance_sampling_ratio/mean": 1.0171968936920166, "sampling/importance_sampling_ratio/min": 0.4456639289855957, "sampling/sampling_logp_difference/max": 0.8085798621177673, "sampling/sampling_logp_difference/mean": 0.006639677099883556, "step": 210, "step_time": 11.400979839000684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.0, "completions/max_terminated_length": 946.0, "completions/mean_length": 735.96875, "completions/mean_terminated_length": 735.96875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.014219694188795984, "epoch": 0.01688, "frac_reward_zero_std": 0.125, "grad_norm": 0.01701798290014267, "kl": 1.3959637433290482, "learning_rate": 4.353009154698771e-05, "loss": 0.001, "num_tokens": 6517238.0, "reward": 1.3403124809265137, "reward_std": 0.11523891985416412, "rewards/rollout_reward_func/mean": 1.3403124809265137, "rewards/rollout_reward_func/std": 0.13487112522125244, "sampling/importance_sampling_ratio/max": 1.3800396919250488, "sampling/importance_sampling_ratio/mean": 1.0152673721313477, "sampling/importance_sampling_ratio/min": 0.934730589389801, "sampling/sampling_logp_difference/max": 0.2805182933807373, "sampling/sampling_logp_difference/mean": 0.0022937562316656113, "step": 211, "step_time": 11.455478573001528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.007582720601931214, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007582720601931214, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 842.71875, "completions/mean_terminated_length": 842.71875, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.012328882497968152, "epoch": 0.01696, "frac_reward_zero_std": 0.375, "grad_norm": 0.008041408844292164, "kl": 0.9642734080553055, "learning_rate": 4.353004618012366e-05, "loss": 0.002, "num_tokens": 6561953.0, "reward": 1.3640625476837158, "reward_std": 0.06969517469406128, "rewards/rollout_reward_func/mean": 1.3640625476837158, "rewards/rollout_reward_func/std": 0.10814743489027023, "sampling/importance_sampling_ratio/max": 1.7417699098587036, "sampling/importance_sampling_ratio/mean": 1.0468037128448486, "sampling/importance_sampling_ratio/min": 0.9994431138038635, "sampling/sampling_logp_difference/max": 0.5522991418838501, "sampling/sampling_logp_difference/mean": 0.004340730141848326, "step": 212, "step_time": 14.013881145998312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 799.25, "completions/mean_terminated_length": 799.25, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.012393098033498973, "epoch": 0.01704, "frac_reward_zero_std": 0.375, "grad_norm": 0.02640676684677601, "kl": 1.1751731634140015, "learning_rate": 4.3530000554801354e-05, "loss": 0.0004, "num_tokens": 6604726.0, "reward": 1.341249942779541, "reward_std": 0.09520867466926575, "rewards/rollout_reward_func/mean": 1.341249942779541, "rewards/rollout_reward_func/std": 0.13439829647541046, "sampling/importance_sampling_ratio/max": 1.5823979377746582, "sampling/importance_sampling_ratio/mean": 0.9979571104049683, "sampling/importance_sampling_ratio/min": 0.37999415397644043, "sampling/sampling_logp_difference/max": 0.9676960110664368, "sampling/sampling_logp_difference/mean": 0.005302423145622015, "step": 213, "step_time": 12.59322357399924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 689.0625, "completions/mean_terminated_length": 689.0625, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.019323122163768858, "epoch": 0.01712, "frac_reward_zero_std": 0.125, "grad_norm": 0.03306151553988457, "kl": 1.2192897498607635, "learning_rate": 4.352995467102151e-05, "loss": -0.001, "num_tokens": 6642607.0, "reward": 1.3068749904632568, "reward_std": 0.14200256764888763, "rewards/rollout_reward_func/mean": 1.3068749904632568, "rewards/rollout_reward_func/std": 0.15736618638038635, "sampling/importance_sampling_ratio/max": 1.0766754150390625, "sampling/importance_sampling_ratio/mean": 0.961956262588501, "sampling/importance_sampling_ratio/min": 0.6119734048843384, "sampling/sampling_logp_difference/max": 0.4913320541381836, "sampling/sampling_logp_difference/mean": 0.0065686386078596115, "step": 214, "step_time": 11.49665304899736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1069.0, "completions/max_terminated_length": 1069.0, "completions/mean_length": 836.5625, "completions/mean_terminated_length": 836.5625, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.005768057948444039, "epoch": 0.0172, "frac_reward_zero_std": 0.25, "grad_norm": 0.02897113561630249, "kl": 0.9899681210517883, "learning_rate": 4.352990852878485e-05, "loss": 0.0047, "num_tokens": 6686862.0, "reward": 1.3424999713897705, "reward_std": 0.11364591866731644, "rewards/rollout_reward_func/mean": 1.3424999713897705, "rewards/rollout_reward_func/std": 0.25525444746017456, "sampling/importance_sampling_ratio/max": 1.0677052736282349, "sampling/importance_sampling_ratio/mean": 0.9963893890380859, "sampling/importance_sampling_ratio/min": 0.7359907031059265, "sampling/sampling_logp_difference/max": 0.3748788833618164, "sampling/sampling_logp_difference/mean": 0.0017382600344717503, "step": 215, "step_time": 12.86151616600091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1069.0, "completions/max_terminated_length": 1069.0, "completions/mean_length": 760.84375, "completions/mean_terminated_length": 760.84375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.007495094643672928, "epoch": 0.01728, "frac_reward_zero_std": 0.375, "grad_norm": 0.0440945066511631, "kl": 1.1108784824609756, "learning_rate": 4.352986212809211e-05, "loss": 0.0005, "num_tokens": 6728092.0, "reward": 1.3753125667572021, "reward_std": 0.04719538241624832, "rewards/rollout_reward_func/mean": 1.3753125667572021, "rewards/rollout_reward_func/std": 0.089369997382164, "sampling/importance_sampling_ratio/max": 1.0533312559127808, "sampling/importance_sampling_ratio/mean": 1.00504469871521, "sampling/importance_sampling_ratio/min": 0.9998946785926819, "sampling/sampling_logp_difference/max": 0.04908720403909683, "sampling/sampling_logp_difference/mean": 0.0005805892869830132, "step": 216, "step_time": 13.988293712000996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 935.0, "completions/mean_terminated_length": 935.0, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.00876357132801786, "epoch": 0.01736, "frac_reward_zero_std": 0.25, "grad_norm": 0.015669899061322212, "kl": 1.0506566725671291, "learning_rate": 4.3529815468944015e-05, "loss": -0.0013, "num_tokens": 6776839.0, "reward": 1.3637499809265137, "reward_std": 0.05189581960439682, "rewards/rollout_reward_func/mean": 1.3637499809265137, "rewards/rollout_reward_func/std": 0.10805465281009674, "sampling/importance_sampling_ratio/max": 1.127084493637085, "sampling/importance_sampling_ratio/mean": 0.989716649055481, "sampling/importance_sampling_ratio/min": 0.476799875497818, "sampling/sampling_logp_difference/max": 0.7395426630973816, "sampling/sampling_logp_difference/mean": 0.0032930090092122555, "step": 217, "step_time": 12.703108485000485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 812.25, "completions/mean_terminated_length": 812.25, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.006011943201883696, "epoch": 0.01744, "frac_reward_zero_std": 0.25, "grad_norm": 0.003371297148987651, "kl": 1.0788130089640617, "learning_rate": 4.3529768551341325e-05, "loss": -0.0001, "num_tokens": 6820077.0, "reward": 1.3637499809265137, "reward_std": 0.05242414027452469, "rewards/rollout_reward_func/mean": 1.3637499809265137, "rewards/rollout_reward_func/std": 0.1091240793466568, "sampling/importance_sampling_ratio/max": 1.2068098783493042, "sampling/importance_sampling_ratio/mean": 1.0084235668182373, "sampling/importance_sampling_ratio/min": 0.9994925856590271, "sampling/sampling_logp_difference/max": 0.18575119972229004, "sampling/sampling_logp_difference/mean": 0.0009199994383379817, "step": 218, "step_time": 12.609826256000815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1070.0, "completions/max_terminated_length": 1070.0, "completions/mean_length": 878.90625, "completions/mean_terminated_length": 878.90625, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.003030878579011187, "epoch": 0.01752, "frac_reward_zero_std": 0.625, "grad_norm": 9.69138418440707e-05, "kl": 0.9021786786615849, "learning_rate": 4.352972137528476e-05, "loss": 0.0004, "num_tokens": 6866547.0, "reward": 1.3875000476837158, "reward_std": 0.024471690878272057, "rewards/rollout_reward_func/mean": 1.3875000476837158, "rewards/rollout_reward_func/std": 0.0653032660484314, "sampling/importance_sampling_ratio/max": 1.016251564025879, "sampling/importance_sampling_ratio/mean": 1.0008690357208252, "sampling/importance_sampling_ratio/min": 0.9994603395462036, "sampling/sampling_logp_difference/max": 0.014313308522105217, "sampling/sampling_logp_difference/mean": 0.00012150093971285969, "step": 219, "step_time": 13.72492781299843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 787.71875, "completions/mean_terminated_length": 787.71875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.004698759017628618, "epoch": 0.0176, "frac_reward_zero_std": 0.375, "grad_norm": 0.0037652584724128246, "kl": 1.1737270131707191, "learning_rate": 4.3529673940775083e-05, "loss": 0.0007, "num_tokens": 6908664.0, "reward": 1.3640625476837158, "reward_std": 0.07052093744277954, "rewards/rollout_reward_func/mean": 1.3640625476837158, "rewards/rollout_reward_func/std": 0.10921595990657806, "sampling/importance_sampling_ratio/max": 1.1063340902328491, "sampling/importance_sampling_ratio/mean": 1.008526086807251, "sampling/importance_sampling_ratio/min": 0.9968012571334839, "sampling/sampling_logp_difference/max": 0.10101655125617981, "sampling/sampling_logp_difference/mean": 0.0010173411574214697, "step": 220, "step_time": 12.994045929001004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 846.21875, "completions/mean_terminated_length": 846.21875, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.003184319706633687, "epoch": 0.01768, "frac_reward_zero_std": 0.25, "grad_norm": 0.0006268086726777256, "kl": 1.1305500492453575, "learning_rate": 4.3529626247813036e-05, "loss": 0.0004, "num_tokens": 6954037.0, "reward": 1.3746874332427979, "reward_std": 0.04843766242265701, "rewards/rollout_reward_func/mean": 1.3746874332427979, "rewards/rollout_reward_func/std": 0.09051766991615295, "sampling/importance_sampling_ratio/max": 1.050655484199524, "sampling/importance_sampling_ratio/mean": 1.0019145011901855, "sampling/importance_sampling_ratio/min": 0.9893781542778015, "sampling/sampling_logp_difference/max": 0.04602622240781784, "sampling/sampling_logp_difference/mean": 0.00030205893563106656, "step": 221, "step_time": 12.752805059998536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 778.09375, "completions/mean_terminated_length": 778.09375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.006974071686272509, "epoch": 0.01776, "frac_reward_zero_std": 0.125, "grad_norm": 0.01005039643496275, "kl": 1.4584176018834114, "learning_rate": 4.3529578296399384e-05, "loss": 0.0017, "num_tokens": 6995561.0, "reward": 1.327187418937683, "reward_std": 0.1187576949596405, "rewards/rollout_reward_func/mean": 1.327187418937683, "rewards/rollout_reward_func/std": 0.14511919021606445, "sampling/importance_sampling_ratio/max": 1.6012929677963257, "sampling/importance_sampling_ratio/mean": 1.0230417251586914, "sampling/importance_sampling_ratio/min": 0.9966906905174255, "sampling/sampling_logp_difference/max": 0.46934860944747925, "sampling/sampling_logp_difference/mean": 0.002297968603670597, "step": 222, "step_time": 12.032114978000209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1070.0, "completions/max_terminated_length": 1070.0, "completions/mean_length": 788.28125, "completions/mean_terminated_length": 788.28125, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.0026757272935356013, "epoch": 0.01784, "frac_reward_zero_std": 0.125, "grad_norm": 0.00017036452481988817, "kl": 0.9798122048377991, "learning_rate": 4.352953008653487e-05, "loss": 0.0001, "num_tokens": 7038266.0, "reward": 1.3737499713897705, "reward_std": 0.04876521974802017, "rewards/rollout_reward_func/mean": 1.3737499713897705, "rewards/rollout_reward_func/std": 0.09029521048069, "sampling/importance_sampling_ratio/max": 1.0036282539367676, "sampling/importance_sampling_ratio/mean": 1.000563383102417, "sampling/importance_sampling_ratio/min": 0.9996352791786194, "sampling/sampling_logp_difference/max": 0.0027123214676976204, "sampling/sampling_logp_difference/mean": 7.249752525240183e-05, "step": 223, "step_time": 13.384920580999278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 754.8125, "completions/mean_terminated_length": 754.8125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.004091307535418309, "epoch": 0.01792, "frac_reward_zero_std": 0.125, "grad_norm": 0.007164743263274431, "kl": 1.1202947795391083, "learning_rate": 4.3529481618220277e-05, "loss": 0.0006, "num_tokens": 7079208.0, "reward": 1.3409374952316284, "reward_std": 0.11481447517871857, "rewards/rollout_reward_func/mean": 1.3409374952316284, "rewards/rollout_reward_func/std": 0.13429288566112518, "sampling/importance_sampling_ratio/max": 1.0368787050247192, "sampling/importance_sampling_ratio/mean": 0.9925178289413452, "sampling/importance_sampling_ratio/min": 0.6944671273231506, "sampling/sampling_logp_difference/max": 0.3741269111633301, "sampling/sampling_logp_difference/mean": 0.0016769724898040295, "step": 224, "step_time": 12.67211695900005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1067.0, "completions/max_terminated_length": 1067.0, "completions/mean_length": 792.96875, "completions/mean_terminated_length": 792.96875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.0035193069197703153, "epoch": 0.018, "frac_reward_zero_std": 0.25, "grad_norm": 0.000708606094121933, "kl": 1.304376482963562, "learning_rate": 4.352943289145636e-05, "loss": -0.0003, "num_tokens": 7122017.0, "reward": 1.329687476158142, "reward_std": 0.09974370896816254, "rewards/rollout_reward_func/mean": 1.329687476158142, "rewards/rollout_reward_func/std": 0.14472408592700958, "sampling/importance_sampling_ratio/max": 1.0082000494003296, "sampling/importance_sampling_ratio/mean": 0.9790818691253662, "sampling/importance_sampling_ratio/min": 0.29551875591278076, "sampling/sampling_logp_difference/max": 1.2205462455749512, "sampling/sampling_logp_difference/mean": 0.00470501184463501, "step": 225, "step_time": 12.586796108001181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 908.5, "completions/mean_terminated_length": 908.5, "completions/min_length": 741.0, "completions/min_terminated_length": 741.0, "entropy": 0.0020592653745552525, "epoch": 0.01808, "frac_reward_zero_std": 0.125, "grad_norm": 9.195167513098568e-05, "kl": 0.8180484175682068, "learning_rate": 4.3529383906243895e-05, "loss": 0.0002, "num_tokens": 7169865.0, "reward": 1.3753125667572021, "reward_std": 0.049375005066394806, "rewards/rollout_reward_func/mean": 1.3753125667572021, "rewards/rollout_reward_func/std": 0.09066010266542435, "sampling/importance_sampling_ratio/max": 1.0027103424072266, "sampling/importance_sampling_ratio/mean": 1.0003072023391724, "sampling/importance_sampling_ratio/min": 0.9997700452804565, "sampling/sampling_logp_difference/max": 0.0016643642447888851, "sampling/sampling_logp_difference/mean": 5.335191235644743e-05, "step": 226, "step_time": 12.662217518000034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1073.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 817.65625, "completions/mean_terminated_length": 817.65625, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.004091012175194919, "epoch": 0.01816, "frac_reward_zero_std": 0.125, "grad_norm": 0.0009728107252158225, "kl": 1.2105174362659454, "learning_rate": 4.352933466258365e-05, "loss": 0.0005, "num_tokens": 7213325.0, "reward": 1.317812442779541, "reward_std": 0.12347325682640076, "rewards/rollout_reward_func/mean": 1.317812442779541, "rewards/rollout_reward_func/std": 0.1547679752111435, "sampling/importance_sampling_ratio/max": 1.0542813539505005, "sampling/importance_sampling_ratio/mean": 1.0031394958496094, "sampling/importance_sampling_ratio/min": 0.9997279644012451, "sampling/sampling_logp_difference/max": 0.05175451189279556, "sampling/sampling_logp_difference/mean": 0.0003900099836755544, "step": 227, "step_time": 12.451788925000983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1074.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 857.59375, "completions/mean_terminated_length": 857.59375, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.003471946489298716, "epoch": 0.01824, "frac_reward_zero_std": 0.25, "grad_norm": 0.0008134921081364155, "kl": 1.1737002730369568, "learning_rate": 4.352928516047641e-05, "loss": 0.0001, "num_tokens": 7258898.0, "reward": 1.3521875143051147, "reward_std": 0.07482745498418808, "rewards/rollout_reward_func/mean": 1.3521875143051147, "rewards/rollout_reward_func/std": 0.12377749383449554, "sampling/importance_sampling_ratio/max": 1.007401704788208, "sampling/importance_sampling_ratio/mean": 1.000411868095398, "sampling/importance_sampling_ratio/min": 0.9902722239494324, "sampling/sampling_logp_difference/max": 0.008140049874782562, "sampling/sampling_logp_difference/mean": 0.0001859504554886371, "step": 228, "step_time": 13.05081267100286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 950.625, "completions/mean_terminated_length": 950.625, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.004122210462810472, "epoch": 0.01832, "frac_reward_zero_std": 0.25, "grad_norm": 0.0014444816624745727, "kl": 0.7899932637810707, "learning_rate": 4.352923539992297e-05, "loss": 0.0003, "num_tokens": 7308123.0, "reward": 1.3643749952316284, "reward_std": 0.07125000655651093, "rewards/rollout_reward_func/mean": 1.3643749952316284, "rewards/rollout_reward_func/std": 0.10930682718753815, "sampling/importance_sampling_ratio/max": 1.0109620094299316, "sampling/importance_sampling_ratio/mean": 1.0003173351287842, "sampling/importance_sampling_ratio/min": 0.9849017858505249, "sampling/sampling_logp_difference/max": 0.018824134021997452, "sampling/sampling_logp_difference/mean": 0.00020106861484237015, "step": 229, "step_time": 12.819134097000642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 714.84375, "completions/mean_terminated_length": 714.84375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.003325498619233258, "epoch": 0.0184, "frac_reward_zero_std": 0.125, "grad_norm": 0.0014230383094400167, "kl": 1.1215359196066856, "learning_rate": 4.35291853809241e-05, "loss": 0.0003, "num_tokens": 7347211.0, "reward": 1.3624999523162842, "reward_std": 0.026249999180436134, "rewards/rollout_reward_func/mean": 1.3624999523162842, "rewards/rollout_reward_func/std": 0.10874652862548828, "sampling/importance_sampling_ratio/max": 1.0188852548599243, "sampling/importance_sampling_ratio/mean": 1.0016140937805176, "sampling/importance_sampling_ratio/min": 0.9997276663780212, "sampling/sampling_logp_difference/max": 0.016084127128124237, "sampling/sampling_logp_difference/mean": 0.00020086145377717912, "step": 230, "step_time": 12.466587757000525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 784.59375, "completions/mean_terminated_length": 784.59375, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.002998186682816595, "epoch": 0.01848, "frac_reward_zero_std": 0.25, "grad_norm": 0.00041479169158264995, "kl": 1.236737608909607, "learning_rate": 4.3529135103480604e-05, "loss": 0.0004, "num_tokens": 7389323.0, "reward": 1.3634374141693115, "reward_std": 0.07104925066232681, "rewards/rollout_reward_func/mean": 1.3634374141693115, "rewards/rollout_reward_func/std": 0.10903119295835495, "sampling/importance_sampling_ratio/max": 1.0123037099838257, "sampling/importance_sampling_ratio/mean": 1.000969648361206, "sampling/importance_sampling_ratio/min": 0.9989936947822571, "sampling/sampling_logp_difference/max": 0.012214427813887596, "sampling/sampling_logp_difference/mean": 0.00013993452012073249, "step": 231, "step_time": 13.160630287000458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 825.0, "completions/max_terminated_length": 825.0, "completions/mean_length": 670.34375, "completions/mean_terminated_length": 670.34375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.001746599176840391, "epoch": 0.01856, "frac_reward_zero_std": 0.0, "grad_norm": 0.0007132685859687626, "kl": 0.9120770432054996, "learning_rate": 4.3529084567593274e-05, "loss": 0.0004, "num_tokens": 7426673.0, "reward": 1.3740625381469727, "reward_std": 0.0502900667488575, "rewards/rollout_reward_func/mean": 1.3740625381469727, "rewards/rollout_reward_func/std": 0.0903705433011055, "sampling/importance_sampling_ratio/max": 1.0069023370742798, "sampling/importance_sampling_ratio/mean": 1.000330924987793, "sampling/importance_sampling_ratio/min": 0.9999498724937439, "sampling/sampling_logp_difference/max": 0.005611442029476166, "sampling/sampling_logp_difference/mean": 4.606381480698474e-05, "step": 232, "step_time": 11.75446410800123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 787.9375, "completions/mean_terminated_length": 787.9375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.004278149863239378, "epoch": 0.01864, "frac_reward_zero_std": 0.375, "grad_norm": 0.00029920486849732697, "kl": 1.2465581446886063, "learning_rate": 4.3529033773262916e-05, "loss": 0.0003, "num_tokens": 7468702.0, "reward": 1.363437533378601, "reward_std": 0.05169506371021271, "rewards/rollout_reward_func/mean": 1.363437533378601, "rewards/rollout_reward_func/std": 0.10903120040893555, "sampling/importance_sampling_ratio/max": 1.0243152379989624, "sampling/importance_sampling_ratio/mean": 1.0023664236068726, "sampling/importance_sampling_ratio/min": 0.9983525276184082, "sampling/sampling_logp_difference/max": 0.019477814435958862, "sampling/sampling_logp_difference/mean": 0.0003059761947952211, "step": 233, "step_time": 12.601653269000963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 775.59375, "completions/mean_terminated_length": 775.59375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.002833324106177315, "epoch": 0.01872, "frac_reward_zero_std": 0.125, "grad_norm": 0.00034557850449346006, "kl": 0.9522892758250237, "learning_rate": 4.352898272049031e-05, "loss": 0.0003, "num_tokens": 7510643.0, "reward": 1.3865625858306885, "reward_std": 0.026875000447034836, "rewards/rollout_reward_func/mean": 1.3865625858306885, "rewards/rollout_reward_func/std": 0.0651850625872612, "sampling/importance_sampling_ratio/max": 1.0073658227920532, "sampling/importance_sampling_ratio/mean": 1.0008758306503296, "sampling/importance_sampling_ratio/min": 0.9992078542709351, "sampling/sampling_logp_difference/max": 0.007297653704881668, "sampling/sampling_logp_difference/mean": 0.00012273859465494752, "step": 234, "step_time": 12.628584027000215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 879.28125, "completions/mean_terminated_length": 879.28125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.003645331642474048, "epoch": 0.0188, "frac_reward_zero_std": 0.375, "grad_norm": 0.0001558636868139729, "kl": 1.0621516108512878, "learning_rate": 4.35289314092763e-05, "loss": -0.0001, "num_tokens": 7556674.0, "reward": 1.3753125667572021, "reward_std": 0.04802093654870987, "rewards/rollout_reward_func/mean": 1.3753125667572021, "rewards/rollout_reward_func/std": 0.09066010266542435, "sampling/importance_sampling_ratio/max": 1.020668864250183, "sampling/importance_sampling_ratio/mean": 1.00249183177948, "sampling/importance_sampling_ratio/min": 0.9991171360015869, "sampling/sampling_logp_difference/max": 0.02011857181787491, "sampling/sampling_logp_difference/mean": 0.00028348679188638926, "step": 235, "step_time": 13.021579721997114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1070.0, "completions/max_terminated_length": 1070.0, "completions/mean_length": 803.90625, "completions/mean_terminated_length": 803.90625, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.0034233445912832394, "epoch": 0.01888, "frac_reward_zero_std": 0.125, "grad_norm": 0.07419255375862122, "kl": 4.308958098292351, "learning_rate": 4.352887983962168e-05, "loss": 0.0015, "num_tokens": 7599572.0, "reward": 1.3643749952316284, "reward_std": 0.07042424380779266, "rewards/rollout_reward_func/mean": 1.3643749952316284, "rewards/rollout_reward_func/std": 0.13923610746860504, "sampling/importance_sampling_ratio/max": 1.0195599794387817, "sampling/importance_sampling_ratio/mean": 1.0016744136810303, "sampling/importance_sampling_ratio/min": 0.9992853999137878, "sampling/sampling_logp_difference/max": 0.018869303166866302, "sampling/sampling_logp_difference/mean": 0.0002004392008529976, "step": 236, "step_time": 13.076398480000535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 821.0625, "completions/mean_terminated_length": 821.0625, "completions/min_length": 623.0, "completions/min_terminated_length": 623.0, "entropy": 0.00393398740561679, "epoch": 0.01896, "frac_reward_zero_std": 0.375, "grad_norm": 0.001819550059735775, "kl": 1.1685934364795685, "learning_rate": 4.3528828011527264e-05, "loss": 0.0004, "num_tokens": 7643212.0, "reward": 1.3409374952316284, "reward_std": 0.09624247997999191, "rewards/rollout_reward_func/mean": 1.3409374952316284, "rewards/rollout_reward_func/std": 0.13598765432834625, "sampling/importance_sampling_ratio/max": 1.0184205770492554, "sampling/importance_sampling_ratio/mean": 1.0008788108825684, "sampling/importance_sampling_ratio/min": 0.9961562156677246, "sampling/sampling_logp_difference/max": 0.013584079220890999, "sampling/sampling_logp_difference/mean": 0.0001679242413956672, "step": 237, "step_time": 12.610900462000245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 861.46875, "completions/mean_terminated_length": 861.46875, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.008929777191951871, "epoch": 0.01904, "frac_reward_zero_std": 0.375, "grad_norm": 0.0018033122178167105, "kl": 0.9158731773495674, "learning_rate": 4.352877592499388e-05, "loss": 0.0, "num_tokens": 7688885.0, "reward": 1.3756250143051147, "reward_std": 0.04822169244289398, "rewards/rollout_reward_func/mean": 1.3756250143051147, "rewards/rollout_reward_func/std": 0.09072957187891006, "sampling/importance_sampling_ratio/max": 1.024948239326477, "sampling/importance_sampling_ratio/mean": 1.0000635385513306, "sampling/importance_sampling_ratio/min": 0.9795880317687988, "sampling/sampling_logp_difference/max": 0.02006128802895546, "sampling/sampling_logp_difference/mean": 0.00047332525718957186, "step": 238, "step_time": 12.697592687998622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1073.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 747.34375, "completions/mean_terminated_length": 747.34375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.007325785234570503, "epoch": 0.01912, "frac_reward_zero_std": 0.5, "grad_norm": 0.002523417118936777, "kl": 0.9120106101036072, "learning_rate": 4.352872358002235e-05, "loss": 0.0005, "num_tokens": 7729610.0, "reward": 1.386875033378601, "reward_std": 0.02489592507481575, "rewards/rollout_reward_func/mean": 1.386875033378601, "rewards/rollout_reward_func/std": 0.06522603332996368, "sampling/importance_sampling_ratio/max": 1.0043797492980957, "sampling/importance_sampling_ratio/mean": 0.9998886585235596, "sampling/importance_sampling_ratio/min": 0.9868246912956238, "sampling/sampling_logp_difference/max": 0.012637123465538025, "sampling/sampling_logp_difference/mean": 0.00020861122175119817, "step": 239, "step_time": 13.246423375999257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 893.21875, "completions/mean_terminated_length": 893.21875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.014964211382903159, "epoch": 0.0192, "frac_reward_zero_std": 0.5, "grad_norm": 0.016695383936166763, "kl": 1.019041657447815, "learning_rate": 4.3528670976613505e-05, "loss": 0.0001, "num_tokens": 7776754.0, "reward": 1.3643748760223389, "reward_std": 0.06907039135694504, "rewards/rollout_reward_func/mean": 1.3643748760223389, "rewards/rollout_reward_func/std": 0.10823920369148254, "sampling/importance_sampling_ratio/max": 1.0064201354980469, "sampling/importance_sampling_ratio/mean": 0.9929094314575195, "sampling/importance_sampling_ratio/min": 0.8456259369850159, "sampling/sampling_logp_difference/max": 0.1480047106742859, "sampling/sampling_logp_difference/mean": 0.0010414570569992065, "step": 240, "step_time": 13.776470687997971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1075.0, "completions/max_terminated_length": 1075.0, "completions/mean_length": 821.03125, "completions/mean_terminated_length": 821.03125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.020121681096497923, "epoch": 0.01928, "frac_reward_zero_std": 0.125, "grad_norm": 0.03309411182999611, "kl": 1.3817816227674484, "learning_rate": 4.3528618114768165e-05, "loss": 0.0023, "num_tokens": 7820536.0, "reward": 1.2521874904632568, "reward_std": 0.22009071707725525, "rewards/rollout_reward_func/mean": 1.2521874904632568, "rewards/rollout_reward_func/std": 0.32300737500190735, "sampling/importance_sampling_ratio/max": 1.6042089462280273, "sampling/importance_sampling_ratio/mean": 1.0293755531311035, "sampling/importance_sampling_ratio/min": 0.9230387210845947, "sampling/sampling_logp_difference/max": 0.4726090431213379, "sampling/sampling_logp_difference/mean": 0.004414524883031845, "step": 241, "step_time": 12.68996941399837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 803.28125, "completions/mean_terminated_length": 803.28125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.019756323075853288, "epoch": 0.01936, "frac_reward_zero_std": 0.375, "grad_norm": 0.010790826752781868, "kl": 1.0403105169534683, "learning_rate": 4.3528564994487184e-05, "loss": 0.0006, "num_tokens": 7863843.0, "reward": 1.3068749904632568, "reward_std": 0.11727352440357208, "rewards/rollout_reward_func/mean": 1.3068749904632568, "rewards/rollout_reward_func/std": 0.1610136777162552, "sampling/importance_sampling_ratio/max": 1.1919512748718262, "sampling/importance_sampling_ratio/mean": 1.0040278434753418, "sampling/importance_sampling_ratio/min": 0.9669156670570374, "sampling/sampling_logp_difference/max": 0.10626080632209778, "sampling/sampling_logp_difference/mean": 0.0014382557710632682, "step": 242, "step_time": 12.82130197999868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1072.0, "completions/max_terminated_length": 1072.0, "completions/mean_length": 825.59375, "completions/mean_terminated_length": 825.59375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.017846475762780756, "epoch": 0.01944, "frac_reward_zero_std": 0.375, "grad_norm": 0.014205086044967175, "kl": 1.1347443908452988, "learning_rate": 4.35285116157714e-05, "loss": -0.0001, "num_tokens": 7907937.0, "reward": 1.3534374237060547, "reward_std": 0.0922994613647461, "rewards/rollout_reward_func/mean": 1.3534374237060547, "rewards/rollout_reward_func/std": 0.12327805906534195, "sampling/importance_sampling_ratio/max": 1.0531905889511108, "sampling/importance_sampling_ratio/mean": 0.9895926117897034, "sampling/importance_sampling_ratio/min": 0.5983666777610779, "sampling/sampling_logp_difference/max": 0.515831708908081, "sampling/sampling_logp_difference/mean": 0.002615257864817977, "step": 243, "step_time": 13.595092517999547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 876.0, "completions/mean_terminated_length": 876.0, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.023155823815613985, "epoch": 0.01952, "frac_reward_zero_std": 0.5, "grad_norm": 0.0029363110661506653, "kl": 0.8656644225120544, "learning_rate": 4.3528457978621654e-05, "loss": 0.0008, "num_tokens": 7954148.0, "reward": 1.3756248950958252, "reward_std": 0.04750000685453415, "rewards/rollout_reward_func/mean": 1.3756248950958252, "rewards/rollout_reward_func/std": 0.09072957187891006, "sampling/importance_sampling_ratio/max": 1.0780805349349976, "sampling/importance_sampling_ratio/mean": 0.995234489440918, "sampling/importance_sampling_ratio/min": 0.9275557994842529, "sampling/sampling_logp_difference/max": 0.07169128209352493, "sampling/sampling_logp_difference/mean": 0.0013344994513317943, "step": 244, "step_time": 13.351626089998717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1074.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 811.9375, "completions/mean_terminated_length": 811.9375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.018343242176342756, "epoch": 0.0196, "frac_reward_zero_std": 0.0, "grad_norm": 0.006820749491453171, "kl": 1.2255051508545876, "learning_rate": 4.35284040830388e-05, "loss": -0.0005, "num_tokens": 7997471.0, "reward": 1.3403124809265137, "reward_std": 0.09786579012870789, "rewards/rollout_reward_func/mean": 1.3403124809265137, "rewards/rollout_reward_func/std": 0.13487112522125244, "sampling/importance_sampling_ratio/max": 1.1795234680175781, "sampling/importance_sampling_ratio/mean": 0.9879854917526245, "sampling/importance_sampling_ratio/min": 0.30893734097480774, "sampling/sampling_logp_difference/max": 1.177883505821228, "sampling/sampling_logp_difference/mean": 0.005922486539930105, "step": 245, "step_time": 12.654412299999422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 833.125, "completions/mean_terminated_length": 833.125, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.015432106680236757, "epoch": 0.01968, "frac_reward_zero_std": 0.125, "grad_norm": 0.016397248953580856, "kl": 0.9301614388823509, "learning_rate": 4.352834992902369e-05, "loss": -0.0, "num_tokens": 8041521.0, "reward": 1.3528125286102295, "reward_std": 0.09259669482707977, "rewards/rollout_reward_func/mean": 1.3528125286102295, "rewards/rollout_reward_func/std": 0.12118326127529144, "sampling/importance_sampling_ratio/max": 1.0044794082641602, "sampling/importance_sampling_ratio/mean": 0.9830595254898071, "sampling/importance_sampling_ratio/min": 0.7344288229942322, "sampling/sampling_logp_difference/max": 0.30916285514831543, "sampling/sampling_logp_difference/mean": 0.0022346044424921274, "step": 246, "step_time": 12.719296387997929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0028409091755747795, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0028409091755747795, "completions/clipped_ratio": 0.0, "completions/max_length": 1076.0, "completions/max_terminated_length": 1076.0, "completions/mean_length": 893.8125, "completions/mean_terminated_length": 893.8125, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.02766610262915492, "epoch": 0.01976, "frac_reward_zero_std": 0.25, "grad_norm": 0.005095380824059248, "kl": 0.9952676370739937, "learning_rate": 4.352829551657718e-05, "loss": 0.0013, "num_tokens": 8088779.0, "reward": 1.3643749952316284, "reward_std": 0.069598488509655, "rewards/rollout_reward_func/mean": 1.3643749952316284, "rewards/rollout_reward_func/std": 0.1071910411119461, "sampling/importance_sampling_ratio/max": 1.2506084442138672, "sampling/importance_sampling_ratio/mean": 1.0112621784210205, "sampling/importance_sampling_ratio/min": 0.9677061438560486, "sampling/sampling_logp_difference/max": 0.22481179237365723, "sampling/sampling_logp_difference/mean": 0.002222137525677681, "step": 247, "step_time": 13.48529559400049 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1073.0, "completions/max_terminated_length": 1073.0, "completions/mean_length": 759.28125, "completions/mean_terminated_length": 759.28125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.016109436168335378, "epoch": 0.01984, "frac_reward_zero_std": 0.375, "grad_norm": 0.00017063402628991753, "kl": 1.184188388288021, "learning_rate": 4.352824084570012e-05, "loss": 0.0003, "num_tokens": 8130054.0, "reward": 1.363124966621399, "reward_std": 0.051293425261974335, "rewards/rollout_reward_func/mean": 1.363124966621399, "rewards/rollout_reward_func/std": 0.10893731564283371, "sampling/importance_sampling_ratio/max": 1.059173583984375, "sampling/importance_sampling_ratio/mean": 0.9990248680114746, "sampling/importance_sampling_ratio/min": 0.9648736119270325, "sampling/sampling_logp_difference/max": 0.04194006323814392, "sampling/sampling_logp_difference/mean": 0.000747383339330554, "step": 248, "step_time": 13.59947278999698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 772.8125, "completions/mean_terminated_length": 772.8125, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "entropy": 0.015488804259803146, "epoch": 0.01992, "frac_reward_zero_std": 0.5, "grad_norm": 0.00736682303249836, "kl": 1.198657363653183, "learning_rate": 4.35281859163934e-05, "loss": 0.0001, "num_tokens": 8171621.0, "reward": 1.3643749952316284, "reward_std": 0.05061747878789902, "rewards/rollout_reward_func/mean": 1.3643749952316284, "rewards/rollout_reward_func/std": 0.10823920369148254, "sampling/importance_sampling_ratio/max": 1.0712273120880127, "sampling/importance_sampling_ratio/mean": 0.9906635284423828, "sampling/importance_sampling_ratio/min": 0.6561650633811951, "sampling/sampling_logp_difference/max": 0.4395906925201416, "sampling/sampling_logp_difference/mean": 0.0023677293211221695, "step": 249, "step_time": 11.561343429999397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 809.375, "completions/mean_terminated_length": 809.375, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.010312289698049426, "epoch": 0.02, "frac_reward_zero_std": 0.375, "grad_norm": 0.0004567661671899259, "kl": 1.0156885758042336, "learning_rate": 4.352813072865787e-05, "loss": 0.0001, "num_tokens": 8214933.0, "reward": 1.3406249284744263, "reward_std": 0.09618936479091644, "rewards/rollout_reward_func/mean": 1.3406249284744263, "rewards/rollout_reward_func/std": 0.13585898280143738, "sampling/importance_sampling_ratio/max": 1.0116921663284302, "sampling/importance_sampling_ratio/mean": 0.9995847940444946, "sampling/importance_sampling_ratio/min": 0.9866081476211548, "sampling/sampling_logp_difference/max": 0.012102082371711731, "sampling/sampling_logp_difference/mean": 0.00030595573480241, "step": 250, "step_time": 12.566381278001245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 948.0, "completions/max_terminated_length": 948.0, "completions/mean_length": 708.46875, "completions/mean_terminated_length": 708.46875, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.013004280743189156, "epoch": 0.02008, "frac_reward_zero_std": 0.0, "grad_norm": 0.000706467020791024, "kl": 1.47422706335783, "learning_rate": 4.352807528249442e-05, "loss": 0.0001, "num_tokens": 8253817.0, "reward": 1.3390624523162842, "reward_std": 0.09811009466648102, "rewards/rollout_reward_func/mean": 1.3390624523162842, "rewards/rollout_reward_func/std": 0.1352025866508484, "sampling/importance_sampling_ratio/max": 1.0191792249679565, "sampling/importance_sampling_ratio/mean": 1.0005251169204712, "sampling/importance_sampling_ratio/min": 0.9858677983283997, "sampling/sampling_logp_difference/max": 0.011849284172058105, "sampling/sampling_logp_difference/mean": 0.0005177630810067058, "step": 251, "step_time": 11.91060841900071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 979.15625, "completions/mean_terminated_length": 979.15625, "completions/min_length": 817.0, "completions/min_terminated_length": 817.0, "entropy": 0.02508464641869068, "epoch": 0.02016, "frac_reward_zero_std": 0.375, "grad_norm": 0.013846738263964653, "kl": 0.9371653497219086, "learning_rate": 4.352801957790391e-05, "loss": -0.001, "num_tokens": 8304101.0, "reward": 1.3428125381469727, "reward_std": 0.08900570869445801, "rewards/rollout_reward_func/mean": 1.3428125381469727, "rewards/rollout_reward_func/std": 0.16024647653102875, "sampling/importance_sampling_ratio/max": 1.1066476106643677, "sampling/importance_sampling_ratio/mean": 0.9810701608657837, "sampling/importance_sampling_ratio/min": 0.6211662888526917, "sampling/sampling_logp_difference/max": 0.4962015151977539, "sampling/sampling_logp_difference/mean": 0.003845494706183672, "step": 252, "step_time": 13.529035380998721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.0, "completions/max_terminated_length": 1078.0, "completions/mean_length": 810.84375, "completions/mean_terminated_length": 810.84375, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.012552122643683106, "epoch": 0.02024, "frac_reward_zero_std": 0.25, "grad_norm": 0.0009351514163427055, "kl": 1.1290166601538658, "learning_rate": 4.352796361488723e-05, "loss": 0.0006, "num_tokens": 8346724.0, "reward": 1.3521875143051147, "reward_std": 0.09343767911195755, "rewards/rollout_reward_func/mean": 1.3521875143051147, "rewards/rollout_reward_func/std": 0.12377749383449554, "sampling/importance_sampling_ratio/max": 1.0175739526748657, "sampling/importance_sampling_ratio/mean": 1.001145362854004, "sampling/importance_sampling_ratio/min": 0.9944302439689636, "sampling/sampling_logp_difference/max": 0.009256923571228981, "sampling/sampling_logp_difference/mean": 0.0003068247460760176, "step": 253, "step_time": 12.59971599399978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1065.0, "completions/max_terminated_length": 1065.0, "completions/mean_length": 786.75, "completions/mean_terminated_length": 786.75, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.01382026867941022, "epoch": 0.02032, "frac_reward_zero_std": 0.125, "grad_norm": 0.009819595143198967, "kl": 1.1624132469296455, "learning_rate": 4.352790739344527e-05, "loss": -0.0002, "num_tokens": 8388739.0, "reward": 1.3409374952316284, "reward_std": 0.09647348523139954, "rewards/rollout_reward_func/mean": 1.3409374952316284, "rewards/rollout_reward_func/std": 0.13342534005641937, "sampling/importance_sampling_ratio/max": 1.1201127767562866, "sampling/importance_sampling_ratio/mean": 0.9927041530609131, "sampling/importance_sampling_ratio/min": 0.6532156467437744, "sampling/sampling_logp_difference/max": 0.43164122104644775, "sampling/sampling_logp_difference/mean": 0.0022482513450086117, "step": 254, "step_time": 12.790489995999451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 761.09375, "completions/mean_terminated_length": 761.09375, "completions/min_length": 590.0, "completions/min_terminated_length": 590.0, "entropy": 0.010383239481598139, "epoch": 0.0204, "frac_reward_zero_std": 0.25, "grad_norm": 0.014241879805922508, "kl": 1.0187014117836952, "learning_rate": 4.352785091357892e-05, "loss": -0.0006, "num_tokens": 8430119.0, "reward": 1.3528125286102295, "reward_std": 0.09123700857162476, "rewards/rollout_reward_func/mean": 1.3528125286102295, "rewards/rollout_reward_func/std": 0.24872881174087524, "sampling/importance_sampling_ratio/max": 1.0183489322662354, "sampling/importance_sampling_ratio/mean": 0.981582522392273, "sampling/importance_sampling_ratio/min": 0.41857296228408813, "sampling/sampling_logp_difference/max": 0.8696680068969727, "sampling/sampling_logp_difference/mean": 0.0036287419497966766, "step": 255, "step_time": 12.529680682000617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 817.0, "completions/mean_terminated_length": 817.0, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.013195579696912318, "epoch": 0.02048, "frac_reward_zero_std": 0.25, "grad_norm": 0.013219167478382587, "kl": 1.4100112989544868, "learning_rate": 4.3527794175289075e-05, "loss": -0.0002, "num_tokens": 8473319.0, "reward": 1.3412500619888306, "reward_std": 0.11336391419172287, "rewards/rollout_reward_func/mean": 1.3412500619888306, "rewards/rollout_reward_func/std": 0.1335555911064148, "sampling/importance_sampling_ratio/max": 1.0355483293533325, "sampling/importance_sampling_ratio/mean": 0.9834560751914978, "sampling/importance_sampling_ratio/min": 0.6080559492111206, "sampling/sampling_logp_difference/max": 0.4989027976989746, "sampling/sampling_logp_difference/mean": 0.003108314238488674, "step": 256, "step_time": 13.105327364000004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1408.0, "completions/max_terminated_length": 1408.0, "completions/mean_length": 1119.03125, "completions/mean_terminated_length": 1119.03125, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.01288627926260233, "epoch": 0.02056, "frac_reward_zero_std": 0.125, "grad_norm": 0.0004936541663482785, "kl": 1.1819181069731712, "learning_rate": 4.3527737178576624e-05, "loss": 0.0005, "num_tokens": 8527234.0, "reward": 1.4678125381469727, "reward_std": 0.06259670853614807, "rewards/rollout_reward_func/mean": 1.4678125381469727, "rewards/rollout_reward_func/std": 0.09414246678352356, "sampling/importance_sampling_ratio/max": 1.0139153003692627, "sampling/importance_sampling_ratio/mean": 0.9997690916061401, "sampling/importance_sampling_ratio/min": 0.9865546822547913, "sampling/sampling_logp_difference/max": 0.007584128528833389, "sampling/sampling_logp_difference/mean": 0.000386849045753479, "step": 257, "step_time": 14.935714080000253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1099.8125, "completions/mean_terminated_length": 1099.8125, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.010245902929455042, "epoch": 0.02064, "frac_reward_zero_std": 0.125, "grad_norm": 0.007684650365263224, "kl": 1.4944283664226532, "learning_rate": 4.3527679923442484e-05, "loss": 0.0004, "num_tokens": 8580249.0, "reward": 1.3712499141693115, "reward_std": 0.17268699407577515, "rewards/rollout_reward_func/mean": 1.3712499141693115, "rewards/rollout_reward_func/std": 0.24705687165260315, "sampling/importance_sampling_ratio/max": 1.1316958665847778, "sampling/importance_sampling_ratio/mean": 1.0044887065887451, "sampling/importance_sampling_ratio/min": 0.9900782704353333, "sampling/sampling_logp_difference/max": 0.12409162521362305, "sampling/sampling_logp_difference/mean": 0.0005648285150527954, "step": 258, "step_time": 15.73785023399978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1409.0, "completions/max_terminated_length": 1409.0, "completions/mean_length": 1168.84375, "completions/mean_terminated_length": 1168.84375, "completions/min_length": 776.0, "completions/min_terminated_length": 776.0, "entropy": 0.013445691554807127, "epoch": 0.02072, "frac_reward_zero_std": 0.25, "grad_norm": 0.008845318108797073, "kl": 1.2297791615128517, "learning_rate": 4.352762240988755e-05, "loss": 0.0013, "num_tokens": 8636121.0, "reward": 1.389687418937683, "reward_std": 0.17009033262729645, "rewards/rollout_reward_func/mean": 1.389687418937683, "rewards/rollout_reward_func/std": 0.205857515335083, "sampling/importance_sampling_ratio/max": 1.796868920326233, "sampling/importance_sampling_ratio/mean": 1.0265977382659912, "sampling/importance_sampling_ratio/min": 0.9894893765449524, "sampling/sampling_logp_difference/max": 0.5804515480995178, "sampling/sampling_logp_difference/mean": 0.0020170523785054684, "step": 259, "step_time": 14.935599012000239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1232.0, "completions/max_terminated_length": 1232.0, "completions/mean_length": 935.34375, "completions/mean_terminated_length": 935.34375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.005989312223391607, "epoch": 0.0208, "frac_reward_zero_std": 0.125, "grad_norm": 0.0032137625385075808, "kl": 1.3615578263998032, "learning_rate": 4.352756463791274e-05, "loss": 0.0013, "num_tokens": 8682599.0, "reward": 1.4090625047683716, "reward_std": 0.11784670501947403, "rewards/rollout_reward_func/mean": 1.4090625047683716, "rewards/rollout_reward_func/std": 0.197506383061409, "sampling/importance_sampling_ratio/max": 1.0175423622131348, "sampling/importance_sampling_ratio/mean": 1.0011472702026367, "sampling/importance_sampling_ratio/min": 0.9966825246810913, "sampling/sampling_logp_difference/max": 0.01055358536541462, "sampling/sampling_logp_difference/mean": 0.00019703229190781713, "step": 260, "step_time": 14.504925056001412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 1100.125, "completions/mean_terminated_length": 1100.125, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.009060867480002344, "epoch": 0.02088, "frac_reward_zero_std": 0.125, "grad_norm": 0.05085044726729393, "kl": 1.1283669844269753, "learning_rate": 4.3527506607518955e-05, "loss": 0.0011, "num_tokens": 8736099.0, "reward": 1.3796875476837158, "reward_std": 0.19496139883995056, "rewards/rollout_reward_func/mean": 1.3796875476837158, "rewards/rollout_reward_func/std": 0.23220519721508026, "sampling/importance_sampling_ratio/max": 1.644400715827942, "sampling/importance_sampling_ratio/mean": 1.019720196723938, "sampling/importance_sampling_ratio/min": 0.9818691611289978, "sampling/sampling_logp_difference/max": 0.49997520446777344, "sampling/sampling_logp_difference/mean": 0.0016554798930883408, "step": 261, "step_time": 14.690663080999911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 983.25, "completions/mean_terminated_length": 983.25, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.007238654245156795, "epoch": 0.02096, "frac_reward_zero_std": 0.0, "grad_norm": 0.0009298570221289992, "kl": 1.6834731996059418, "learning_rate": 4.3527448318707135e-05, "loss": 0.0005, "num_tokens": 8784424.0, "reward": 1.3590624332427979, "reward_std": 0.1753281056880951, "rewards/rollout_reward_func/mean": 1.3590624332427979, "rewards/rollout_reward_func/std": 0.21059608459472656, "sampling/importance_sampling_ratio/max": 1.0274535417556763, "sampling/importance_sampling_ratio/mean": 1.0009706020355225, "sampling/importance_sampling_ratio/min": 0.9930329322814941, "sampling/sampling_logp_difference/max": 0.026969633996486664, "sampling/sampling_logp_difference/mean": 0.0002653336268849671, "step": 262, "step_time": 15.380581524998888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0027173913549631834, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0027173913549631834, "completions/clipped_ratio": 0.0, "completions/max_length": 1389.0, "completions/max_terminated_length": 1389.0, "completions/mean_length": 1018.96875, "completions/mean_terminated_length": 1018.96875, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.007735161139862612, "epoch": 0.02104, "frac_reward_zero_std": 0.0, "grad_norm": 0.0030210965778678656, "kl": 1.3236347436904907, "learning_rate": 4.352738977147818e-05, "loss": 0.0005, "num_tokens": 8834323.0, "reward": 1.3887500762939453, "reward_std": 0.17983748018741608, "rewards/rollout_reward_func/mean": 1.3887500762939453, "rewards/rollout_reward_func/std": 0.18966440856456757, "sampling/importance_sampling_ratio/max": 1.0111273527145386, "sampling/importance_sampling_ratio/mean": 0.9981800317764282, "sampling/importance_sampling_ratio/min": 0.8840491771697998, "sampling/sampling_logp_difference/max": 0.12092113494873047, "sampling/sampling_logp_difference/mean": 0.0006835178937762976, "step": 263, "step_time": 14.421692077999069 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1240.0, "completions/max_terminated_length": 1240.0, "completions/mean_length": 897.8125, "completions/mean_terminated_length": 897.8125, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.0039858446980360895, "epoch": 0.02112, "frac_reward_zero_std": 0.0, "grad_norm": 0.0005928917089477181, "kl": 1.3688715919852257, "learning_rate": 4.3527330965833046e-05, "loss": 0.0009, "num_tokens": 8879622.0, "reward": 1.3590624332427979, "reward_std": 0.17768147587776184, "rewards/rollout_reward_func/mean": 1.3590624332427979, "rewards/rollout_reward_func/std": 0.21059606969356537, "sampling/importance_sampling_ratio/max": 1.0217485427856445, "sampling/importance_sampling_ratio/mean": 1.0011656284332275, "sampling/importance_sampling_ratio/min": 0.9985518455505371, "sampling/sampling_logp_difference/max": 0.019565103575587273, "sampling/sampling_logp_difference/mean": 0.0001498066121712327, "step": 264, "step_time": 14.460604645998501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1395.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 924.1875, "completions/mean_terminated_length": 924.1875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.004289799777325243, "epoch": 0.0212, "frac_reward_zero_std": 0.0, "grad_norm": 0.00018133035337086767, "kl": 1.3957815617322922, "learning_rate": 4.352727190177264e-05, "loss": 0.0003, "num_tokens": 8925862.0, "reward": 1.3690624237060547, "reward_std": 0.1890537589788437, "rewards/rollout_reward_func/mean": 1.3690624237060547, "rewards/rollout_reward_func/std": 0.22372736036777496, "sampling/importance_sampling_ratio/max": 1.0046107769012451, "sampling/importance_sampling_ratio/mean": 1.0005857944488525, "sampling/importance_sampling_ratio/min": 0.9973756670951843, "sampling/sampling_logp_difference/max": 0.00405377522110939, "sampling/sampling_logp_difference/mean": 0.00010226925951428711, "step": 265, "step_time": 14.123372251000546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 1088.90625, "completions/mean_terminated_length": 1088.90625, "completions/min_length": 759.0, "completions/min_terminated_length": 759.0, "entropy": 0.0039648443926125765, "epoch": 0.02128, "frac_reward_zero_std": 0.25, "grad_norm": 0.00030852711643092334, "kl": 1.0767375752329826, "learning_rate": 4.3527212579297906e-05, "loss": 0.0005, "num_tokens": 8978740.0, "reward": 1.4284374713897705, "reward_std": 0.1222282350063324, "rewards/rollout_reward_func/mean": 1.4284374713897705, "rewards/rollout_reward_func/std": 0.15508811175823212, "sampling/importance_sampling_ratio/max": 1.0149312019348145, "sampling/importance_sampling_ratio/mean": 1.0010687112808228, "sampling/importance_sampling_ratio/min": 0.9994909763336182, "sampling/sampling_logp_difference/max": 0.011709963902831078, "sampling/sampling_logp_difference/mean": 0.0001218499819515273, "step": 266, "step_time": 15.6646998110009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1399.0, "completions/max_terminated_length": 1399.0, "completions/mean_length": 1091.8125, "completions/mean_terminated_length": 1091.8125, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.007119086978491396, "epoch": 0.02136, "frac_reward_zero_std": 0.125, "grad_norm": 0.009929779917001724, "kl": 1.665942832827568, "learning_rate": 4.3527152998409784e-05, "loss": 0.0005, "num_tokens": 9031881.0, "reward": 1.290624976158142, "reward_std": 0.1715102344751358, "rewards/rollout_reward_func/mean": 1.290624976158142, "rewards/rollout_reward_func/std": 0.2427920699119568, "sampling/importance_sampling_ratio/max": 1.0537097454071045, "sampling/importance_sampling_ratio/mean": 1.0005769729614258, "sampling/importance_sampling_ratio/min": 0.8828297853469849, "sampling/sampling_logp_difference/max": 0.1249542236328125, "sampling/sampling_logp_difference/mean": 0.0007696831016801298, "step": 267, "step_time": 14.668183650998799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1237.0, "completions/max_terminated_length": 1237.0, "completions/mean_length": 1039.65625, "completions/mean_terminated_length": 1039.65625, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.005156323342816904, "epoch": 0.02144, "frac_reward_zero_std": 0.125, "grad_norm": 0.004163484554737806, "kl": 1.450408011674881, "learning_rate": 4.352709315910921e-05, "loss": 0.0011, "num_tokens": 9082372.0, "reward": 1.3700000047683716, "reward_std": 0.16798052191734314, "rewards/rollout_reward_func/mean": 1.3700000047683716, "rewards/rollout_reward_func/std": 0.22382310032844543, "sampling/importance_sampling_ratio/max": 1.032687783241272, "sampling/importance_sampling_ratio/mean": 0.9768401980400085, "sampling/importance_sampling_ratio/min": 0.18174850940704346, "sampling/sampling_logp_difference/max": 1.705451250076294, "sampling/sampling_logp_difference/mean": 0.005355492699891329, "step": 268, "step_time": 14.707516569999825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1406.0, "completions/max_terminated_length": 1406.0, "completions/mean_length": 1045.4375, "completions/mean_terminated_length": 1045.4375, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.0034229234734084457, "epoch": 0.02152, "frac_reward_zero_std": 0.0, "grad_norm": 0.00024435375235043466, "kl": 1.482334889471531, "learning_rate": 4.3527033061397145e-05, "loss": 0.001, "num_tokens": 9133476.0, "reward": 1.3874999284744263, "reward_std": 0.17062141001224518, "rewards/rollout_reward_func/mean": 1.3874999284744263, "rewards/rollout_reward_func/std": 0.18893590569496155, "sampling/importance_sampling_ratio/max": 1.0109573602676392, "sampling/importance_sampling_ratio/mean": 1.0013325214385986, "sampling/importance_sampling_ratio/min": 0.9990901947021484, "sampling/sampling_logp_difference/max": 0.008873144164681435, "sampling/sampling_logp_difference/mean": 0.0001453204604331404, "step": 269, "step_time": 14.489621971000815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 927.0625, "completions/mean_terminated_length": 927.0625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0029610454221256077, "epoch": 0.0216, "frac_reward_zero_std": 0.25, "grad_norm": 0.00025934429140761495, "kl": 1.5116874426603317, "learning_rate": 4.352697270527453e-05, "loss": 0.0005, "num_tokens": 9179404.0, "reward": 1.3793749809265137, "reward_std": 0.12967877089977264, "rewards/rollout_reward_func/mean": 1.3793749809265137, "rewards/rollout_reward_func/std": 0.20790721476078033, "sampling/importance_sampling_ratio/max": 1.0258089303970337, "sampling/importance_sampling_ratio/mean": 1.0024495124816895, "sampling/importance_sampling_ratio/min": 0.9998931288719177, "sampling/sampling_logp_difference/max": 0.019726015627384186, "sampling/sampling_logp_difference/mean": 0.00023909012088552117, "step": 270, "step_time": 15.032327041002645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1406.0, "completions/max_terminated_length": 1406.0, "completions/mean_length": 1135.09375, "completions/mean_terminated_length": 1135.09375, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.011294258147245273, "epoch": 0.02168, "frac_reward_zero_std": 0.125, "grad_norm": 0.02470974251627922, "kl": 1.262643314898014, "learning_rate": 4.3526912090742326e-05, "loss": -0.0014, "num_tokens": 9234193.0, "reward": 1.3712499141693115, "reward_std": 0.16654174029827118, "rewards/rollout_reward_func/mean": 1.3712499141693115, "rewards/rollout_reward_func/std": 0.20790429413318634, "sampling/importance_sampling_ratio/max": 1.2832297086715698, "sampling/importance_sampling_ratio/mean": 0.9702903628349304, "sampling/importance_sampling_ratio/min": 0.17007799446582794, "sampling/sampling_logp_difference/max": 1.7724655866622925, "sampling/sampling_logp_difference/mean": 0.007879002951085567, "step": 271, "step_time": 14.617936108999857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1004.65625, "completions/mean_terminated_length": 1004.65625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.003545735336956568, "epoch": 0.02176, "frac_reward_zero_std": 0.0, "grad_norm": 0.001063665491528809, "kl": 1.4363198280334473, "learning_rate": 4.352685121780149e-05, "loss": 0.001, "num_tokens": 9283442.0, "reward": 1.3981249332427979, "reward_std": 0.11498142033815384, "rewards/rollout_reward_func/mean": 1.3981249332427979, "rewards/rollout_reward_func/std": 0.18638217449188232, "sampling/importance_sampling_ratio/max": 1.1485286951065063, "sampling/importance_sampling_ratio/mean": 1.0060712099075317, "sampling/importance_sampling_ratio/min": 0.9995570778846741, "sampling/sampling_logp_difference/max": 0.11990290880203247, "sampling/sampling_logp_difference/mean": 0.0005382323870435357, "step": 272, "step_time": 14.820025227001679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1402.0, "completions/max_terminated_length": 1402.0, "completions/mean_length": 1091.6875, "completions/mean_terminated_length": 1091.6875, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.00465562756289728, "epoch": 0.02184, "frac_reward_zero_std": 0.125, "grad_norm": 0.003528676461428404, "kl": 1.3634295165538788, "learning_rate": 4.3526790086452984e-05, "loss": 0.0016, "num_tokens": 9336482.0, "reward": 1.3681249618530273, "reward_std": 0.17218083143234253, "rewards/rollout_reward_func/mean": 1.3681249618530273, "rewards/rollout_reward_func/std": 0.20882834494113922, "sampling/importance_sampling_ratio/max": 1.220862627029419, "sampling/importance_sampling_ratio/mean": 1.0088647603988647, "sampling/importance_sampling_ratio/min": 0.9992243647575378, "sampling/sampling_logp_difference/max": 0.19686201214790344, "sampling/sampling_logp_difference/mean": 0.0007493819575756788, "step": 273, "step_time": 14.648696356999608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1396.0, "completions/max_terminated_length": 1396.0, "completions/mean_length": 1085.78125, "completions/mean_terminated_length": 1085.78125, "completions/min_length": 701.0, "completions/min_terminated_length": 701.0, "entropy": 0.0031133781012613326, "epoch": 0.02192, "frac_reward_zero_std": 0.0, "grad_norm": 0.00025524981901980937, "kl": 1.2089229673147202, "learning_rate": 4.352672869669779e-05, "loss": 0.0004, "num_tokens": 9388880.0, "reward": 1.4081250429153442, "reward_std": 0.15721899271011353, "rewards/rollout_reward_func/mean": 1.4081250429153442, "rewards/rollout_reward_func/std": 0.1828526258468628, "sampling/importance_sampling_ratio/max": 1.012891411781311, "sampling/importance_sampling_ratio/mean": 1.0015082359313965, "sampling/importance_sampling_ratio/min": 0.9998921155929565, "sampling/sampling_logp_difference/max": 0.010347716510295868, "sampling/sampling_logp_difference/mean": 0.00013984122779220343, "step": 274, "step_time": 15.341402130999995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1387.0, "completions/max_terminated_length": 1387.0, "completions/mean_length": 1013.6875, "completions/mean_terminated_length": 1013.6875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.00274747604271397, "epoch": 0.022, "frac_reward_zero_std": 0.125, "grad_norm": 0.0002794708707369864, "kl": 1.684442512691021, "learning_rate": 4.352666704853686e-05, "loss": 0.0012, "num_tokens": 9438467.0, "reward": 1.3781249523162842, "reward_std": 0.19803780317306519, "rewards/rollout_reward_func/mean": 1.3781249523162842, "rewards/rollout_reward_func/std": 0.22164250910282135, "sampling/importance_sampling_ratio/max": 1.0170491933822632, "sampling/importance_sampling_ratio/mean": 1.0015946626663208, "sampling/importance_sampling_ratio/min": 0.9997260570526123, "sampling/sampling_logp_difference/max": 0.0165861863642931, "sampling/sampling_logp_difference/mean": 0.00015983052435331047, "step": 275, "step_time": 14.412089000002197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 1109.21875, "completions/mean_terminated_length": 1109.21875, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.007127594668418169, "epoch": 0.02208, "frac_reward_zero_std": 0.0, "grad_norm": 0.011279938742518425, "kl": 1.4085028767585754, "learning_rate": 4.352660514197118e-05, "loss": 0.0006, "num_tokens": 9491951.0, "reward": 1.3790624141693115, "reward_std": 0.2027800977230072, "rewards/rollout_reward_func/mean": 1.3790624141693115, "rewards/rollout_reward_func/std": 0.19118238985538483, "sampling/importance_sampling_ratio/max": 1.0552219152450562, "sampling/importance_sampling_ratio/mean": 0.9952234625816345, "sampling/importance_sampling_ratio/min": 0.7536978125572205, "sampling/sampling_logp_difference/max": 0.2903429865837097, "sampling/sampling_logp_difference/mean": 0.001094974111765623, "step": 276, "step_time": 15.599566595000397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1237.0, "completions/max_terminated_length": 1237.0, "completions/mean_length": 1015.09375, "completions/mean_terminated_length": 1015.09375, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.0031858347065281123, "epoch": 0.02216, "frac_reward_zero_std": 0.0, "grad_norm": 0.001688950345851481, "kl": 1.486321173608303, "learning_rate": 4.352654297700173e-05, "loss": 0.0007, "num_tokens": 9541987.0, "reward": 1.3778125047683716, "reward_std": 0.17704704403877258, "rewards/rollout_reward_func/mean": 1.3778125047683716, "rewards/rollout_reward_func/std": 0.2070021778345108, "sampling/importance_sampling_ratio/max": 1.0568755865097046, "sampling/importance_sampling_ratio/mean": 1.0023021697998047, "sampling/importance_sampling_ratio/min": 0.999120831489563, "sampling/sampling_logp_difference/max": 0.05377529561519623, "sampling/sampling_logp_difference/mean": 0.0002184143813792616, "step": 277, "step_time": 14.740778651000255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1162.8125, "completions/mean_terminated_length": 1162.8125, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.0031103099463507533, "epoch": 0.02224, "frac_reward_zero_std": 0.0, "grad_norm": 0.00018695999460760504, "kl": 1.0019804015755653, "learning_rate": 4.35264805536295e-05, "loss": 0.0008, "num_tokens": 9597544.0, "reward": 1.467812418937683, "reward_std": 0.06331837922334671, "rewards/rollout_reward_func/mean": 1.467812418937683, "rewards/rollout_reward_func/std": 0.12273061275482178, "sampling/importance_sampling_ratio/max": 1.0046443939208984, "sampling/importance_sampling_ratio/mean": 1.0007374286651611, "sampling/importance_sampling_ratio/min": 0.9994370937347412, "sampling/sampling_logp_difference/max": 0.0027993139810860157, "sampling/sampling_logp_difference/mean": 8.135155076161027e-05, "step": 278, "step_time": 14.747987871999612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1396.0, "completions/max_terminated_length": 1396.0, "completions/mean_length": 1143.9375, "completions/mean_terminated_length": 1143.9375, "completions/min_length": 777.0, "completions/min_terminated_length": 777.0, "entropy": 0.0043777875835075974, "epoch": 0.02232, "frac_reward_zero_std": 0.25, "grad_norm": 0.0006338707753457129, "kl": 1.4038750380277634, "learning_rate": 4.352641787185546e-05, "loss": 0.0008, "num_tokens": 9651976.0, "reward": 1.369999885559082, "reward_std": 0.13540147244930267, "rewards/rollout_reward_func/mean": 1.369999885559082, "rewards/rollout_reward_func/std": 0.22426941990852356, "sampling/importance_sampling_ratio/max": 1.1279278993606567, "sampling/importance_sampling_ratio/mean": 1.0052225589752197, "sampling/importance_sampling_ratio/min": 0.9998675584793091, "sampling/sampling_logp_difference/max": 0.10152699053287506, "sampling/sampling_logp_difference/mean": 0.000454612891189754, "step": 279, "step_time": 14.683554675999403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1403.0, "completions/max_terminated_length": 1403.0, "completions/mean_length": 988.84375, "completions/mean_terminated_length": 988.84375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0026005300460383296, "epoch": 0.0224, "frac_reward_zero_std": 0.0, "grad_norm": 0.000320942112011835, "kl": 1.6481316983699799, "learning_rate": 4.352635493168063e-05, "loss": 0.0005, "num_tokens": 9700933.0, "reward": 1.3287500143051147, "reward_std": 0.2177070677280426, "rewards/rollout_reward_func/mean": 1.3287500143051147, "rewards/rollout_reward_func/std": 0.22515587508678436, "sampling/importance_sampling_ratio/max": 1.0111550092697144, "sampling/importance_sampling_ratio/mean": 1.0014656782150269, "sampling/importance_sampling_ratio/min": 0.9995223879814148, "sampling/sampling_logp_difference/max": 0.01101747713983059, "sampling/sampling_logp_difference/mean": 0.00013987431884743273, "step": 280, "step_time": 15.031700139999884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1402.0, "completions/max_terminated_length": 1402.0, "completions/mean_length": 993.125, "completions/mean_terminated_length": 993.125, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0023896704078651965, "epoch": 0.02248, "frac_reward_zero_std": 0.125, "grad_norm": 0.0003611746651586145, "kl": 1.324975922703743, "learning_rate": 4.3526291733105985e-05, "loss": 0.0003, "num_tokens": 9750098.0, "reward": 1.3496873378753662, "reward_std": 0.17636257410049438, "rewards/rollout_reward_func/mean": 1.3496873378753662, "rewards/rollout_reward_func/std": 0.22589585185050964, "sampling/importance_sampling_ratio/max": 1.0039739608764648, "sampling/importance_sampling_ratio/mean": 1.0006616115570068, "sampling/importance_sampling_ratio/min": 1.0000085830688477, "sampling/sampling_logp_difference/max": 0.0028261281549930573, "sampling/sampling_logp_difference/mean": 6.444350583478808e-05, "step": 281, "step_time": 15.411456243999964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1079.5625, "completions/mean_terminated_length": 1079.5625, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.0025500940973870456, "epoch": 0.02256, "frac_reward_zero_std": 0.0, "grad_norm": 0.00013335788389667869, "kl": 1.436171941459179, "learning_rate": 4.352622827613253e-05, "loss": 0.001, "num_tokens": 9802677.0, "reward": 1.3878124952316284, "reward_std": 0.122791588306427, "rewards/rollout_reward_func/mean": 1.3878124952316284, "rewards/rollout_reward_func/std": 0.20485611259937286, "sampling/importance_sampling_ratio/max": 1.0058237314224243, "sampling/importance_sampling_ratio/mean": 1.0010042190551758, "sampling/importance_sampling_ratio/min": 0.9988491535186768, "sampling/sampling_logp_difference/max": 0.004842483904212713, "sampling/sampling_logp_difference/mean": 0.00010834991553565487, "step": 282, "step_time": 14.621012292001978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1393.0, "completions/max_terminated_length": 1393.0, "completions/mean_length": 1001.78125, "completions/mean_terminated_length": 1001.78125, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.0025823600153671578, "epoch": 0.02264, "frac_reward_zero_std": 0.25, "grad_norm": 0.011004217900335789, "kl": 1.279792569577694, "learning_rate": 4.3526164560761275e-05, "loss": -0.0008, "num_tokens": 9851715.0, "reward": 1.3899999856948853, "reward_std": 0.15547162294387817, "rewards/rollout_reward_func/mean": 1.3899999856948853, "rewards/rollout_reward_func/std": 0.20504917204380035, "sampling/importance_sampling_ratio/max": 1.0033406019210815, "sampling/importance_sampling_ratio/mean": 0.9765082001686096, "sampling/importance_sampling_ratio/min": 0.23104307055473328, "sampling/sampling_logp_difference/max": 1.4664194583892822, "sampling/sampling_logp_difference/mean": 0.0044484627433121204, "step": 283, "step_time": 14.549764078000408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 1198.0625, "completions/mean_terminated_length": 1198.0625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.003079408605117351, "epoch": 0.02272, "frac_reward_zero_std": 0.25, "grad_norm": 0.0002884472196456045, "kl": 1.3950769901275635, "learning_rate": 4.352610058699323e-05, "loss": 0.0005, "num_tokens": 9909167.0, "reward": 1.3406250476837158, "reward_std": 0.1900569647550583, "rewards/rollout_reward_func/mean": 1.3406250476837158, "rewards/rollout_reward_func/std": 0.2526368796825409, "sampling/importance_sampling_ratio/max": 1.0103665590286255, "sampling/importance_sampling_ratio/mean": 1.0008618831634521, "sampling/importance_sampling_ratio/min": 0.9988930821418762, "sampling/sampling_logp_difference/max": 0.009973946958780289, "sampling/sampling_logp_difference/mean": 0.00010576923523331061, "step": 284, "step_time": 15.396554936999564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 1100.8125, "completions/mean_terminated_length": 1100.8125, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.002971002017147839, "epoch": 0.0228, "frac_reward_zero_std": 0.0, "grad_norm": 0.0005178764113225043, "kl": 1.303280234336853, "learning_rate": 4.352603635482941e-05, "loss": 0.0002, "num_tokens": 9962857.0, "reward": 1.349062442779541, "reward_std": 0.24459488689899445, "rewards/rollout_reward_func/mean": 1.349062442779541, "rewards/rollout_reward_func/std": 0.23883263766765594, "sampling/importance_sampling_ratio/max": 1.0130797624588013, "sampling/importance_sampling_ratio/mean": 1.0014615058898926, "sampling/importance_sampling_ratio/min": 0.9997479915618896, "sampling/sampling_logp_difference/max": 0.012918243184685707, "sampling/sampling_logp_difference/mean": 0.00014228842337615788, "step": 285, "step_time": 15.3264213500006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1397.0, "completions/max_terminated_length": 1397.0, "completions/mean_length": 965.5, "completions/mean_terminated_length": 965.5, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0020862015007878654, "epoch": 0.02288, "frac_reward_zero_std": 0.0, "grad_norm": 0.00031141223735176027, "kl": 1.4680378884077072, "learning_rate": 4.352597186427082e-05, "loss": 0.0012, "num_tokens": 10010731.0, "reward": 1.3878124952316284, "reward_std": 0.1836380809545517, "rewards/rollout_reward_func/mean": 1.3878124952316284, "rewards/rollout_reward_func/std": 0.1891191005706787, "sampling/importance_sampling_ratio/max": 1.0229754447937012, "sampling/importance_sampling_ratio/mean": 1.0015562772750854, "sampling/importance_sampling_ratio/min": 0.9998933672904968, "sampling/sampling_logp_difference/max": 0.021344412118196487, "sampling/sampling_logp_difference/mean": 0.00015177251771092415, "step": 286, "step_time": 14.379162742999142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 961.3125, "completions/mean_terminated_length": 961.3125, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.002120313758496195, "epoch": 0.02296, "frac_reward_zero_std": 0.125, "grad_norm": 0.0001521996018709615, "kl": 1.5824646577239037, "learning_rate": 4.352590711531848e-05, "loss": 0.0008, "num_tokens": 10058424.0, "reward": 1.3590624332427979, "reward_std": 0.19394317269325256, "rewards/rollout_reward_func/mean": 1.3590624332427979, "rewards/rollout_reward_func/std": 0.2248348444700241, "sampling/importance_sampling_ratio/max": 1.008632779121399, "sampling/importance_sampling_ratio/mean": 1.0009452104568481, "sampling/importance_sampling_ratio/min": 0.9996870756149292, "sampling/sampling_logp_difference/max": 0.007715284824371338, "sampling/sampling_logp_difference/mean": 9.561354818288237e-05, "step": 287, "step_time": 14.542267005001122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1231.0, "completions/max_terminated_length": 1231.0, "completions/mean_length": 872.75, "completions/mean_terminated_length": 872.75, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0015078156429808587, "epoch": 0.02304, "frac_reward_zero_std": 0.125, "grad_norm": 0.00019142174278385937, "kl": 1.436690904200077, "learning_rate": 4.3525842107973435e-05, "loss": 0.0007, "num_tokens": 10102704.0, "reward": 1.3981249332427979, "reward_std": 0.1592741161584854, "rewards/rollout_reward_func/mean": 1.3981249332427979, "rewards/rollout_reward_func/std": 0.18638217449188232, "sampling/importance_sampling_ratio/max": 1.005138635635376, "sampling/importance_sampling_ratio/mean": 1.000516414642334, "sampling/importance_sampling_ratio/min": 0.9999203085899353, "sampling/sampling_logp_difference/max": 0.006823747418820858, "sampling/sampling_logp_difference/mean": 6.862651935080066e-05, "step": 288, "step_time": 14.281335515999672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1053.53125, "completions/mean_terminated_length": 1053.53125, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0033906081516761333, "epoch": 0.02312, "frac_reward_zero_std": 0.0, "grad_norm": 0.0003199540660716593, "kl": 1.8884027898311615, "learning_rate": 4.35257768422367e-05, "loss": -0.0001, "num_tokens": 10153785.0, "reward": 1.2893750667572021, "reward_std": 0.2368914783000946, "rewards/rollout_reward_func/mean": 1.2893750667572021, "rewards/rollout_reward_func/std": 0.2465364933013916, "sampling/importance_sampling_ratio/max": 1.0235637426376343, "sampling/importance_sampling_ratio/mean": 1.0021393299102783, "sampling/importance_sampling_ratio/min": 0.9997169971466064, "sampling/sampling_logp_difference/max": 0.02245708554983139, "sampling/sampling_logp_difference/mean": 0.0002081571874441579, "step": 289, "step_time": 15.469989959001396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 1065.0625, "completions/mean_terminated_length": 1065.0625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0023097002558643, "epoch": 0.0232, "frac_reward_zero_std": 0.0, "grad_norm": 0.0003482460742816329, "kl": 1.218141533434391, "learning_rate": 4.352571131810931e-05, "loss": 0.0003, "num_tokens": 10204972.0, "reward": 1.4084374904632568, "reward_std": 0.15711796283721924, "rewards/rollout_reward_func/mean": 1.4084374904632568, "rewards/rollout_reward_func/std": 0.18300555646419525, "sampling/importance_sampling_ratio/max": 1.0053390264511108, "sampling/importance_sampling_ratio/mean": 1.0008487701416016, "sampling/importance_sampling_ratio/min": 0.9997925162315369, "sampling/sampling_logp_difference/max": 0.0034016119316220284, "sampling/sampling_logp_difference/mean": 8.488541061524302e-05, "step": 290, "step_time": 14.523790367999936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 1177.875, "completions/mean_terminated_length": 1177.875, "completions/min_length": 987.0, "completions/min_terminated_length": 987.0, "entropy": 0.0034381559962639585, "epoch": 0.02328, "frac_reward_zero_std": 0.125, "grad_norm": 0.0003598316106945276, "kl": 1.1930644065141678, "learning_rate": 4.352564553559231e-05, "loss": 0.0004, "num_tokens": 10261133.0, "reward": 1.4071874618530273, "reward_std": 0.14033816754817963, "rewards/rollout_reward_func/mean": 1.4071874618530273, "rewards/rollout_reward_func/std": 0.1823897808790207, "sampling/importance_sampling_ratio/max": 1.0175366401672363, "sampling/importance_sampling_ratio/mean": 1.0017327070236206, "sampling/importance_sampling_ratio/min": 0.9993559718132019, "sampling/sampling_logp_difference/max": 0.013427533209323883, "sampling/sampling_logp_difference/mean": 0.00016510578279849142, "step": 291, "step_time": 14.639463846999206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1239.0, "completions/max_terminated_length": 1239.0, "completions/mean_length": 1021.71875, "completions/mean_terminated_length": 1021.71875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.002389333603787236, "epoch": 0.02336, "frac_reward_zero_std": 0.125, "grad_norm": 0.015959294512867928, "kl": 1.1408146172761917, "learning_rate": 4.352557949468673e-05, "loss": 0.0008, "num_tokens": 10310882.0, "reward": 1.3990625143051147, "reward_std": 0.1598682552576065, "rewards/rollout_reward_func/mean": 1.3990625143051147, "rewards/rollout_reward_func/std": 0.18688204884529114, "sampling/importance_sampling_ratio/max": 1.0039191246032715, "sampling/importance_sampling_ratio/mean": 0.9938333630561829, "sampling/importance_sampling_ratio/min": 0.7854521870613098, "sampling/sampling_logp_difference/max": 0.24147558212280273, "sampling/sampling_logp_difference/mean": 0.0007541939849033952, "step": 292, "step_time": 14.517383200998665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1392.0, "completions/max_terminated_length": 1392.0, "completions/mean_length": 958.90625, "completions/mean_terminated_length": 958.90625, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.0018838260584743693, "epoch": 0.02344, "frac_reward_zero_std": 0.0, "grad_norm": 0.000329512229654938, "kl": 1.3868747055530548, "learning_rate": 4.352551319539362e-05, "loss": 0.0014, "num_tokens": 10358426.0, "reward": 1.3590624332427979, "reward_std": 0.19227752089500427, "rewards/rollout_reward_func/mean": 1.3590624332427979, "rewards/rollout_reward_func/std": 0.2248348444700241, "sampling/importance_sampling_ratio/max": 1.0070171356201172, "sampling/importance_sampling_ratio/mean": 1.0005909204483032, "sampling/importance_sampling_ratio/min": 0.9997692108154297, "sampling/sampling_logp_difference/max": 0.0056536756455898285, "sampling/sampling_logp_difference/mean": 6.506861245725304e-05, "step": 293, "step_time": 15.238301209999008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1395.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 968.09375, "completions/mean_terminated_length": 968.09375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0020268762891646475, "epoch": 0.02352, "frac_reward_zero_std": 0.0, "grad_norm": 0.00021339833620004356, "kl": 1.4166330471634865, "learning_rate": 4.352544663771404e-05, "loss": 0.0006, "num_tokens": 10406217.0, "reward": 1.377500057220459, "reward_std": 0.17397193610668182, "rewards/rollout_reward_func/mean": 1.377500057220459, "rewards/rollout_reward_func/std": 0.20681925117969513, "sampling/importance_sampling_ratio/max": 1.0048887729644775, "sampling/importance_sampling_ratio/mean": 1.0004796981811523, "sampling/importance_sampling_ratio/min": 0.9993647336959839, "sampling/sampling_logp_difference/max": 0.0028124856762588024, "sampling/sampling_logp_difference/mean": 5.55791484657675e-05, "step": 294, "step_time": 14.481231163001212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1395.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 914.9375, "completions/mean_terminated_length": 914.9375, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.0017815238170442171, "epoch": 0.0236, "frac_reward_zero_std": 0.125, "grad_norm": 5.0119888328481466e-05, "kl": 1.4391477033495903, "learning_rate": 4.352537982164903e-05, "loss": 0.0006, "num_tokens": 10451613.0, "reward": 1.427187442779541, "reward_std": 0.10012329369783401, "rewards/rollout_reward_func/mean": 1.427187442779541, "rewards/rollout_reward_func/std": 0.132962167263031, "sampling/importance_sampling_ratio/max": 1.003832459449768, "sampling/importance_sampling_ratio/mean": 1.0005431175231934, "sampling/importance_sampling_ratio/min": 0.9999297261238098, "sampling/sampling_logp_difference/max": 0.0020615519024431705, "sampling/sampling_logp_difference/mean": 5.550234345719218e-05, "step": 295, "step_time": 15.075040312000965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 1095.0625, "completions/mean_terminated_length": 1095.0625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.002572660116129555, "epoch": 0.02368, "frac_reward_zero_std": 0.125, "grad_norm": 0.0006346157169900835, "kl": 1.2457918897271156, "learning_rate": 4.3525312747199646e-05, "loss": 0.0007, "num_tokens": 10504237.0, "reward": 1.4384374618530273, "reward_std": 0.09920263290405273, "rewards/rollout_reward_func/mean": 1.4384374618530273, "rewards/rollout_reward_func/std": 0.14873899519443512, "sampling/importance_sampling_ratio/max": 1.0132149457931519, "sampling/importance_sampling_ratio/mean": 1.001011610031128, "sampling/importance_sampling_ratio/min": 0.99959397315979, "sampling/sampling_logp_difference/max": 0.008996929973363876, "sampling/sampling_logp_difference/mean": 9.888988279271871e-05, "step": 296, "step_time": 14.63860159700016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1408.0, "completions/max_terminated_length": 1408.0, "completions/mean_length": 1095.5625, "completions/mean_terminated_length": 1095.5625, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.0026569287438178435, "epoch": 0.02376, "frac_reward_zero_std": 0.0, "grad_norm": 0.0004906331305392087, "kl": 1.281331643462181, "learning_rate": 4.352524541436698e-05, "loss": 0.0008, "num_tokens": 10557401.0, "reward": 1.3790624141693115, "reward_std": 0.15363040566444397, "rewards/rollout_reward_func/mean": 1.3790624141693115, "rewards/rollout_reward_func/std": 0.20772749185562134, "sampling/importance_sampling_ratio/max": 1.0175577402114868, "sampling/importance_sampling_ratio/mean": 1.0013713836669922, "sampling/importance_sampling_ratio/min": 0.9998760223388672, "sampling/sampling_logp_difference/max": 0.013637684285640717, "sampling/sampling_logp_difference/mean": 0.00013039841724094003, "step": 297, "step_time": 15.504451380002138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 1096.5625, "completions/mean_terminated_length": 1096.5625, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.0022385425836546347, "epoch": 0.02384, "frac_reward_zero_std": 0.0, "grad_norm": 0.00013270735507830977, "kl": 1.306058093905449, "learning_rate": 4.352517782315205e-05, "loss": 0.0003, "num_tokens": 10610241.0, "reward": 1.3890624046325684, "reward_std": 0.16278724372386932, "rewards/rollout_reward_func/mean": 1.3890624046325684, "rewards/rollout_reward_func/std": 0.1898447722196579, "sampling/importance_sampling_ratio/max": 1.0045454502105713, "sampling/importance_sampling_ratio/mean": 1.0006787776947021, "sampling/importance_sampling_ratio/min": 0.9999034404754639, "sampling/sampling_logp_difference/max": 0.003211792092770338, "sampling/sampling_logp_difference/mean": 6.957697041798383e-05, "step": 298, "step_time": 14.593442251000852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1071.0, "completions/max_terminated_length": 1071.0, "completions/mean_length": 871.84375, "completions/mean_terminated_length": 871.84375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.00139855602174066, "epoch": 0.02392, "frac_reward_zero_std": 0.125, "grad_norm": 0.00015406249440275133, "kl": 1.6075129956007004, "learning_rate": 4.3525109973555965e-05, "loss": 0.0009, "num_tokens": 10653922.0, "reward": 1.3981249332427979, "reward_std": 0.10269489139318466, "rewards/rollout_reward_func/mean": 1.3981249332427979, "rewards/rollout_reward_func/std": 0.18638215959072113, "sampling/importance_sampling_ratio/max": 1.0036144256591797, "sampling/importance_sampling_ratio/mean": 1.0006210803985596, "sampling/importance_sampling_ratio/min": 0.9999150037765503, "sampling/sampling_logp_difference/max": 0.00332455150783062, "sampling/sampling_logp_difference/mean": 6.170924461912364e-05, "step": 299, "step_time": 13.480941136999718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1392.0, "completions/max_terminated_length": 1392.0, "completions/mean_length": 931.9375, "completions/mean_terminated_length": 931.9375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0016009823739295825, "epoch": 0.024, "frac_reward_zero_std": 0.125, "grad_norm": 7.371027459157631e-05, "kl": 1.488294206559658, "learning_rate": 4.3525041865579784e-05, "loss": 0.0007, "num_tokens": 10700340.0, "reward": 1.3303124904632568, "reward_std": 0.2005147635936737, "rewards/rollout_reward_func/mean": 1.3303124904632568, "rewards/rollout_reward_func/std": 0.22633810341358185, "sampling/importance_sampling_ratio/max": 1.0026512145996094, "sampling/importance_sampling_ratio/mean": 1.0004723072052002, "sampling/importance_sampling_ratio/min": 0.9999507069587708, "sampling/sampling_logp_difference/max": 0.001108462456613779, "sampling/sampling_logp_difference/mean": 4.841598274651915e-05, "step": 300, "step_time": 14.348326801997246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1397.0, "completions/max_terminated_length": 1397.0, "completions/mean_length": 944.875, "completions/mean_terminated_length": 944.875, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.001861830780399032, "epoch": 0.02408, "frac_reward_zero_std": 0.0, "grad_norm": 0.00011311442358419299, "kl": 1.7470774203538895, "learning_rate": 4.352497349922459e-05, "loss": 0.0009, "num_tokens": 10747125.0, "reward": 1.3678125143051147, "reward_std": 0.16257210075855255, "rewards/rollout_reward_func/mean": 1.3678125143051147, "rewards/rollout_reward_func/std": 0.1764293760061264, "sampling/importance_sampling_ratio/max": 1.0024003982543945, "sampling/importance_sampling_ratio/mean": 1.0003498792648315, "sampling/importance_sampling_ratio/min": 0.9995083212852478, "sampling/sampling_logp_difference/max": 0.0023103938437998295, "sampling/sampling_logp_difference/mean": 4.4392189010977745e-05, "step": 301, "step_time": 15.207507338998766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1396.0, "completions/max_terminated_length": 1396.0, "completions/mean_length": 1030.625, "completions/mean_terminated_length": 1030.625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0018984269700013101, "epoch": 0.02416, "frac_reward_zero_std": 0.125, "grad_norm": 0.0001631418999750167, "kl": 1.5144736170768738, "learning_rate": 4.352490487449147e-05, "loss": 0.0006, "num_tokens": 10797826.0, "reward": 1.3103125095367432, "reward_std": 0.13674750924110413, "rewards/rollout_reward_func/mean": 1.3103125095367432, "rewards/rollout_reward_func/std": 0.26270902156829834, "sampling/importance_sampling_ratio/max": 1.002581000328064, "sampling/importance_sampling_ratio/mean": 1.0004303455352783, "sampling/importance_sampling_ratio/min": 0.9998255372047424, "sampling/sampling_logp_difference/max": 0.0023903250694274902, "sampling/sampling_logp_difference/mean": 5.009027518099174e-05, "step": 302, "step_time": 14.571414621001168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1240.0, "completions/max_terminated_length": 1240.0, "completions/mean_length": 921.65625, "completions/mean_terminated_length": 921.65625, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.0013581342791439965, "epoch": 0.02424, "frac_reward_zero_std": 0.0, "grad_norm": 4.424104918143712e-05, "kl": 1.471993513405323, "learning_rate": 4.35248359913815e-05, "loss": 0.0004, "num_tokens": 10843581.0, "reward": 1.3781249523162842, "reward_std": 0.15046776831150055, "rewards/rollout_reward_func/mean": 1.3781249523162842, "rewards/rollout_reward_func/std": 0.20718446373939514, "sampling/importance_sampling_ratio/max": 1.0014572143554688, "sampling/importance_sampling_ratio/mean": 1.0003595352172852, "sampling/importance_sampling_ratio/min": 0.9999315142631531, "sampling/sampling_logp_difference/max": 0.0009319223463535309, "sampling/sampling_logp_difference/mean": 3.837828990072012e-05, "step": 303, "step_time": 14.306532604001404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1395.0, "completions/max_terminated_length": 1395.0, "completions/mean_length": 959.15625, "completions/mean_terminated_length": 959.15625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0014844699471723288, "epoch": 0.02432, "frac_reward_zero_std": 0.0, "grad_norm": 8.381879888474941e-05, "kl": 1.524976372718811, "learning_rate": 4.352476684989577e-05, "loss": 0.0008, "num_tokens": 10890729.0, "reward": 1.3878124952316284, "reward_std": 0.17414933443069458, "rewards/rollout_reward_func/mean": 1.3878124952316284, "rewards/rollout_reward_func/std": 0.1891191005706787, "sampling/importance_sampling_ratio/max": 1.005253791809082, "sampling/importance_sampling_ratio/mean": 1.000572919845581, "sampling/importance_sampling_ratio/min": 0.9999451637268066, "sampling/sampling_logp_difference/max": 0.0017119564581662416, "sampling/sampling_logp_difference/mean": 5.8744073612615466e-05, "step": 304, "step_time": 14.29154242200093 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1238.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 982.59375, "completions/mean_terminated_length": 982.59375, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.003179191378876567, "epoch": 0.0244, "frac_reward_zero_std": 0.0, "grad_norm": 0.007014547940343618, "kl": 1.7429795861244202, "learning_rate": 4.3524697450035387e-05, "loss": 0.0011, "num_tokens": 10938534.0, "reward": 1.3787500858306885, "reward_std": 0.18229946494102478, "rewards/rollout_reward_func/mean": 1.3787500858306885, "rewards/rollout_reward_func/std": 0.19203075766563416, "sampling/importance_sampling_ratio/max": 1.3547965288162231, "sampling/importance_sampling_ratio/mean": 1.0122910737991333, "sampling/importance_sampling_ratio/min": 0.9999843239784241, "sampling/sampling_logp_difference/max": 0.30230382084846497, "sampling/sampling_logp_difference/mean": 0.001058527734130621, "step": 305, "step_time": 14.285866606001036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1237.0, "completions/max_terminated_length": 1237.0, "completions/mean_length": 961.4375, "completions/mean_terminated_length": 961.4375, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.00230067728261929, "epoch": 0.02448, "frac_reward_zero_std": 0.125, "grad_norm": 0.0002202680625487119, "kl": 1.687927484512329, "learning_rate": 4.3524627791801446e-05, "loss": 0.0014, "num_tokens": 10985613.0, "reward": 1.3681249618530273, "reward_std": 0.14854896068572998, "rewards/rollout_reward_func/mean": 1.3681249618530273, "rewards/rollout_reward_func/std": 0.19341479241847992, "sampling/importance_sampling_ratio/max": 1.0714207887649536, "sampling/importance_sampling_ratio/mean": 1.0032967329025269, "sampling/importance_sampling_ratio/min": 0.9996800422668457, "sampling/sampling_logp_difference/max": 0.06747142225503922, "sampling/sampling_logp_difference/mean": 0.0003068496589548886, "step": 306, "step_time": 13.675839747999817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1402.0, "completions/max_terminated_length": 1402.0, "completions/mean_length": 1172.125, "completions/mean_terminated_length": 1172.125, "completions/min_length": 986.0, "completions/min_terminated_length": 986.0, "entropy": 0.0019008369272341952, "epoch": 0.02456, "frac_reward_zero_std": 0.125, "grad_norm": 8.696097211213782e-05, "kl": 1.0740288607776165, "learning_rate": 4.352455787519504e-05, "loss": -0.0, "num_tokens": 11041802.0, "reward": 1.4487500190734863, "reward_std": 0.08476945757865906, "rewards/rollout_reward_func/mean": 1.4487500190734863, "rewards/rollout_reward_func/std": 0.11754891276359558, "sampling/importance_sampling_ratio/max": 1.0069996118545532, "sampling/importance_sampling_ratio/mean": 1.0005407333374023, "sampling/importance_sampling_ratio/min": 0.9996516704559326, "sampling/sampling_logp_difference/max": 0.003280839417129755, "sampling/sampling_logp_difference/mean": 5.469562893267721e-05, "step": 307, "step_time": 15.533062751998841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1404.0, "completions/max_terminated_length": 1404.0, "completions/mean_length": 1128.53125, "completions/mean_terminated_length": 1128.53125, "completions/min_length": 898.0, "completions/min_terminated_length": 898.0, "entropy": 0.00238486826128792, "epoch": 0.02464, "frac_reward_zero_std": 0.125, "grad_norm": 0.00012374477228149772, "kl": 1.1467287689447403, "learning_rate": 4.3524487700217286e-05, "loss": 0.0002, "num_tokens": 11095310.0, "reward": 1.439062476158142, "reward_std": 0.10398891568183899, "rewards/rollout_reward_func/mean": 1.439062476158142, "rewards/rollout_reward_func/std": 0.14898277819156647, "sampling/importance_sampling_ratio/max": 1.0040817260742188, "sampling/importance_sampling_ratio/mean": 1.0008351802825928, "sampling/importance_sampling_ratio/min": 0.9993410110473633, "sampling/sampling_logp_difference/max": 0.003461932297796011, "sampling/sampling_logp_difference/mean": 8.358842751476914e-05, "step": 308, "step_time": 14.608692377998523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1397.0, "completions/max_terminated_length": 1397.0, "completions/mean_length": 1074.6875, "completions/mean_terminated_length": 1074.6875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0028102786891395226, "epoch": 0.02472, "frac_reward_zero_std": 0.125, "grad_norm": 0.00019298835832159966, "kl": 1.2968119233846664, "learning_rate": 4.3524417266869296e-05, "loss": -0.0001, "num_tokens": 11147733.0, "reward": 1.378749966621399, "reward_std": 0.18152371048927307, "rewards/rollout_reward_func/mean": 1.378749966621399, "rewards/rollout_reward_func/std": 0.207547128200531, "sampling/importance_sampling_ratio/max": 1.008417010307312, "sampling/importance_sampling_ratio/mean": 1.0010756254196167, "sampling/importance_sampling_ratio/min": 0.9994828104972839, "sampling/sampling_logp_difference/max": 0.004324738867580891, "sampling/sampling_logp_difference/mean": 0.00010654746438376606, "step": 309, "step_time": 15.643517381001402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 1058.21875, "completions/mean_terminated_length": 1058.21875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0023156637034844607, "epoch": 0.0248, "frac_reward_zero_std": 0.375, "grad_norm": 0.00017605569155421108, "kl": 1.292950190603733, "learning_rate": 4.3524346575152175e-05, "loss": 0.0011, "num_tokens": 11199176.0, "reward": 1.3796875476837158, "reward_std": 0.16439618170261383, "rewards/rollout_reward_func/mean": 1.3796875476837158, "rewards/rollout_reward_func/std": 0.2224857360124588, "sampling/importance_sampling_ratio/max": 1.0055351257324219, "sampling/importance_sampling_ratio/mean": 1.0007262229919434, "sampling/importance_sampling_ratio/min": 0.9990553855895996, "sampling/sampling_logp_difference/max": 0.004991581663489342, "sampling/sampling_logp_difference/mean": 8.17785257822834e-05, "step": 310, "step_time": 14.576996252000754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1405.0, "completions/max_terminated_length": 1405.0, "completions/mean_length": 1058.90625, "completions/mean_terminated_length": 1058.90625, "completions/min_length": 700.0, "completions/min_terminated_length": 700.0, "entropy": 0.003638004622189328, "epoch": 0.02488, "frac_reward_zero_std": 0.25, "grad_norm": 0.0008163711172528565, "kl": 1.2638441622257233, "learning_rate": 4.352427562506706e-05, "loss": 0.0008, "num_tokens": 11250760.0, "reward": 1.3987500667572021, "reward_std": 0.11353804916143417, "rewards/rollout_reward_func/mean": 1.3987500667572021, "rewards/rollout_reward_func/std": 0.18671612441539764, "sampling/importance_sampling_ratio/max": 1.0256770849227905, "sampling/importance_sampling_ratio/mean": 1.0025192499160767, "sampling/importance_sampling_ratio/min": 0.9996899962425232, "sampling/sampling_logp_difference/max": 0.024086691439151764, "sampling/sampling_logp_difference/mean": 0.00024389015743508935, "step": 311, "step_time": 15.649319667999407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1408.0, "completions/max_terminated_length": 1408.0, "completions/mean_length": 1074.375, "completions/mean_terminated_length": 1074.375, "completions/min_length": 758.0, "completions/min_terminated_length": 758.0, "entropy": 0.002730246400460601, "epoch": 0.02496, "frac_reward_zero_std": 0.0, "grad_norm": 0.0020308021921664476, "kl": 1.259428583085537, "learning_rate": 4.3524204416615056e-05, "loss": -0.0002, "num_tokens": 11302793.0, "reward": 1.3790624141693115, "reward_std": 0.1496875137090683, "rewards/rollout_reward_func/mean": 1.3790624141693115, "rewards/rollout_reward_func/std": 0.234002485871315, "sampling/importance_sampling_ratio/max": 1.0046217441558838, "sampling/importance_sampling_ratio/mean": 0.9801512956619263, "sampling/importance_sampling_ratio/min": 0.34196656942367554, "sampling/sampling_logp_difference/max": 1.0721805095672607, "sampling/sampling_logp_difference/mean": 0.0031848216895014048, "step": 312, "step_time": 14.850039277001088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1401.0, "completions/max_terminated_length": 1401.0, "completions/mean_length": 975.0, "completions/mean_terminated_length": 975.0, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.0025660441897343844, "epoch": 0.02504, "frac_reward_zero_std": 0.0, "grad_norm": 0.00034504837822169065, "kl": 1.4808623641729355, "learning_rate": 4.35241329497973e-05, "loss": 0.0001, "num_tokens": 11350664.0, "reward": 1.3506250381469727, "reward_std": 0.23211465775966644, "rewards/rollout_reward_func/mean": 1.3506250381469727, "rewards/rollout_reward_func/std": 0.2265155166387558, "sampling/importance_sampling_ratio/max": 1.0111159086227417, "sampling/importance_sampling_ratio/mean": 1.0015347003936768, "sampling/importance_sampling_ratio/min": 0.9989986419677734, "sampling/sampling_logp_difference/max": 0.009005140513181686, "sampling/sampling_logp_difference/mean": 0.0001532700116513297, "step": 313, "step_time": 15.128601263000746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1239.0, "completions/max_terminated_length": 1239.0, "completions/mean_length": 891.6875, "completions/mean_terminated_length": 891.6875, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.001816475771192927, "epoch": 0.02512, "frac_reward_zero_std": 0.0, "grad_norm": 0.008241352625191212, "kl": 1.8158975541591644, "learning_rate": 4.352406122461493e-05, "loss": -0.0006, "num_tokens": 11395672.0, "reward": 1.2718749046325684, "reward_std": 0.2329370379447937, "rewards/rollout_reward_func/mean": 1.2718749046325684, "rewards/rollout_reward_func/std": 0.27508869767189026, "sampling/importance_sampling_ratio/max": 1.0112429857254028, "sampling/importance_sampling_ratio/mean": 0.9813331365585327, "sampling/importance_sampling_ratio/min": 0.3767866790294647, "sampling/sampling_logp_difference/max": 0.9786195755004883, "sampling/sampling_logp_difference/mean": 0.003090068930760026, "step": 314, "step_time": 13.606914277000214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1386.0, "completions/max_terminated_length": 1386.0, "completions/mean_length": 952.75, "completions/mean_terminated_length": 952.75, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.002330313087441027, "epoch": 0.0252, "frac_reward_zero_std": 0.0, "grad_norm": 0.38245177268981934, "kl": 9.639879435300827, "learning_rate": 4.352398924106907e-05, "loss": 0.0054, "num_tokens": 11442829.0, "reward": 1.3409374952316284, "reward_std": 0.22507403790950775, "rewards/rollout_reward_func/mean": 1.3409374952316284, "rewards/rollout_reward_func/std": 0.3132798373699188, "sampling/importance_sampling_ratio/max": 1.0013846158981323, "sampling/importance_sampling_ratio/mean": 0.9964277744293213, "sampling/importance_sampling_ratio/min": 0.8763317465782166, "sampling/sampling_logp_difference/max": 0.18738746643066406, "sampling/sampling_logp_difference/mean": 0.0007187107694335282, "step": 315, "step_time": 15.15053471400006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1400.0, "completions/max_terminated_length": 1400.0, "completions/mean_length": 974.0, "completions/mean_terminated_length": 974.0, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.0030339003424160182, "epoch": 0.02528, "frac_reward_zero_std": 0.125, "grad_norm": 0.008363212458789349, "kl": 1.5484594777226448, "learning_rate": 4.352391699916086e-05, "loss": 0.0, "num_tokens": 11491119.0, "reward": 1.3503124713897705, "reward_std": 0.1838027536869049, "rewards/rollout_reward_func/mean": 1.3503124713897705, "rewards/rollout_reward_func/std": 0.2093690186738968, "sampling/importance_sampling_ratio/max": 1.0063698291778564, "sampling/importance_sampling_ratio/mean": 0.9909765124320984, "sampling/importance_sampling_ratio/min": 0.6887931227684021, "sampling/sampling_logp_difference/max": 0.3734774589538574, "sampling/sampling_logp_difference/mean": 0.0012330904137343168, "step": 316, "step_time": 15.438087376001022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1398.0, "completions/max_terminated_length": 1398.0, "completions/mean_length": 1118.03125, "completions/mean_terminated_length": 1118.03125, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.015473175328224897, "epoch": 0.02536, "frac_reward_zero_std": 0.125, "grad_norm": 0.021545276045799255, "kl": 1.346261776983738, "learning_rate": 4.3523844498891454e-05, "loss": -0.0021, "num_tokens": 11545243.0, "reward": 1.350000023841858, "reward_std": 0.19404742121696472, "rewards/rollout_reward_func/mean": 1.350000023841858, "rewards/rollout_reward_func/std": 0.21052928268909454, "sampling/importance_sampling_ratio/max": 1.0114686489105225, "sampling/importance_sampling_ratio/mean": 0.9488431215286255, "sampling/importance_sampling_ratio/min": 0.24018579721450806, "sampling/sampling_logp_difference/max": 1.4701623916625977, "sampling/sampling_logp_difference/mean": 0.008239943534135818, "step": 317, "step_time": 14.435779884000112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1398.0, "completions/max_terminated_length": 1398.0, "completions/mean_length": 1065.90625, "completions/mean_terminated_length": 1065.90625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.03361464408226311, "epoch": 0.02544, "frac_reward_zero_std": 0.125, "grad_norm": 0.020390303805470467, "kl": 1.2107831984758377, "learning_rate": 4.352377174026199e-05, "loss": 0.0012, "num_tokens": 11596766.0, "reward": 1.3428125381469727, "reward_std": 0.24000835418701172, "rewards/rollout_reward_func/mean": 1.3428125381469727, "rewards/rollout_reward_func/std": 0.39348921179771423, "sampling/importance_sampling_ratio/max": 1.3156139850616455, "sampling/importance_sampling_ratio/mean": 0.9956793189048767, "sampling/importance_sampling_ratio/min": 0.7030260562896729, "sampling/sampling_logp_difference/max": 0.49698829650878906, "sampling/sampling_logp_difference/mean": 0.004403485916554928, "step": 318, "step_time": 14.62994858699949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0029761905316263437, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029761905316263437, "completions/clipped_ratio": 0.0, "completions/max_length": 1237.0, "completions/max_terminated_length": 1237.0, "completions/mean_length": 966.5625, "completions/mean_terminated_length": 966.5625, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.05006060074083507, "epoch": 0.02552, "frac_reward_zero_std": 0.0, "grad_norm": 0.022741060703992844, "kl": 1.370442807674408, "learning_rate": 4.3523698723273633e-05, "loss": 0.0008, "num_tokens": 11644294.0, "reward": 1.1828124523162842, "reward_std": 0.4750112295150757, "rewards/rollout_reward_func/mean": 1.1828124523162842, "rewards/rollout_reward_func/std": 0.5119931697845459, "sampling/importance_sampling_ratio/max": 1.4119722843170166, "sampling/importance_sampling_ratio/mean": 0.9966235756874084, "sampling/importance_sampling_ratio/min": 0.6402744650840759, "sampling/sampling_logp_difference/max": 0.44452810287475586, "sampling/sampling_logp_difference/mean": 0.005956551991403103, "step": 319, "step_time": 14.74385449699912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008275691885501146, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008275691885501146, "completions/clipped_ratio": 0.0, "completions/max_length": 1238.0, "completions/max_terminated_length": 1238.0, "completions/mean_length": 1193.15625, "completions/mean_terminated_length": 1193.15625, "completions/min_length": 1009.0, "completions/min_terminated_length": 1009.0, "entropy": 0.12373029813170433, "epoch": 0.0256, "frac_reward_zero_std": 0.125, "grad_norm": 0.040515754371881485, "kl": 1.4507074803113937, "learning_rate": 4.352362544792752e-05, "loss": 0.0043, "num_tokens": 11701207.0, "reward": 0.9962500333786011, "reward_std": 0.5119329690933228, "rewards/rollout_reward_func/mean": 0.9962500333786011, "rewards/rollout_reward_func/std": 0.6602479815483093, "sampling/importance_sampling_ratio/max": 1.957187533378601, "sampling/importance_sampling_ratio/mean": 1.0538028478622437, "sampling/importance_sampling_ratio/min": 0.7880349159240723, "sampling/sampling_logp_difference/max": 0.6715010404586792, "sampling/sampling_logp_difference/mean": 0.014758694916963577, "step": 320, "step_time": 14.272954180999477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1494.0, "completions/max_terminated_length": 1494.0, "completions/mean_length": 1202.4375, "completions/mean_terminated_length": 1202.4375, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.11645952751860023, "epoch": 0.02568, "frac_reward_zero_std": 0.0, "grad_norm": 0.03537426143884659, "kl": 2.206043779850006, "learning_rate": 4.352355191422483e-05, "loss": -0.0009, "num_tokens": 11756454.0, "reward": 1.1177083253860474, "reward_std": 0.48546552658081055, "rewards/rollout_reward_func/mean": 1.1177083253860474, "rewards/rollout_reward_func/std": 0.5172649621963501, "sampling/importance_sampling_ratio/max": 1.3944413661956787, "sampling/importance_sampling_ratio/mean": 0.9679909944534302, "sampling/importance_sampling_ratio/min": 0.213487446308136, "sampling/sampling_logp_difference/max": 1.0565942525863647, "sampling/sampling_logp_difference/mean": 0.019914956763386726, "step": 321, "step_time": 15.743595232000189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1705.0, "completions/max_terminated_length": 1705.0, "completions/mean_length": 1157.5, "completions/mean_terminated_length": 1157.5, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.10209901724010706, "epoch": 0.02576, "frac_reward_zero_std": 0.0, "grad_norm": 0.040049273520708084, "kl": 1.5929739028215408, "learning_rate": 4.352347812216672e-05, "loss": 0.0026, "num_tokens": 11810282.0, "reward": 1.1137499809265137, "reward_std": 0.4387747049331665, "rewards/rollout_reward_func/mean": 1.1137499809265137, "rewards/rollout_reward_func/std": 0.5836449265480042, "sampling/importance_sampling_ratio/max": 1.1318328380584717, "sampling/importance_sampling_ratio/mean": 0.9540979862213135, "sampling/importance_sampling_ratio/min": 0.5336391925811768, "sampling/sampling_logp_difference/max": 0.6350715160369873, "sampling/sampling_logp_difference/mean": 0.011897875927388668, "step": 322, "step_time": 16.745308953999483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 1162.5625, "completions/mean_terminated_length": 1162.5625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.10315238311886787, "epoch": 0.02584, "frac_reward_zero_std": 0.0, "grad_norm": 0.02198137529194355, "kl": 1.8563614934682846, "learning_rate": 4.352340407175436e-05, "loss": -0.0023, "num_tokens": 11864511.0, "reward": 1.2103126049041748, "reward_std": 0.3911898136138916, "rewards/rollout_reward_func/mean": 1.2103126049041748, "rewards/rollout_reward_func/std": 0.433576762676239, "sampling/importance_sampling_ratio/max": 1.2460194826126099, "sampling/importance_sampling_ratio/mean": 0.988715648651123, "sampling/importance_sampling_ratio/min": 0.3440873920917511, "sampling/sampling_logp_difference/max": 0.7444963455200195, "sampling/sampling_logp_difference/mean": 0.012276777997612953, "step": 323, "step_time": 18.18370322299961 }, { "clip_ratio/high_max": 0.004807692486792803, "clip_ratio/high_mean": 0.0024038462433964014, "clip_ratio/low_mean": 0.002016128972172737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0044199752155691385, "completions/clipped_ratio": 0.0, "completions/max_length": 1706.0, "completions/max_terminated_length": 1706.0, "completions/mean_length": 1248.34375, "completions/mean_terminated_length": 1248.34375, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.15853185672312975, "epoch": 0.02592, "frac_reward_zero_std": 0.0, "grad_norm": 0.0445425920188427, "kl": 1.7290366441011429, "learning_rate": 4.3523329762988905e-05, "loss": 0.0015, "num_tokens": 11922135.0, "reward": 1.02385413646698, "reward_std": 0.4840511977672577, "rewards/rollout_reward_func/mean": 1.02385413646698, "rewards/rollout_reward_func/std": 0.5155983567237854, "sampling/importance_sampling_ratio/max": 2.3378312587738037, "sampling/importance_sampling_ratio/mean": 1.0035336017608643, "sampling/importance_sampling_ratio/min": 0.11611832678318024, "sampling/sampling_logp_difference/max": 2.294131278991699, "sampling/sampling_logp_difference/mean": 0.02312912978231907, "step": 324, "step_time": 17.88831883399962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1712.0, "completions/max_terminated_length": 1712.0, "completions/mean_length": 1295.03125, "completions/mean_terminated_length": 1295.03125, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.13334195874631405, "epoch": 0.026, "frac_reward_zero_std": 0.0, "grad_norm": 0.5655775666236877, "kl": 9.70188283920288, "learning_rate": 4.3523255195871565e-05, "loss": 0.0024, "num_tokens": 11980944.0, "reward": 1.195833444595337, "reward_std": 0.3606184720993042, "rewards/rollout_reward_func/mean": 1.195833444595337, "rewards/rollout_reward_func/std": 0.47339922189712524, "sampling/importance_sampling_ratio/max": 1.5267651081085205, "sampling/importance_sampling_ratio/mean": 1.0199445486068726, "sampling/importance_sampling_ratio/min": 0.3952530026435852, "sampling/sampling_logp_difference/max": 1.036156415939331, "sampling/sampling_logp_difference/mean": 0.018237780779600143, "step": 325, "step_time": 16.98113548999936 }, { "clip_ratio/high_max": 0.011305894237011671, "clip_ratio/high_mean": 0.0056529471185058355, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0056529471185058355, "completions/clipped_ratio": 0.0, "completions/max_length": 1519.0, "completions/max_terminated_length": 1519.0, "completions/mean_length": 1180.8125, "completions/mean_terminated_length": 1180.8125, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.27728694304823875, "epoch": 0.02608, "frac_reward_zero_std": 0.0, "grad_norm": 0.15635909140110016, "kl": 1.4338090270757675, "learning_rate": 4.35231803704035e-05, "loss": -0.0039, "num_tokens": 12035686.0, "reward": 1.0779166221618652, "reward_std": 0.45532697439193726, "rewards/rollout_reward_func/mean": 1.0779166221618652, "rewards/rollout_reward_func/std": 0.4720167815685272, "sampling/importance_sampling_ratio/max": 1.76887047290802, "sampling/importance_sampling_ratio/mean": 1.0160675048828125, "sampling/importance_sampling_ratio/min": 0.6883763670921326, "sampling/sampling_logp_difference/max": 0.5690274238586426, "sampling/sampling_logp_difference/mean": 0.023085942491889, "step": 326, "step_time": 16.017784890000257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1718.0, "completions/max_terminated_length": 1718.0, "completions/mean_length": 1362.09375, "completions/mean_terminated_length": 1362.09375, "completions/min_length": 868.0, "completions/min_terminated_length": 868.0, "entropy": 0.3653669059276581, "epoch": 0.02616, "frac_reward_zero_std": 0.0, "grad_norm": 0.11234799772500992, "kl": 1.6105803847312927, "learning_rate": 4.352310528658589e-05, "loss": -0.0029, "num_tokens": 12097262.0, "reward": 0.8739583492279053, "reward_std": 0.6031307578086853, "rewards/rollout_reward_func/mean": 0.8739583492279053, "rewards/rollout_reward_func/std": 0.5885523557662964, "sampling/importance_sampling_ratio/max": 1.65760338306427, "sampling/importance_sampling_ratio/mean": 0.9718091487884521, "sampling/importance_sampling_ratio/min": 0.5822751522064209, "sampling/sampling_logp_difference/max": 0.432680606842041, "sampling/sampling_logp_difference/mean": 0.023493019863963127, "step": 327, "step_time": 16.984133383999506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1710.0, "completions/max_terminated_length": 1710.0, "completions/mean_length": 1211.125, "completions/mean_terminated_length": 1211.125, "completions/min_length": 774.0, "completions/min_terminated_length": 774.0, "entropy": 0.3852327782660723, "epoch": 0.02624, "frac_reward_zero_std": 0.0, "grad_norm": 0.06201113387942314, "kl": 1.128612607717514, "learning_rate": 4.352302994441994e-05, "loss": -0.0016, "num_tokens": 12152857.0, "reward": 0.7505208253860474, "reward_std": 0.5072071552276611, "rewards/rollout_reward_func/mean": 0.7505208253860474, "rewards/rollout_reward_func/std": 0.5755409598350525, "sampling/importance_sampling_ratio/max": 1.7917141914367676, "sampling/importance_sampling_ratio/mean": 1.0745790004730225, "sampling/importance_sampling_ratio/min": 0.5414091944694519, "sampling/sampling_logp_difference/max": 0.5405113697052002, "sampling/sampling_logp_difference/mean": 0.023343343287706375, "step": 328, "step_time": 17.371139249999942 }, { "clip_ratio/high_max": 0.014136905316263437, "clip_ratio/high_mean": 0.009300595615059137, "clip_ratio/low_mean": 0.004999999888241291, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.014300595270469785, "completions/clipped_ratio": 0.0, "completions/max_length": 1713.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 1365.9375, "completions/mean_terminated_length": 1365.9375, "completions/min_length": 849.0, "completions/min_terminated_length": 849.0, "entropy": 0.4193199388682842, "epoch": 0.02632, "frac_reward_zero_std": 0.0, "grad_norm": 0.0497148223221302, "kl": 1.0424881502985954, "learning_rate": 4.352295434390683e-05, "loss": 0.0012, "num_tokens": 12214580.0, "reward": 0.8514583110809326, "reward_std": 0.5297824144363403, "rewards/rollout_reward_func/mean": 0.8514583110809326, "rewards/rollout_reward_func/std": 0.5194565057754517, "sampling/importance_sampling_ratio/max": 1.501612663269043, "sampling/importance_sampling_ratio/mean": 0.9670021533966064, "sampling/importance_sampling_ratio/min": 0.6143242716789246, "sampling/sampling_logp_difference/max": 0.4510568380355835, "sampling/sampling_logp_difference/mean": 0.022711243480443954, "step": 329, "step_time": 16.972004039998865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0048363097012043, "clip_ratio/low_min": 0.004464285913854837, "clip_ratio/region_mean": 0.0048363097012043, "completions/clipped_ratio": 0.0, "completions/max_length": 1509.0, "completions/max_terminated_length": 1509.0, "completions/mean_length": 1195.125, "completions/mean_terminated_length": 1195.125, "completions/min_length": 912.0, "completions/min_terminated_length": 912.0, "entropy": 0.4052167013287544, "epoch": 0.0264, "frac_reward_zero_std": 0.0, "grad_norm": 0.045991797000169754, "kl": 1.0619773417711258, "learning_rate": 4.352287848504777e-05, "loss": 0.0013, "num_tokens": 12269542.0, "reward": 0.8684375286102295, "reward_std": 0.4174615442752838, "rewards/rollout_reward_func/mean": 0.8684375286102295, "rewards/rollout_reward_func/std": 0.4729287922382355, "sampling/importance_sampling_ratio/max": 1.5503970384597778, "sampling/importance_sampling_ratio/mean": 1.0029687881469727, "sampling/importance_sampling_ratio/min": 0.5991948246955872, "sampling/sampling_logp_difference/max": 0.3720242977142334, "sampling/sampling_logp_difference/mean": 0.018947504460811615, "step": 330, "step_time": 15.99639298999955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0021551724057644606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 1707.0, "completions/max_terminated_length": 1707.0, "completions/mean_length": 1252.78125, "completions/mean_terminated_length": 1252.78125, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.3955857902765274, "epoch": 0.02648, "frac_reward_zero_std": 0.0, "grad_norm": 0.04664609208703041, "kl": 0.919258676469326, "learning_rate": 4.352280236784396e-05, "loss": -0.0001, "num_tokens": 12326258.0, "reward": 0.8700000047683716, "reward_std": 0.38242220878601074, "rewards/rollout_reward_func/mean": 0.8700000047683716, "rewards/rollout_reward_func/std": 0.43409690260887146, "sampling/importance_sampling_ratio/max": 1.3299115896224976, "sampling/importance_sampling_ratio/mean": 0.9932869672775269, "sampling/importance_sampling_ratio/min": 0.7277578115463257, "sampling/sampling_logp_difference/max": 0.39430952072143555, "sampling/sampling_logp_difference/mean": 0.015269611030817032, "step": 331, "step_time": 16.76456870000129 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 1719.0, "completions/max_terminated_length": 1719.0, "completions/mean_length": 1267.9375, "completions/mean_terminated_length": 1267.9375, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.404892660677433, "epoch": 0.02656, "frac_reward_zero_std": 0.0, "grad_norm": 0.0622086226940155, "kl": 1.4579979479312897, "learning_rate": 4.352272599229658e-05, "loss": -0.0002, "num_tokens": 12383674.0, "reward": 0.9190625548362732, "reward_std": 0.4583616554737091, "rewards/rollout_reward_func/mean": 0.9190625548362732, "rewards/rollout_reward_func/std": 0.48632869124412537, "sampling/importance_sampling_ratio/max": 1.2798994779586792, "sampling/importance_sampling_ratio/mean": 1.0218572616577148, "sampling/importance_sampling_ratio/min": 0.7131215929985046, "sampling/sampling_logp_difference/max": 0.3581733703613281, "sampling/sampling_logp_difference/mean": 0.016207275912165642, "step": 332, "step_time": 17.67153544699977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 1370.15625, "completions/mean_terminated_length": 1370.15625, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.3117890451103449, "epoch": 0.02664, "frac_reward_zero_std": 0.0, "grad_norm": 0.04014379158616066, "kl": 1.1019662767648697, "learning_rate": 4.352264935840688e-05, "loss": -0.0001, "num_tokens": 12445508.0, "reward": 1.0203125476837158, "reward_std": 0.4061201214790344, "rewards/rollout_reward_func/mean": 1.0203125476837158, "rewards/rollout_reward_func/std": 0.4554515480995178, "sampling/importance_sampling_ratio/max": 1.4196443557739258, "sampling/importance_sampling_ratio/mean": 1.006690502166748, "sampling/importance_sampling_ratio/min": 0.7101880311965942, "sampling/sampling_logp_difference/max": 0.32967329025268555, "sampling/sampling_logp_difference/mean": 0.012734422460198402, "step": 333, "step_time": 16.94337937599994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1718.0, "completions/max_terminated_length": 1718.0, "completions/mean_length": 1272.90625, "completions/mean_terminated_length": 1272.90625, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.2772346958518028, "epoch": 0.02672, "frac_reward_zero_std": 0.0, "grad_norm": 0.06779488176107407, "kl": 1.3149583600461483, "learning_rate": 4.352257246617604e-05, "loss": -0.0011, "num_tokens": 12503962.0, "reward": 1.07770836353302, "reward_std": 0.42652925848960876, "rewards/rollout_reward_func/mean": 1.07770836353302, "rewards/rollout_reward_func/std": 0.41706642508506775, "sampling/importance_sampling_ratio/max": 1.580692172050476, "sampling/importance_sampling_ratio/mean": 0.9755028486251831, "sampling/importance_sampling_ratio/min": 0.4750201404094696, "sampling/sampling_logp_difference/max": 0.4240999221801758, "sampling/sampling_logp_difference/mean": 0.015434881672263145, "step": 334, "step_time": 17.821035622000636 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1392.5625, "completions/mean_terminated_length": 1392.5625, "completions/min_length": 1026.0, "completions/min_terminated_length": 1026.0, "entropy": 0.23976965621113777, "epoch": 0.0268, "frac_reward_zero_std": 0.0, "grad_norm": 3.7565720081329346, "kl": 16.413961358368397, "learning_rate": 4.352249531560529e-05, "loss": 0.0083, "num_tokens": 12566562.0, "reward": 1.1597917079925537, "reward_std": 0.49342256784439087, "rewards/rollout_reward_func/mean": 1.1597917079925537, "rewards/rollout_reward_func/std": 0.5045697689056396, "sampling/importance_sampling_ratio/max": 1.9389302730560303, "sampling/importance_sampling_ratio/mean": 0.9594783782958984, "sampling/importance_sampling_ratio/min": 1.179496522337941e-17, "sampling/sampling_logp_difference/max": 25.664409637451172, "sampling/sampling_logp_difference/mean": 0.1084611639380455, "step": 335, "step_time": 17.24413768499926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1292.5625, "completions/mean_terminated_length": 1292.5625, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.20167534425854683, "epoch": 0.02688, "frac_reward_zero_std": 0.0, "grad_norm": 0.040485601872205734, "kl": 1.0903836339712143, "learning_rate": 4.352241790669585e-05, "loss": 0.0002, "num_tokens": 12625290.0, "reward": 1.1449999809265137, "reward_std": 0.4654574990272522, "rewards/rollout_reward_func/mean": 1.1449999809265137, "rewards/rollout_reward_func/std": 0.494474858045578, "sampling/importance_sampling_ratio/max": 1.3251538276672363, "sampling/importance_sampling_ratio/mean": 1.026557207107544, "sampling/importance_sampling_ratio/min": 0.7794497609138489, "sampling/sampling_logp_difference/max": 0.28444862365722656, "sampling/sampling_logp_difference/mean": 0.012347551062703133, "step": 336, "step_time": 17.487867061000543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 1721.0, "completions/max_terminated_length": 1721.0, "completions/mean_length": 1300.71875, "completions/mean_terminated_length": 1300.71875, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.12485662661492825, "epoch": 0.02696, "frac_reward_zero_std": 0.125, "grad_norm": 0.04138629138469696, "kl": 0.8710551932454109, "learning_rate": 4.352234023944895e-05, "loss": 0.0108, "num_tokens": 12684398.0, "reward": 1.2542709112167358, "reward_std": 0.3879399597644806, "rewards/rollout_reward_func/mean": 1.2542709112167358, "rewards/rollout_reward_func/std": 0.4424324035644531, "sampling/importance_sampling_ratio/max": 1.3264662027359009, "sampling/importance_sampling_ratio/mean": 1.0084706544876099, "sampling/importance_sampling_ratio/min": 0.6194111108779907, "sampling/sampling_logp_difference/max": 0.4649994373321533, "sampling/sampling_logp_difference/mean": 0.007206466980278492, "step": 337, "step_time": 17.08761501800018 }, { "clip_ratio/high_max": 0.0016891892300918698, "clip_ratio/high_mean": 0.0008445946150459349, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008445946150459349, "completions/clipped_ratio": 0.0, "completions/max_length": 1709.0, "completions/max_terminated_length": 1709.0, "completions/mean_length": 1293.5625, "completions/mean_terminated_length": 1293.5625, "completions/min_length": 834.0, "completions/min_terminated_length": 834.0, "entropy": 0.15367809496819973, "epoch": 0.02704, "frac_reward_zero_std": 0.0, "grad_norm": 0.03370169550180435, "kl": 1.35938660800457, "learning_rate": 4.352226231386582e-05, "loss": 0.0011, "num_tokens": 12743087.0, "reward": 1.1156249046325684, "reward_std": 0.40081098675727844, "rewards/rollout_reward_func/mean": 1.1156249046325684, "rewards/rollout_reward_func/std": 0.48784834146499634, "sampling/importance_sampling_ratio/max": 1.4100341796875, "sampling/importance_sampling_ratio/mean": 1.0114266872406006, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.1074142456054688, "sampling/sampling_logp_difference/mean": 0.014980400912463665, "step": 338, "step_time": 17.75621947200034 }, { "clip_ratio/high_max": 0.004629629664123058, "clip_ratio/high_mean": 0.002314814832061529, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "completions/clipped_ratio": 0.0, "completions/max_length": 1713.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 1205.15625, "completions/mean_terminated_length": 1205.15625, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.10817605350166559, "epoch": 0.02712, "frac_reward_zero_std": 0.0, "grad_norm": 0.028273746371269226, "kl": 1.2366398200392723, "learning_rate": 4.352218412994769e-05, "loss": 0.0029, "num_tokens": 12798323.0, "reward": 1.2471874952316284, "reward_std": 0.2949613630771637, "rewards/rollout_reward_func/mean": 1.2471874952316284, "rewards/rollout_reward_func/std": 0.3250884711742401, "sampling/importance_sampling_ratio/max": 1.8471603393554688, "sampling/importance_sampling_ratio/mean": 0.9846133589744568, "sampling/importance_sampling_ratio/min": 0.40606752038002014, "sampling/sampling_logp_difference/max": 0.5822393894195557, "sampling/sampling_logp_difference/mean": 0.009146168828010559, "step": 339, "step_time": 17.121054860000186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1720.0, "completions/max_terminated_length": 1720.0, "completions/mean_length": 1218.0625, "completions/mean_terminated_length": 1218.0625, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.09801414562389255, "epoch": 0.0272, "frac_reward_zero_std": 0.0, "grad_norm": 0.03325510397553444, "kl": 1.1109517589211464, "learning_rate": 4.35221056876958e-05, "loss": 0.0001, "num_tokens": 12854140.0, "reward": 1.3273957967758179, "reward_std": 0.30563104152679443, "rewards/rollout_reward_func/mean": 1.3273957967758179, "rewards/rollout_reward_func/std": 0.36204591393470764, "sampling/importance_sampling_ratio/max": 1.284331202507019, "sampling/importance_sampling_ratio/mean": 0.9898492097854614, "sampling/importance_sampling_ratio/min": 0.7037913203239441, "sampling/sampling_logp_difference/max": 0.35213518142700195, "sampling/sampling_logp_difference/mean": 0.00794786587357521, "step": 340, "step_time": 17.747169034999388 }, { "clip_ratio/high_max": 0.00937500037252903, "clip_ratio/high_mean": 0.004687500186264515, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004687500186264515, "completions/clipped_ratio": 0.0, "completions/max_length": 1717.0, "completions/max_terminated_length": 1717.0, "completions/mean_length": 1170.40625, "completions/mean_terminated_length": 1170.40625, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.09109024796634912, "epoch": 0.02728, "frac_reward_zero_std": 0.0, "grad_norm": 0.13352710008621216, "kl": 3.2971304804086685, "learning_rate": 4.352202698711139e-05, "loss": 0.0002, "num_tokens": 12908761.0, "reward": 1.251145839691162, "reward_std": 0.3656933009624481, "rewards/rollout_reward_func/mean": 1.251145839691162, "rewards/rollout_reward_func/std": 0.42825278639793396, "sampling/importance_sampling_ratio/max": 1.1685794591903687, "sampling/importance_sampling_ratio/mean": 0.9332302212715149, "sampling/importance_sampling_ratio/min": 0.5167968273162842, "sampling/sampling_logp_difference/max": 0.7262272834777832, "sampling/sampling_logp_difference/mean": 0.0102533008903265, "step": 341, "step_time": 17.01586440400024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002314814832061529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "completions/clipped_ratio": 0.0, "completions/max_length": 1755.0, "completions/max_terminated_length": 1755.0, "completions/mean_length": 1318.875, "completions/mean_terminated_length": 1318.875, "completions/min_length": 867.0, "completions/min_terminated_length": 867.0, "entropy": 0.09542487189173698, "epoch": 0.02736, "frac_reward_zero_std": 0.0, "grad_norm": 0.04378607124090195, "kl": 0.9792122319340706, "learning_rate": 4.352194802819571e-05, "loss": 0.0376, "num_tokens": 12969093.0, "reward": 1.0809375047683716, "reward_std": 0.44543275237083435, "rewards/rollout_reward_func/mean": 1.0809375047683716, "rewards/rollout_reward_func/std": 0.4744203984737396, "sampling/importance_sampling_ratio/max": 2.442532539367676, "sampling/importance_sampling_ratio/mean": 1.035360336303711, "sampling/importance_sampling_ratio/min": 0.647396981716156, "sampling/sampling_logp_difference/max": 0.8608934879302979, "sampling/sampling_logp_difference/mean": 0.008088693022727966, "step": 342, "step_time": 18.61463137299961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0020833334419876337, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020833334419876337, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 1310.25, "completions/mean_terminated_length": 1310.25, "completions/min_length": 823.0, "completions/min_terminated_length": 823.0, "entropy": 0.07468834868632257, "epoch": 0.02744, "frac_reward_zero_std": 0.0, "grad_norm": 0.04567530006170273, "kl": 1.0874993205070496, "learning_rate": 4.352186881095001e-05, "loss": -0.0021, "num_tokens": 13028720.0, "reward": 1.3903125524520874, "reward_std": 0.27689051628112793, "rewards/rollout_reward_func/mean": 1.3903125524520874, "rewards/rollout_reward_func/std": 0.29182204604148865, "sampling/importance_sampling_ratio/max": 1.1298450231552124, "sampling/importance_sampling_ratio/mean": 0.935370683670044, "sampling/importance_sampling_ratio/min": 0.5263364911079407, "sampling/sampling_logp_difference/max": 0.7008383274078369, "sampling/sampling_logp_difference/mean": 0.009211380034685135, "step": 343, "step_time": 17.06135498400181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1507.0, "completions/max_terminated_length": 1507.0, "completions/mean_length": 1241.96875, "completions/mean_terminated_length": 1241.96875, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.09442544309422374, "epoch": 0.02752, "frac_reward_zero_std": 0.0, "grad_norm": 0.05574144795536995, "kl": 1.2134367898106575, "learning_rate": 4.352178933537554e-05, "loss": 0.0015, "num_tokens": 13085351.0, "reward": 1.2538542747497559, "reward_std": 0.3148723542690277, "rewards/rollout_reward_func/mean": 1.2538542747497559, "rewards/rollout_reward_func/std": 0.38628655672073364, "sampling/importance_sampling_ratio/max": 1.2705038785934448, "sampling/importance_sampling_ratio/mean": 1.0385420322418213, "sampling/importance_sampling_ratio/min": 0.6461852192878723, "sampling/sampling_logp_difference/max": 0.44155240058898926, "sampling/sampling_logp_difference/mean": 0.0097053162753582, "step": 344, "step_time": 16.627580508000392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0039408867014572024, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0039408867014572024, "completions/clipped_ratio": 0.0, "completions/max_length": 1724.0, "completions/max_terminated_length": 1724.0, "completions/mean_length": 1380.375, "completions/mean_terminated_length": 1380.375, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.06200834002811462, "epoch": 0.0276, "frac_reward_zero_std": 0.0, "grad_norm": 0.05271453037858009, "kl": 0.9990547671914101, "learning_rate": 4.352170960147356e-05, "loss": 0.0068, "num_tokens": 13147892.0, "reward": 1.267187476158142, "reward_std": 0.3931194245815277, "rewards/rollout_reward_func/mean": 1.267187476158142, "rewards/rollout_reward_func/std": 0.4408993124961853, "sampling/importance_sampling_ratio/max": 2.1962788105010986, "sampling/importance_sampling_ratio/mean": 1.0351104736328125, "sampling/importance_sampling_ratio/min": 0.5635271072387695, "sampling/sampling_logp_difference/max": 0.7765555381774902, "sampling/sampling_logp_difference/mean": 0.005809242837131023, "step": 345, "step_time": 17.12209170299866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1708.0, "completions/max_terminated_length": 1708.0, "completions/mean_length": 1306.71875, "completions/mean_terminated_length": 1306.71875, "completions/min_length": 867.0, "completions/min_terminated_length": 867.0, "entropy": 0.07165526738390326, "epoch": 0.02768, "frac_reward_zero_std": 0.0, "grad_norm": 0.0463113859295845, "kl": 1.0555335879325867, "learning_rate": 4.352162960924533e-05, "loss": -0.0011, "num_tokens": 13207455.0, "reward": 1.3181250095367432, "reward_std": 0.3428001403808594, "rewards/rollout_reward_func/mean": 1.3181250095367432, "rewards/rollout_reward_func/std": 0.39015933871269226, "sampling/importance_sampling_ratio/max": 1.4012701511383057, "sampling/importance_sampling_ratio/mean": 1.0103687047958374, "sampling/importance_sampling_ratio/min": 0.5512271523475647, "sampling/sampling_logp_difference/max": 0.594733476638794, "sampling/sampling_logp_difference/mean": 0.006423857994377613, "step": 346, "step_time": 17.59926497000015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1703.0, "completions/max_terminated_length": 1703.0, "completions/mean_length": 1110.5, "completions/mean_terminated_length": 1110.5, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.0822213371284306, "epoch": 0.02776, "frac_reward_zero_std": 0.0, "grad_norm": 0.030949028208851814, "kl": 1.3897902518510818, "learning_rate": 4.3521549358692135e-05, "loss": -0.0014, "num_tokens": 13259372.0, "reward": 1.2412500381469727, "reward_std": 0.31364837288856506, "rewards/rollout_reward_func/mean": 1.2412500381469727, "rewards/rollout_reward_func/std": 0.4226751923561096, "sampling/importance_sampling_ratio/max": 1.4595236778259277, "sampling/importance_sampling_ratio/mean": 0.9748409986495972, "sampling/importance_sampling_ratio/min": 2.7791039826339267e-16, "sampling/sampling_logp_difference/max": 35.66115188598633, "sampling/sampling_logp_difference/mean": 0.0925031304359436, "step": 347, "step_time": 16.495583671999157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0021551724057644606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 1711.0, "completions/max_terminated_length": 1711.0, "completions/mean_length": 1314.9375, "completions/mean_terminated_length": 1314.9375, "completions/min_length": 869.0, "completions/min_terminated_length": 869.0, "entropy": 0.09827141091227531, "epoch": 0.02784, "frac_reward_zero_std": 0.0, "grad_norm": 0.036135464906692505, "kl": 1.5863417759537697, "learning_rate": 4.3521468849815226e-05, "loss": 0.0027, "num_tokens": 13318913.0, "reward": 1.1761457920074463, "reward_std": 0.49571555852890015, "rewards/rollout_reward_func/mean": 1.1761457920074463, "rewards/rollout_reward_func/std": 0.5305661559104919, "sampling/importance_sampling_ratio/max": 1.3427156209945679, "sampling/importance_sampling_ratio/mean": 1.0138609409332275, "sampling/importance_sampling_ratio/min": 0.7402840852737427, "sampling/sampling_logp_difference/max": 0.36693572998046875, "sampling/sampling_logp_difference/mean": 0.006480918265879154, "step": 348, "step_time": 17.58347384800072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0034420291194692254, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034420291194692254, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 1388.4375, "completions/mean_terminated_length": 1388.4375, "completions/min_length": 1020.0, "completions/min_terminated_length": 1020.0, "entropy": 0.08081163559108973, "epoch": 0.02792, "frac_reward_zero_std": 0.0, "grad_norm": 0.05909596383571625, "kl": 1.1988626196980476, "learning_rate": 4.3521388082615874e-05, "loss": 0.0116, "num_tokens": 13381381.0, "reward": 1.3427083492279053, "reward_std": 0.3518395721912384, "rewards/rollout_reward_func/mean": 1.3427083492279053, "rewards/rollout_reward_func/std": 0.39086562395095825, "sampling/importance_sampling_ratio/max": 2.3991494178771973, "sampling/importance_sampling_ratio/mean": 1.0414245128631592, "sampling/importance_sampling_ratio/min": 0.6907925605773926, "sampling/sampling_logp_difference/max": 0.5988783836364746, "sampling/sampling_logp_difference/mean": 0.009158501401543617, "step": 349, "step_time": 17.21198544300023 }, { "clip_ratio/high_max": 0.0027173913549631834, "clip_ratio/high_mean": 0.0013586956774815917, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038586956216022372, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1272.8125, "completions/mean_terminated_length": 1272.8125, "completions/min_length": 809.0, "completions/min_terminated_length": 809.0, "entropy": 0.09519756399095058, "epoch": 0.028, "frac_reward_zero_std": 0.0, "grad_norm": 420.8794860839844, "kl": 1362.42623334378, "learning_rate": 4.352130705709538e-05, "loss": 1.2212, "num_tokens": 13439525.0, "reward": 1.307083249092102, "reward_std": 0.33360785245895386, "rewards/rollout_reward_func/mean": 1.307083249092102, "rewards/rollout_reward_func/std": 0.3460838496685028, "sampling/importance_sampling_ratio/max": 1.3958418369293213, "sampling/importance_sampling_ratio/mean": 0.9161311388015747, "sampling/importance_sampling_ratio/min": 4.6450518252544215e-27, "sampling/sampling_logp_difference/max": 35.07139205932617, "sampling/sampling_logp_difference/mean": 0.22121793031692505, "step": 350, "step_time": 17.864303822001602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0026041667442768812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "completions/clipped_ratio": 0.0, "completions/max_length": 1716.0, "completions/max_terminated_length": 1716.0, "completions/mean_length": 1239.875, "completions/mean_terminated_length": 1239.875, "completions/min_length": 810.0, "completions/min_terminated_length": 810.0, "entropy": 0.0911985281854868, "epoch": 0.02808, "frac_reward_zero_std": 0.0, "grad_norm": 0.4715736210346222, "kl": 5.429608404636383, "learning_rate": 4.3521225773255e-05, "loss": 0.0046, "num_tokens": 13496542.0, "reward": 1.3512499332427979, "reward_std": 0.23143547773361206, "rewards/rollout_reward_func/mean": 1.3512499332427979, "rewards/rollout_reward_func/std": 0.2634371221065521, "sampling/importance_sampling_ratio/max": 1.3497503995895386, "sampling/importance_sampling_ratio/mean": 0.9565280675888062, "sampling/importance_sampling_ratio/min": 2.100634471360305e-18, "sampling/sampling_logp_difference/max": 30.643905639648438, "sampling/sampling_logp_difference/mean": 0.19630080461502075, "step": 351, "step_time": 17.45963009099978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1723.0, "completions/max_terminated_length": 1723.0, "completions/mean_length": 1337.46875, "completions/mean_terminated_length": 1337.46875, "completions/min_length": 868.0, "completions/min_terminated_length": 868.0, "entropy": 0.06239241920411587, "epoch": 0.02816, "frac_reward_zero_std": 0.0, "grad_norm": 0.034518223255872726, "kl": 1.1109143123030663, "learning_rate": 4.352114423109605e-05, "loss": 0.0003, "num_tokens": 13557162.0, "reward": 1.3601042032241821, "reward_std": 0.3164181113243103, "rewards/rollout_reward_func/mean": 1.3601042032241821, "rewards/rollout_reward_func/std": 0.3594016134738922, "sampling/importance_sampling_ratio/max": 1.2307181358337402, "sampling/importance_sampling_ratio/mean": 1.001114845275879, "sampling/importance_sampling_ratio/min": 0.7933324575424194, "sampling/sampling_logp_difference/max": 0.2617335319519043, "sampling/sampling_logp_difference/mean": 0.004346120171248913, "step": 352, "step_time": 16.972440388000905 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 1394.4375, "completions/mean_terminated_length": 1394.4375, "completions/min_length": 965.0, "completions/min_terminated_length": 965.0, "entropy": 0.08904736582189798, "epoch": 0.02824, "frac_reward_zero_std": 0.0, "grad_norm": 0.06758604943752289, "kl": 1.2682492658495903, "learning_rate": 4.35210624306198e-05, "loss": 0.0029, "num_tokens": 13620180.0, "reward": 1.3508334159851074, "reward_std": 0.2964782416820526, "rewards/rollout_reward_func/mean": 1.3508334159851074, "rewards/rollout_reward_func/std": 0.33647340536117554, "sampling/importance_sampling_ratio/max": 1.2379772663116455, "sampling/importance_sampling_ratio/mean": 1.0063295364379883, "sampling/importance_sampling_ratio/min": 0.7679148316383362, "sampling/sampling_logp_difference/max": 0.36278870701789856, "sampling/sampling_logp_difference/mean": 0.006252596620470285, "step": 353, "step_time": 17.896027205999417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024038462433964014, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024038462433964014, "completions/clipped_ratio": 0.0, "completions/max_length": 1710.0, "completions/max_terminated_length": 1710.0, "completions/mean_length": 1235.1875, "completions/mean_terminated_length": 1235.1875, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.10806149570271373, "epoch": 0.02832, "frac_reward_zero_std": 0.0, "grad_norm": 0.03877844661474228, "kl": 1.3933938443660736, "learning_rate": 4.352098037182756e-05, "loss": 0.0006, "num_tokens": 13676309.0, "reward": 1.1748957633972168, "reward_std": 0.44876259565353394, "rewards/rollout_reward_func/mean": 1.1748957633972168, "rewards/rollout_reward_func/std": 0.5190569162368774, "sampling/importance_sampling_ratio/max": 1.182686686515808, "sampling/importance_sampling_ratio/mean": 0.9663407802581787, "sampling/importance_sampling_ratio/min": 0.7425516843795776, "sampling/sampling_logp_difference/max": 0.30616116523742676, "sampling/sampling_logp_difference/mean": 0.006878938991576433, "step": 354, "step_time": 16.92320221799946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.008535879664123058, "clip_ratio/low_min": 0.00390625, "clip_ratio/region_mean": 0.008535879664123058, "completions/clipped_ratio": 0.0, "completions/max_length": 1722.0, "completions/max_terminated_length": 1722.0, "completions/mean_length": 1398.25, "completions/mean_terminated_length": 1398.25, "completions/min_length": 1030.0, "completions/min_terminated_length": 1030.0, "entropy": 0.10463481862097979, "epoch": 0.0284, "frac_reward_zero_std": 0.125, "grad_norm": 0.03712048381567001, "kl": 1.1513079702854156, "learning_rate": 4.352089805472061e-05, "loss": 0.0013, "num_tokens": 13738974.0, "reward": 1.3796875476837158, "reward_std": 0.27943238615989685, "rewards/rollout_reward_func/mean": 1.3796875476837158, "rewards/rollout_reward_func/std": 0.30768340826034546, "sampling/importance_sampling_ratio/max": 1.7142870426177979, "sampling/importance_sampling_ratio/mean": 1.011326551437378, "sampling/importance_sampling_ratio/min": 5.939741498650619e-08, "sampling/sampling_logp_difference/max": 17.28327178955078, "sampling/sampling_logp_difference/mean": 0.04818720370531082, "step": 355, "step_time": 17.840288657999736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 1313.40625, "completions/mean_terminated_length": 1313.40625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.07071215892210603, "epoch": 0.02848, "frac_reward_zero_std": 0.0, "grad_norm": 0.024140482768416405, "kl": 1.1442812159657478, "learning_rate": 4.3520815479300274e-05, "loss": 0.0003, "num_tokens": 13798941.0, "reward": 1.3668750524520874, "reward_std": 0.15893539786338806, "rewards/rollout_reward_func/mean": 1.3668750524520874, "rewards/rollout_reward_func/std": 0.3011090159416199, "sampling/importance_sampling_ratio/max": 1.46834135055542, "sampling/importance_sampling_ratio/mean": 1.0040292739868164, "sampling/importance_sampling_ratio/min": 0.7974897027015686, "sampling/sampling_logp_difference/max": 0.2882571220397949, "sampling/sampling_logp_difference/mean": 0.005079442635178566, "step": 356, "step_time": 15.783080119999795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1707.0, "completions/max_terminated_length": 1707.0, "completions/mean_length": 1173.6875, "completions/mean_terminated_length": 1173.6875, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.06927677989006042, "epoch": 0.02856, "frac_reward_zero_std": 0.0, "grad_norm": 0.02704356424510479, "kl": 1.2285664975643158, "learning_rate": 4.352073264556785e-05, "loss": 0.0003, "num_tokens": 13853002.0, "reward": 1.4270833730697632, "reward_std": 0.19016316533088684, "rewards/rollout_reward_func/mean": 1.4270833730697632, "rewards/rollout_reward_func/std": 0.24420708417892456, "sampling/importance_sampling_ratio/max": 1.1432957649230957, "sampling/importance_sampling_ratio/mean": 1.010627031326294, "sampling/importance_sampling_ratio/min": 0.9052286744117737, "sampling/sampling_logp_difference/max": 0.12285137176513672, "sampling/sampling_logp_difference/mean": 0.002778419991955161, "step": 357, "step_time": 17.88173982799981 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020833334419876337, "completions/clipped_ratio": 0.0, "completions/max_length": 1715.0, "completions/max_terminated_length": 1715.0, "completions/mean_length": 1318.90625, "completions/mean_terminated_length": 1318.90625, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.09761294443160295, "epoch": 0.02864, "frac_reward_zero_std": 0.0, "grad_norm": 0.09925169497728348, "kl": 2.009278364479542, "learning_rate": 4.3520649553524646e-05, "loss": 0.0269, "num_tokens": 13912767.0, "reward": 1.2360416650772095, "reward_std": 0.3886907994747162, "rewards/rollout_reward_func/mean": 1.2360416650772095, "rewards/rollout_reward_func/std": 0.42481574416160583, "sampling/importance_sampling_ratio/max": 2.615985870361328, "sampling/importance_sampling_ratio/mean": 1.0160441398620605, "sampling/importance_sampling_ratio/min": 1.0338647271623813e-08, "sampling/sampling_logp_difference/max": 18.476581573486328, "sampling/sampling_logp_difference/mean": 0.0511816143989563, "step": 358, "step_time": 17.319016177000776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1506.0, "completions/max_terminated_length": 1506.0, "completions/mean_length": 1061.6875, "completions/mean_terminated_length": 1061.6875, "completions/min_length": 821.0, "completions/min_terminated_length": 821.0, "entropy": 0.09180882526561618, "epoch": 0.02872, "frac_reward_zero_std": 0.0, "grad_norm": 0.0629788488149643, "kl": 1.4210215210914612, "learning_rate": 4.3520566203171984e-05, "loss": -0.0036, "num_tokens": 13962738.0, "reward": 1.25697922706604, "reward_std": 0.344196081161499, "rewards/rollout_reward_func/mean": 1.25697922706604, "rewards/rollout_reward_func/std": 0.321320503950119, "sampling/importance_sampling_ratio/max": 2.821002960205078, "sampling/importance_sampling_ratio/mean": 1.0251867771148682, "sampling/importance_sampling_ratio/min": 0.640070378780365, "sampling/sampling_logp_difference/max": 1.0382603406906128, "sampling/sampling_logp_difference/mean": 0.009464966133236885, "step": 359, "step_time": 16.23085396299939 }, { "clip_ratio/high_max": 0.009464285802096128, "clip_ratio/high_mean": 0.004732142901048064, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004732142901048064, "completions/clipped_ratio": 0.0, "completions/max_length": 1504.0, "completions/max_terminated_length": 1504.0, "completions/mean_length": 1232.1875, "completions/mean_terminated_length": 1232.1875, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.07700601685792208, "epoch": 0.0288, "frac_reward_zero_std": 0.0, "grad_norm": 0.02596249245107174, "kl": 1.2199073508381844, "learning_rate": 4.352048259451119e-05, "loss": 0.0007, "num_tokens": 14019507.0, "reward": 1.2502083778381348, "reward_std": 0.2370825856924057, "rewards/rollout_reward_func/mean": 1.2502083778381348, "rewards/rollout_reward_func/std": 0.3083008825778961, "sampling/importance_sampling_ratio/max": 1.1728909015655518, "sampling/importance_sampling_ratio/mean": 0.9925214052200317, "sampling/importance_sampling_ratio/min": 0.7087746262550354, "sampling/sampling_logp_difference/max": 0.3200467824935913, "sampling/sampling_logp_difference/mean": 0.0047745793126523495, "step": 360, "step_time": 15.57783333399857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002314814832061529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "completions/clipped_ratio": 0.0, "completions/max_length": 1716.0, "completions/max_terminated_length": 1716.0, "completions/mean_length": 1301.28125, "completions/mean_terminated_length": 1301.28125, "completions/min_length": 867.0, "completions/min_terminated_length": 867.0, "entropy": 0.09387564053758979, "epoch": 0.02888, "frac_reward_zero_std": 0.0, "grad_norm": 0.07039353251457214, "kl": 3.3397975116968155, "learning_rate": 4.352039872754357e-05, "loss": 0.0015, "num_tokens": 14078706.0, "reward": 1.098645806312561, "reward_std": 0.36396825313568115, "rewards/rollout_reward_func/mean": 1.098645806312561, "rewards/rollout_reward_func/std": 0.44209495186805725, "sampling/importance_sampling_ratio/max": 1.27671217918396, "sampling/importance_sampling_ratio/mean": 1.0001639127731323, "sampling/importance_sampling_ratio/min": 0.7914810180664062, "sampling/sampling_logp_difference/max": 0.23616981506347656, "sampling/sampling_logp_difference/mean": 0.005045760422945023, "step": 361, "step_time": 17.545285806997526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.0, "completions/max_length": 1503.0, "completions/max_terminated_length": 1503.0, "completions/mean_length": 1192.46875, "completions/mean_terminated_length": 1192.46875, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.08537824125960469, "epoch": 0.02896, "frac_reward_zero_std": 0.0, "grad_norm": 0.03708412125706673, "kl": 1.3644461780786514, "learning_rate": 4.3520314602270474e-05, "loss": 0.0013, "num_tokens": 14133512.0, "reward": 1.2921874523162842, "reward_std": 0.31878477334976196, "rewards/rollout_reward_func/mean": 1.2921874523162842, "rewards/rollout_reward_func/std": 0.3900589346885681, "sampling/importance_sampling_ratio/max": 1.3159561157226562, "sampling/importance_sampling_ratio/mean": 1.002734661102295, "sampling/importance_sampling_ratio/min": 0.7749403119087219, "sampling/sampling_logp_difference/max": 0.2846531867980957, "sampling/sampling_logp_difference/mean": 0.004866480827331543, "step": 362, "step_time": 15.636170574998687 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1134.21875, "completions/mean_terminated_length": 1134.21875, "completions/min_length": 809.0, "completions/min_terminated_length": 809.0, "entropy": 0.0879516857676208, "epoch": 0.02904, "frac_reward_zero_std": 0.0, "grad_norm": 0.026215925812721252, "kl": 1.402923583984375, "learning_rate": 4.352023021869321e-05, "loss": 0.0009, "num_tokens": 14186522.0, "reward": 1.2579166889190674, "reward_std": 0.31184834241867065, "rewards/rollout_reward_func/mean": 1.2579166889190674, "rewards/rollout_reward_func/std": 0.3601122498512268, "sampling/importance_sampling_ratio/max": 1.3541795015335083, "sampling/importance_sampling_ratio/mean": 0.9721823334693909, "sampling/importance_sampling_ratio/min": 0.6422971487045288, "sampling/sampling_logp_difference/max": 0.35497403144836426, "sampling/sampling_logp_difference/mean": 0.006820994429290295, "step": 363, "step_time": 17.302361278998433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1707.0, "completions/max_terminated_length": 1707.0, "completions/mean_length": 1178.34375, "completions/mean_terminated_length": 1178.34375, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.0602953233756125, "epoch": 0.02912, "frac_reward_zero_std": 0.0, "grad_norm": 0.017147168517112732, "kl": 1.45445716381073, "learning_rate": 4.3520145576813133e-05, "loss": -0.0004, "num_tokens": 14240946.0, "reward": 1.4011459350585938, "reward_std": 0.2112138271331787, "rewards/rollout_reward_func/mean": 1.4011459350585938, "rewards/rollout_reward_func/std": 0.26684215664863586, "sampling/importance_sampling_ratio/max": 1.3180420398712158, "sampling/importance_sampling_ratio/mean": 0.9844136238098145, "sampling/importance_sampling_ratio/min": 0.7359997630119324, "sampling/sampling_logp_difference/max": 0.2997981309890747, "sampling/sampling_logp_difference/mean": 0.004418114200234413, "step": 364, "step_time": 16.803370065001218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1713.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 1284.15625, "completions/mean_terminated_length": 1284.15625, "completions/min_length": 809.0, "completions/min_terminated_length": 809.0, "entropy": 0.07712499890476465, "epoch": 0.0292, "frac_reward_zero_std": 0.0, "grad_norm": 0.03857647255063057, "kl": 1.175073005259037, "learning_rate": 4.352006067663158e-05, "loss": 0.0005, "num_tokens": 14299714.0, "reward": 1.3264583349227905, "reward_std": 0.30575335025787354, "rewards/rollout_reward_func/mean": 1.3264583349227905, "rewards/rollout_reward_func/std": 0.3669460713863373, "sampling/importance_sampling_ratio/max": 1.1384637355804443, "sampling/importance_sampling_ratio/mean": 1.0113253593444824, "sampling/importance_sampling_ratio/min": 0.8846418261528015, "sampling/sampling_logp_difference/max": 0.2600116729736328, "sampling/sampling_logp_difference/mean": 0.0035732537508010864, "step": 365, "step_time": 17.195817992998855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1504.0, "completions/max_terminated_length": 1504.0, "completions/mean_length": 1088.1875, "completions/mean_terminated_length": 1088.1875, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.08965331455692649, "epoch": 0.02928, "frac_reward_zero_std": 0.0, "grad_norm": 0.028532933443784714, "kl": 1.711681067943573, "learning_rate": 4.3519975518149896e-05, "loss": 0.0016, "num_tokens": 14350416.0, "reward": 1.3252084255218506, "reward_std": 0.322020024061203, "rewards/rollout_reward_func/mean": 1.3252084255218506, "rewards/rollout_reward_func/std": 0.3415009379386902, "sampling/importance_sampling_ratio/max": 1.1109697818756104, "sampling/importance_sampling_ratio/mean": 0.9975960850715637, "sampling/importance_sampling_ratio/min": 0.8416164517402649, "sampling/sampling_logp_difference/max": 0.21628892421722412, "sampling/sampling_logp_difference/mean": 0.004843099508434534, "step": 366, "step_time": 15.26562132700019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1712.0, "completions/max_terminated_length": 1712.0, "completions/mean_length": 1076.21875, "completions/mean_terminated_length": 1076.21875, "completions/min_length": 834.0, "completions/min_terminated_length": 834.0, "entropy": 0.08263562759384513, "epoch": 0.02936, "frac_reward_zero_std": 0.125, "grad_norm": 0.02415214292705059, "kl": 1.5377716422080994, "learning_rate": 4.351989010136942e-05, "loss": 0.0016, "num_tokens": 14400520.0, "reward": 1.3406250476837158, "reward_std": 0.23055405914783478, "rewards/rollout_reward_func/mean": 1.3406250476837158, "rewards/rollout_reward_func/std": 0.3727015256881714, "sampling/importance_sampling_ratio/max": 1.250171184539795, "sampling/importance_sampling_ratio/mean": 1.0046801567077637, "sampling/importance_sampling_ratio/min": 0.7805756330490112, "sampling/sampling_logp_difference/max": 0.24811533093452454, "sampling/sampling_logp_difference/mean": 0.004280885681509972, "step": 367, "step_time": 17.3321683260001 }, { "clip_ratio/high_max": 0.004629629664123058, "clip_ratio/high_mean": 0.002314814832061529, "clip_ratio/low_mean": 0.002016128972172737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004330943804234266, "completions/clipped_ratio": 0.0, "completions/max_length": 1702.0, "completions/max_terminated_length": 1702.0, "completions/mean_length": 1219.625, "completions/mean_terminated_length": 1219.625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.09326597023755312, "epoch": 0.02944, "frac_reward_zero_std": 0.0, "grad_norm": 0.04260089993476868, "kl": 1.3377633392810822, "learning_rate": 4.35198044262915e-05, "loss": 0.0017, "num_tokens": 14456601.0, "reward": 1.3076040744781494, "reward_std": 0.3317175507545471, "rewards/rollout_reward_func/mean": 1.3076040744781494, "rewards/rollout_reward_func/std": 0.3791333734989166, "sampling/importance_sampling_ratio/max": 1.521978497505188, "sampling/importance_sampling_ratio/mean": 1.0403764247894287, "sampling/importance_sampling_ratio/min": 0.9085478782653809, "sampling/sampling_logp_difference/max": 0.4901766777038574, "sampling/sampling_logp_difference/mean": 0.005632284563034773, "step": 368, "step_time": 16.733283728999595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002314814832061529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "completions/clipped_ratio": 0.0, "completions/max_length": 1703.0, "completions/max_terminated_length": 1703.0, "completions/mean_length": 1252.5625, "completions/mean_terminated_length": 1252.5625, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.08957207761704922, "epoch": 0.02952, "frac_reward_zero_std": 0.0, "grad_norm": 0.05407310649752617, "kl": 1.155344769358635, "learning_rate": 4.351971849291753e-05, "loss": 0.0025, "num_tokens": 14514095.0, "reward": 1.3332291841506958, "reward_std": 0.2586555480957031, "rewards/rollout_reward_func/mean": 1.3332291841506958, "rewards/rollout_reward_func/std": 0.2611094117164612, "sampling/importance_sampling_ratio/max": 1.3760251998901367, "sampling/importance_sampling_ratio/mean": 1.0266987085342407, "sampling/importance_sampling_ratio/min": 0.8590778112411499, "sampling/sampling_logp_difference/max": 0.2240445613861084, "sampling/sampling_logp_difference/mean": 0.004480384290218353, "step": 369, "step_time": 17.84523157900003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002314814832061529, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002314814832061529, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1450.71875, "completions/mean_terminated_length": 1450.71875, "completions/min_length": 1008.0, "completions/min_terminated_length": 1008.0, "entropy": 0.1162331709638238, "epoch": 0.0296, "frac_reward_zero_std": 0.0, "grad_norm": 0.03459122031927109, "kl": 1.3834916576743126, "learning_rate": 4.351963230124883e-05, "loss": 0.0032, "num_tokens": 14579345.0, "reward": 1.286250114440918, "reward_std": 0.36404678225517273, "rewards/rollout_reward_func/mean": 1.286250114440918, "rewards/rollout_reward_func/std": 0.40112990140914917, "sampling/importance_sampling_ratio/max": 1.5118522644042969, "sampling/importance_sampling_ratio/mean": 1.0009479522705078, "sampling/importance_sampling_ratio/min": 0.7502236366271973, "sampling/sampling_logp_difference/max": 0.4166536331176758, "sampling/sampling_logp_difference/mean": 0.008170945569872856, "step": 370, "step_time": 17.376009576999422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1705.0, "completions/max_terminated_length": 1705.0, "completions/mean_length": 1221.5625, "completions/mean_terminated_length": 1221.5625, "completions/min_length": 867.0, "completions/min_terminated_length": 867.0, "entropy": 0.08482339978218079, "epoch": 0.02968, "frac_reward_zero_std": 0.0, "grad_norm": 0.029663683846592903, "kl": 1.027076505124569, "learning_rate": 4.351954585128677e-05, "loss": 0.0013, "num_tokens": 14635250.0, "reward": 1.2440625429153442, "reward_std": 0.3457430303096771, "rewards/rollout_reward_func/mean": 1.2440625429153442, "rewards/rollout_reward_func/std": 0.44896355271339417, "sampling/importance_sampling_ratio/max": 1.2111306190490723, "sampling/importance_sampling_ratio/mean": 0.993769109249115, "sampling/importance_sampling_ratio/min": 0.7997491955757141, "sampling/sampling_logp_difference/max": 0.21321916580200195, "sampling/sampling_logp_difference/mean": 0.004747485276311636, "step": 371, "step_time": 17.713880446002804 }, { "clip_ratio/high_max": 0.0016447368543595076, "clip_ratio/high_mean": 0.0008223684271797538, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008223684271797538, "completions/clipped_ratio": 0.0, "completions/max_length": 1726.0, "completions/max_terminated_length": 1726.0, "completions/mean_length": 1326.90625, "completions/mean_terminated_length": 1326.90625, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.1301589417271316, "epoch": 0.02976, "frac_reward_zero_std": 0.0, "grad_norm": 0.038507502526044846, "kl": 1.2337740063667297, "learning_rate": 4.351945914303275e-05, "loss": -0.0008, "num_tokens": 14695532.0, "reward": 1.3304166793823242, "reward_std": 0.25983041524887085, "rewards/rollout_reward_func/mean": 1.3304166793823242, "rewards/rollout_reward_func/std": 0.26984497904777527, "sampling/importance_sampling_ratio/max": 1.1626516580581665, "sampling/importance_sampling_ratio/mean": 0.9876977801322937, "sampling/importance_sampling_ratio/min": 0.5363976359367371, "sampling/sampling_logp_difference/max": 0.26091575622558594, "sampling/sampling_logp_difference/mean": 0.007123067043721676, "step": 372, "step_time": 17.913086962001216 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.004032257944345474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006187430350109935, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 1355.1875, "completions/mean_terminated_length": 1355.1875, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.10239329282194376, "epoch": 0.02984, "frac_reward_zero_std": 0.0, "grad_norm": 0.02715749852359295, "kl": 1.3833892941474915, "learning_rate": 4.351937217648811e-05, "loss": 0.001, "num_tokens": 14756982.0, "reward": 1.1779167652130127, "reward_std": 0.4399711489677429, "rewards/rollout_reward_func/mean": 1.1779167652130127, "rewards/rollout_reward_func/std": 0.46091988682746887, "sampling/importance_sampling_ratio/max": 1.1953136920928955, "sampling/importance_sampling_ratio/mean": 0.9718055725097656, "sampling/importance_sampling_ratio/min": 0.7101711630821228, "sampling/sampling_logp_difference/max": 0.4137594699859619, "sampling/sampling_logp_difference/mean": 0.007291782647371292, "step": 373, "step_time": 17.816319883999313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024999999441206455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024999999441206455, "completions/clipped_ratio": 0.0, "completions/max_length": 1508.0, "completions/max_terminated_length": 1508.0, "completions/mean_length": 1165.90625, "completions/mean_terminated_length": 1165.90625, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "entropy": 0.08281758520752192, "epoch": 0.02992, "frac_reward_zero_std": 0.0, "grad_norm": 0.026055721566081047, "kl": 1.1931406408548355, "learning_rate": 4.351928495165423e-05, "loss": 0.0, "num_tokens": 14811055.0, "reward": 1.2925000190734863, "reward_std": 0.4106175899505615, "rewards/rollout_reward_func/mean": 1.2925000190734863, "rewards/rollout_reward_func/std": 0.4563710689544678, "sampling/importance_sampling_ratio/max": 1.1212685108184814, "sampling/importance_sampling_ratio/mean": 0.982858419418335, "sampling/importance_sampling_ratio/min": 0.5128992795944214, "sampling/sampling_logp_difference/max": 0.24790048599243164, "sampling/sampling_logp_difference/mean": 0.005383657291531563, "step": 374, "step_time": 15.73733054799959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1511.0, "completions/max_terminated_length": 1511.0, "completions/mean_length": 1157.65625, "completions/mean_terminated_length": 1157.65625, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.1311346720904112, "epoch": 0.03, "frac_reward_zero_std": 0.0, "grad_norm": 0.062497060745954514, "kl": 1.270805224776268, "learning_rate": 4.3519197468532514e-05, "loss": 0.0028, "num_tokens": 14865220.0, "reward": 1.305833339691162, "reward_std": 0.24293756484985352, "rewards/rollout_reward_func/mean": 1.305833339691162, "rewards/rollout_reward_func/std": 0.28027382493019104, "sampling/importance_sampling_ratio/max": 1.0894625186920166, "sampling/importance_sampling_ratio/mean": 0.9685341119766235, "sampling/importance_sampling_ratio/min": 0.7514563798904419, "sampling/sampling_logp_difference/max": 0.6248531341552734, "sampling/sampling_logp_difference/mean": 0.008436138741672039, "step": 375, "step_time": 16.82074028399984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0013586956774815917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013586956774815917, "completions/clipped_ratio": 0.0, "completions/max_length": 1713.0, "completions/max_terminated_length": 1713.0, "completions/mean_length": 1331.4375, "completions/mean_terminated_length": 1331.4375, "completions/min_length": 809.0, "completions/min_terminated_length": 809.0, "entropy": 0.1323856762610376, "epoch": 0.03008, "frac_reward_zero_std": 0.0, "grad_norm": 0.06945398449897766, "kl": 1.0373866632580757, "learning_rate": 4.351910972712433e-05, "loss": 0.0006, "num_tokens": 14925647.0, "reward": 1.3588541746139526, "reward_std": 0.3061314821243286, "rewards/rollout_reward_func/mean": 1.3588541746139526, "rewards/rollout_reward_func/std": 0.35433679819107056, "sampling/importance_sampling_ratio/max": 1.080748438835144, "sampling/importance_sampling_ratio/mean": 0.9760987162590027, "sampling/importance_sampling_ratio/min": 0.49252068996429443, "sampling/sampling_logp_difference/max": 0.5783686637878418, "sampling/sampling_logp_difference/mean": 0.007453630678355694, "step": 376, "step_time": 17.340093591000368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004559018649160862, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004559018649160862, "completions/clipped_ratio": 0.0, "completions/max_length": 1714.0, "completions/max_terminated_length": 1714.0, "completions/mean_length": 1342.6875, "completions/mean_terminated_length": 1342.6875, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.09266566787846386, "epoch": 0.03016, "frac_reward_zero_std": 0.0, "grad_norm": 0.055598244071006775, "kl": 1.8891348466277122, "learning_rate": 4.3519021727431074e-05, "loss": 0.0028, "num_tokens": 14986553.0, "reward": 1.2711458206176758, "reward_std": 0.33851662278175354, "rewards/rollout_reward_func/mean": 1.2711458206176758, "rewards/rollout_reward_func/std": 0.35368579626083374, "sampling/importance_sampling_ratio/max": 1.3240599632263184, "sampling/importance_sampling_ratio/mean": 0.9939791560173035, "sampling/importance_sampling_ratio/min": 0.6927686929702759, "sampling/sampling_logp_difference/max": 0.36653947830200195, "sampling/sampling_logp_difference/mean": 0.0064801909029483795, "step": 377, "step_time": 17.46869726099885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0026041667442768812, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026041667442768812, "completions/clipped_ratio": 0.0, "completions/max_length": 1718.0, "completions/max_terminated_length": 1718.0, "completions/mean_length": 1251.25, "completions/mean_terminated_length": 1251.25, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.13174276147037745, "epoch": 0.03024, "frac_reward_zero_std": 0.0, "grad_norm": 0.034299470484256744, "kl": 1.1499724984169006, "learning_rate": 4.3518933469454124e-05, "loss": 0.0002, "num_tokens": 15043720.0, "reward": 1.24958336353302, "reward_std": 0.43141067028045654, "rewards/rollout_reward_func/mean": 1.24958336353302, "rewards/rollout_reward_func/std": 0.45436927676200867, "sampling/importance_sampling_ratio/max": 1.5337663888931274, "sampling/importance_sampling_ratio/mean": 1.004024624824524, "sampling/importance_sampling_ratio/min": 0.6565076112747192, "sampling/sampling_logp_difference/max": 0.431746244430542, "sampling/sampling_logp_difference/mean": 0.007876993156969547, "step": 378, "step_time": 16.85271898099836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1705.0, "completions/max_terminated_length": 1705.0, "completions/mean_length": 1261.6875, "completions/mean_terminated_length": 1261.6875, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.12376785976812243, "epoch": 0.03032, "frac_reward_zero_std": 0.0, "grad_norm": 0.03475338593125343, "kl": 1.8483332693576813, "learning_rate": 4.35188449531949e-05, "loss": -0.0001, "num_tokens": 15101340.0, "reward": 1.2248958349227905, "reward_std": 0.35945191979408264, "rewards/rollout_reward_func/mean": 1.2248958349227905, "rewards/rollout_reward_func/std": 0.4390336573123932, "sampling/importance_sampling_ratio/max": 1.1599044799804688, "sampling/importance_sampling_ratio/mean": 0.9691849946975708, "sampling/importance_sampling_ratio/min": 0.5103804469108582, "sampling/sampling_logp_difference/max": 0.4613311290740967, "sampling/sampling_logp_difference/mean": 0.008111555129289627, "step": 379, "step_time": 17.863230562000354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1704.0, "completions/max_terminated_length": 1704.0, "completions/mean_length": 1309.71875, "completions/mean_terminated_length": 1309.71875, "completions/min_length": 856.0, "completions/min_terminated_length": 856.0, "entropy": 0.10195940639823675, "epoch": 0.0304, "frac_reward_zero_std": 0.0, "grad_norm": 0.04176342114806175, "kl": 1.2277474626898766, "learning_rate": 4.351875617865479e-05, "loss": 0.0005, "num_tokens": 15161189.0, "reward": 1.2591667175292969, "reward_std": 0.2851460576057434, "rewards/rollout_reward_func/mean": 1.2591667175292969, "rewards/rollout_reward_func/std": 0.40939244627952576, "sampling/importance_sampling_ratio/max": 1.2473872900009155, "sampling/importance_sampling_ratio/mean": 1.0173866748809814, "sampling/importance_sampling_ratio/min": 0.828081488609314, "sampling/sampling_logp_difference/max": 0.2541961669921875, "sampling/sampling_logp_difference/mean": 0.0050089298747479916, "step": 380, "step_time": 16.92356729400035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1491.0, "completions/max_terminated_length": 1491.0, "completions/mean_length": 1079.40625, "completions/mean_terminated_length": 1079.40625, "completions/min_length": 808.0, "completions/min_terminated_length": 808.0, "entropy": 0.11516939010471106, "epoch": 0.03048, "frac_reward_zero_std": 0.0, "grad_norm": 0.03276447206735611, "kl": 1.4176598489284515, "learning_rate": 4.3518667145835204e-05, "loss": 0.0008, "num_tokens": 15212394.0, "reward": 1.2733334302902222, "reward_std": 0.2991836369037628, "rewards/rollout_reward_func/mean": 1.2733334302902222, "rewards/rollout_reward_func/std": 0.33769726753234863, "sampling/importance_sampling_ratio/max": 1.186470627784729, "sampling/importance_sampling_ratio/mean": 0.9842174649238586, "sampling/importance_sampling_ratio/min": 0.7534291744232178, "sampling/sampling_logp_difference/max": 0.3054252862930298, "sampling/sampling_logp_difference/mean": 0.00696407537907362, "step": 381, "step_time": 16.33374701499997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1718.0, "completions/max_terminated_length": 1718.0, "completions/mean_length": 1434.0, "completions/mean_terminated_length": 1434.0, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.15649321302771568, "epoch": 0.03056, "frac_reward_zero_std": 0.0, "grad_norm": 0.08845416456460953, "kl": 1.1474775448441505, "learning_rate": 4.351857785473755e-05, "loss": 0.002, "num_tokens": 15276680.0, "reward": 1.1675000190734863, "reward_std": 0.4211508333683014, "rewards/rollout_reward_func/mean": 1.1675000190734863, "rewards/rollout_reward_func/std": 0.5062229037284851, "sampling/importance_sampling_ratio/max": 1.342994213104248, "sampling/importance_sampling_ratio/mean": 1.0028210878372192, "sampling/importance_sampling_ratio/min": 0.803119421005249, "sampling/sampling_logp_difference/max": 0.32015109062194824, "sampling/sampling_logp_difference/mean": 0.008382948115468025, "step": 382, "step_time": 16.985979784998563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0024038462433964014, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024038462433964014, "completions/clipped_ratio": 0.0, "completions/max_length": 1710.0, "completions/max_terminated_length": 1710.0, "completions/mean_length": 1445.5, "completions/mean_terminated_length": 1445.5, "completions/min_length": 968.0, "completions/min_terminated_length": 968.0, "entropy": 0.11425340734422207, "epoch": 0.03064, "frac_reward_zero_std": 0.0, "grad_norm": 0.04125145822763443, "kl": 1.269750788807869, "learning_rate": 4.351848830536325e-05, "loss": 0.0004, "num_tokens": 15341310.0, "reward": 1.260729193687439, "reward_std": 0.3894973397254944, "rewards/rollout_reward_func/mean": 1.260729193687439, "rewards/rollout_reward_func/std": 0.5272092819213867, "sampling/importance_sampling_ratio/max": 1.2831430435180664, "sampling/importance_sampling_ratio/mean": 0.9950462579727173, "sampling/importance_sampling_ratio/min": 0.7872307300567627, "sampling/sampling_logp_difference/max": 0.3305473327636719, "sampling/sampling_logp_difference/mean": 0.005943463183939457, "step": 383, "step_time": 17.935340926001118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004903846187517047, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004903846187517047, "completions/clipped_ratio": 0.0, "completions/max_length": 1504.0, "completions/max_terminated_length": 1504.0, "completions/mean_length": 1248.03125, "completions/mean_terminated_length": 1248.03125, "completions/min_length": 858.0, "completions/min_terminated_length": 858.0, "entropy": 0.09486775053665042, "epoch": 0.03072, "frac_reward_zero_std": 0.0, "grad_norm": 0.055712293833494186, "kl": 1.062335543334484, "learning_rate": 4.351839849771371e-05, "loss": 0.004, "num_tokens": 15397826.0, "reward": 1.3597917556762695, "reward_std": 0.32021769881248474, "rewards/rollout_reward_func/mean": 1.3597917556762695, "rewards/rollout_reward_func/std": 0.42264649271965027, "sampling/importance_sampling_ratio/max": 1.913156270980835, "sampling/importance_sampling_ratio/mean": 1.0493799448013306, "sampling/importance_sampling_ratio/min": 0.874789297580719, "sampling/sampling_logp_difference/max": 0.6467399597167969, "sampling/sampling_logp_difference/mean": 0.005974984727799892, "step": 384, "step_time": 15.607371300001432 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 2005.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1501.09375, "completions/mean_terminated_length": 1501.09375, "completions/min_length": 998.0, "completions/min_terminated_length": 998.0, "entropy": 0.10185459535568953, "epoch": 0.0308, "frac_reward_zero_std": 0.0, "grad_norm": 0.05421164631843567, "kl": 1.2741950377821922, "learning_rate": 4.351830843179036e-05, "loss": 0.0007, "num_tokens": 15463536.0, "reward": 1.250133991241455, "reward_std": 0.38298264145851135, "rewards/rollout_reward_func/mean": 1.250133991241455, "rewards/rollout_reward_func/std": 0.4777206778526306, "sampling/importance_sampling_ratio/max": 1.5989691019058228, "sampling/importance_sampling_ratio/mean": 1.0044381618499756, "sampling/importance_sampling_ratio/min": 0.7376886606216431, "sampling/sampling_logp_difference/max": 0.4480748772621155, "sampling/sampling_logp_difference/mean": 0.00551580497995019, "step": 385, "step_time": 19.871555921000436 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 1752.0, "completions/max_terminated_length": 1752.0, "completions/mean_length": 1276.90625, "completions/mean_terminated_length": 1276.90625, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.13616743125021458, "epoch": 0.03088, "frac_reward_zero_std": 0.0, "grad_norm": 0.03135287016630173, "kl": 1.4195483103394508, "learning_rate": 4.351821810759462e-05, "loss": -0.0002, "num_tokens": 15520635.0, "reward": 1.2242411375045776, "reward_std": 0.43393582105636597, "rewards/rollout_reward_func/mean": 1.2242411375045776, "rewards/rollout_reward_func/std": 0.4152339994907379, "sampling/importance_sampling_ratio/max": 1.5646860599517822, "sampling/importance_sampling_ratio/mean": 1.0069856643676758, "sampling/importance_sampling_ratio/min": 0.5982756018638611, "sampling/sampling_logp_difference/max": 0.3932235836982727, "sampling/sampling_logp_difference/mean": 0.007780009415000677, "step": 386, "step_time": 18.301943077999567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1996.0, "completions/max_terminated_length": 1996.0, "completions/mean_length": 1397.875, "completions/mean_terminated_length": 1397.875, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.11557755619287491, "epoch": 0.03096, "frac_reward_zero_std": 0.0, "grad_norm": 0.02693888731300831, "kl": 1.5959184914827347, "learning_rate": 4.351812752512793e-05, "loss": 0.0007, "num_tokens": 15581897.0, "reward": 1.3377678394317627, "reward_std": 0.3032139539718628, "rewards/rollout_reward_func/mean": 1.3377678394317627, "rewards/rollout_reward_func/std": 0.3128412663936615, "sampling/importance_sampling_ratio/max": 1.1638623476028442, "sampling/importance_sampling_ratio/mean": 1.0008968114852905, "sampling/importance_sampling_ratio/min": 0.8342984318733215, "sampling/sampling_logp_difference/max": 0.1826918125152588, "sampling/sampling_logp_difference/mean": 0.004302058834582567, "step": 387, "step_time": 18.849604537998857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0022321429569274187, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 2018.0, "completions/max_terminated_length": 2018.0, "completions/mean_length": 1497.71875, "completions/mean_terminated_length": 1497.71875, "completions/min_length": 952.0, "completions/min_terminated_length": 952.0, "entropy": 0.10269700177013874, "epoch": 0.03104, "frac_reward_zero_std": 0.0, "grad_norm": 0.024481987580657005, "kl": 1.4167095720767975, "learning_rate": 4.3518036684391716e-05, "loss": -0.0011, "num_tokens": 15647214.0, "reward": 1.2570981979370117, "reward_std": 0.37303075194358826, "rewards/rollout_reward_func/mean": 1.2570981979370117, "rewards/rollout_reward_func/std": 0.43470728397369385, "sampling/importance_sampling_ratio/max": 1.6113234758377075, "sampling/importance_sampling_ratio/mean": 0.9832198619842529, "sampling/importance_sampling_ratio/min": 0.37167832255363464, "sampling/sampling_logp_difference/max": 0.9382085800170898, "sampling/sampling_logp_difference/mean": 0.0077951024286448956, "step": 388, "step_time": 19.163982818001386 }, { "clip_ratio/high_max": 0.004310344811528921, "clip_ratio/high_mean": 0.0021551724057644606, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021551724057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1444.1875, "completions/mean_terminated_length": 1444.1875, "completions/min_length": 997.0, "completions/min_terminated_length": 997.0, "entropy": 0.08002800587564707, "epoch": 0.03112, "frac_reward_zero_std": 0.0, "grad_norm": 0.024939125403761864, "kl": 1.5021373927593231, "learning_rate": 4.351794558538741e-05, "loss": 0.0003, "num_tokens": 15710720.0, "reward": 1.3586608171463013, "reward_std": 0.25470495223999023, "rewards/rollout_reward_func/mean": 1.3586608171463013, "rewards/rollout_reward_func/std": 0.2862400710582733, "sampling/importance_sampling_ratio/max": 1.1827646493911743, "sampling/importance_sampling_ratio/mean": 0.9941922426223755, "sampling/importance_sampling_ratio/min": 0.7870350480079651, "sampling/sampling_logp_difference/max": 0.24129796028137207, "sampling/sampling_logp_difference/mean": 0.0032134163193404675, "step": 389, "step_time": 19.97311166899908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0041082974057644606, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041082974057644606, "completions/clipped_ratio": 0.0, "completions/max_length": 2011.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1461.0625, "completions/mean_terminated_length": 1461.0625, "completions/min_length": 989.0, "completions/min_terminated_length": 989.0, "entropy": 0.11741241998970509, "epoch": 0.0312, "frac_reward_zero_std": 0.0, "grad_norm": 0.035224054008722305, "kl": 1.460279256105423, "learning_rate": 4.3517854228116476e-05, "loss": 0.0012, "num_tokens": 15774839.0, "reward": 1.2305803298950195, "reward_std": 0.3470849394798279, "rewards/rollout_reward_func/mean": 1.2305803298950195, "rewards/rollout_reward_func/std": 0.33403095602989197, "sampling/importance_sampling_ratio/max": 1.4164410829544067, "sampling/importance_sampling_ratio/mean": 1.0246703624725342, "sampling/importance_sampling_ratio/min": 0.7914348244667053, "sampling/sampling_logp_difference/max": 0.34709328413009644, "sampling/sampling_logp_difference/mean": 0.005629987455904484, "step": 390, "step_time": 19.367711700001564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002016128972172737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002016128972172737, "completions/clipped_ratio": 0.0, "completions/max_length": 1753.0, "completions/max_terminated_length": 1753.0, "completions/mean_length": 1345.21875, "completions/mean_terminated_length": 1345.21875, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.09966882830485702, "epoch": 0.03128, "frac_reward_zero_std": 0.0, "grad_norm": 0.06100916862487793, "kl": 1.978619895875454, "learning_rate": 4.351776261258035e-05, "loss": 0.0006, "num_tokens": 15834460.0, "reward": 1.2327678203582764, "reward_std": 0.32978057861328125, "rewards/rollout_reward_func/mean": 1.2327678203582764, "rewards/rollout_reward_func/std": 0.42957043647766113, "sampling/importance_sampling_ratio/max": 1.3301268815994263, "sampling/importance_sampling_ratio/mean": 1.0050439834594727, "sampling/importance_sampling_ratio/min": 0.6559979319572449, "sampling/sampling_logp_difference/max": 0.4842514991760254, "sampling/sampling_logp_difference/mean": 0.005505594424903393, "step": 391, "step_time": 17.473636302997875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1493.21875, "completions/mean_terminated_length": 1493.21875, "completions/min_length": 917.0, "completions/min_terminated_length": 917.0, "entropy": 0.10191400302574039, "epoch": 0.03136, "frac_reward_zero_std": 0.0, "grad_norm": 0.037474215030670166, "kl": 1.3089447915554047, "learning_rate": 4.351767073878047e-05, "loss": -0.0014, "num_tokens": 15899348.0, "reward": 1.4095089435577393, "reward_std": 0.29044100642204285, "rewards/rollout_reward_func/mean": 1.4095089435577393, "rewards/rollout_reward_func/std": 0.35301050543785095, "sampling/importance_sampling_ratio/max": 1.2233799695968628, "sampling/importance_sampling_ratio/mean": 0.9716734290122986, "sampling/importance_sampling_ratio/min": 0.43156298995018005, "sampling/sampling_logp_difference/max": 0.5866320133209229, "sampling/sampling_logp_difference/mean": 0.0059129297733306885, "step": 392, "step_time": 19.36185482900055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2010.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 1517.9375, "completions/mean_terminated_length": 1517.9375, "completions/min_length": 1175.0, "completions/min_terminated_length": 1175.0, "entropy": 0.10565304663032293, "epoch": 0.03144, "frac_reward_zero_std": 0.0, "grad_norm": 0.028801077976822853, "kl": 1.1449712812900543, "learning_rate": 4.3517578606718314e-05, "loss": 0.0023, "num_tokens": 15965171.0, "reward": 1.3314285278320312, "reward_std": 0.3658674359321594, "rewards/rollout_reward_func/mean": 1.3314285278320312, "rewards/rollout_reward_func/std": 0.3640679717063904, "sampling/importance_sampling_ratio/max": 1.1952705383300781, "sampling/importance_sampling_ratio/mean": 1.0161319971084595, "sampling/importance_sampling_ratio/min": 0.8022790551185608, "sampling/sampling_logp_difference/max": 0.24917984008789062, "sampling/sampling_logp_difference/mean": 0.00443237042054534, "step": 393, "step_time": 18.961374678999164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2013.0, "completions/max_terminated_length": 2013.0, "completions/mean_length": 1724.78125, "completions/mean_terminated_length": 1724.78125, "completions/min_length": 1199.0, "completions/min_terminated_length": 1199.0, "entropy": 0.11421716259792447, "epoch": 0.03152, "frac_reward_zero_std": 0.0, "grad_norm": 0.024188455194234848, "kl": 1.1718277484178543, "learning_rate": 4.351748621639532e-05, "loss": 0.0008, "num_tokens": 16039074.0, "reward": 1.387678623199463, "reward_std": 0.2607552409172058, "rewards/rollout_reward_func/mean": 1.387678623199463, "rewards/rollout_reward_func/std": 0.30559900403022766, "sampling/importance_sampling_ratio/max": 1.1956379413604736, "sampling/importance_sampling_ratio/mean": 1.0023425817489624, "sampling/importance_sampling_ratio/min": 0.771578848361969, "sampling/sampling_logp_difference/max": 0.24651384353637695, "sampling/sampling_logp_difference/mean": 0.0039917477406561375, "step": 394, "step_time": 19.689614613999765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1991.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1300.125, "completions/mean_terminated_length": 1300.125, "completions/min_length": 965.0, "completions/min_terminated_length": 965.0, "entropy": 0.12842487916350365, "epoch": 0.0316, "frac_reward_zero_std": 0.0, "grad_norm": 0.037752993404865265, "kl": 1.4499746561050415, "learning_rate": 4.3517393567812966e-05, "loss": 0.0007, "num_tokens": 16096989.0, "reward": 1.2002232074737549, "reward_std": 0.3352164924144745, "rewards/rollout_reward_func/mean": 1.2002232074737549, "rewards/rollout_reward_func/std": 0.33648163080215454, "sampling/importance_sampling_ratio/max": 1.2686258554458618, "sampling/importance_sampling_ratio/mean": 0.9874203205108643, "sampling/importance_sampling_ratio/min": 0.8107923269271851, "sampling/sampling_logp_difference/max": 0.2142200469970703, "sampling/sampling_logp_difference/mean": 0.005989481694996357, "step": 395, "step_time": 18.91636544799985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1994.0, "completions/max_terminated_length": 1994.0, "completions/mean_length": 1624.0625, "completions/mean_terminated_length": 1624.0625, "completions/min_length": 1164.0, "completions/min_terminated_length": 1164.0, "entropy": 0.1143784960731864, "epoch": 0.03168, "frac_reward_zero_std": 0.0, "grad_norm": 0.07085367292165756, "kl": 2.2139622643589973, "learning_rate": 4.351730066097271e-05, "loss": 0.0038, "num_tokens": 16166975.0, "reward": 1.2947322130203247, "reward_std": 0.38576534390449524, "rewards/rollout_reward_func/mean": 1.2947322130203247, "rewards/rollout_reward_func/std": 0.42919424176216125, "sampling/importance_sampling_ratio/max": 1.7032363414764404, "sampling/importance_sampling_ratio/mean": 1.0334141254425049, "sampling/importance_sampling_ratio/min": 0.7799621820449829, "sampling/sampling_logp_difference/max": 0.41777706146240234, "sampling/sampling_logp_difference/mean": 0.004620470106601715, "step": 396, "step_time": 20.051171920001252 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0018939394503831863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0038470644503831863, "completions/clipped_ratio": 0.0, "completions/max_length": 2013.0, "completions/max_terminated_length": 2013.0, "completions/mean_length": 1714.96875, "completions/mean_terminated_length": 1714.96875, "completions/min_length": 1189.0, "completions/min_terminated_length": 1189.0, "entropy": 0.11265545524656773, "epoch": 0.03176, "frac_reward_zero_std": 0.0, "grad_norm": 0.016771715134382248, "kl": 1.0999369695782661, "learning_rate": 4.3517207495876035e-05, "loss": 0.0001, "num_tokens": 16240944.0, "reward": 1.332991123199463, "reward_std": 0.28439366817474365, "rewards/rollout_reward_func/mean": 1.332991123199463, "rewards/rollout_reward_func/std": 0.3308113217353821, "sampling/importance_sampling_ratio/max": 1.2937415838241577, "sampling/importance_sampling_ratio/mean": 0.9932774901390076, "sampling/importance_sampling_ratio/min": 0.7487343549728394, "sampling/sampling_logp_difference/max": 0.2873861789703369, "sampling/sampling_logp_difference/mean": 0.004606327507644892, "step": 397, "step_time": 19.27818380199915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2015.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1522.25, "completions/mean_terminated_length": 1522.25, "completions/min_length": 965.0, "completions/min_terminated_length": 965.0, "entropy": 0.11292540142312646, "epoch": 0.03184, "frac_reward_zero_std": 0.0, "grad_norm": 0.030631281435489655, "kl": 1.4515981376171112, "learning_rate": 4.35171140725244e-05, "loss": 0.0012, "num_tokens": 16307238.0, "reward": 1.3156249523162842, "reward_std": 0.3033728003501892, "rewards/rollout_reward_func/mean": 1.3156249523162842, "rewards/rollout_reward_func/std": 0.3433370292186737, "sampling/importance_sampling_ratio/max": 1.1943674087524414, "sampling/importance_sampling_ratio/mean": 1.0095311403274536, "sampling/importance_sampling_ratio/min": 0.7864588499069214, "sampling/sampling_logp_difference/max": 0.1746225357055664, "sampling/sampling_logp_difference/mean": 0.004183834884315729, "step": 398, "step_time": 20.044419360000575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1342.125, "completions/mean_terminated_length": 1342.125, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.13102778792381287, "epoch": 0.03192, "frac_reward_zero_std": 0.0, "grad_norm": 0.03096434660255909, "kl": 1.4250099062919617, "learning_rate": 4.351702039091928e-05, "loss": 0.0016, "num_tokens": 16367069.0, "reward": 1.2479465007781982, "reward_std": 0.33253055810928345, "rewards/rollout_reward_func/mean": 1.2479465007781982, "rewards/rollout_reward_func/std": 0.3212151825428009, "sampling/importance_sampling_ratio/max": 1.2048758268356323, "sampling/importance_sampling_ratio/mean": 0.9917153120040894, "sampling/importance_sampling_ratio/min": 0.7215684056282043, "sampling/sampling_logp_difference/max": 0.2488112449645996, "sampling/sampling_logp_difference/mean": 0.005188476759940386, "step": 399, "step_time": 19.00160939899979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1762.0, "completions/max_terminated_length": 1762.0, "completions/mean_length": 1518.53125, "completions/mean_terminated_length": 1518.53125, "completions/min_length": 1164.0, "completions/min_terminated_length": 1164.0, "entropy": 0.11011755093932152, "epoch": 0.032, "frac_reward_zero_std": 0.0, "grad_norm": 0.12310519069433212, "kl": 5.904455900192261, "learning_rate": 4.351692645106219e-05, "loss": 0.0038, "num_tokens": 16433101.0, "reward": 1.3870534896850586, "reward_std": 0.32266470789909363, "rewards/rollout_reward_func/mean": 1.3870534896850586, "rewards/rollout_reward_func/std": 0.3311446011066437, "sampling/importance_sampling_ratio/max": 1.2097538709640503, "sampling/importance_sampling_ratio/mean": 0.9954119324684143, "sampling/importance_sampling_ratio/min": 0.886063277721405, "sampling/sampling_logp_difference/max": 0.1313490867614746, "sampling/sampling_logp_difference/mean": 0.0028611463494598866, "step": 400, "step_time": 18.536976002998927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018939394503831863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018939394503831863, "completions/clipped_ratio": 0.0, "completions/max_length": 2009.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 1532.65625, "completions/mean_terminated_length": 1532.65625, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.16469209175556898, "epoch": 0.03208, "frac_reward_zero_std": 0.0, "grad_norm": 0.03193681687116623, "kl": 1.3408787995576859, "learning_rate": 4.351683225295459e-05, "loss": 0.0003, "num_tokens": 16499682.0, "reward": 1.2024106979370117, "reward_std": 0.3586331605911255, "rewards/rollout_reward_func/mean": 1.2024106979370117, "rewards/rollout_reward_func/std": 0.40373721718788147, "sampling/importance_sampling_ratio/max": 1.3515583276748657, "sampling/importance_sampling_ratio/mean": 1.0118014812469482, "sampling/importance_sampling_ratio/min": 0.7749048471450806, "sampling/sampling_logp_difference/max": 0.25551748275756836, "sampling/sampling_logp_difference/mean": 0.0059585669077932835, "step": 401, "step_time": 19.00786445600079 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1743.0, "completions/max_terminated_length": 1743.0, "completions/mean_length": 1284.40625, "completions/mean_terminated_length": 1284.40625, "completions/min_length": 996.0, "completions/min_terminated_length": 996.0, "entropy": 0.11700951680541039, "epoch": 0.03216, "frac_reward_zero_std": 0.0, "grad_norm": 0.024214085191488266, "kl": 1.4004377648234367, "learning_rate": 4.351673779659799e-05, "loss": 0.0009, "num_tokens": 16557187.0, "reward": 1.3083481788635254, "reward_std": 0.3267161548137665, "rewards/rollout_reward_func/mean": 1.3083481788635254, "rewards/rollout_reward_func/std": 0.3318912982940674, "sampling/importance_sampling_ratio/max": 1.2611801624298096, "sampling/importance_sampling_ratio/mean": 0.9981577396392822, "sampling/importance_sampling_ratio/min": 0.7390158176422119, "sampling/sampling_logp_difference/max": 0.24982118606567383, "sampling/sampling_logp_difference/mean": 0.004812702536582947, "step": 402, "step_time": 17.58199221399991 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.004036458441987634, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006268601398915052, "completions/clipped_ratio": 0.0, "completions/max_length": 1893.0, "completions/max_terminated_length": 1893.0, "completions/mean_length": 1365.9375, "completions/mean_terminated_length": 1365.9375, "completions/min_length": 953.0, "completions/min_terminated_length": 953.0, "entropy": 0.1517460886389017, "epoch": 0.03224, "frac_reward_zero_std": 0.0, "grad_norm": 0.03093048743903637, "kl": 1.4676104635000229, "learning_rate": 4.351664308199386e-05, "loss": 0.0005, "num_tokens": 16617425.0, "reward": 1.0794196128845215, "reward_std": 0.38626646995544434, "rewards/rollout_reward_func/mean": 1.0794196128845215, "rewards/rollout_reward_func/std": 0.49277570843696594, "sampling/importance_sampling_ratio/max": 1.6248277425765991, "sampling/importance_sampling_ratio/mean": 1.004536747932434, "sampling/importance_sampling_ratio/min": 0.8171869516372681, "sampling/sampling_logp_difference/max": 0.3099743127822876, "sampling/sampling_logp_difference/mean": 0.006892438977956772, "step": 403, "step_time": 17.874606272000165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1992.0, "completions/max_terminated_length": 1992.0, "completions/mean_length": 1605.53125, "completions/mean_terminated_length": 1605.53125, "completions/min_length": 1201.0, "completions/min_terminated_length": 1201.0, "entropy": 0.15379546768963337, "epoch": 0.03232, "frac_reward_zero_std": 0.0, "grad_norm": 0.03863086178898811, "kl": 1.2399758398532867, "learning_rate": 4.3516548109143726e-05, "loss": 0.0019, "num_tokens": 16687220.0, "reward": 1.2956695556640625, "reward_std": 0.3125684857368469, "rewards/rollout_reward_func/mean": 1.2956695556640625, "rewards/rollout_reward_func/std": 0.390508770942688, "sampling/importance_sampling_ratio/max": 1.2892779111862183, "sampling/importance_sampling_ratio/mean": 1.0143754482269287, "sampling/importance_sampling_ratio/min": 0.882417619228363, "sampling/sampling_logp_difference/max": 0.26941728591918945, "sampling/sampling_logp_difference/mean": 0.004763694480061531, "step": 404, "step_time": 19.691675340000074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2019.0, "completions/max_terminated_length": 2019.0, "completions/mean_length": 1603.8125, "completions/mean_terminated_length": 1603.8125, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.1362717691808939, "epoch": 0.0324, "frac_reward_zero_std": 0.0, "grad_norm": 0.030788173899054527, "kl": 1.3668439388275146, "learning_rate": 4.351645287804908e-05, "loss": 0.0011, "num_tokens": 16756650.0, "reward": 1.242232084274292, "reward_std": 0.4849998950958252, "rewards/rollout_reward_func/mean": 1.242232084274292, "rewards/rollout_reward_func/std": 0.4810883104801178, "sampling/importance_sampling_ratio/max": 1.2008061408996582, "sampling/importance_sampling_ratio/mean": 1.0026133060455322, "sampling/importance_sampling_ratio/min": 0.8341912627220154, "sampling/sampling_logp_difference/max": 0.17972040176391602, "sampling/sampling_logp_difference/mean": 0.004910994321107864, "step": 405, "step_time": 19.024919379000494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2015.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1640.0625, "completions/mean_terminated_length": 1640.0625, "completions/min_length": 953.0, "completions/min_terminated_length": 953.0, "entropy": 0.16947399359196424, "epoch": 0.03248, "frac_reward_zero_std": 0.0, "grad_norm": 0.03171316161751747, "kl": 1.322145126760006, "learning_rate": 4.3516357388711434e-05, "loss": 0.0001, "num_tokens": 16827457.0, "reward": 1.2264286279678345, "reward_std": 0.3592597246170044, "rewards/rollout_reward_func/mean": 1.2264286279678345, "rewards/rollout_reward_func/std": 0.4089851677417755, "sampling/importance_sampling_ratio/max": 1.3821918964385986, "sampling/importance_sampling_ratio/mean": 0.986224889755249, "sampling/importance_sampling_ratio/min": 0.7992033958435059, "sampling/sampling_logp_difference/max": 0.24425268173217773, "sampling/sampling_logp_difference/mean": 0.006246502511203289, "step": 406, "step_time": 20.137381045999064 }, { "clip_ratio/high_max": 0.004032257944345474, "clip_ratio/high_mean": 0.002016128972172737, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002016128972172737, "completions/clipped_ratio": 0.0, "completions/max_length": 1767.0, "completions/max_terminated_length": 1767.0, "completions/mean_length": 1421.1875, "completions/mean_terminated_length": 1421.1875, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.1917640808969736, "epoch": 0.03256, "frac_reward_zero_std": 0.0, "grad_norm": 0.02409077249467373, "kl": 1.8117220997810364, "learning_rate": 4.35162616411323e-05, "loss": 0.0041, "num_tokens": 16889704.0, "reward": 1.1881695985794067, "reward_std": 0.26586073637008667, "rewards/rollout_reward_func/mean": 1.1881695985794067, "rewards/rollout_reward_func/std": 0.31011056900024414, "sampling/importance_sampling_ratio/max": 1.2541096210479736, "sampling/importance_sampling_ratio/mean": 0.9706884622573853, "sampling/importance_sampling_ratio/min": 6.845068867278314e-08, "sampling/sampling_logp_difference/max": 16.075927734375, "sampling/sampling_logp_difference/mean": 0.04156479239463806, "step": 407, "step_time": 17.40800011399915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2009.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 1445.09375, "completions/mean_terminated_length": 1445.09375, "completions/min_length": 951.0, "completions/min_terminated_length": 951.0, "entropy": 0.17019114643335342, "epoch": 0.03264, "frac_reward_zero_std": 0.0, "grad_norm": 0.023700011894106865, "kl": 1.3820624276995659, "learning_rate": 4.351616563531318e-05, "loss": 0.0015, "num_tokens": 16952764.0, "reward": 1.3472321033477783, "reward_std": 0.31562888622283936, "rewards/rollout_reward_func/mean": 1.3472321033477783, "rewards/rollout_reward_func/std": 0.3274456560611725, "sampling/importance_sampling_ratio/max": 1.2002990245819092, "sampling/importance_sampling_ratio/mean": 1.0002837181091309, "sampling/importance_sampling_ratio/min": 0.7171620726585388, "sampling/sampling_logp_difference/max": 0.23921126127243042, "sampling/sampling_logp_difference/mean": 0.005154905375093222, "step": 408, "step_time": 19.437838337000358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2010.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 1583.1875, "completions/mean_terminated_length": 1583.1875, "completions/min_length": 976.0, "completions/min_terminated_length": 976.0, "entropy": 0.20041520707309246, "epoch": 0.03272, "frac_reward_zero_std": 0.0, "grad_norm": 0.02834649384021759, "kl": 1.7760061770677567, "learning_rate": 4.3516069371255615e-05, "loss": -0.0011, "num_tokens": 17021344.0, "reward": 1.1204910278320312, "reward_std": 0.3436295986175537, "rewards/rollout_reward_func/mean": 1.1204910278320312, "rewards/rollout_reward_func/std": 0.4666310250759125, "sampling/importance_sampling_ratio/max": 1.4008840322494507, "sampling/importance_sampling_ratio/mean": 0.9827120304107666, "sampling/importance_sampling_ratio/min": 0.6324211955070496, "sampling/sampling_logp_difference/max": 0.4781184196472168, "sampling/sampling_logp_difference/mean": 0.0074058398604393005, "step": 409, "step_time": 19.210802878999857 }, { "clip_ratio/high_max": 0.004464285913854837, "clip_ratio/high_mean": 0.0022321429569274187, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022321429569274187, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1282.0625, "completions/mean_terminated_length": 1282.0625, "completions/min_length": 951.0, "completions/min_terminated_length": 951.0, "entropy": 0.21531999856233597, "epoch": 0.0328, "frac_reward_zero_std": 0.0, "grad_norm": 0.03925439342856407, "kl": 1.4878593534231186, "learning_rate": 4.3515972848961124e-05, "loss": 0.001, "num_tokens": 17078846.0, "reward": 1.124955415725708, "reward_std": 0.28008079528808594, "rewards/rollout_reward_func/mean": 1.124955415725708, "rewards/rollout_reward_func/std": 0.34490400552749634, "sampling/importance_sampling_ratio/max": 1.2706680297851562, "sampling/importance_sampling_ratio/mean": 1.012557029724121, "sampling/importance_sampling_ratio/min": 0.6129598021507263, "sampling/sampling_logp_difference/max": 0.321958065032959, "sampling/sampling_logp_difference/mean": 0.00951363891363144, "step": 410, "step_time": 19.329919856999368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2011.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1418.625, "completions/mean_terminated_length": 1418.625, "completions/min_length": 965.0, "completions/min_terminated_length": 965.0, "entropy": 0.1741889799013734, "epoch": 0.03288, "frac_reward_zero_std": 0.0, "grad_norm": 0.04840390011668205, "kl": 1.4625470861792564, "learning_rate": 4.3515876068431225e-05, "loss": 0.004, "num_tokens": 17140891.0, "reward": 1.2182142734527588, "reward_std": 0.35829177498817444, "rewards/rollout_reward_func/mean": 1.2182142734527588, "rewards/rollout_reward_func/std": 0.43561041355133057, "sampling/importance_sampling_ratio/max": 1.7605849504470825, "sampling/importance_sampling_ratio/mean": 1.028963565826416, "sampling/importance_sampling_ratio/min": 0.8346232175827026, "sampling/sampling_logp_difference/max": 0.24656391143798828, "sampling/sampling_logp_difference/mean": 0.0076140714809298515, "step": 411, "step_time": 19.001938796001014 }, { "clip_ratio/high_max": 0.00825216481462121, "clip_ratio/high_mean": 0.004126082407310605, "clip_ratio/low_mean": 0.004032257944345474, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00815834035165608, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1323.125, "completions/mean_terminated_length": 1323.125, "completions/min_length": 989.0, "completions/min_terminated_length": 989.0, "entropy": 0.16383494902402163, "epoch": 0.03296, "frac_reward_zero_std": 0.0, "grad_norm": 0.02945910021662712, "kl": 1.4696750193834305, "learning_rate": 4.3515779029667455e-05, "loss": 0.0003, "num_tokens": 17199590.0, "reward": 1.2334821224212646, "reward_std": 0.3364136517047882, "rewards/rollout_reward_func/mean": 1.2334821224212646, "rewards/rollout_reward_func/std": 0.38044580817222595, "sampling/importance_sampling_ratio/max": 1.253343105316162, "sampling/importance_sampling_ratio/mean": 0.9874032139778137, "sampling/importance_sampling_ratio/min": 0.49538084864616394, "sampling/sampling_logp_difference/max": 0.3492014408111572, "sampling/sampling_logp_difference/mean": 0.006908235140144825, "step": 412, "step_time": 19.548285316000147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1626.3125, "completions/mean_terminated_length": 1626.3125, "completions/min_length": 1161.0, "completions/min_terminated_length": 1161.0, "entropy": 0.1738692494109273, "epoch": 0.03304, "frac_reward_zero_std": 0.0, "grad_norm": 0.029299898073077202, "kl": 1.089780442416668, "learning_rate": 4.3515681732671365e-05, "loss": 0.0017, "num_tokens": 17269838.0, "reward": 1.2716517448425293, "reward_std": 0.35179081559181213, "rewards/rollout_reward_func/mean": 1.2716517448425293, "rewards/rollout_reward_func/std": 0.37599533796310425, "sampling/importance_sampling_ratio/max": 1.3319426774978638, "sampling/importance_sampling_ratio/mean": 1.0067682266235352, "sampling/importance_sampling_ratio/min": 0.8342869877815247, "sampling/sampling_logp_difference/max": 0.19209718704223633, "sampling/sampling_logp_difference/mean": 0.005746249575167894, "step": 413, "step_time": 19.217791426001895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1462.21875, "completions/mean_terminated_length": 1462.21875, "completions/min_length": 944.0, "completions/min_terminated_length": 944.0, "entropy": 0.1480067577213049, "epoch": 0.03312, "frac_reward_zero_std": 0.0, "grad_norm": 0.028857281431555748, "kl": 1.3536845594644547, "learning_rate": 4.351558417744448e-05, "loss": 0.0006, "num_tokens": 17333851.0, "reward": 1.2245535850524902, "reward_std": 0.35397857427597046, "rewards/rollout_reward_func/mean": 1.2245535850524902, "rewards/rollout_reward_func/std": 0.42036953568458557, "sampling/importance_sampling_ratio/max": 1.2734744548797607, "sampling/importance_sampling_ratio/mean": 0.988574743270874, "sampling/importance_sampling_ratio/min": 0.7200579643249512, "sampling/sampling_logp_difference/max": 0.3194620609283447, "sampling/sampling_logp_difference/mean": 0.00529476348310709, "step": 414, "step_time": 18.769877648001057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 2005.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1742.75, "completions/mean_terminated_length": 1742.75, "completions/min_length": 1210.0, "completions/min_terminated_length": 1210.0, "entropy": 0.16278051026165485, "epoch": 0.0332, "frac_reward_zero_std": 0.0, "grad_norm": 0.020142098888754845, "kl": 1.4707140773534775, "learning_rate": 4.351548636398835e-05, "loss": 0.002, "num_tokens": 17407737.0, "reward": 1.195446491241455, "reward_std": 0.36340975761413574, "rewards/rollout_reward_func/mean": 1.195446491241455, "rewards/rollout_reward_func/std": 0.3587248623371124, "sampling/importance_sampling_ratio/max": 1.4520175457000732, "sampling/importance_sampling_ratio/mean": 1.023461103439331, "sampling/importance_sampling_ratio/min": 0.7210068702697754, "sampling/sampling_logp_difference/max": 0.2993950843811035, "sampling/sampling_logp_difference/mean": 0.007059973198920488, "step": 415, "step_time": 19.757469224999113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.002016128972172737, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002016128972172737, "completions/clipped_ratio": 0.0, "completions/max_length": 2005.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1497.6875, "completions/mean_terminated_length": 1497.6875, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.09867626521736383, "epoch": 0.03328, "frac_reward_zero_std": 0.0, "grad_norm": 0.03114388883113861, "kl": 1.139742910861969, "learning_rate": 4.351538829230452e-05, "loss": 0.0013, "num_tokens": 17473291.0, "reward": 1.2959821224212646, "reward_std": 0.40789923071861267, "rewards/rollout_reward_func/mean": 1.2959821224212646, "rewards/rollout_reward_func/std": 0.44843366742134094, "sampling/importance_sampling_ratio/max": 1.2757599353790283, "sampling/importance_sampling_ratio/mean": 0.9916281700134277, "sampling/importance_sampling_ratio/min": 0.764074444770813, "sampling/sampling_logp_difference/max": 0.26970362663269043, "sampling/sampling_logp_difference/mean": 0.004366945009678602, "step": 416, "step_time": 19.07112825100012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1989.0, "completions/max_terminated_length": 1989.0, "completions/mean_length": 1486.03125, "completions/mean_terminated_length": 1486.03125, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.11016158387064934, "epoch": 0.03336, "frac_reward_zero_std": 0.0, "grad_norm": 0.037343092262744904, "kl": 1.7246194183826447, "learning_rate": 4.3515289962394555e-05, "loss": 0.0007, "num_tokens": 17538258.0, "reward": 1.3171875476837158, "reward_std": 0.25831934809684753, "rewards/rollout_reward_func/mean": 1.3171875476837158, "rewards/rollout_reward_func/std": 0.2879869341850281, "sampling/importance_sampling_ratio/max": 1.282310128211975, "sampling/importance_sampling_ratio/mean": 1.0108277797698975, "sampling/importance_sampling_ratio/min": 0.8269407153129578, "sampling/sampling_logp_difference/max": 0.23927855491638184, "sampling/sampling_logp_difference/mean": 0.004565322771668434, "step": 417, "step_time": 19.926530552999793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1984.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 1418.5, "completions/mean_terminated_length": 1418.5, "completions/min_length": 979.0, "completions/min_terminated_length": 979.0, "entropy": 0.13993806764483452, "epoch": 0.03344, "frac_reward_zero_std": 0.0, "grad_norm": 0.029917331412434578, "kl": 1.3509763330221176, "learning_rate": 4.351519137426e-05, "loss": 0.003, "num_tokens": 17600823.0, "reward": 1.188020944595337, "reward_std": 0.3242608904838562, "rewards/rollout_reward_func/mean": 1.188020944595337, "rewards/rollout_reward_func/std": 0.3450336754322052, "sampling/importance_sampling_ratio/max": 1.2973617315292358, "sampling/importance_sampling_ratio/mean": 1.0066996812820435, "sampling/importance_sampling_ratio/min": 0.8276716470718384, "sampling/sampling_logp_difference/max": 0.30675792694091797, "sampling/sampling_logp_difference/mean": 0.005851969122886658, "step": 418, "step_time": 18.95404483099992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1407.5, "completions/mean_terminated_length": 1407.5, "completions/min_length": 1008.0, "completions/min_terminated_length": 1008.0, "entropy": 0.10549632459878922, "epoch": 0.03352, "frac_reward_zero_std": 0.0, "grad_norm": 0.023580625653266907, "kl": 1.4711807072162628, "learning_rate": 4.351509252790241e-05, "loss": 0.0003, "num_tokens": 17662559.0, "reward": 1.2694642543792725, "reward_std": 0.25323885679244995, "rewards/rollout_reward_func/mean": 1.2694642543792725, "rewards/rollout_reward_func/std": 0.3454189896583557, "sampling/importance_sampling_ratio/max": 1.108425259590149, "sampling/importance_sampling_ratio/mean": 0.9653444886207581, "sampling/importance_sampling_ratio/min": 0.7673803567886353, "sampling/sampling_logp_difference/max": 0.27285194396972656, "sampling/sampling_logp_difference/mean": 0.004450447857379913, "step": 419, "step_time": 19.55636043999857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004036458441987634, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004036458441987634, "completions/clipped_ratio": 0.0, "completions/max_length": 2009.0, "completions/max_terminated_length": 2009.0, "completions/mean_length": 1375.625, "completions/mean_terminated_length": 1375.625, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.10738167446106672, "epoch": 0.0336, "frac_reward_zero_std": 0.0, "grad_norm": 0.048171669244766235, "kl": 3.47794496268034, "learning_rate": 4.3514993423323364e-05, "loss": 0.0025, "num_tokens": 17723226.0, "reward": 1.1941964626312256, "reward_std": 0.36261245608329773, "rewards/rollout_reward_func/mean": 1.1941964626312256, "rewards/rollout_reward_func/std": 0.3865503966808319, "sampling/importance_sampling_ratio/max": 1.1881119012832642, "sampling/importance_sampling_ratio/mean": 1.0155229568481445, "sampling/importance_sampling_ratio/min": 0.7569072246551514, "sampling/sampling_logp_difference/max": 0.2822704315185547, "sampling/sampling_logp_difference/mean": 0.00426518777385354, "step": 420, "step_time": 19.055002493001666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2017.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1536.46875, "completions/mean_terminated_length": 1536.46875, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.08695069793611765, "epoch": 0.03368, "frac_reward_zero_std": 0.0, "grad_norm": 0.02462403103709221, "kl": 1.4096513092517853, "learning_rate": 4.351489406052443e-05, "loss": -0.0007, "num_tokens": 17789785.0, "reward": 1.34879469871521, "reward_std": 0.37546390295028687, "rewards/rollout_reward_func/mean": 1.34879469871521, "rewards/rollout_reward_func/std": 0.3921813368797302, "sampling/importance_sampling_ratio/max": 1.1123865842819214, "sampling/importance_sampling_ratio/mean": 0.9802084565162659, "sampling/importance_sampling_ratio/min": 0.6871497631072998, "sampling/sampling_logp_difference/max": 0.3623694181442261, "sampling/sampling_logp_difference/mean": 0.004458474926650524, "step": 421, "step_time": 20.193978274000074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2018.0, "completions/max_terminated_length": 2018.0, "completions/mean_length": 1596.4375, "completions/mean_terminated_length": 1596.4375, "completions/min_length": 1159.0, "completions/min_terminated_length": 1159.0, "entropy": 0.07802481343969703, "epoch": 0.03376, "frac_reward_zero_std": 0.0, "grad_norm": 0.03669828921556473, "kl": 1.2077355831861496, "learning_rate": 4.351479443950718e-05, "loss": 0.001, "num_tokens": 17859026.0, "reward": 1.3263392448425293, "reward_std": 0.42604172229766846, "rewards/rollout_reward_func/mean": 1.3263392448425293, "rewards/rollout_reward_func/std": 0.44234994053840637, "sampling/importance_sampling_ratio/max": 1.1175730228424072, "sampling/importance_sampling_ratio/mean": 1.0128930807113647, "sampling/importance_sampling_ratio/min": 0.9036186933517456, "sampling/sampling_logp_difference/max": 0.12484550476074219, "sampling/sampling_logp_difference/mean": 0.0025576383341103792, "step": 422, "step_time": 19.16358807800043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2005.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1666.75, "completions/mean_terminated_length": 1666.75, "completions/min_length": 1195.0, "completions/min_terminated_length": 1195.0, "entropy": 0.09428122779354453, "epoch": 0.03384, "frac_reward_zero_std": 0.0, "grad_norm": 0.019514644518494606, "kl": 1.055082656443119, "learning_rate": 4.3514694560273184e-05, "loss": 0.0003, "num_tokens": 17930641.0, "reward": 1.409196376800537, "reward_std": 0.2480936348438263, "rewards/rollout_reward_func/mean": 1.409196376800537, "rewards/rollout_reward_func/std": 0.3075895309448242, "sampling/importance_sampling_ratio/max": 1.1017780303955078, "sampling/importance_sampling_ratio/mean": 0.9998123645782471, "sampling/importance_sampling_ratio/min": 0.855816125869751, "sampling/sampling_logp_difference/max": 0.1539233922958374, "sampling/sampling_logp_difference/mean": 0.002993072383105755, "step": 423, "step_time": 20.03841195900077 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1384.625, "completions/mean_terminated_length": 1384.625, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.10552145959809422, "epoch": 0.03392, "frac_reward_zero_std": 0.0, "grad_norm": 0.02530786208808422, "kl": 1.7276794612407684, "learning_rate": 4.3514594422824036e-05, "loss": 0.0021, "num_tokens": 17991620.0, "reward": 1.1945090293884277, "reward_std": 0.3398154377937317, "rewards/rollout_reward_func/mean": 1.1945090293884277, "rewards/rollout_reward_func/std": 0.37065979838371277, "sampling/importance_sampling_ratio/max": 1.2307108640670776, "sampling/importance_sampling_ratio/mean": 0.9989968538284302, "sampling/importance_sampling_ratio/min": 0.7894630432128906, "sampling/sampling_logp_difference/max": 0.2368316650390625, "sampling/sampling_logp_difference/mean": 0.004551183898001909, "step": 424, "step_time": 19.125172461999682 }, { "clip_ratio/high_max": 0.004166666883975267, "clip_ratio/high_mean": 0.0020833334419876337, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020833334419876337, "completions/clipped_ratio": 0.0, "completions/max_length": 1759.0, "completions/max_terminated_length": 1759.0, "completions/mean_length": 1425.46875, "completions/mean_terminated_length": 1425.46875, "completions/min_length": 998.0, "completions/min_terminated_length": 998.0, "entropy": 0.12842916510999203, "epoch": 0.034, "frac_reward_zero_std": 0.0, "grad_norm": 0.02739594876766205, "kl": 1.2570633441209793, "learning_rate": 4.351449402716132e-05, "loss": 0.0014, "num_tokens": 18054099.0, "reward": 1.3004465103149414, "reward_std": 0.21949994564056396, "rewards/rollout_reward_func/mean": 1.3004465103149414, "rewards/rollout_reward_func/std": 0.27063634991645813, "sampling/importance_sampling_ratio/max": 1.2682249546051025, "sampling/importance_sampling_ratio/mean": 1.007074236869812, "sampling/importance_sampling_ratio/min": 0.8463065028190613, "sampling/sampling_logp_difference/max": 0.2265791893005371, "sampling/sampling_logp_difference/mean": 0.004874488338828087, "step": 425, "step_time": 18.312763593000454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1998.0, "completions/max_terminated_length": 1998.0, "completions/mean_length": 1378.75, "completions/mean_terminated_length": 1378.75, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.10711577069014311, "epoch": 0.03408, "frac_reward_zero_std": 0.0, "grad_norm": 0.022950436919927597, "kl": 1.4831666871905327, "learning_rate": 4.351439337328661e-05, "loss": -0.0007, "num_tokens": 18115151.0, "reward": 1.2713392972946167, "reward_std": 0.3097730875015259, "rewards/rollout_reward_func/mean": 1.2713392972946167, "rewards/rollout_reward_func/std": 0.3585454523563385, "sampling/importance_sampling_ratio/max": 1.1835638284683228, "sampling/importance_sampling_ratio/mean": 0.9969443082809448, "sampling/importance_sampling_ratio/min": 0.6871855854988098, "sampling/sampling_logp_difference/max": 0.3538026809692383, "sampling/sampling_logp_difference/mean": 0.004290965385735035, "step": 426, "step_time": 18.860186455000076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2015.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1635.40625, "completions/mean_terminated_length": 1635.40625, "completions/min_length": 1146.0, "completions/min_terminated_length": 1146.0, "entropy": 0.11844338662922382, "epoch": 0.03416, "frac_reward_zero_std": 0.0, "grad_norm": 0.029191043227910995, "kl": 1.4002943858504295, "learning_rate": 4.351429246120152e-05, "loss": 0.001, "num_tokens": 18185595.0, "reward": 1.2953572273254395, "reward_std": 0.3885155916213989, "rewards/rollout_reward_func/mean": 1.2953572273254395, "rewards/rollout_reward_func/std": 0.37968283891677856, "sampling/importance_sampling_ratio/max": 1.1812409162521362, "sampling/importance_sampling_ratio/mean": 0.9996167421340942, "sampling/importance_sampling_ratio/min": 0.657285749912262, "sampling/sampling_logp_difference/max": 0.41747021675109863, "sampling/sampling_logp_difference/mean": 0.0051177325658500195, "step": 427, "step_time": 19.890075419000823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2008.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 1317.5, "completions/mean_terminated_length": 1317.5, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.1195755647495389, "epoch": 0.03424, "frac_reward_zero_std": 0.0, "grad_norm": 0.019143415614962578, "kl": 1.4650322496891022, "learning_rate": 4.351419129090764e-05, "loss": -0.0007, "num_tokens": 18244209.0, "reward": 1.2087500095367432, "reward_std": 0.3009724020957947, "rewards/rollout_reward_func/mean": 1.2087500095367432, "rewards/rollout_reward_func/std": 0.32787036895751953, "sampling/importance_sampling_ratio/max": 1.1791809797286987, "sampling/importance_sampling_ratio/mean": 0.9879035949707031, "sampling/importance_sampling_ratio/min": 0.7166708707809448, "sampling/sampling_logp_difference/max": 0.32955431938171387, "sampling/sampling_logp_difference/mean": 0.004283321090042591, "step": 428, "step_time": 18.849618232998182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1993.0, "completions/max_terminated_length": 1993.0, "completions/mean_length": 1512.125, "completions/mean_terminated_length": 1512.125, "completions/min_length": 1200.0, "completions/min_terminated_length": 1200.0, "entropy": 0.1111707603558898, "epoch": 0.03432, "frac_reward_zero_std": 0.0, "grad_norm": 0.02224867232143879, "kl": 1.0960643440485, "learning_rate": 4.351408986240657e-05, "loss": 0.0002, "num_tokens": 18310131.0, "reward": 1.3184375762939453, "reward_std": 0.28711965680122375, "rewards/rollout_reward_func/mean": 1.3184375762939453, "rewards/rollout_reward_func/std": 0.3107913136482239, "sampling/importance_sampling_ratio/max": 1.1098463535308838, "sampling/importance_sampling_ratio/mean": 0.9895414710044861, "sampling/importance_sampling_ratio/min": 0.8289187550544739, "sampling/sampling_logp_difference/max": 0.1757497787475586, "sampling/sampling_logp_difference/mean": 0.0041455538012087345, "step": 429, "step_time": 19.566366455001116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1767.0, "completions/max_terminated_length": 1767.0, "completions/mean_length": 1336.875, "completions/mean_terminated_length": 1336.875, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.07651641685515642, "epoch": 0.0344, "frac_reward_zero_std": 0.0, "grad_norm": 0.02379908598959446, "kl": 1.3860501945018768, "learning_rate": 4.351398817569991e-05, "loss": 0.0013, "num_tokens": 18369436.0, "reward": 1.410133957862854, "reward_std": 0.27028533816337585, "rewards/rollout_reward_func/mean": 1.410133957862854, "rewards/rollout_reward_func/std": 0.34279000759124756, "sampling/importance_sampling_ratio/max": 1.2376017570495605, "sampling/importance_sampling_ratio/mean": 1.0079046487808228, "sampling/importance_sampling_ratio/min": 0.8543627858161926, "sampling/sampling_logp_difference/max": 0.21372365951538086, "sampling/sampling_logp_difference/mean": 0.0028606595005840063, "step": 430, "step_time": 17.348939376001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1740.0, "completions/max_terminated_length": 1740.0, "completions/mean_length": 1426.78125, "completions/mean_terminated_length": 1426.78125, "completions/min_length": 966.0, "completions/min_terminated_length": 966.0, "entropy": 0.11709891166538, "epoch": 0.03448, "frac_reward_zero_std": 0.0, "grad_norm": 0.020691199228167534, "kl": 1.3303712978959084, "learning_rate": 4.3513886230789284e-05, "loss": 0.0007, "num_tokens": 18431767.0, "reward": 1.2779910564422607, "reward_std": 0.2544879913330078, "rewards/rollout_reward_func/mean": 1.2779910564422607, "rewards/rollout_reward_func/std": 0.2952415943145752, "sampling/importance_sampling_ratio/max": 1.117701768875122, "sampling/importance_sampling_ratio/mean": 1.0012716054916382, "sampling/importance_sampling_ratio/min": 0.8708288073539734, "sampling/sampling_logp_difference/max": 0.12551474571228027, "sampling/sampling_logp_difference/mean": 0.0031481836922466755, "step": 431, "step_time": 18.37306998800159 }, { "clip_ratio/high_max": 0.004032257944345474, "clip_ratio/high_mean": 0.002016128972172737, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003969253972172737, "completions/clipped_ratio": 0.0, "completions/max_length": 2007.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1437.9375, "completions/mean_terminated_length": 1437.9375, "completions/min_length": 996.0, "completions/min_terminated_length": 996.0, "entropy": 0.14306320063769817, "epoch": 0.03456, "frac_reward_zero_std": 0.0, "grad_norm": 0.025802303105592728, "kl": 1.3111656159162521, "learning_rate": 4.3513784027676306e-05, "loss": 0.0009, "num_tokens": 18494528.0, "reward": 1.247633934020996, "reward_std": 0.3073995113372803, "rewards/rollout_reward_func/mean": 1.247633934020996, "rewards/rollout_reward_func/std": 0.3273521065711975, "sampling/importance_sampling_ratio/max": 1.2076414823532104, "sampling/importance_sampling_ratio/mean": 1.0050315856933594, "sampling/importance_sampling_ratio/min": 0.739372193813324, "sampling/sampling_logp_difference/max": 0.20773863792419434, "sampling/sampling_logp_difference/mean": 0.005362858530133963, "step": 432, "step_time": 19.106649470000775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2011.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1574.4375, "completions/mean_terminated_length": 1574.4375, "completions/min_length": 945.0, "completions/min_terminated_length": 945.0, "entropy": 0.1255772616714239, "epoch": 0.03464, "frac_reward_zero_std": 0.0, "grad_norm": 0.06164729595184326, "kl": 1.288819707930088, "learning_rate": 4.351368156636258e-05, "loss": 0.0036, "num_tokens": 18562804.0, "reward": 1.3013839721679688, "reward_std": 0.35672318935394287, "rewards/rollout_reward_func/mean": 1.3013839721679688, "rewards/rollout_reward_func/std": 0.39492326974868774, "sampling/importance_sampling_ratio/max": 1.7386871576309204, "sampling/importance_sampling_ratio/mean": 1.0283199548721313, "sampling/importance_sampling_ratio/min": 0.855311930179596, "sampling/sampling_logp_difference/max": 0.4815669059753418, "sampling/sampling_logp_difference/mean": 0.005880427546799183, "step": 433, "step_time": 19.97610494500077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1515.0, "completions/max_terminated_length": 1515.0, "completions/mean_length": 1162.21875, "completions/mean_terminated_length": 1162.21875, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.12322713527828455, "epoch": 0.03472, "frac_reward_zero_std": 0.0, "grad_norm": 0.02245832420885563, "kl": 1.7110685855150223, "learning_rate": 4.351357884684974e-05, "loss": 0.0022, "num_tokens": 18615704.0, "reward": 1.2308928966522217, "reward_std": 0.3085770606994629, "rewards/rollout_reward_func/mean": 1.2308928966522217, "rewards/rollout_reward_func/std": 0.31039056181907654, "sampling/importance_sampling_ratio/max": 1.2779377698898315, "sampling/importance_sampling_ratio/mean": 0.9819905161857605, "sampling/importance_sampling_ratio/min": 1.734670185271625e-08, "sampling/sampling_logp_difference/max": 17.639698028564453, "sampling/sampling_logp_difference/mean": 0.042796485126018524, "step": 434, "step_time": 16.07746595200024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2008.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 1470.75, "completions/mean_terminated_length": 1470.75, "completions/min_length": 995.0, "completions/min_terminated_length": 995.0, "entropy": 0.10298708779737353, "epoch": 0.0348, "frac_reward_zero_std": 0.0, "grad_norm": 0.025740597397089005, "kl": 1.685651957988739, "learning_rate": 4.3513475869139404e-05, "loss": 0.0013, "num_tokens": 18679800.0, "reward": 1.3782142400741577, "reward_std": 0.21197101473808289, "rewards/rollout_reward_func/mean": 1.3782142400741577, "rewards/rollout_reward_func/std": 0.3264513909816742, "sampling/importance_sampling_ratio/max": 1.201029896736145, "sampling/importance_sampling_ratio/mean": 0.9998208284378052, "sampling/importance_sampling_ratio/min": 0.9078076481819153, "sampling/sampling_logp_difference/max": 0.1515902280807495, "sampling/sampling_logp_difference/mean": 0.002852079691365361, "step": 435, "step_time": 19.846371037002427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1902.0, "completions/max_terminated_length": 1902.0, "completions/mean_length": 1364.15625, "completions/mean_terminated_length": 1364.15625, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.11826783418655396, "epoch": 0.03488, "frac_reward_zero_std": 0.0, "grad_norm": 0.02533641643822193, "kl": 1.9646610468626022, "learning_rate": 4.3513372633233225e-05, "loss": 0.003, "num_tokens": 18740005.0, "reward": 1.3086607456207275, "reward_std": 0.30162328481674194, "rewards/rollout_reward_func/mean": 1.3086607456207275, "rewards/rollout_reward_func/std": 0.3599502742290497, "sampling/importance_sampling_ratio/max": 1.465258240699768, "sampling/importance_sampling_ratio/mean": 1.0255091190338135, "sampling/importance_sampling_ratio/min": 0.7720927596092224, "sampling/sampling_logp_difference/max": 0.2583122253417969, "sampling/sampling_logp_difference/mean": 0.004235124681144953, "step": 436, "step_time": 18.49279846600075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2008.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 1575.21875, "completions/mean_terminated_length": 1575.21875, "completions/min_length": 963.0, "completions/min_terminated_length": 963.0, "entropy": 0.10395689401775599, "epoch": 0.03496, "frac_reward_zero_std": 0.0, "grad_norm": 0.03237058222293854, "kl": 1.3916453048586845, "learning_rate": 4.351326913913281e-05, "loss": 0.0012, "num_tokens": 18807831.0, "reward": 1.4003571271896362, "reward_std": 0.2501515746116638, "rewards/rollout_reward_func/mean": 1.4003571271896362, "rewards/rollout_reward_func/std": 0.2764086127281189, "sampling/importance_sampling_ratio/max": 1.6518012285232544, "sampling/importance_sampling_ratio/mean": 1.012318730354309, "sampling/importance_sampling_ratio/min": 0.8594926595687866, "sampling/sampling_logp_difference/max": 0.30870628356933594, "sampling/sampling_logp_difference/mean": 0.0034519191831350327, "step": 437, "step_time": 19.923624370999278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1996.0, "completions/max_terminated_length": 1996.0, "completions/mean_length": 1311.5, "completions/mean_terminated_length": 1311.5, "completions/min_length": 952.0, "completions/min_terminated_length": 952.0, "entropy": 0.1045112619176507, "epoch": 0.03504, "frac_reward_zero_std": 0.0, "grad_norm": 0.017320280894637108, "kl": 1.5624884217977524, "learning_rate": 4.3513165386839804e-05, "loss": 0.0013, "num_tokens": 18866039.0, "reward": 1.3508334159851074, "reward_std": 0.25639843940734863, "rewards/rollout_reward_func/mean": 1.3508334159851074, "rewards/rollout_reward_func/std": 0.3077371418476105, "sampling/importance_sampling_ratio/max": 1.0936716794967651, "sampling/importance_sampling_ratio/mean": 0.9354032278060913, "sampling/importance_sampling_ratio/min": 3.5559732936008004e-08, "sampling/sampling_logp_difference/max": 17.009506225585938, "sampling/sampling_logp_difference/mean": 0.07660629600286484, "step": 438, "step_time": 18.844263283000146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1506.3125, "completions/mean_terminated_length": 1506.3125, "completions/min_length": 996.0, "completions/min_terminated_length": 996.0, "entropy": 0.13235563971102238, "epoch": 0.03512, "frac_reward_zero_std": 0.0, "grad_norm": 0.031413234770298004, "kl": 1.8649556785821915, "learning_rate": 4.351306137635586e-05, "loss": 0.0019, "num_tokens": 18931823.0, "reward": 1.2691518068313599, "reward_std": 0.24997474253177643, "rewards/rollout_reward_func/mean": 1.2691518068313599, "rewards/rollout_reward_func/std": 0.27614477276802063, "sampling/importance_sampling_ratio/max": 1.1445410251617432, "sampling/importance_sampling_ratio/mean": 1.0019164085388184, "sampling/importance_sampling_ratio/min": 0.8828837871551514, "sampling/sampling_logp_difference/max": 0.292957067489624, "sampling/sampling_logp_difference/mean": 0.0044544776901602745, "step": 439, "step_time": 19.95472260400402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2003.0, "completions/max_terminated_length": 2003.0, "completions/mean_length": 1661.90625, "completions/mean_terminated_length": 1661.90625, "completions/min_length": 1009.0, "completions/min_terminated_length": 1009.0, "entropy": 0.1168982875533402, "epoch": 0.0352, "frac_reward_zero_std": 0.0, "grad_norm": 0.06630422174930573, "kl": 1.353127807378769, "learning_rate": 4.351295710768263e-05, "loss": 0.0022, "num_tokens": 19003307.0, "reward": 1.3994196653366089, "reward_std": 0.31752511858940125, "rewards/rollout_reward_func/mean": 1.3994196653366089, "rewards/rollout_reward_func/std": 0.3110869526863098, "sampling/importance_sampling_ratio/max": 2.0901682376861572, "sampling/importance_sampling_ratio/mean": 1.0337252616882324, "sampling/importance_sampling_ratio/min": 0.8203296065330505, "sampling/sampling_logp_difference/max": 0.6611959934234619, "sampling/sampling_logp_difference/mean": 0.004697342403233051, "step": 440, "step_time": 20.55567447100111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1987.0, "completions/max_terminated_length": 1987.0, "completions/mean_length": 1303.71875, "completions/mean_terminated_length": 1303.71875, "completions/min_length": 951.0, "completions/min_terminated_length": 951.0, "entropy": 0.12714618351310492, "epoch": 0.03528, "frac_reward_zero_std": 0.0, "grad_norm": 0.03497716784477234, "kl": 1.248435065150261, "learning_rate": 4.351285258082175e-05, "loss": 0.001, "num_tokens": 19061907.0, "reward": 1.2544643878936768, "reward_std": 0.3255124092102051, "rewards/rollout_reward_func/mean": 1.2544643878936768, "rewards/rollout_reward_func/std": 0.3226786255836487, "sampling/importance_sampling_ratio/max": 1.1105939149856567, "sampling/importance_sampling_ratio/mean": 0.9828460216522217, "sampling/importance_sampling_ratio/min": 0.7734447121620178, "sampling/sampling_logp_difference/max": 0.37151408195495605, "sampling/sampling_logp_difference/mean": 0.005027194507420063, "step": 441, "step_time": 18.806081024999003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2000.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1637.1875, "completions/mean_terminated_length": 1637.1875, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.10764246946200728, "epoch": 0.03536, "frac_reward_zero_std": 0.0, "grad_norm": 0.026830436661839485, "kl": 0.9796565100550652, "learning_rate": 4.351274779577488e-05, "loss": 0.0007, "num_tokens": 19132622.0, "reward": 1.380089282989502, "reward_std": 0.3041646182537079, "rewards/rollout_reward_func/mean": 1.380089282989502, "rewards/rollout_reward_func/std": 0.37029460072517395, "sampling/importance_sampling_ratio/max": 1.2288217544555664, "sampling/importance_sampling_ratio/mean": 1.0071978569030762, "sampling/importance_sampling_ratio/min": 0.7454875707626343, "sampling/sampling_logp_difference/max": 0.24756324291229248, "sampling/sampling_logp_difference/mean": 0.004656828939914703, "step": 442, "step_time": 19.32207883599949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2004.0, "completions/max_terminated_length": 2004.0, "completions/mean_length": 1522.9375, "completions/mean_terminated_length": 1522.9375, "completions/min_length": 1199.0, "completions/min_terminated_length": 1199.0, "entropy": 0.10407107695937157, "epoch": 0.03544, "frac_reward_zero_std": 0.0, "grad_norm": 0.022910773754119873, "kl": 1.1446568071842194, "learning_rate": 4.351264275254368e-05, "loss": -0.0, "num_tokens": 19198890.0, "reward": 1.4246875047683716, "reward_std": 0.22621680796146393, "rewards/rollout_reward_func/mean": 1.4246875047683716, "rewards/rollout_reward_func/std": 0.2892785370349884, "sampling/importance_sampling_ratio/max": 2.4350717067718506, "sampling/importance_sampling_ratio/mean": 1.0357922315597534, "sampling/importance_sampling_ratio/min": 0.8219593167304993, "sampling/sampling_logp_difference/max": 0.42475366592407227, "sampling/sampling_logp_difference/mean": 0.005304509773850441, "step": 443, "step_time": 20.240618418998565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2015.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 1412.625, "completions/mean_terminated_length": 1412.625, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.12327547650784254, "epoch": 0.03552, "frac_reward_zero_std": 0.0, "grad_norm": 0.01965952478349209, "kl": 1.4333181828260422, "learning_rate": 4.351253745112982e-05, "loss": 0.0003, "num_tokens": 19261028.0, "reward": 1.2027232646942139, "reward_std": 0.3745243549346924, "rewards/rollout_reward_func/mean": 1.2027232646942139, "rewards/rollout_reward_func/std": 0.38339436054229736, "sampling/importance_sampling_ratio/max": 1.247205376625061, "sampling/importance_sampling_ratio/mean": 0.9975084662437439, "sampling/importance_sampling_ratio/min": 0.8557063341140747, "sampling/sampling_logp_difference/max": 0.2080078125, "sampling/sampling_logp_difference/mean": 0.004128223285079002, "step": 444, "step_time": 19.97043932699853 }, { "clip_ratio/high_max": 0.0037878789007663727, "clip_ratio/high_mean": 0.0018939394503831863, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018939394503831863, "completions/clipped_ratio": 0.0, "completions/max_length": 2007.0, "completions/max_terminated_length": 2007.0, "completions/mean_length": 1536.9375, "completions/mean_terminated_length": 1536.9375, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.1479588495567441, "epoch": 0.0356, "frac_reward_zero_std": 0.0, "grad_norm": 0.02587660774588585, "kl": 1.3854539841413498, "learning_rate": 4.351243189153495e-05, "loss": 0.0002, "num_tokens": 19327770.0, "reward": 1.300758957862854, "reward_std": 0.3115580379962921, "rewards/rollout_reward_func/mean": 1.300758957862854, "rewards/rollout_reward_func/std": 0.33613359928131104, "sampling/importance_sampling_ratio/max": 1.1194759607315063, "sampling/importance_sampling_ratio/mean": 0.9832184314727783, "sampling/importance_sampling_ratio/min": 0.8204161524772644, "sampling/sampling_logp_difference/max": 0.1801905632019043, "sampling/sampling_logp_difference/mean": 0.004841884132474661, "step": 445, "step_time": 19.076110300000437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2008.0, "completions/max_terminated_length": 2008.0, "completions/mean_length": 1631.59375, "completions/mean_terminated_length": 1631.59375, "completions/min_length": 1160.0, "completions/min_terminated_length": 1160.0, "entropy": 0.15464814379811287, "epoch": 0.03568, "frac_reward_zero_std": 0.0, "grad_norm": 0.026106417179107666, "kl": 1.2157297059893608, "learning_rate": 4.351232607376076e-05, "loss": 0.0002, "num_tokens": 19398450.0, "reward": 1.3301784992218018, "reward_std": 0.2951168417930603, "rewards/rollout_reward_func/mean": 1.3301784992218018, "rewards/rollout_reward_func/std": 0.2956228256225586, "sampling/importance_sampling_ratio/max": 1.1191374063491821, "sampling/importance_sampling_ratio/mean": 0.9668092727661133, "sampling/importance_sampling_ratio/min": 0.608216404914856, "sampling/sampling_logp_difference/max": 0.6167263984680176, "sampling/sampling_logp_difference/mean": 0.007042648736387491, "step": 446, "step_time": 20.384870130001218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2005.0, "completions/max_terminated_length": 2005.0, "completions/mean_length": 1562.875, "completions/mean_terminated_length": 1562.875, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.13539850758388638, "epoch": 0.03576, "frac_reward_zero_std": 0.0, "grad_norm": 0.04132156819105148, "kl": 1.369851976633072, "learning_rate": 4.351221999780892e-05, "loss": 0.0031, "num_tokens": 19466807.0, "reward": 1.2928571701049805, "reward_std": 0.40943554043769836, "rewards/rollout_reward_func/mean": 1.2928571701049805, "rewards/rollout_reward_func/std": 0.3985946476459503, "sampling/importance_sampling_ratio/max": 1.4449394941329956, "sampling/importance_sampling_ratio/mean": 1.0008090734481812, "sampling/importance_sampling_ratio/min": 0.8020069599151611, "sampling/sampling_logp_difference/max": 0.19983983039855957, "sampling/sampling_logp_difference/mean": 0.005730110686272383, "step": 447, "step_time": 19.079979527999967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2011.0, "completions/max_terminated_length": 2011.0, "completions/mean_length": 1424.21875, "completions/mean_terminated_length": 1424.21875, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.13628586754202843, "epoch": 0.03584, "frac_reward_zero_std": 0.0, "grad_norm": 0.023519694805145264, "kl": 1.2665162086486816, "learning_rate": 4.35121136636811e-05, "loss": 0.0028, "num_tokens": 19530030.0, "reward": 1.291607141494751, "reward_std": 0.28190988302230835, "rewards/rollout_reward_func/mean": 1.291607141494751, "rewards/rollout_reward_func/std": 0.34222403168678284, "sampling/importance_sampling_ratio/max": 1.2409650087356567, "sampling/importance_sampling_ratio/mean": 1.0157532691955566, "sampling/importance_sampling_ratio/min": 0.8777416348457336, "sampling/sampling_logp_difference/max": 0.22675871849060059, "sampling/sampling_logp_difference/mean": 0.004183557350188494, "step": 448, "step_time": 19.862230442997316 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 2299.0, "completions/max_terminated_length": 2299.0, "completions/mean_length": 1480.5, "completions/mean_terminated_length": 1480.5, "completions/min_length": 1107.0, "completions/min_terminated_length": 1107.0, "entropy": 0.14688823092728853, "epoch": 0.03592, "frac_reward_zero_std": 0.0, "grad_norm": 0.05728699639439583, "kl": 2.5422574877738953, "learning_rate": 4.3512007071379004e-05, "loss": 0.0014, "num_tokens": 19594192.0, "reward": 1.2021093368530273, "reward_std": 0.30396372079849243, "rewards/rollout_reward_func/mean": 1.2021093368530273, "rewards/rollout_reward_func/std": 0.394738107919693, "sampling/importance_sampling_ratio/max": 1.1540701389312744, "sampling/importance_sampling_ratio/mean": 0.9906277656555176, "sampling/importance_sampling_ratio/min": 0.7503366470336914, "sampling/sampling_logp_difference/max": 0.23612570762634277, "sampling/sampling_logp_difference/mean": 0.006921189837157726, "step": 449, "step_time": 21.07240093099972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.0, "completions/max_length": 2175.0, "completions/max_terminated_length": 2175.0, "completions/mean_length": 1397.625, "completions/mean_terminated_length": 1397.625, "completions/min_length": 1049.0, "completions/min_terminated_length": 1049.0, "entropy": 0.17279407754540443, "epoch": 0.036, "frac_reward_zero_std": 0.0, "grad_norm": 0.036582861095666885, "kl": 1.611095204949379, "learning_rate": 4.35119002209043e-05, "loss": 0.0007, "num_tokens": 19654844.0, "reward": 1.107031226158142, "reward_std": 0.3132411241531372, "rewards/rollout_reward_func/mean": 1.107031226158142, "rewards/rollout_reward_func/std": 0.36538171768188477, "sampling/importance_sampling_ratio/max": 1.3820340633392334, "sampling/importance_sampling_ratio/mean": 1.0127546787261963, "sampling/importance_sampling_ratio/min": 0.6920953392982483, "sampling/sampling_logp_difference/max": 0.40668272972106934, "sampling/sampling_logp_difference/mean": 0.007298831827938557, "step": 450, "step_time": 20.819073458000275 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 2255.0, "completions/max_terminated_length": 2255.0, "completions/mean_length": 1600.0625, "completions/mean_terminated_length": 1600.0625, "completions/min_length": 1055.0, "completions/min_terminated_length": 1055.0, "entropy": 0.16251110937446356, "epoch": 0.03608, "frac_reward_zero_std": 0.0, "grad_norm": 0.03864491730928421, "kl": 1.3900231271982193, "learning_rate": 4.351179311225869e-05, "loss": -0.0014, "num_tokens": 19722910.0, "reward": 1.3239063024520874, "reward_std": 0.2845610976219177, "rewards/rollout_reward_func/mean": 1.3239063024520874, "rewards/rollout_reward_func/std": 0.32594212889671326, "sampling/importance_sampling_ratio/max": 1.238783597946167, "sampling/importance_sampling_ratio/mean": 0.9645678997039795, "sampling/importance_sampling_ratio/min": 0.5839642286300659, "sampling/sampling_logp_difference/max": 0.3585357666015625, "sampling/sampling_logp_difference/mean": 0.006515017710626125, "step": 451, "step_time": 20.784086424000634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 1987.0, "completions/max_terminated_length": 1987.0, "completions/mean_length": 1382.15625, "completions/mean_terminated_length": 1382.15625, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.1465105377137661, "epoch": 0.03616, "frac_reward_zero_std": 0.0, "grad_norm": 0.01978950947523117, "kl": 1.7588499933481216, "learning_rate": 4.3511685745443866e-05, "loss": 0.0015, "num_tokens": 19783279.0, "reward": 1.1714062690734863, "reward_std": 0.3317691683769226, "rewards/rollout_reward_func/mean": 1.1714062690734863, "rewards/rollout_reward_func/std": 0.3523331880569458, "sampling/importance_sampling_ratio/max": 1.1903308629989624, "sampling/importance_sampling_ratio/mean": 1.0118590593338013, "sampling/importance_sampling_ratio/min": 0.8770582675933838, "sampling/sampling_logp_difference/max": 0.10892641544342041, "sampling/sampling_logp_difference/mean": 0.003840208053588867, "step": 452, "step_time": 20.3049050110003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2277.0, "completions/max_terminated_length": 2277.0, "completions/mean_length": 1769.40625, "completions/mean_terminated_length": 1769.40625, "completions/min_length": 1325.0, "completions/min_terminated_length": 1325.0, "entropy": 0.15119088906794786, "epoch": 0.03624, "frac_reward_zero_std": 0.0, "grad_norm": 0.03152238950133324, "kl": 1.3112496212124825, "learning_rate": 4.351157812046153e-05, "loss": 0.003, "num_tokens": 19857960.0, "reward": 1.3429687023162842, "reward_std": 0.31747859716415405, "rewards/rollout_reward_func/mean": 1.3429687023162842, "rewards/rollout_reward_func/std": 0.32948604226112366, "sampling/importance_sampling_ratio/max": 1.2797363996505737, "sampling/importance_sampling_ratio/mean": 1.0099296569824219, "sampling/importance_sampling_ratio/min": 0.8436086177825928, "sampling/sampling_logp_difference/max": 0.19930505752563477, "sampling/sampling_logp_difference/mean": 0.004180118907243013, "step": 453, "step_time": 21.16426409199812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2269.0, "completions/max_terminated_length": 2269.0, "completions/mean_length": 1618.4375, "completions/mean_terminated_length": 1618.4375, "completions/min_length": 1047.0, "completions/min_terminated_length": 1047.0, "entropy": 0.18973787501454353, "epoch": 0.03632, "frac_reward_zero_std": 0.0, "grad_norm": 0.025750862434506416, "kl": 2.020817667245865, "learning_rate": 4.351147023731339e-05, "loss": 0.0003, "num_tokens": 19926972.0, "reward": 1.1725780963897705, "reward_std": 0.2742801606655121, "rewards/rollout_reward_func/mean": 1.1725780963897705, "rewards/rollout_reward_func/std": 0.3577378988265991, "sampling/importance_sampling_ratio/max": 1.2680680751800537, "sampling/importance_sampling_ratio/mean": 0.948244571685791, "sampling/importance_sampling_ratio/min": 0.5121569633483887, "sampling/sampling_logp_difference/max": 0.3251817226409912, "sampling/sampling_logp_difference/mean": 0.008640282787382603, "step": 454, "step_time": 21.92666780199943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2253.0, "completions/max_terminated_length": 2253.0, "completions/mean_length": 1913.96875, "completions/mean_terminated_length": 1913.96875, "completions/min_length": 1574.0, "completions/min_terminated_length": 1574.0, "entropy": 0.17033606581389904, "epoch": 0.0364, "frac_reward_zero_std": 0.0, "grad_norm": 0.025179319083690643, "kl": 1.3243939876556396, "learning_rate": 4.3511362096001157e-05, "loss": 0.0039, "num_tokens": 20006454.0, "reward": 1.2890625, "reward_std": 0.32188642024993896, "rewards/rollout_reward_func/mean": 1.2890625, "rewards/rollout_reward_func/std": 0.34329622983932495, "sampling/importance_sampling_ratio/max": 1.2443270683288574, "sampling/importance_sampling_ratio/mean": 1.014451265335083, "sampling/importance_sampling_ratio/min": 0.7116398215293884, "sampling/sampling_logp_difference/max": 0.31006956100463867, "sampling/sampling_logp_difference/mean": 0.005285301245748997, "step": 455, "step_time": 21.29109275900373 }, { "clip_ratio/high_max": 0.00390625, "clip_ratio/high_mean": 0.001953125, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001953125, "completions/clipped_ratio": 0.0, "completions/max_length": 2270.0, "completions/max_terminated_length": 2270.0, "completions/mean_length": 1452.40625, "completions/mean_terminated_length": 1452.40625, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.13888521119952202, "epoch": 0.03648, "frac_reward_zero_std": 0.0, "grad_norm": 0.01606239564716816, "kl": 1.355677291750908, "learning_rate": 4.351125369652653e-05, "loss": 0.0014, "num_tokens": 20069169.0, "reward": 1.3115625381469727, "reward_std": 0.3648865520954132, "rewards/rollout_reward_func/mean": 1.3115625381469727, "rewards/rollout_reward_func/std": 0.37190696597099304, "sampling/importance_sampling_ratio/max": 1.1802500486373901, "sampling/importance_sampling_ratio/mean": 1.0177788734436035, "sampling/importance_sampling_ratio/min": 0.9052064418792725, "sampling/sampling_logp_difference/max": 0.13881611824035645, "sampling/sampling_logp_difference/mean": 0.003884486388415098, "step": 456, "step_time": 21.66388492400074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2296.0, "completions/max_terminated_length": 2296.0, "completions/mean_length": 1580.40625, "completions/mean_terminated_length": 1580.40625, "completions/min_length": 1048.0, "completions/min_terminated_length": 1048.0, "entropy": 0.15925555117428303, "epoch": 0.03656, "frac_reward_zero_std": 0.0, "grad_norm": 0.027490703389048576, "kl": 1.534289911389351, "learning_rate": 4.3511145038891234e-05, "loss": 0.0025, "num_tokens": 20136961.0, "reward": 1.2362499237060547, "reward_std": 0.24991071224212646, "rewards/rollout_reward_func/mean": 1.2362499237060547, "rewards/rollout_reward_func/std": 0.34371957182884216, "sampling/importance_sampling_ratio/max": 1.3156007528305054, "sampling/importance_sampling_ratio/mean": 1.0020053386688232, "sampling/importance_sampling_ratio/min": 0.8159122467041016, "sampling/sampling_logp_difference/max": 0.279962420463562, "sampling/sampling_logp_difference/mean": 0.0054258918389678, "step": 457, "step_time": 21.048957628001517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2283.0, "completions/max_terminated_length": 2283.0, "completions/mean_length": 1737.90625, "completions/mean_terminated_length": 1737.90625, "completions/min_length": 1094.0, "completions/min_terminated_length": 1094.0, "entropy": 0.1693255677819252, "epoch": 0.03664, "frac_reward_zero_std": 0.0, "grad_norm": 0.036008626222610474, "kl": 1.3187776952981949, "learning_rate": 4.351103612309698e-05, "loss": 0.0027, "num_tokens": 20210110.0, "reward": 1.1349999904632568, "reward_std": 0.42027705907821655, "rewards/rollout_reward_func/mean": 1.1349999904632568, "rewards/rollout_reward_func/std": 0.44531068205833435, "sampling/importance_sampling_ratio/max": 1.3201963901519775, "sampling/importance_sampling_ratio/mean": 0.9837891459465027, "sampling/importance_sampling_ratio/min": 0.8385423421859741, "sampling/sampling_logp_difference/max": 0.1688218116760254, "sampling/sampling_logp_difference/mean": 0.004514969885349274, "step": 458, "step_time": 21.767060453999875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.0, "completions/max_length": 1958.0, "completions/max_terminated_length": 1958.0, "completions/mean_length": 1636.46875, "completions/mean_terminated_length": 1636.46875, "completions/min_length": 1085.0, "completions/min_terminated_length": 1085.0, "entropy": 0.1545517686754465, "epoch": 0.03672, "frac_reward_zero_std": 0.0, "grad_norm": 0.027378764003515244, "kl": 1.7394763976335526, "learning_rate": 4.351092694914552e-05, "loss": 0.0024, "num_tokens": 20280079.0, "reward": 1.2003124952316284, "reward_std": 0.3061574101448059, "rewards/rollout_reward_func/mean": 1.2003124952316284, "rewards/rollout_reward_func/std": 0.35424885153770447, "sampling/importance_sampling_ratio/max": 1.14541757106781, "sampling/importance_sampling_ratio/mean": 1.0065100193023682, "sampling/importance_sampling_ratio/min": 0.8319900631904602, "sampling/sampling_logp_difference/max": 0.12372541427612305, "sampling/sampling_logp_difference/mean": 0.003971042577177286, "step": 459, "step_time": 19.643436166001266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016891892300918698, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016891892300918698, "completions/clipped_ratio": 0.0, "completions/max_length": 2282.0, "completions/max_terminated_length": 2282.0, "completions/mean_length": 1848.3125, "completions/mean_terminated_length": 1848.3125, "completions/min_length": 1105.0, "completions/min_terminated_length": 1105.0, "entropy": 0.15818947553634644, "epoch": 0.0368, "frac_reward_zero_std": 0.0, "grad_norm": 0.02810801938176155, "kl": 1.2819881215691566, "learning_rate": 4.351081751703856e-05, "loss": 0.0004, "num_tokens": 20357718.0, "reward": 1.286149501800537, "reward_std": 0.35905665159225464, "rewards/rollout_reward_func/mean": 1.286149501800537, "rewards/rollout_reward_func/std": 0.36788707971572876, "sampling/importance_sampling_ratio/max": 1.1593621969223022, "sampling/importance_sampling_ratio/mean": 0.9685060977935791, "sampling/importance_sampling_ratio/min": 0.6839935779571533, "sampling/sampling_logp_difference/max": 0.3226926326751709, "sampling/sampling_logp_difference/mean": 0.0048732515424489975, "step": 460, "step_time": 21.929809391998788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2263.0, "completions/max_terminated_length": 2263.0, "completions/mean_length": 1606.25, "completions/mean_terminated_length": 1606.25, "completions/min_length": 1054.0, "completions/min_terminated_length": 1054.0, "entropy": 0.1669622678309679, "epoch": 0.03688, "frac_reward_zero_std": 0.0, "grad_norm": 0.022904695942997932, "kl": 1.5764231830835342, "learning_rate": 4.351070782677783e-05, "loss": 0.003, "num_tokens": 20426030.0, "reward": 1.2538281679153442, "reward_std": 0.3722527027130127, "rewards/rollout_reward_func/mean": 1.2538281679153442, "rewards/rollout_reward_func/std": 0.37429338693618774, "sampling/importance_sampling_ratio/max": 1.1590495109558105, "sampling/importance_sampling_ratio/mean": 0.9956021308898926, "sampling/importance_sampling_ratio/min": 0.7509719729423523, "sampling/sampling_logp_difference/max": 0.24529743194580078, "sampling/sampling_logp_difference/mean": 0.004810416139662266, "step": 461, "step_time": 21.19246551499964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2296.0, "completions/max_terminated_length": 2296.0, "completions/mean_length": 1806.28125, "completions/mean_terminated_length": 1806.28125, "completions/min_length": 1072.0, "completions/min_terminated_length": 1072.0, "entropy": 0.15030423924326897, "epoch": 0.03696, "frac_reward_zero_std": 0.0, "grad_norm": 0.023910850286483765, "kl": 1.7444822564721107, "learning_rate": 4.3510597878365084e-05, "loss": 0.0035, "num_tokens": 20501703.0, "reward": 1.3188281059265137, "reward_std": 0.3302443027496338, "rewards/rollout_reward_func/mean": 1.3188281059265137, "rewards/rollout_reward_func/std": 0.3698370158672333, "sampling/importance_sampling_ratio/max": 1.3466730117797852, "sampling/importance_sampling_ratio/mean": 1.0261965990066528, "sampling/importance_sampling_ratio/min": 0.773158073425293, "sampling/sampling_logp_difference/max": 0.2341247797012329, "sampling/sampling_logp_difference/mean": 0.004519036039710045, "step": 462, "step_time": 22.175456629003747 }, { "clip_ratio/high_max": 0.0035714285913854837, "clip_ratio/high_mean": 0.0017857142956927419, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017857142956927419, "completions/clipped_ratio": 0.0, "completions/max_length": 2277.0, "completions/max_terminated_length": 2277.0, "completions/mean_length": 1736.5, "completions/mean_terminated_length": 1736.5, "completions/min_length": 1026.0, "completions/min_terminated_length": 1026.0, "entropy": 0.17292192485183477, "epoch": 0.03704, "frac_reward_zero_std": 0.0, "grad_norm": 0.028145743533968925, "kl": 1.3880857229232788, "learning_rate": 4.3510487671802045e-05, "loss": 0.002, "num_tokens": 20575211.0, "reward": 1.3361718654632568, "reward_std": 0.3306407928466797, "rewards/rollout_reward_func/mean": 1.3361718654632568, "rewards/rollout_reward_func/std": 0.3906399607658386, "sampling/importance_sampling_ratio/max": 1.2223674058914185, "sampling/importance_sampling_ratio/mean": 0.9965864419937134, "sampling/importance_sampling_ratio/min": 0.7275149822235107, "sampling/sampling_logp_difference/max": 0.24979400634765625, "sampling/sampling_logp_difference/mean": 0.005266452208161354, "step": 463, "step_time": 21.00476123399858 }, { "clip_ratio/high_max": 0.0032051282469183207, "clip_ratio/high_mean": 0.0016025641234591603, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0035556891234591603, "completions/clipped_ratio": 0.0, "completions/max_length": 2262.0, "completions/max_terminated_length": 2262.0, "completions/mean_length": 1733.90625, "completions/mean_terminated_length": 1733.90625, "completions/min_length": 1305.0, "completions/min_terminated_length": 1305.0, "entropy": 0.17973755672574043, "epoch": 0.03712, "frac_reward_zero_std": 0.0, "grad_norm": 0.19356437027454376, "kl": 8.77133321762085, "learning_rate": 4.3510377207090475e-05, "loss": 0.0083, "num_tokens": 20648208.0, "reward": 1.1471874713897705, "reward_std": 0.4077732563018799, "rewards/rollout_reward_func/mean": 1.1471874713897705, "rewards/rollout_reward_func/std": 0.4333532452583313, "sampling/importance_sampling_ratio/max": 1.2450696229934692, "sampling/importance_sampling_ratio/mean": 0.9891819953918457, "sampling/importance_sampling_ratio/min": 0.6340681910514832, "sampling/sampling_logp_difference/max": 0.40570545196533203, "sampling/sampling_logp_difference/mean": 0.0041664037853479385, "step": 464, "step_time": 21.710010430000693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2288.0, "completions/max_terminated_length": 2288.0, "completions/mean_length": 1688.4375, "completions/mean_terminated_length": 1688.4375, "completions/min_length": 1028.0, "completions/min_terminated_length": 1028.0, "entropy": 0.21302555315196514, "epoch": 0.0372, "frac_reward_zero_std": 0.0, "grad_norm": 0.029307162389159203, "kl": 1.784995049238205, "learning_rate": 4.351026648423211e-05, "loss": 0.001, "num_tokens": 20719630.0, "reward": 1.201328158378601, "reward_std": 0.3347158432006836, "rewards/rollout_reward_func/mean": 1.201328158378601, "rewards/rollout_reward_func/std": 0.36622968316078186, "sampling/importance_sampling_ratio/max": 1.1954693794250488, "sampling/importance_sampling_ratio/mean": 0.9961185455322266, "sampling/importance_sampling_ratio/min": 0.661683201789856, "sampling/sampling_logp_difference/max": 0.2202625274658203, "sampling/sampling_logp_difference/mean": 0.005831114947795868, "step": 465, "step_time": 21.005544434998228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2267.0, "completions/max_terminated_length": 2267.0, "completions/mean_length": 1605.5625, "completions/mean_terminated_length": 1605.5625, "completions/min_length": 1059.0, "completions/min_terminated_length": 1059.0, "entropy": 0.22792821004986763, "epoch": 0.03728, "frac_reward_zero_std": 0.0, "grad_norm": 0.027634654194116592, "kl": 1.6688253581523895, "learning_rate": 4.3510155503228704e-05, "loss": 0.0004, "num_tokens": 20787755.0, "reward": 1.209531307220459, "reward_std": 0.35501375794410706, "rewards/rollout_reward_func/mean": 1.209531307220459, "rewards/rollout_reward_func/std": 0.36799007654190063, "sampling/importance_sampling_ratio/max": 1.1875765323638916, "sampling/importance_sampling_ratio/mean": 0.9919424653053284, "sampling/importance_sampling_ratio/min": 0.706575334072113, "sampling/sampling_logp_difference/max": 0.19767546653747559, "sampling/sampling_logp_difference/mean": 0.0061794971115887165, "step": 466, "step_time": 21.507248660000187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2286.0, "completions/max_terminated_length": 2286.0, "completions/mean_length": 1796.65625, "completions/mean_terminated_length": 1796.65625, "completions/min_length": 1272.0, "completions/min_terminated_length": 1272.0, "entropy": 0.2604329325258732, "epoch": 0.03736, "frac_reward_zero_std": 0.0, "grad_norm": 0.026333356276154518, "kl": 1.9852266013622284, "learning_rate": 4.3510044264082026e-05, "loss": 0.0017, "num_tokens": 20862716.0, "reward": 1.1906249523162842, "reward_std": 0.3338077664375305, "rewards/rollout_reward_func/mean": 1.1906249523162842, "rewards/rollout_reward_func/std": 0.38301804661750793, "sampling/importance_sampling_ratio/max": 1.1201250553131104, "sampling/importance_sampling_ratio/mean": 1.0055842399597168, "sampling/importance_sampling_ratio/min": 0.8867427706718445, "sampling/sampling_logp_difference/max": 0.12251591682434082, "sampling/sampling_logp_difference/mean": 0.004749735817313194, "step": 467, "step_time": 21.97904465799911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2282.0, "completions/max_terminated_length": 2282.0, "completions/mean_length": 1712.3125, "completions/mean_terminated_length": 1712.3125, "completions/min_length": 1254.0, "completions/min_terminated_length": 1254.0, "entropy": 0.30082870461046696, "epoch": 0.03744, "frac_reward_zero_std": 0.0, "grad_norm": 0.022102246060967445, "kl": 1.2094616815447807, "learning_rate": 4.350993276679383e-05, "loss": 0.0008, "num_tokens": 20935117.0, "reward": 1.2450000047683716, "reward_std": 0.3933524489402771, "rewards/rollout_reward_func/mean": 1.2450000047683716, "rewards/rollout_reward_func/std": 0.4167829751968384, "sampling/importance_sampling_ratio/max": 1.12843918800354, "sampling/importance_sampling_ratio/mean": 0.9759523272514343, "sampling/importance_sampling_ratio/min": 0.7744077444076538, "sampling/sampling_logp_difference/max": 0.19882917404174805, "sampling/sampling_logp_difference/mean": 0.007227214053273201, "step": 468, "step_time": 20.973747544001526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2264.0, "completions/max_terminated_length": 2264.0, "completions/mean_length": 1510.5, "completions/mean_terminated_length": 1510.5, "completions/min_length": 1047.0, "completions/min_terminated_length": 1047.0, "entropy": 0.4518680088222027, "epoch": 0.03752, "frac_reward_zero_std": 0.0, "grad_norm": 0.027814773842692375, "kl": 2.0430333763360977, "learning_rate": 4.350982101136588e-05, "loss": -0.0001, "num_tokens": 20999695.0, "reward": 1.021328091621399, "reward_std": 0.39135342836380005, "rewards/rollout_reward_func/mean": 1.021328091621399, "rewards/rollout_reward_func/std": 0.5141741037368774, "sampling/importance_sampling_ratio/max": 1.294985294342041, "sampling/importance_sampling_ratio/mean": 0.9690108299255371, "sampling/importance_sampling_ratio/min": 3.944446234527277e-06, "sampling/sampling_logp_difference/max": 10.349905014038086, "sampling/sampling_logp_difference/mean": 0.027196917682886124, "step": 469, "step_time": 22.010063381998407 }, { "clip_ratio/high_max": 0.0035714285913854837, "clip_ratio/high_mean": 0.0017857142956927419, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017857142956927419, "completions/clipped_ratio": 0.0, "completions/max_length": 2277.0, "completions/max_terminated_length": 2277.0, "completions/mean_length": 1537.5625, "completions/mean_terminated_length": 1537.5625, "completions/min_length": 1067.0, "completions/min_terminated_length": 1067.0, "entropy": 0.34851862862706184, "epoch": 0.0376, "frac_reward_zero_std": 0.0, "grad_norm": 0.03174252435564995, "kl": 1.7732892334461212, "learning_rate": 4.350970899779995e-05, "loss": -0.0003, "num_tokens": 21065639.0, "reward": 1.0758593082427979, "reward_std": 0.3413071036338806, "rewards/rollout_reward_func/mean": 1.0758593082427979, "rewards/rollout_reward_func/std": 0.4304005205631256, "sampling/importance_sampling_ratio/max": 1.2619812488555908, "sampling/importance_sampling_ratio/mean": 0.9835142493247986, "sampling/importance_sampling_ratio/min": 0.7752272486686707, "sampling/sampling_logp_difference/max": 0.19443702697753906, "sampling/sampling_logp_difference/mean": 0.00876557920128107, "step": 470, "step_time": 20.998156521000055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2285.0, "completions/max_terminated_length": 2285.0, "completions/mean_length": 1662.9375, "completions/mean_terminated_length": 1662.9375, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.35199451074004173, "epoch": 0.03768, "frac_reward_zero_std": 0.0, "grad_norm": 0.031805116683244705, "kl": 1.6608392894268036, "learning_rate": 4.350959672609781e-05, "loss": 0.0005, "num_tokens": 21136172.0, "reward": 1.0600000619888306, "reward_std": 0.32763946056365967, "rewards/rollout_reward_func/mean": 1.0600000619888306, "rewards/rollout_reward_func/std": 0.4073517918586731, "sampling/importance_sampling_ratio/max": 1.2795802354812622, "sampling/importance_sampling_ratio/mean": 1.0136668682098389, "sampling/importance_sampling_ratio/min": 0.8718326687812805, "sampling/sampling_logp_difference/max": 0.12455940246582031, "sampling/sampling_logp_difference/mean": 0.008277669548988342, "step": 471, "step_time": 21.684552179998718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017361111240461469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.0, "completions/max_length": 2310.0, "completions/max_terminated_length": 2310.0, "completions/mean_length": 1560.125, "completions/mean_terminated_length": 1560.125, "completions/min_length": 1067.0, "completions/min_terminated_length": 1067.0, "entropy": 0.40191906318068504, "epoch": 0.03776, "frac_reward_zero_std": 0.0, "grad_norm": 0.03674622252583504, "kl": 2.108980357646942, "learning_rate": 4.350948419626124e-05, "loss": 0.0048, "num_tokens": 21202626.0, "reward": 1.018125057220459, "reward_std": 0.4354340434074402, "rewards/rollout_reward_func/mean": 1.018125057220459, "rewards/rollout_reward_func/std": 0.4403952658176422, "sampling/importance_sampling_ratio/max": 1.3709548711776733, "sampling/importance_sampling_ratio/mean": 0.9928151369094849, "sampling/importance_sampling_ratio/min": 0.8447038531303406, "sampling/sampling_logp_difference/max": 0.361217737197876, "sampling/sampling_logp_difference/mean": 0.010566790588200092, "step": 472, "step_time": 21.13077892000183 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017361111240461469, "completions/clipped_ratio": 0.0, "completions/max_length": 2286.0, "completions/max_terminated_length": 2286.0, "completions/mean_length": 1703.75, "completions/mean_terminated_length": 1703.75, "completions/min_length": 1062.0, "completions/min_terminated_length": 1062.0, "entropy": 0.40351075306534767, "epoch": 0.03784, "frac_reward_zero_std": 0.0, "grad_norm": 0.034398533403873444, "kl": 1.644257128238678, "learning_rate": 4.350937140829203e-05, "loss": 0.0033, "num_tokens": 21274348.0, "reward": 0.8982812166213989, "reward_std": 0.44988879561424255, "rewards/rollout_reward_func/mean": 0.8982812166213989, "rewards/rollout_reward_func/std": 0.47038525342941284, "sampling/importance_sampling_ratio/max": 1.328954815864563, "sampling/importance_sampling_ratio/mean": 0.984686017036438, "sampling/importance_sampling_ratio/min": 0.8103592395782471, "sampling/sampling_logp_difference/max": 0.2278759479522705, "sampling/sampling_logp_difference/mean": 0.009880313649773598, "step": 473, "step_time": 21.160430014002486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2197.0, "completions/max_terminated_length": 2197.0, "completions/mean_length": 1715.46875, "completions/mean_terminated_length": 1715.46875, "completions/min_length": 1294.0, "completions/min_terminated_length": 1294.0, "entropy": 0.40747233107686043, "epoch": 0.03792, "frac_reward_zero_std": 0.0, "grad_norm": 0.033851757645606995, "kl": 1.9112671613693237, "learning_rate": 4.350925836219194e-05, "loss": 0.0004, "num_tokens": 21346945.0, "reward": 1.1095311641693115, "reward_std": 0.5657190084457397, "rewards/rollout_reward_func/mean": 1.1095311641693115, "rewards/rollout_reward_func/std": 0.5655303001403809, "sampling/importance_sampling_ratio/max": 1.3002556562423706, "sampling/importance_sampling_ratio/mean": 0.9567885994911194, "sampling/importance_sampling_ratio/min": 2.719808662732248e-06, "sampling/sampling_logp_difference/max": 12.512527465820312, "sampling/sampling_logp_difference/mean": 0.03192385658621788, "step": 474, "step_time": 21.853002795000066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1942.0, "completions/max_terminated_length": 1942.0, "completions/mean_length": 1263.6875, "completions/mean_terminated_length": 1263.6875, "completions/min_length": 1018.0, "completions/min_terminated_length": 1018.0, "entropy": 0.39922328665852547, "epoch": 0.038, "frac_reward_zero_std": 0.0, "grad_norm": 0.030855298042297363, "kl": 2.355858474969864, "learning_rate": 4.350914505796279e-05, "loss": 0.0014, "num_tokens": 21403067.0, "reward": 0.8353571891784668, "reward_std": 0.2875702381134033, "rewards/rollout_reward_func/mean": 0.8353571891784668, "rewards/rollout_reward_func/std": 0.3397657573223114, "sampling/importance_sampling_ratio/max": 1.2884502410888672, "sampling/importance_sampling_ratio/mean": 1.011977195739746, "sampling/importance_sampling_ratio/min": 0.6395496129989624, "sampling/sampling_logp_difference/max": 0.25211429595947266, "sampling/sampling_logp_difference/mean": 0.010995794087648392, "step": 475, "step_time": 20.377302775998032 }, { "clip_ratio/high_max": 0.0033783784601837397, "clip_ratio/high_mean": 0.0016891892300918698, "clip_ratio/low_mean": 0.001953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00364231423009187, "completions/clipped_ratio": 0.0, "completions/max_length": 2294.0, "completions/max_terminated_length": 2294.0, "completions/mean_length": 1546.1875, "completions/mean_terminated_length": 1546.1875, "completions/min_length": 1041.0, "completions/min_terminated_length": 1041.0, "entropy": 0.40282924845814705, "epoch": 0.03808, "frac_reward_zero_std": 0.0, "grad_norm": 0.032898399978876114, "kl": 1.815461128950119, "learning_rate": 4.3509031495606355e-05, "loss": 0.0017, "num_tokens": 21469550.0, "reward": 1.0751898288726807, "reward_std": 0.33780813217163086, "rewards/rollout_reward_func/mean": 1.0751898288726807, "rewards/rollout_reward_func/std": 0.37129542231559753, "sampling/importance_sampling_ratio/max": 1.3354887962341309, "sampling/importance_sampling_ratio/mean": 0.9789102077484131, "sampling/importance_sampling_ratio/min": 0.6543655395507812, "sampling/sampling_logp_difference/max": 0.2943298816680908, "sampling/sampling_logp_difference/mean": 0.013163464143872261, "step": 476, "step_time": 22.00155049100067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1985.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1457.125, "completions/mean_terminated_length": 1457.125, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.3353220038115978, "epoch": 0.03816, "frac_reward_zero_std": 0.0, "grad_norm": 0.044596560299396515, "kl": 1.9562983065843582, "learning_rate": 4.350891767512445e-05, "loss": 0.0023, "num_tokens": 21532489.0, "reward": 1.0796763896942139, "reward_std": 0.35564035177230835, "rewards/rollout_reward_func/mean": 1.0796763896942139, "rewards/rollout_reward_func/std": 0.48986926674842834, "sampling/importance_sampling_ratio/max": 1.180814266204834, "sampling/importance_sampling_ratio/mean": 0.9819018244743347, "sampling/importance_sampling_ratio/min": 0.7710828185081482, "sampling/sampling_logp_difference/max": 0.18579626083374023, "sampling/sampling_logp_difference/mean": 0.008297894150018692, "step": 477, "step_time": 19.734708554999088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2006.0, "completions/max_terminated_length": 2006.0, "completions/mean_length": 1382.625, "completions/mean_terminated_length": 1382.625, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.3149391748011112, "epoch": 0.03824, "frac_reward_zero_std": 0.0, "grad_norm": 0.037862420082092285, "kl": 2.3352505564689636, "learning_rate": 4.350880359651886e-05, "loss": 0.0035, "num_tokens": 21592447.0, "reward": 1.0750782489776611, "reward_std": 0.3833707869052887, "rewards/rollout_reward_func/mean": 1.0750782489776611, "rewards/rollout_reward_func/std": 0.40945306420326233, "sampling/importance_sampling_ratio/max": 1.1858710050582886, "sampling/importance_sampling_ratio/mean": 0.976517915725708, "sampling/importance_sampling_ratio/min": 0.7941383719444275, "sampling/sampling_logp_difference/max": 0.2101764678955078, "sampling/sampling_logp_difference/mean": 0.00930311344563961, "step": 478, "step_time": 19.72718857000109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018939394503831863, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018939394503831863, "completions/clipped_ratio": 0.0, "completions/max_length": 2271.0, "completions/max_terminated_length": 2271.0, "completions/mean_length": 1644.84375, "completions/mean_terminated_length": 1644.84375, "completions/min_length": 1249.0, "completions/min_terminated_length": 1249.0, "entropy": 0.3089934177696705, "epoch": 0.03832, "frac_reward_zero_std": 0.0, "grad_norm": 0.05829310417175293, "kl": 1.7170538902282715, "learning_rate": 4.3508689259791404e-05, "loss": 0.0034, "num_tokens": 21662616.0, "reward": 1.0362277030944824, "reward_std": 0.5265812873840332, "rewards/rollout_reward_func/mean": 1.0362277030944824, "rewards/rollout_reward_func/std": 0.5158542394638062, "sampling/importance_sampling_ratio/max": 1.2906343936920166, "sampling/importance_sampling_ratio/mean": 1.0282336473464966, "sampling/importance_sampling_ratio/min": 0.8140368461608887, "sampling/sampling_logp_difference/max": 0.32518601417541504, "sampling/sampling_logp_difference/mean": 0.010529391467571259, "step": 479, "step_time": 21.81783216200165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0018382353009656072, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018382353009656072, "completions/clipped_ratio": 0.0, "completions/max_length": 2282.0, "completions/max_terminated_length": 2282.0, "completions/mean_length": 1667.96875, "completions/mean_terminated_length": 1667.96875, "completions/min_length": 1028.0, "completions/min_terminated_length": 1028.0, "entropy": 0.2555731702595949, "epoch": 0.0384, "frac_reward_zero_std": 0.0, "grad_norm": 0.04218175634741783, "kl": 1.4886293560266495, "learning_rate": 4.3508574664943884e-05, "loss": 0.0021, "num_tokens": 21733666.0, "reward": 1.2613615989685059, "reward_std": 0.4492562413215637, "rewards/rollout_reward_func/mean": 1.2613615989685059, "rewards/rollout_reward_func/std": 0.4695952236652374, "sampling/importance_sampling_ratio/max": 1.2968169450759888, "sampling/importance_sampling_ratio/mean": 0.9986293315887451, "sampling/importance_sampling_ratio/min": 0.7888181805610657, "sampling/sampling_logp_difference/max": 0.2057943344116211, "sampling/sampling_logp_difference/mean": 0.006158492993563414, "step": 480, "step_time": 20.97417339000276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2280.0, "completions/max_terminated_length": 2280.0, "completions/mean_length": 1575.96875, "completions/mean_terminated_length": 1575.96875, "completions/min_length": 1025.0, "completions/min_terminated_length": 1025.0, "entropy": 0.24822546914219856, "epoch": 0.03848, "frac_reward_zero_std": 0.0, "grad_norm": 0.0392855666577816, "kl": 2.161859467625618, "learning_rate": 4.3508459811978116e-05, "loss": 0.003, "num_tokens": 21801840.0, "reward": 1.0489063262939453, "reward_std": 0.4236152172088623, "rewards/rollout_reward_func/mean": 1.0489063262939453, "rewards/rollout_reward_func/std": 0.44897863268852234, "sampling/importance_sampling_ratio/max": 1.452493667602539, "sampling/importance_sampling_ratio/mean": 1.0024800300598145, "sampling/importance_sampling_ratio/min": 0.7890416383743286, "sampling/sampling_logp_difference/max": 0.22674179077148438, "sampling/sampling_logp_difference/mean": 0.006946468260139227, "step": 481, "step_time": 20.852223577998302 }, { "clip_ratio/high_max": 0.0034722222480922937, "clip_ratio/high_mean": 0.0017361111240461469, "clip_ratio/low_mean": 0.00390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005642361124046147, "completions/clipped_ratio": 0.0, "completions/max_length": 2286.0, "completions/max_terminated_length": 2286.0, "completions/mean_length": 1636.0, "completions/mean_terminated_length": 1636.0, "completions/min_length": 1018.0, "completions/min_terminated_length": 1018.0, "entropy": 0.21233047731220722, "epoch": 0.03856, "frac_reward_zero_std": 0.0, "grad_norm": 0.025900932028889656, "kl": 1.9071004092693329, "learning_rate": 4.3508344700895923e-05, "loss": 0.0015, "num_tokens": 21871343.0, "reward": 1.2575000524520874, "reward_std": 0.3066626787185669, "rewards/rollout_reward_func/mean": 1.2575000524520874, "rewards/rollout_reward_func/std": 0.38621610403060913, "sampling/importance_sampling_ratio/max": 1.4933018684387207, "sampling/importance_sampling_ratio/mean": 0.9910992383956909, "sampling/importance_sampling_ratio/min": 0.7080161571502686, "sampling/sampling_logp_difference/max": 0.3618510961532593, "sampling/sampling_logp_difference/mean": 0.006462990306317806, "step": 482, "step_time": 21.522876353998072 }, { "clip_ratio/high_max": 0.00034722223062999547, "clip_ratio/high_mean": 0.00017361111531499773, "clip_ratio/low_mean": 0.0016447368543595076, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018183479696745053, "completions/clipped_ratio": 0.0, "completions/max_length": 2575.0, "completions/max_terminated_length": 2575.0, "completions/mean_length": 1619.75, "completions/mean_terminated_length": 1619.75, "completions/min_length": 1225.0, "completions/min_terminated_length": 1225.0, "entropy": 0.27162860706448555, "epoch": 0.03864, "frac_reward_zero_std": 0.0, "grad_norm": 0.06861855089664459, "kl": 1.68471097946167, "learning_rate": 4.350822933169913e-05, "loss": -0.04, "num_tokens": 21940658.0, "reward": 1.2098438739776611, "reward_std": 0.33211296796798706, "rewards/rollout_reward_func/mean": 1.2098438739776611, "rewards/rollout_reward_func/std": 0.3743395209312439, "sampling/importance_sampling_ratio/max": 1.5144727230072021, "sampling/importance_sampling_ratio/mean": 1.0108520984649658, "sampling/importance_sampling_ratio/min": 0.698194146156311, "sampling/sampling_logp_difference/max": 0.3865842819213867, "sampling/sampling_logp_difference/mean": 0.010698549449443817, "step": 483, "step_time": 25.426996454996697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1712.0, "completions/max_terminated_length": 1712.0, "completions/mean_length": 1431.0625, "completions/mean_terminated_length": 1431.0625, "completions/min_length": 1049.0, "completions/min_terminated_length": 1049.0, "entropy": 0.23142492584884167, "epoch": 0.03872, "frac_reward_zero_std": 0.0, "grad_norm": 0.03638770431280136, "kl": 1.8820354342460632, "learning_rate": 4.350811370438956e-05, "loss": 0.0024, "num_tokens": 22002383.0, "reward": 1.161250114440918, "reward_std": 0.34516477584838867, "rewards/rollout_reward_func/mean": 1.161250114440918, "rewards/rollout_reward_func/std": 0.34153589606285095, "sampling/importance_sampling_ratio/max": 1.4858440160751343, "sampling/importance_sampling_ratio/mean": 1.012505054473877, "sampling/importance_sampling_ratio/min": 0.7980007529258728, "sampling/sampling_logp_difference/max": 0.33208632469177246, "sampling/sampling_logp_difference/mean": 0.008955961093306541, "step": 484, "step_time": 18.142083656997784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2268.0, "completions/max_terminated_length": 2268.0, "completions/mean_length": 1586.21875, "completions/mean_terminated_length": 1586.21875, "completions/min_length": 1018.0, "completions/min_terminated_length": 1018.0, "entropy": 0.3589001316577196, "epoch": 0.0388, "frac_reward_zero_std": 0.0, "grad_norm": 0.07500310242176056, "kl": 2.144585609436035, "learning_rate": 4.350799781896905e-05, "loss": -0.0102, "num_tokens": 22070673.0, "reward": 1.2537834644317627, "reward_std": 0.3008209764957428, "rewards/rollout_reward_func/mean": 1.2537834644317627, "rewards/rollout_reward_func/std": 0.3618324398994446, "sampling/importance_sampling_ratio/max": 1.245313048362732, "sampling/importance_sampling_ratio/mean": 0.9873307943344116, "sampling/importance_sampling_ratio/min": 0.7736237049102783, "sampling/sampling_logp_difference/max": 0.2588846683502197, "sampling/sampling_logp_difference/mean": 0.01129869744181633, "step": 485, "step_time": 21.120636174999163 }, { "clip_ratio/high_max": 0.0013586956774815917, "clip_ratio/high_mean": 0.0006793478387407959, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006793478387407959, "completions/clipped_ratio": 0.0, "completions/max_length": 2185.0, "completions/max_terminated_length": 2185.0, "completions/mean_length": 1561.25, "completions/mean_terminated_length": 1561.25, "completions/min_length": 1034.0, "completions/min_terminated_length": 1034.0, "entropy": 0.46727394685149193, "epoch": 0.03888, "frac_reward_zero_std": 0.0, "grad_norm": 0.4043847322463989, "kl": 15.223807379603386, "learning_rate": 4.350788167543942e-05, "loss": -0.0018, "num_tokens": 22137592.0, "reward": 1.2983890771865845, "reward_std": 0.30991891026496887, "rewards/rollout_reward_func/mean": 1.2983890771865845, "rewards/rollout_reward_func/std": 0.31112346053123474, "sampling/importance_sampling_ratio/max": 1.8562908172607422, "sampling/importance_sampling_ratio/mean": 1.0479333400726318, "sampling/importance_sampling_ratio/min": 0.7416137456893921, "sampling/sampling_logp_difference/max": 0.2993452548980713, "sampling/sampling_logp_difference/mean": 0.015729406848549843, "step": 486, "step_time": 22.04229079200195 }, { "clip_ratio/high_max": 0.004567149328067899, "clip_ratio/high_mean": 0.0022835746640339494, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022835746640339494, "completions/clipped_ratio": 0.0, "completions/max_length": 2305.0, "completions/max_terminated_length": 2305.0, "completions/mean_length": 1867.34375, "completions/mean_terminated_length": 1867.34375, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.4684232324361801, "epoch": 0.03896, "frac_reward_zero_std": 0.0, "grad_norm": 0.05821802094578743, "kl": 1.7616848051548004, "learning_rate": 4.350776527380253e-05, "loss": 0.0012, "num_tokens": 22215199.0, "reward": 1.44921875, "reward_std": 0.2955199182033539, "rewards/rollout_reward_func/mean": 1.44921875, "rewards/rollout_reward_func/std": 0.3479388654232025, "sampling/importance_sampling_ratio/max": 1.2801240682601929, "sampling/importance_sampling_ratio/mean": 0.9727150201797485, "sampling/importance_sampling_ratio/min": 0.7425044775009155, "sampling/sampling_logp_difference/max": 0.30663585662841797, "sampling/sampling_logp_difference/mean": 0.012603990733623505, "step": 487, "step_time": 23.46598453000115 }, { "clip_ratio/high_max": 0.00025667352019809186, "clip_ratio/high_mean": 0.00012833676009904593, "clip_ratio/low_mean": 0.0005274261347949505, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006557628948939964, "completions/clipped_ratio": 0.0, "completions/max_length": 2468.0, "completions/max_terminated_length": 2468.0, "completions/mean_length": 1539.34375, "completions/mean_terminated_length": 1539.34375, "completions/min_length": 967.0, "completions/min_terminated_length": 967.0, "entropy": 0.4489026218652725, "epoch": 0.03904, "frac_reward_zero_std": 0.0, "grad_norm": 0.058680932968854904, "kl": 2.1054976508021355, "learning_rate": 4.35076486140602e-05, "loss": 0.0057, "num_tokens": 22280963.0, "reward": 1.2243006229400635, "reward_std": 0.3178659677505493, "rewards/rollout_reward_func/mean": 1.2243006229400635, "rewards/rollout_reward_func/std": 0.40663912892341614, "sampling/importance_sampling_ratio/max": 1.2117995023727417, "sampling/importance_sampling_ratio/mean": 0.8971771001815796, "sampling/importance_sampling_ratio/min": 3.3081926410297705e-18, "sampling/sampling_logp_difference/max": 15.810070037841797, "sampling/sampling_logp_difference/mean": 0.03928075730800629, "step": 488, "step_time": 27.165604240002722 }, { "clip_ratio/high_max": 0.006257151137106121, "clip_ratio/high_mean": 0.0031285755685530603, "clip_ratio/low_mean": 0.0018805928993970156, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005009168467950076, "completions/clipped_ratio": 0.0, "completions/max_length": 2270.0, "completions/max_terminated_length": 2270.0, "completions/mean_length": 1592.3125, "completions/mean_terminated_length": 1592.3125, "completions/min_length": 1069.0, "completions/min_terminated_length": 1069.0, "entropy": 0.6235471926629543, "epoch": 0.03912, "frac_reward_zero_std": 0.0, "grad_norm": 0.12222673743963242, "kl": 1.5263077020645142, "learning_rate": 4.350753169621429e-05, "loss": 0.0191, "num_tokens": 22348827.0, "reward": 1.302053451538086, "reward_std": 0.39894431829452515, "rewards/rollout_reward_func/mean": 1.302053451538086, "rewards/rollout_reward_func/std": 0.4515552818775177, "sampling/importance_sampling_ratio/max": 2.495269298553467, "sampling/importance_sampling_ratio/mean": 0.9968598484992981, "sampling/importance_sampling_ratio/min": 0.4195743799209595, "sampling/sampling_logp_difference/max": 0.48882532119750977, "sampling/sampling_logp_difference/mean": 0.017229344695806503, "step": 489, "step_time": 28.955380232999232 }, { "clip_ratio/high_max": 0.0038389242836274207, "clip_ratio/high_mean": 0.0019194621418137103, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019194621418137103, "completions/clipped_ratio": 0.0, "completions/max_length": 2292.0, "completions/max_terminated_length": 2292.0, "completions/mean_length": 1501.71875, "completions/mean_terminated_length": 1501.71875, "completions/min_length": 1033.0, "completions/min_terminated_length": 1033.0, "entropy": 0.560648400336504, "epoch": 0.0392, "frac_reward_zero_std": 0.0, "grad_norm": 0.12264908105134964, "kl": 1.730082631111145, "learning_rate": 4.350741452026665e-05, "loss": 0.0356, "num_tokens": 22412932.0, "reward": 1.32924485206604, "reward_std": 0.3472854793071747, "rewards/rollout_reward_func/mean": 1.32924485206604, "rewards/rollout_reward_func/std": 0.42214691638946533, "sampling/importance_sampling_ratio/max": 2.1108505725860596, "sampling/importance_sampling_ratio/mean": 0.9567543268203735, "sampling/importance_sampling_ratio/min": 0.5376855731010437, "sampling/sampling_logp_difference/max": 0.414121150970459, "sampling/sampling_logp_difference/mean": 0.014563169330358505, "step": 490, "step_time": 25.551508224001736 }, { "clip_ratio/high_max": 0.0032327587250620127, "clip_ratio/high_mean": 0.0016163793625310063, "clip_ratio/low_mean": 0.002480020688381046, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004096400050912052, "completions/clipped_ratio": 0.0, "completions/max_length": 2294.0, "completions/max_terminated_length": 2294.0, "completions/mean_length": 1748.0, "completions/mean_terminated_length": 1748.0, "completions/min_length": 1033.0, "completions/min_terminated_length": 1033.0, "entropy": 0.6692380756139755, "epoch": 0.03928, "frac_reward_zero_std": 0.0, "grad_norm": 0.07657730579376221, "kl": 1.852459579706192, "learning_rate": 4.3507297086219146e-05, "loss": 0.0121, "num_tokens": 22486404.0, "reward": 1.402782678604126, "reward_std": 0.3506074845790863, "rewards/rollout_reward_func/mean": 1.402782678604126, "rewards/rollout_reward_func/std": 0.3573211431503296, "sampling/importance_sampling_ratio/max": 1.6660906076431274, "sampling/importance_sampling_ratio/mean": 0.9458626508712769, "sampling/importance_sampling_ratio/min": 0.4045098125934601, "sampling/sampling_logp_difference/max": 0.5263748168945312, "sampling/sampling_logp_difference/mean": 0.020242689177393913, "step": 491, "step_time": 24.687113252000927 }, { "clip_ratio/high_max": 0.0018939394503831863, "clip_ratio/high_mean": 0.0009469697251915932, "clip_ratio/low_mean": 0.0013147438439773396, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022617135691689327, "completions/clipped_ratio": 0.03125, "completions/max_length": 2287.0, "completions/max_terminated_length": 2287.0, "completions/mean_length": 1646.03125, "completions/mean_terminated_length": 1646.6773681640625, "completions/min_length": 1283.0, "completions/min_terminated_length": 1283.0, "entropy": 0.6534626297652721, "epoch": 0.03936, "frac_reward_zero_std": 0.0, "grad_norm": 0.09518894553184509, "kl": 1.2786764055490494, "learning_rate": 4.350717939407362e-05, "loss": -0.0485, "num_tokens": 22556202.0, "reward": 1.379717230796814, "reward_std": 0.4167005717754364, "rewards/rollout_reward_func/mean": 1.379717230796814, "rewards/rollout_reward_func/std": 0.453245609998703, "sampling/importance_sampling_ratio/max": 1.8996371030807495, "sampling/importance_sampling_ratio/mean": 0.9357471466064453, "sampling/importance_sampling_ratio/min": 0.35783326625823975, "sampling/sampling_logp_difference/max": 0.46391773223876953, "sampling/sampling_logp_difference/mean": 0.016672402620315552, "step": 492, "step_time": 27.617428686000494 }, { "clip_ratio/high_max": 0.006472550972830504, "clip_ratio/high_mean": 0.0034488605160731822, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0034488605160731822, "completions/clipped_ratio": 0.03125, "completions/max_length": 2497.0, "completions/max_terminated_length": 2497.0, "completions/mean_length": 1809.875, "completions/mean_terminated_length": 1788.9354248046875, "completions/min_length": 1005.0, "completions/min_terminated_length": 1005.0, "entropy": 0.5601083301007748, "epoch": 0.03944, "frac_reward_zero_std": 0.0, "grad_norm": 0.12390033900737762, "kl": 1.3503845408558846, "learning_rate": 4.3507061443831944e-05, "loss": -0.0708, "num_tokens": 22631773.0, "reward": 1.1002156734466553, "reward_std": 0.544530987739563, "rewards/rollout_reward_func/mean": 1.1002156734466553, "rewards/rollout_reward_func/std": 0.6276413798332214, "sampling/importance_sampling_ratio/max": 2.1586251258850098, "sampling/importance_sampling_ratio/mean": 0.9894818663597107, "sampling/importance_sampling_ratio/min": 4.433411415760702e-09, "sampling/sampling_logp_difference/max": 10.72472095489502, "sampling/sampling_logp_difference/mean": 0.016700491309165955, "step": 493, "step_time": 34.366338636000364 }, { "clip_ratio/high_max": 0.0030127070494927466, "clip_ratio/high_mean": 0.0017022782994899899, "clip_ratio/low_mean": 0.0010822586009453516, "clip_ratio/low_min": 0.0008333333535119891, "clip_ratio/region_mean": 0.0027845370168506633, "completions/clipped_ratio": 0.03125, "completions/max_length": 2818.0, "completions/max_terminated_length": 2696.0, "completions/mean_length": 1797.71875, "completions/mean_terminated_length": 1764.806396484375, "completions/min_length": 1096.0, "completions/min_terminated_length": 1096.0, "entropy": 0.5301521606743336, "epoch": 0.03952, "frac_reward_zero_std": 0.0, "grad_norm": 0.10699569433927536, "kl": 1.359189823269844, "learning_rate": 4.350694323549599e-05, "loss": 0.1561, "num_tokens": 22706904.0, "reward": 1.2543973922729492, "reward_std": 0.5552285313606262, "rewards/rollout_reward_func/mean": 1.2543973922729492, "rewards/rollout_reward_func/std": 0.5705404877662659, "sampling/importance_sampling_ratio/max": 1.860823631286621, "sampling/importance_sampling_ratio/mean": 0.9157564640045166, "sampling/importance_sampling_ratio/min": 5.093529154009957e-09, "sampling/sampling_logp_difference/max": 12.060123443603516, "sampling/sampling_logp_difference/mean": 0.019018394872546196, "step": 494, "step_time": 45.62132661999931 }, { "clip_ratio/high_max": 0.0029354136786423624, "clip_ratio/high_mean": 0.0018577223818283528, "clip_ratio/low_mean": 0.00047272423398680985, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0023304466158151627, "completions/clipped_ratio": 0.0625, "completions/max_length": 2000.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 1523.4375, "completions/mean_terminated_length": 1497.6334228515625, "completions/min_length": 1027.0, "completions/min_terminated_length": 1027.0, "entropy": 0.6856301687657833, "epoch": 0.0396, "frac_reward_zero_std": 0.0, "grad_norm": 0.10666242241859436, "kl": 1.5170733854174614, "learning_rate": 4.3506824769067624e-05, "loss": 0.0357, "num_tokens": 22772345.0, "reward": 1.2804241180419922, "reward_std": 0.41068795323371887, "rewards/rollout_reward_func/mean": 1.2804241180419922, "rewards/rollout_reward_func/std": 0.4344504475593567, "sampling/importance_sampling_ratio/max": 2.024656057357788, "sampling/importance_sampling_ratio/mean": 0.9166590571403503, "sampling/importance_sampling_ratio/min": 2.373608865013921e-09, "sampling/sampling_logp_difference/max": 15.507872581481934, "sampling/sampling_logp_difference/mean": 0.02008155919611454, "step": 495, "step_time": 34.62832755600175 }, { "clip_ratio/high_max": 0.002711061155423522, "clip_ratio/high_mean": 0.001355530577711761, "clip_ratio/low_mean": 0.0011556378449313343, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0025111684226430953, "completions/clipped_ratio": 0.0, "completions/max_length": 2274.0, "completions/max_terminated_length": 2274.0, "completions/mean_length": 1675.28125, "completions/mean_terminated_length": 1675.28125, "completions/min_length": 1053.0, "completions/min_terminated_length": 1053.0, "entropy": 0.643932331353426, "epoch": 0.03968, "frac_reward_zero_std": 0.0, "grad_norm": 0.04430066794157028, "kl": 1.8674080669879913, "learning_rate": 4.350670604454872e-05, "loss": -0.0101, "num_tokens": 22843607.0, "reward": 1.3182365894317627, "reward_std": 0.35693663358688354, "rewards/rollout_reward_func/mean": 1.3182365894317627, "rewards/rollout_reward_func/std": 0.40382662415504456, "sampling/importance_sampling_ratio/max": 1.3574124574661255, "sampling/importance_sampling_ratio/mean": 0.8929771184921265, "sampling/importance_sampling_ratio/min": 1.1582916615474672e-22, "sampling/sampling_logp_difference/max": 17.498937606811523, "sampling/sampling_logp_difference/mean": 0.06887960433959961, "step": 496, "step_time": 24.83917410200047 }, { "clip_ratio/high_max": 0.0032739418384153396, "clip_ratio/high_mean": 0.0016369709192076698, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016369709192076698, "completions/clipped_ratio": 0.0, "completions/max_length": 2140.0, "completions/max_terminated_length": 2140.0, "completions/mean_length": 1364.3125, "completions/mean_terminated_length": 1364.3125, "completions/min_length": 1033.0, "completions/min_terminated_length": 1033.0, "entropy": 0.6230824962258339, "epoch": 0.03976, "frac_reward_zero_std": 0.0, "grad_norm": 0.07468285411596298, "kl": 2.0376343801617622, "learning_rate": 4.350658706194116e-05, "loss": 0.0032, "num_tokens": 22903191.0, "reward": 1.40386164188385, "reward_std": 0.23951232433319092, "rewards/rollout_reward_func/mean": 1.40386164188385, "rewards/rollout_reward_func/std": 0.2742568552494049, "sampling/importance_sampling_ratio/max": 1.8042473793029785, "sampling/importance_sampling_ratio/mean": 0.9940310120582581, "sampling/importance_sampling_ratio/min": 0.2603985071182251, "sampling/sampling_logp_difference/max": 0.3728055953979492, "sampling/sampling_logp_difference/mean": 0.016477739438414574, "step": 497, "step_time": 26.067979766999997 }, { "clip_ratio/high_max": 0.0005364806856960058, "clip_ratio/high_mean": 0.0002682403428480029, "clip_ratio/low_mean": 0.000590405281400308, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008586456242483109, "completions/clipped_ratio": 0.0, "completions/max_length": 2284.0, "completions/max_terminated_length": 2284.0, "completions/mean_length": 1664.28125, "completions/mean_terminated_length": 1664.28125, "completions/min_length": 1237.0, "completions/min_terminated_length": 1237.0, "entropy": 0.41426438465714455, "epoch": 0.03984, "frac_reward_zero_std": 0.0, "grad_norm": 0.08341458439826965, "kl": 1.653488963842392, "learning_rate": 4.3506467821246836e-05, "loss": 0.016, "num_tokens": 22974457.0, "reward": 1.3178497552871704, "reward_std": 0.48561912775039673, "rewards/rollout_reward_func/mean": 1.3178497552871704, "rewards/rollout_reward_func/std": 0.49577853083610535, "sampling/importance_sampling_ratio/max": 1.8128371238708496, "sampling/importance_sampling_ratio/mean": 1.001407265663147, "sampling/importance_sampling_ratio/min": 0.3840939700603485, "sampling/sampling_logp_difference/max": 0.6277437210083008, "sampling/sampling_logp_difference/mean": 0.012829242274165154, "step": 498, "step_time": 25.84110838900051 }, { "clip_ratio/high_max": 0.003419612883590162, "clip_ratio/high_mean": 0.001709806441795081, "clip_ratio/low_mean": 0.000724002078641206, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002433808520436287, "completions/clipped_ratio": 0.03125, "completions/max_length": 2281.0, "completions/max_terminated_length": 2281.0, "completions/mean_length": 1829.1875, "completions/mean_terminated_length": 1818.4193115234375, "completions/min_length": 1252.0, "completions/min_terminated_length": 1252.0, "entropy": 0.5101872272789478, "epoch": 0.03992, "frac_reward_zero_std": 0.0, "grad_norm": 0.07856221497058868, "kl": 1.4723606333136559, "learning_rate": 4.3506348322467623e-05, "loss": 0.0263, "num_tokens": 23051365.0, "reward": 1.3964955806732178, "reward_std": 0.32469648122787476, "rewards/rollout_reward_func/mean": 1.3964955806732178, "rewards/rollout_reward_func/std": 0.4110803008079529, "sampling/importance_sampling_ratio/max": 2.1059021949768066, "sampling/importance_sampling_ratio/mean": 0.9591854810714722, "sampling/importance_sampling_ratio/min": 7.893564274441545e-11, "sampling/sampling_logp_difference/max": 10.743935585021973, "sampling/sampling_logp_difference/mean": 0.03841501474380493, "step": 499, "step_time": 27.721470726000916 }, { "clip_ratio/high_max": 0.0030161692411638796, "clip_ratio/high_mean": 0.002532674843678251, "clip_ratio/low_mean": 0.00042517005931586027, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002957844902994111, "completions/clipped_ratio": 0.0, "completions/max_length": 1441.0, "completions/max_terminated_length": 1441.0, "completions/mean_length": 1240.25, "completions/mean_terminated_length": 1240.25, "completions/min_length": 1040.0, "completions/min_terminated_length": 1040.0, "entropy": 0.6175453625619411, "epoch": 0.04, "frac_reward_zero_std": 0.0, "grad_norm": 0.07520284503698349, "kl": 1.4589449763298035, "learning_rate": 4.350622856560543e-05, "loss": 0.0294, "num_tokens": 23106426.0, "reward": 1.4589731693267822, "reward_std": 0.1463196724653244, "rewards/rollout_reward_func/mean": 1.4589731693267822, "rewards/rollout_reward_func/std": 0.15818847715854645, "sampling/importance_sampling_ratio/max": 1.3244643211364746, "sampling/importance_sampling_ratio/mean": 0.8772373795509338, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 0.5778894424438477, "sampling/sampling_logp_difference/mean": 0.01799154467880726, "step": 500, "step_time": 21.573001303002457 }, { "clip_ratio/high_max": 0.01432722236495465, "clip_ratio/high_mean": 0.007779050036333501, "clip_ratio/low_mean": 0.0012917947606183589, "clip_ratio/low_min": 0.00030788176809437573, "clip_ratio/region_mean": 0.00907084479695186, "completions/clipped_ratio": 0.03125, "completions/max_length": 1942.0, "completions/max_terminated_length": 1942.0, "completions/mean_length": 1409.15625, "completions/mean_terminated_length": 1404.290283203125, "completions/min_length": 1092.0, "completions/min_terminated_length": 1092.0, "entropy": 0.5971322543919086, "epoch": 0.04008, "frac_reward_zero_std": 0.0, "grad_norm": 0.10661690682172775, "kl": 1.5821243524551392, "learning_rate": 4.350610855066214e-05, "loss": 0.0667, "num_tokens": 23166685.0, "reward": 1.266369104385376, "reward_std": 0.3391287922859192, "rewards/rollout_reward_func/mean": 1.266369104385376, "rewards/rollout_reward_func/std": 0.4666331708431244, "sampling/importance_sampling_ratio/max": 2.102701425552368, "sampling/importance_sampling_ratio/mean": 0.9252525568008423, "sampling/importance_sampling_ratio/min": 1.114021052598009e-13, "sampling/sampling_logp_difference/max": 15.636421203613281, "sampling/sampling_logp_difference/mean": 0.02847488597035408, "step": 501, "step_time": 27.581571579999945 }, { "clip_ratio/high_max": 0.008988345507532358, "clip_ratio/high_mean": 0.004763569333590567, "clip_ratio/low_mean": 0.00011425960110500455, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004877828934695572, "completions/clipped_ratio": 0.03125, "completions/max_length": 2295.0, "completions/max_terminated_length": 2295.0, "completions/mean_length": 1658.84375, "completions/mean_terminated_length": 1642.7095947265625, "completions/min_length": 1087.0, "completions/min_terminated_length": 1087.0, "entropy": 0.3855232298374176, "epoch": 0.04016, "frac_reward_zero_std": 0.0, "grad_norm": 0.10639430582523346, "kl": 1.1418011039495468, "learning_rate": 4.3505988277639653e-05, "loss": 0.0279, "num_tokens": 23237681.0, "reward": 1.395937442779541, "reward_std": 0.35362929105758667, "rewards/rollout_reward_func/mean": 1.395937442779541, "rewards/rollout_reward_func/std": 0.3754296898841858, "sampling/importance_sampling_ratio/max": 2.115565061569214, "sampling/importance_sampling_ratio/mean": 0.9957630038261414, "sampling/importance_sampling_ratio/min": 9.518031142446748e-17, "sampling/sampling_logp_difference/max": 18.987701416015625, "sampling/sampling_logp_difference/mean": 0.0307175163179636, "step": 502, "step_time": 28.312254919002953 }, { "clip_ratio/high_max": 0.004442117176949978, "clip_ratio/high_mean": 0.002221058588474989, "clip_ratio/low_mean": 0.0007711262733209878, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029921848326921463, "completions/clipped_ratio": 0.0, "completions/max_length": 2279.0, "completions/max_terminated_length": 2279.0, "completions/mean_length": 1525.09375, "completions/mean_terminated_length": 1525.09375, "completions/min_length": 1083.0, "completions/min_terminated_length": 1083.0, "entropy": 0.48789268732070923, "epoch": 0.04024, "frac_reward_zero_std": 0.0, "grad_norm": 0.16678857803344727, "kl": 1.4060129225254059, "learning_rate": 4.3505867746539884e-05, "loss": 0.0971, "num_tokens": 23302607.0, "reward": 1.4134821891784668, "reward_std": 0.30940279364585876, "rewards/rollout_reward_func/mean": 1.4134821891784668, "rewards/rollout_reward_func/std": 0.3326520621776581, "sampling/importance_sampling_ratio/max": 2.9251396656036377, "sampling/importance_sampling_ratio/mean": 0.975304126739502, "sampling/importance_sampling_ratio/min": 2.7933248717249384e-16, "sampling/sampling_logp_difference/max": 16.44134521484375, "sampling/sampling_logp_difference/mean": 0.03095058724284172, "step": 503, "step_time": 27.594967230999828 }, { "clip_ratio/high_max": 0.006187409802805632, "clip_ratio/high_mean": 0.003093704901402816, "clip_ratio/low_mean": 0.000559535576030612, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036532405065372586, "completions/clipped_ratio": 0.03125, "completions/max_length": 2318.0, "completions/max_terminated_length": 2318.0, "completions/mean_length": 1705.96875, "completions/mean_terminated_length": 1695.258056640625, "completions/min_length": 1061.0, "completions/min_terminated_length": 1061.0, "entropy": 0.4241067972034216, "epoch": 0.04032, "frac_reward_zero_std": 0.0, "grad_norm": 0.09703473746776581, "kl": 1.017072319984436, "learning_rate": 4.3505746957364726e-05, "loss": 0.1493, "num_tokens": 23374278.0, "reward": 1.4853646755218506, "reward_std": 0.2326396107673645, "rewards/rollout_reward_func/mean": 1.4853646755218506, "rewards/rollout_reward_func/std": 0.3183010220527649, "sampling/importance_sampling_ratio/max": 1.7187223434448242, "sampling/importance_sampling_ratio/mean": 0.9930965900421143, "sampling/importance_sampling_ratio/min": 0.43530339002609253, "sampling/sampling_logp_difference/max": 0.7342112064361572, "sampling/sampling_logp_difference/mean": 0.009972582571208477, "step": 504, "step_time": 31.500328318001266 }, { "clip_ratio/high_max": 0.0016166174900718033, "clip_ratio/high_mean": 0.0008083087450359017, "clip_ratio/low_mean": 0.0004537512650131248, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001262060039152857, "completions/clipped_ratio": 0.03125, "completions/max_length": 3469.0, "completions/max_terminated_length": 2131.0, "completions/mean_length": 1819.9375, "completions/mean_terminated_length": 1766.741943359375, "completions/min_length": 1315.0, "completions/min_terminated_length": 1315.0, "entropy": 0.462808845564723, "epoch": 0.0404, "frac_reward_zero_std": 0.0, "grad_norm": 0.12099472433328629, "kl": 1.3920458629727364, "learning_rate": 4.3505625910116106e-05, "loss": 0.1532, "num_tokens": 23450291.0, "reward": 1.4665625095367432, "reward_std": 0.27314886450767517, "rewards/rollout_reward_func/mean": 1.4665625095367432, "rewards/rollout_reward_func/std": 0.27578461170196533, "sampling/importance_sampling_ratio/max": 1.6756906509399414, "sampling/importance_sampling_ratio/mean": 0.9754071831703186, "sampling/importance_sampling_ratio/min": 0.2710478603839874, "sampling/sampling_logp_difference/max": 0.376953125, "sampling/sampling_logp_difference/mean": 0.015871919691562653, "step": 505, "step_time": 40.119996000001265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001950226112967357, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001950226112967357, "completions/clipped_ratio": 0.09375, "completions/max_length": 2240.0, "completions/max_terminated_length": 2240.0, "completions/mean_length": 1628.71875, "completions/mean_terminated_length": 1614.82763671875, "completions/min_length": 1085.0, "completions/min_terminated_length": 1085.0, "entropy": 0.338390639051795, "epoch": 0.04048, "frac_reward_zero_std": 0.0, "grad_norm": 0.06772942841053009, "kl": 1.168497160077095, "learning_rate": 4.350550460479594e-05, "loss": 0.0728, "num_tokens": 23519656.0, "reward": 1.5026562213897705, "reward_std": 0.14926777780056, "rewards/rollout_reward_func/mean": 1.5026562213897705, "rewards/rollout_reward_func/std": 0.17454077303409576, "sampling/importance_sampling_ratio/max": 2.423433542251587, "sampling/importance_sampling_ratio/mean": 0.9937686920166016, "sampling/importance_sampling_ratio/min": 1.5719482280606621e-16, "sampling/sampling_logp_difference/max": 15.180965423583984, "sampling/sampling_logp_difference/mean": 0.029709359630942345, "step": 506, "step_time": 30.217873743000382 }, { "clip_ratio/high_max": 0.005972222192212939, "clip_ratio/high_mean": 0.0029861110961064696, "clip_ratio/low_mean": 0.001558315590955317, "clip_ratio/low_min": 0.0005656108842231333, "clip_ratio/region_mean": 0.004544426687061787, "completions/clipped_ratio": 0.03125, "completions/max_length": 2001.0, "completions/max_terminated_length": 2001.0, "completions/mean_length": 1511.03125, "completions/mean_terminated_length": 1501.0966796875, "completions/min_length": 1004.0, "completions/min_terminated_length": 1004.0, "entropy": 0.5005101040005684, "epoch": 0.04056, "frac_reward_zero_std": 0.0, "grad_norm": 0.09722433239221573, "kl": 1.6681914776563644, "learning_rate": 4.350538304140614e-05, "loss": 0.0169, "num_tokens": 23584629.0, "reward": 1.4251562356948853, "reward_std": 0.2580384612083435, "rewards/rollout_reward_func/mean": 1.4251562356948853, "rewards/rollout_reward_func/std": 0.3490297198295593, "sampling/importance_sampling_ratio/max": 1.4233440160751343, "sampling/importance_sampling_ratio/mean": 0.8637205362319946, "sampling/importance_sampling_ratio/min": 4.1477118931899454e-17, "sampling/sampling_logp_difference/max": 14.682512283325195, "sampling/sampling_logp_difference/mean": 0.044047679752111435, "step": 507, "step_time": 29.572940561003634 }, { "clip_ratio/high_max": 0.0029761905316263437, "clip_ratio/high_mean": 0.0014880952658131719, "clip_ratio/low_mean": 0.0007510172144975513, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002239112480310723, "completions/clipped_ratio": 0.09375, "completions/max_length": 1980.0, "completions/max_terminated_length": 1980.0, "completions/mean_length": 1487.84375, "completions/mean_terminated_length": 1454.6551513671875, "completions/min_length": 1027.0, "completions/min_terminated_length": 1027.0, "entropy": 0.6475102603435516, "epoch": 0.04064, "frac_reward_zero_std": 0.0, "grad_norm": 0.07231185585260391, "kl": 1.7034412026405334, "learning_rate": 4.350526121994864e-05, "loss": 0.035, "num_tokens": 23648458.0, "reward": 1.4634374380111694, "reward_std": 0.22385495901107788, "rewards/rollout_reward_func/mean": 1.4634374380111694, "rewards/rollout_reward_func/std": 0.24913591146469116, "sampling/importance_sampling_ratio/max": 1.7437782287597656, "sampling/importance_sampling_ratio/mean": 0.8663283586502075, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 13.800265312194824, "sampling/sampling_logp_difference/mean": 0.09208094328641891, "step": 508, "step_time": 27.569974226005797 }, { "clip_ratio/high_max": 0.0057204426266252995, "clip_ratio/high_mean": 0.0028602213133126497, "clip_ratio/low_mean": 0.00013913444854551926, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002999355761858169, "completions/clipped_ratio": 0.03125, "completions/max_length": 2756.0, "completions/max_terminated_length": 2271.0, "completions/mean_length": 1621.625, "completions/mean_terminated_length": 1585.0322265625, "completions/min_length": 1052.0, "completions/min_terminated_length": 1052.0, "entropy": 0.3369912039488554, "epoch": 0.04072, "frac_reward_zero_std": 0.0, "grad_norm": 0.0562746487557888, "kl": 1.5537941604852676, "learning_rate": 4.350513914042537e-05, "loss": 0.0534, "num_tokens": 23717404.0, "reward": 1.4193750619888306, "reward_std": 0.2755410373210907, "rewards/rollout_reward_func/mean": 1.4193750619888306, "rewards/rollout_reward_func/std": 0.35761696100234985, "sampling/importance_sampling_ratio/max": 1.5411032438278198, "sampling/importance_sampling_ratio/mean": 0.8889001607894897, "sampling/importance_sampling_ratio/min": 1.0538855345151671e-20, "sampling/sampling_logp_difference/max": 15.450027465820312, "sampling/sampling_logp_difference/mean": 0.031983572989702225, "step": 509, "step_time": 36.589883524997276 }, { "clip_ratio/high_max": 0.004589372081682086, "clip_ratio/high_mean": 0.002294686040841043, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002294686040841043, "completions/clipped_ratio": 0.03125, "completions/max_length": 2270.0, "completions/max_terminated_length": 2270.0, "completions/mean_length": 1552.8125, "completions/mean_terminated_length": 1529.9031982421875, "completions/min_length": 1005.0, "completions/min_terminated_length": 1005.0, "entropy": 0.28424483723938465, "epoch": 0.0408, "frac_reward_zero_std": 0.0, "grad_norm": 0.06055377796292305, "kl": 1.714779183268547, "learning_rate": 4.3505016802838246e-05, "loss": 0.0342, "num_tokens": 23784196.0, "reward": 1.4289062023162842, "reward_std": 0.18314948678016663, "rewards/rollout_reward_func/mean": 1.4289062023162842, "rewards/rollout_reward_func/std": 0.26214295625686646, "sampling/importance_sampling_ratio/max": 1.576898455619812, "sampling/importance_sampling_ratio/mean": 0.9390487670898438, "sampling/importance_sampling_ratio/min": 2.4380653305424143e-15, "sampling/sampling_logp_difference/max": 14.760703086853027, "sampling/sampling_logp_difference/mean": 0.05079027637839317, "step": 510, "step_time": 25.40751060399998 }, { "clip_ratio/high_max": 0.0021929824724793434, "clip_ratio/high_mean": 0.0010964912362396717, "clip_ratio/low_mean": 0.0012715065677184612, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002367997803958133, "completions/clipped_ratio": 0.09375, "completions/max_length": 2525.0, "completions/max_terminated_length": 2280.0, "completions/mean_length": 1676.4375, "completions/mean_terminated_length": 1641.6551513671875, "completions/min_length": 1057.0, "completions/min_terminated_length": 1057.0, "entropy": 0.13392251078039408, "epoch": 0.04088, "frac_reward_zero_std": 0.0, "grad_norm": 0.04450288414955139, "kl": 1.3426841646432877, "learning_rate": 4.350489420718923e-05, "loss": 0.1588, "num_tokens": 23855422.0, "reward": 1.5368750095367432, "reward_std": 0.18531948328018188, "rewards/rollout_reward_func/mean": 1.5368750095367432, "rewards/rollout_reward_func/std": 0.18132908642292023, "sampling/importance_sampling_ratio/max": 1.385297417640686, "sampling/importance_sampling_ratio/mean": 0.9369755983352661, "sampling/importance_sampling_ratio/min": 1.2281152972827907e-26, "sampling/sampling_logp_difference/max": 12.580143928527832, "sampling/sampling_logp_difference/mean": 0.03022354282438755, "step": 511, "step_time": 33.655016212997 }, { "clip_ratio/high_max": 0.001953125, "clip_ratio/high_mean": 0.0009765625, "clip_ratio/low_mean": 0.001179245300590992, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002155807800590992, "completions/clipped_ratio": 0.0, "completions/max_length": 1699.0, "completions/max_terminated_length": 1699.0, "completions/mean_length": 1318.625, "completions/mean_terminated_length": 1318.625, "completions/min_length": 1033.0, "completions/min_terminated_length": 1033.0, "entropy": 0.20298829581588507, "epoch": 0.04096, "frac_reward_zero_std": 0.0, "grad_norm": 0.0416649654507637, "kl": 2.037236586213112, "learning_rate": 4.3504771353480243e-05, "loss": 0.0112, "num_tokens": 23913354.0, "reward": 1.493281364440918, "reward_std": 0.09879617393016815, "rewards/rollout_reward_func/mean": 1.493281364440918, "rewards/rollout_reward_func/std": 0.11646408587694168, "sampling/importance_sampling_ratio/max": 1.4748355150222778, "sampling/importance_sampling_ratio/mean": 0.9174975156784058, "sampling/importance_sampling_ratio/min": 0.240130215883255, "sampling/sampling_logp_difference/max": 1.4854216575622559, "sampling/sampling_logp_difference/mean": 0.009785616770386696, "step": 512, "step_time": 19.76890178999929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0007621950935572386, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007621950935572386, "completions/clipped_ratio": 0.03125, "completions/max_length": 2761.0, "completions/max_terminated_length": 2536.0, "completions/mean_length": 1629.8125, "completions/mean_terminated_length": 1593.322509765625, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.1628968962468207, "epoch": 0.04104, "frac_reward_zero_std": 0.0, "grad_norm": 0.06553251296281815, "kl": 1.9513276666402817, "learning_rate": 4.350464824171324e-05, "loss": 0.0108, "num_tokens": 23981965.0, "reward": 1.486875057220459, "reward_std": 0.23460665345191956, "rewards/rollout_reward_func/mean": 1.486875057220459, "rewards/rollout_reward_func/std": 0.2478349506855011, "sampling/importance_sampling_ratio/max": 1.3429099321365356, "sampling/importance_sampling_ratio/mean": 1.0080060958862305, "sampling/importance_sampling_ratio/min": 0.9049611687660217, "sampling/sampling_logp_difference/max": 0.24567890167236328, "sampling/sampling_logp_difference/mean": 0.004327686503529549, "step": 513, "step_time": 28.366088677003063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019409938249737024, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00019409938249737024, "completions/clipped_ratio": 0.0625, "completions/max_length": 2555.0, "completions/max_terminated_length": 2555.0, "completions/mean_length": 1861.6875, "completions/mean_terminated_length": 1896.166748046875, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.19653951283544302, "epoch": 0.04112, "frac_reward_zero_std": 0.0, "grad_norm": 0.0471162311732769, "kl": 1.5657988265156746, "learning_rate": 4.350452487189017e-05, "loss": 0.0283, "num_tokens": 24058982.0, "reward": 1.5130207538604736, "reward_std": 0.1848791241645813, "rewards/rollout_reward_func/mean": 1.5130207538604736, "rewards/rollout_reward_func/std": 0.18249040842056274, "sampling/importance_sampling_ratio/max": 1.5085501670837402, "sampling/importance_sampling_ratio/mean": 0.9721100330352783, "sampling/importance_sampling_ratio/min": 2.1619301256915423e-27, "sampling/sampling_logp_difference/max": 11.786213874816895, "sampling/sampling_logp_difference/mean": 0.0569412037730217, "step": 514, "step_time": 28.94727277700258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2567.0, "completions/max_terminated_length": 2567.0, "completions/mean_length": 1778.6875, "completions/mean_terminated_length": 1778.6875, "completions/min_length": 1384.0, "completions/min_terminated_length": 1384.0, "entropy": 0.050210417713969946, "epoch": 0.0412, "frac_reward_zero_std": 0.0, "grad_norm": 0.018490012735128403, "kl": 2.019573286175728, "learning_rate": 4.3504401244012986e-05, "loss": 0.0058, "num_tokens": 24133434.0, "reward": 1.543125033378601, "reward_std": 0.15059369802474976, "rewards/rollout_reward_func/mean": 1.543125033378601, "rewards/rollout_reward_func/std": 0.15903815627098083, "sampling/importance_sampling_ratio/max": 1.198436975479126, "sampling/importance_sampling_ratio/mean": 1.0056877136230469, "sampling/importance_sampling_ratio/min": 0.9603427648544312, "sampling/sampling_logp_difference/max": 0.18580245971679688, "sampling/sampling_logp_difference/mean": 0.001020009396597743, "step": 515, "step_time": 23.580387451997012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2531.0, "completions/max_terminated_length": 2531.0, "completions/mean_length": 1740.71875, "completions/mean_terminated_length": 1740.71875, "completions/min_length": 1058.0, "completions/min_terminated_length": 1058.0, "entropy": 0.04883858608081937, "epoch": 0.04128, "frac_reward_zero_std": 0.0, "grad_norm": 0.01405002549290657, "kl": 2.056405708193779, "learning_rate": 4.3504277358083646e-05, "loss": 0.0038, "num_tokens": 24205908.0, "reward": 1.5665278434753418, "reward_std": 0.16984900832176208, "rewards/rollout_reward_func/mean": 1.5665278434753418, "rewards/rollout_reward_func/std": 0.20020094513893127, "sampling/importance_sampling_ratio/max": 1.4442834854125977, "sampling/importance_sampling_ratio/mean": 1.021162986755371, "sampling/importance_sampling_ratio/min": 0.9573802947998047, "sampling/sampling_logp_difference/max": 0.3726768493652344, "sampling/sampling_logp_difference/mean": 0.001927885226905346, "step": 516, "step_time": 23.492766293997192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2541.0, "completions/max_terminated_length": 2541.0, "completions/mean_length": 1713.375, "completions/mean_terminated_length": 1713.375, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.0383181138895452, "epoch": 0.04136, "frac_reward_zero_std": 0.0, "grad_norm": 0.0076510109938681126, "kl": 2.044002577662468, "learning_rate": 4.3504153214104105e-05, "loss": 0.0055, "num_tokens": 24278243.0, "reward": 1.5303125381469727, "reward_std": 0.16197480261325836, "rewards/rollout_reward_func/mean": 1.5303125381469727, "rewards/rollout_reward_func/std": 0.17867763340473175, "sampling/importance_sampling_ratio/max": 1.0912551879882812, "sampling/importance_sampling_ratio/mean": 1.003246545791626, "sampling/importance_sampling_ratio/min": 0.9895550608634949, "sampling/sampling_logp_difference/max": 0.09385490417480469, "sampling/sampling_logp_difference/mean": 0.0006502366159111261, "step": 517, "step_time": 22.98754146799729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2208.0, "completions/max_terminated_length": 2208.0, "completions/mean_length": 1417.28125, "completions/mean_terminated_length": 1417.28125, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.03757387469522655, "epoch": 0.04144, "frac_reward_zero_std": 0.0, "grad_norm": 0.00765629205852747, "kl": 2.2071541845798492, "learning_rate": 4.3504028812076344e-05, "loss": 0.0054, "num_tokens": 24339758.0, "reward": 1.5121874809265137, "reward_std": 0.15868999063968658, "rewards/rollout_reward_func/mean": 1.5121874809265137, "rewards/rollout_reward_func/std": 0.1920640468597412, "sampling/importance_sampling_ratio/max": 1.0070819854736328, "sampling/importance_sampling_ratio/mean": 0.9891082048416138, "sampling/importance_sampling_ratio/min": 0.8226378560066223, "sampling/sampling_logp_difference/max": 0.1251678466796875, "sampling/sampling_logp_difference/mean": 0.0007353025721386075, "step": 518, "step_time": 22.54787183100052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2439.0, "completions/max_terminated_length": 2439.0, "completions/mean_length": 1671.0625, "completions/mean_terminated_length": 1671.0625, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.0454859035089612, "epoch": 0.04152, "frac_reward_zero_std": 0.0, "grad_norm": 0.00999575387686491, "kl": 2.208328351378441, "learning_rate": 4.350390415200232e-05, "loss": 0.0043, "num_tokens": 24409667.0, "reward": 1.534806489944458, "reward_std": 0.19760221242904663, "rewards/rollout_reward_func/mean": 1.534806489944458, "rewards/rollout_reward_func/std": 0.28948885202407837, "sampling/importance_sampling_ratio/max": 1.0719252824783325, "sampling/importance_sampling_ratio/mean": 0.9877877831459045, "sampling/importance_sampling_ratio/min": 0.7685434818267822, "sampling/sampling_logp_difference/max": 0.16373538970947266, "sampling/sampling_logp_difference/mean": 0.0015270914882421494, "step": 519, "step_time": 22.95220928100389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2450.0, "completions/max_terminated_length": 2450.0, "completions/mean_length": 1622.0, "completions/mean_terminated_length": 1622.0, "completions/min_length": 1149.0, "completions/min_terminated_length": 1149.0, "entropy": 0.03262764238752425, "epoch": 0.0416, "frac_reward_zero_std": 0.0, "grad_norm": 0.006582471076399088, "kl": 2.285018652677536, "learning_rate": 4.3503779233884016e-05, "loss": 0.0047, "num_tokens": 24478291.0, "reward": 1.4805208444595337, "reward_std": 0.22146454453468323, "rewards/rollout_reward_func/mean": 1.4805208444595337, "rewards/rollout_reward_func/std": 0.2760396897792816, "sampling/importance_sampling_ratio/max": 1.0131319761276245, "sampling/importance_sampling_ratio/mean": 0.9962026476860046, "sampling/importance_sampling_ratio/min": 0.8742657899856567, "sampling/sampling_logp_difference/max": 0.12454843521118164, "sampling/sampling_logp_difference/mean": 0.0004992596805095673, "step": 520, "step_time": 22.927029056998435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0014204545877873898, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014204545877873898, "completions/clipped_ratio": 0.0, "completions/max_length": 2502.0, "completions/max_terminated_length": 2502.0, "completions/mean_length": 1761.46875, "completions/mean_terminated_length": 1761.46875, "completions/min_length": 1375.0, "completions/min_terminated_length": 1375.0, "entropy": 0.04620742052793503, "epoch": 0.04168, "frac_reward_zero_std": 0.0, "grad_norm": 0.02017592452466488, "kl": 2.0255625396966934, "learning_rate": 4.35036540577234e-05, "loss": 0.0053, "num_tokens": 24551812.0, "reward": 1.4631249904632568, "reward_std": 0.1514846235513687, "rewards/rollout_reward_func/mean": 1.4631249904632568, "rewards/rollout_reward_func/std": 0.19093042612075806, "sampling/importance_sampling_ratio/max": 1.1209155321121216, "sampling/importance_sampling_ratio/mean": 1.005683183670044, "sampling/importance_sampling_ratio/min": 0.9921849370002747, "sampling/sampling_logp_difference/max": 0.2162160873413086, "sampling/sampling_logp_difference/mean": 0.0011951872147619724, "step": 521, "step_time": 23.855281564996403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2184.0, "completions/max_terminated_length": 2184.0, "completions/mean_length": 1510.96875, "completions/mean_terminated_length": 1510.96875, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.027321241097524762, "epoch": 0.04176, "frac_reward_zero_std": 0.0, "grad_norm": 0.001442065229639411, "kl": 2.392174005508423, "learning_rate": 4.3503528623522465e-05, "loss": 0.0054, "num_tokens": 24616520.0, "reward": 1.5221874713897705, "reward_std": 0.14080938696861267, "rewards/rollout_reward_func/mean": 1.5221874713897705, "rewards/rollout_reward_func/std": 0.15955042839050293, "sampling/importance_sampling_ratio/max": 1.0072994232177734, "sampling/importance_sampling_ratio/mean": 0.9996162056922913, "sampling/importance_sampling_ratio/min": 0.9935511350631714, "sampling/sampling_logp_difference/max": 0.006880562752485275, "sampling/sampling_logp_difference/mean": 0.0002627854701131582, "step": 522, "step_time": 21.944140899999184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2466.0, "completions/max_terminated_length": 2466.0, "completions/mean_length": 1904.0, "completions/mean_terminated_length": 1904.0, "completions/min_length": 1385.0, "completions/min_terminated_length": 1385.0, "entropy": 0.02743429341353476, "epoch": 0.04184, "frac_reward_zero_std": 0.0, "grad_norm": 0.0016106716357171535, "kl": 1.747746005654335, "learning_rate": 4.350340293128319e-05, "loss": 0.0054, "num_tokens": 24695581.0, "reward": 1.5778124332427979, "reward_std": 0.1759704053401947, "rewards/rollout_reward_func/mean": 1.5778124332427979, "rewards/rollout_reward_func/std": 0.18456107378005981, "sampling/importance_sampling_ratio/max": 1.0069468021392822, "sampling/importance_sampling_ratio/mean": 0.9995313882827759, "sampling/importance_sampling_ratio/min": 0.9917240738868713, "sampling/sampling_logp_difference/max": 0.006625514477491379, "sampling/sampling_logp_difference/mean": 0.00025621330132707953, "step": 523, "step_time": 23.028798597997593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2563.0, "completions/max_terminated_length": 2563.0, "completions/mean_length": 1838.03125, "completions/mean_terminated_length": 1838.03125, "completions/min_length": 1385.0, "completions/min_terminated_length": 1385.0, "entropy": 0.02574685658328235, "epoch": 0.04192, "frac_reward_zero_std": 0.0, "grad_norm": 0.0019158594077453017, "kl": 1.7318956702947617, "learning_rate": 4.350327698100757e-05, "loss": 0.0066, "num_tokens": 24771815.0, "reward": 1.5815625190734863, "reward_std": 0.2231055647134781, "rewards/rollout_reward_func/mean": 1.5815625190734863, "rewards/rollout_reward_func/std": 0.2131803333759308, "sampling/importance_sampling_ratio/max": 1.0174572467803955, "sampling/importance_sampling_ratio/mean": 1.0011612176895142, "sampling/importance_sampling_ratio/min": 0.994476318359375, "sampling/sampling_logp_difference/max": 0.012592941522598267, "sampling/sampling_logp_difference/mean": 0.0002547933254390955, "step": 524, "step_time": 23.92261946599865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2471.0, "completions/max_terminated_length": 2471.0, "completions/mean_length": 1728.375, "completions/mean_terminated_length": 1728.375, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.045267217326909304, "epoch": 0.042, "frac_reward_zero_std": 0.0, "grad_norm": 0.006997913122177124, "kl": 1.9633321017026901, "learning_rate": 4.3503150772697584e-05, "loss": 0.0064, "num_tokens": 24844323.0, "reward": 1.441249966621399, "reward_std": 0.28684788942337036, "rewards/rollout_reward_func/mean": 1.441249966621399, "rewards/rollout_reward_func/std": 0.33891427516937256, "sampling/importance_sampling_ratio/max": 1.0320792198181152, "sampling/importance_sampling_ratio/mean": 0.9959279894828796, "sampling/importance_sampling_ratio/min": 0.8420370817184448, "sampling/sampling_logp_difference/max": 0.18280506134033203, "sampling/sampling_logp_difference/mean": 0.0013428464299067855, "step": 525, "step_time": 23.026867676997426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2497.0, "completions/max_terminated_length": 2497.0, "completions/mean_length": 1682.9375, "completions/mean_terminated_length": 1682.9375, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.0320795823354274, "epoch": 0.04208, "frac_reward_zero_std": 0.0, "grad_norm": 0.020697196945548058, "kl": 2.026650086045265, "learning_rate": 4.3503024306355255e-05, "loss": 0.0061, "num_tokens": 24914946.0, "reward": 1.519687533378601, "reward_std": 0.20263424515724182, "rewards/rollout_reward_func/mean": 1.519687533378601, "rewards/rollout_reward_func/std": 0.1971651166677475, "sampling/importance_sampling_ratio/max": 1.0125550031661987, "sampling/importance_sampling_ratio/mean": 0.9977290630340576, "sampling/importance_sampling_ratio/min": 0.9450276494026184, "sampling/sampling_logp_difference/max": 0.05903339385986328, "sampling/sampling_logp_difference/mean": 0.0004674694500863552, "step": 526, "step_time": 22.991187001996877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2524.0, "completions/max_terminated_length": 2524.0, "completions/mean_length": 1510.4375, "completions/mean_terminated_length": 1510.4375, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.03604439157061279, "epoch": 0.04216, "frac_reward_zero_std": 0.0, "grad_norm": 0.014208417385816574, "kl": 2.087314233183861, "learning_rate": 4.3502897581982574e-05, "loss": 0.0053, "num_tokens": 24979447.0, "reward": 1.4768749475479126, "reward_std": 0.15547645092010498, "rewards/rollout_reward_func/mean": 1.4768749475479126, "rewards/rollout_reward_func/std": 0.1991180032491684, "sampling/importance_sampling_ratio/max": 1.0822861194610596, "sampling/importance_sampling_ratio/mean": 1.0059757232666016, "sampling/importance_sampling_ratio/min": 0.9794338345527649, "sampling/sampling_logp_difference/max": 0.10004568099975586, "sampling/sampling_logp_difference/mean": 0.0006784912548027933, "step": 527, "step_time": 22.92427075500018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2545.0, "completions/max_terminated_length": 2545.0, "completions/mean_length": 1728.71875, "completions/mean_terminated_length": 1728.71875, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.06620468641631305, "epoch": 0.04224, "frac_reward_zero_std": 0.0, "grad_norm": 0.04850616678595543, "kl": 1.9304713159799576, "learning_rate": 4.350277059958155e-05, "loss": 0.0151, "num_tokens": 25051798.0, "reward": 1.5595312118530273, "reward_std": 0.18295279145240784, "rewards/rollout_reward_func/mean": 1.5595312118530273, "rewards/rollout_reward_func/std": 0.19758360087871552, "sampling/importance_sampling_ratio/max": 1.0873664617538452, "sampling/importance_sampling_ratio/mean": 0.9896705150604248, "sampling/importance_sampling_ratio/min": 0.5517197251319885, "sampling/sampling_logp_difference/max": 0.45821523666381836, "sampling/sampling_logp_difference/mean": 0.003306081984192133, "step": 528, "step_time": 24.008410720998654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2517.0, "completions/max_terminated_length": 2517.0, "completions/mean_length": 1763.96875, "completions/mean_terminated_length": 1763.96875, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.0326174208894372, "epoch": 0.04232, "frac_reward_zero_std": 0.0, "grad_norm": 0.01426589023321867, "kl": 2.048347055912018, "learning_rate": 4.3502643359154184e-05, "loss": 0.0059, "num_tokens": 25125659.0, "reward": 1.5204167366027832, "reward_std": 0.14559946954250336, "rewards/rollout_reward_func/mean": 1.5204167366027832, "rewards/rollout_reward_func/std": 0.1844700425863266, "sampling/importance_sampling_ratio/max": 1.014338493347168, "sampling/importance_sampling_ratio/mean": 0.9942537546157837, "sampling/importance_sampling_ratio/min": 0.8177960515022278, "sampling/sampling_logp_difference/max": 0.34892845153808594, "sampling/sampling_logp_difference/mean": 0.0009875374380499125, "step": 529, "step_time": 23.600430205995508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2537.0, "completions/max_terminated_length": 2537.0, "completions/mean_length": 1841.5, "completions/mean_terminated_length": 1841.5, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.036848613526672125, "epoch": 0.0424, "frac_reward_zero_std": 0.0, "grad_norm": 0.013457071967422962, "kl": 1.898549661040306, "learning_rate": 4.350251586070251e-05, "loss": 0.0063, "num_tokens": 25201743.0, "reward": 1.5606250762939453, "reward_std": 0.15666446089744568, "rewards/rollout_reward_func/mean": 1.5606250762939453, "rewards/rollout_reward_func/std": 0.19668109714984894, "sampling/importance_sampling_ratio/max": 1.0236923694610596, "sampling/importance_sampling_ratio/mean": 0.9967292547225952, "sampling/importance_sampling_ratio/min": 0.8953792452812195, "sampling/sampling_logp_difference/max": 0.14081192016601562, "sampling/sampling_logp_difference/mean": 0.000998748349957168, "step": 530, "step_time": 23.189002225004515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2111.0, "completions/max_terminated_length": 2111.0, "completions/mean_length": 1668.28125, "completions/mean_terminated_length": 1668.28125, "completions/min_length": 1380.0, "completions/min_terminated_length": 1380.0, "entropy": 0.04741775966249406, "epoch": 0.04248, "frac_reward_zero_std": 0.0, "grad_norm": 0.017987962812185287, "kl": 2.1011915057897568, "learning_rate": 4.350238810422853e-05, "loss": 0.0068, "num_tokens": 25272633.0, "reward": 1.5162500143051147, "reward_std": 0.19294676184654236, "rewards/rollout_reward_func/mean": 1.5162500143051147, "rewards/rollout_reward_func/std": 0.18932390213012695, "sampling/importance_sampling_ratio/max": 1.2200642824172974, "sampling/importance_sampling_ratio/mean": 1.0005600452423096, "sampling/importance_sampling_ratio/min": 0.8397864103317261, "sampling/sampling_logp_difference/max": 0.13243359327316284, "sampling/sampling_logp_difference/mean": 0.0015937354182824492, "step": 531, "step_time": 21.444081631996596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2536.0, "completions/max_terminated_length": 2536.0, "completions/mean_length": 1859.53125, "completions/mean_terminated_length": 1859.53125, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.026396954897791147, "epoch": 0.04256, "frac_reward_zero_std": 0.0, "grad_norm": 0.011700834147632122, "kl": 1.8505584448575974, "learning_rate": 4.350226008973427e-05, "loss": 0.0043, "num_tokens": 25349842.0, "reward": 1.5465278625488281, "reward_std": 0.17041140794754028, "rewards/rollout_reward_func/mean": 1.5465278625488281, "rewards/rollout_reward_func/std": 0.21283406019210815, "sampling/importance_sampling_ratio/max": 1.0564652681350708, "sampling/importance_sampling_ratio/mean": 1.0022985935211182, "sampling/importance_sampling_ratio/min": 0.98643958568573, "sampling/sampling_logp_difference/max": 0.12457942962646484, "sampling/sampling_logp_difference/mean": 0.000546099676284939, "step": 532, "step_time": 23.899236032997578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2118.0, "completions/max_terminated_length": 2118.0, "completions/mean_length": 1551.65625, "completions/mean_terminated_length": 1551.65625, "completions/min_length": 1146.0, "completions/min_terminated_length": 1146.0, "entropy": 0.051479209680110216, "epoch": 0.04264, "frac_reward_zero_std": 0.0, "grad_norm": 0.031867314130067825, "kl": 2.409143105149269, "learning_rate": 4.350213181722176e-05, "loss": 0.0097, "num_tokens": 25415879.0, "reward": 1.4557812213897705, "reward_std": 0.16032671928405762, "rewards/rollout_reward_func/mean": 1.4557812213897705, "rewards/rollout_reward_func/std": 0.1805417537689209, "sampling/importance_sampling_ratio/max": 1.143579363822937, "sampling/importance_sampling_ratio/mean": 0.9666060209274292, "sampling/importance_sampling_ratio/min": 7.625167164349023e-11, "sampling/sampling_logp_difference/max": 19.015134811401367, "sampling/sampling_logp_difference/mean": 0.03186348080635071, "step": 533, "step_time": 21.842748194003434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2502.0, "completions/max_terminated_length": 2502.0, "completions/mean_length": 1790.09375, "completions/mean_terminated_length": 1790.09375, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.026098836679011583, "epoch": 0.04272, "frac_reward_zero_std": 0.0, "grad_norm": 0.0021260175853967667, "kl": 1.8844581544399261, "learning_rate": 4.350200328669304e-05, "loss": 0.0055, "num_tokens": 25490245.0, "reward": 1.5262500047683716, "reward_std": 0.15318962931632996, "rewards/rollout_reward_func/mean": 1.5262500047683716, "rewards/rollout_reward_func/std": 0.19567498564720154, "sampling/importance_sampling_ratio/max": 1.015159249305725, "sampling/importance_sampling_ratio/mean": 1.001082420349121, "sampling/importance_sampling_ratio/min": 0.9927798509597778, "sampling/sampling_logp_difference/max": 0.011455483734607697, "sampling/sampling_logp_difference/mean": 0.0002638774458318949, "step": 534, "step_time": 22.94502389900117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0014534883666783571, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014534883666783571, "completions/clipped_ratio": 0.0, "completions/max_length": 2514.0, "completions/max_terminated_length": 2514.0, "completions/mean_length": 1587.46875, "completions/mean_terminated_length": 1587.46875, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.02830645372159779, "epoch": 0.0428, "frac_reward_zero_std": 0.0, "grad_norm": 0.0022946796379983425, "kl": 1.9954386353492737, "learning_rate": 4.350187449815013e-05, "loss": 0.0056, "num_tokens": 25557262.0, "reward": 1.532812476158142, "reward_std": 0.15233224630355835, "rewards/rollout_reward_func/mean": 1.532812476158142, "rewards/rollout_reward_func/std": 0.17291118204593658, "sampling/importance_sampling_ratio/max": 1.2940713167190552, "sampling/importance_sampling_ratio/mean": 1.0078811645507812, "sampling/importance_sampling_ratio/min": 0.9832314848899841, "sampling/sampling_logp_difference/max": 0.24964570999145508, "sampling/sampling_logp_difference/mean": 0.0007296758703887463, "step": 535, "step_time": 22.638667481000084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2067.0, "completions/max_terminated_length": 2067.0, "completions/mean_length": 1520.03125, "completions/mean_terminated_length": 1520.03125, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.04499837849289179, "epoch": 0.04288, "frac_reward_zero_std": 0.0, "grad_norm": 0.024165090173482895, "kl": 2.108418256044388, "learning_rate": 4.3501745451595074e-05, "loss": 0.0103, "num_tokens": 25622048.0, "reward": 1.4603126049041748, "reward_std": 0.14107036590576172, "rewards/rollout_reward_func/mean": 1.4603126049041748, "rewards/rollout_reward_func/std": 0.12912815809249878, "sampling/importance_sampling_ratio/max": 1.1976165771484375, "sampling/importance_sampling_ratio/mean": 0.9985228776931763, "sampling/importance_sampling_ratio/min": 0.751528799533844, "sampling/sampling_logp_difference/max": 0.12377643585205078, "sampling/sampling_logp_difference/mean": 0.0016350043006241322, "step": 536, "step_time": 22.204628977997345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2542.0, "completions/max_terminated_length": 2542.0, "completions/mean_length": 1908.75, "completions/mean_terminated_length": 1908.75, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.04285839991644025, "epoch": 0.04296, "frac_reward_zero_std": 0.0, "grad_norm": 0.026015760377049446, "kl": 1.8152785152196884, "learning_rate": 4.3501616147029925e-05, "loss": 0.0039, "num_tokens": 25700834.0, "reward": 1.5706250667572021, "reward_std": 0.18400920927524567, "rewards/rollout_reward_func/mean": 1.5706250667572021, "rewards/rollout_reward_func/std": 0.20075559616088867, "sampling/importance_sampling_ratio/max": 1.1581038236618042, "sampling/importance_sampling_ratio/mean": 1.0062812566757202, "sampling/importance_sampling_ratio/min": 0.9911812543869019, "sampling/sampling_logp_difference/max": 0.11508321762084961, "sampling/sampling_logp_difference/mean": 0.0010693692602217197, "step": 537, "step_time": 23.646109009998327 }, { "clip_ratio/high_max": 0.005067567341029644, "clip_ratio/high_mean": 0.002533783670514822, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002533783670514822, "completions/clipped_ratio": 0.0, "completions/max_length": 2336.0, "completions/max_terminated_length": 2336.0, "completions/mean_length": 1642.5625, "completions/mean_terminated_length": 1642.5625, "completions/min_length": 1053.0, "completions/min_terminated_length": 1053.0, "entropy": 0.05471584014594555, "epoch": 0.04304, "frac_reward_zero_std": 0.0, "grad_norm": 0.02208041585981846, "kl": 1.8992971926927567, "learning_rate": 4.3501486584456717e-05, "loss": 0.0075, "num_tokens": 25770377.0, "reward": 1.5256249904632568, "reward_std": 0.18349377810955048, "rewards/rollout_reward_func/mean": 1.5256249904632568, "rewards/rollout_reward_func/std": 0.18236680328845978, "sampling/importance_sampling_ratio/max": 1.3486698865890503, "sampling/importance_sampling_ratio/mean": 0.985505998134613, "sampling/importance_sampling_ratio/min": 0.4401036500930786, "sampling/sampling_logp_difference/max": 0.57818603515625, "sampling/sampling_logp_difference/mean": 0.004465297795832157, "step": 538, "step_time": 22.98309795100249 }, { "clip_ratio/high_max": 0.0022321429569274187, "clip_ratio/high_mean": 0.0011160714784637094, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011160714784637094, "completions/clipped_ratio": 0.0, "completions/max_length": 2139.0, "completions/max_terminated_length": 2139.0, "completions/mean_length": 1570.59375, "completions/mean_terminated_length": 1570.59375, "completions/min_length": 1381.0, "completions/min_terminated_length": 1381.0, "entropy": 0.04669005796313286, "epoch": 0.04312, "frac_reward_zero_std": 0.0, "grad_norm": 0.05847304314374924, "kl": 2.1686124205589294, "learning_rate": 4.350135676387751e-05, "loss": 0.004, "num_tokens": 25837257.0, "reward": 1.5028274059295654, "reward_std": 0.18949373066425323, "rewards/rollout_reward_func/mean": 1.5028274059295654, "rewards/rollout_reward_func/std": 0.1884104609489441, "sampling/importance_sampling_ratio/max": 1.2997839450836182, "sampling/importance_sampling_ratio/mean": 0.9912744760513306, "sampling/importance_sampling_ratio/min": 0.6143444180488586, "sampling/sampling_logp_difference/max": 0.4856724739074707, "sampling/sampling_logp_difference/mean": 0.002252718899399042, "step": 539, "step_time": 21.438865599000565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2454.0, "completions/max_terminated_length": 2454.0, "completions/mean_length": 1637.71875, "completions/mean_terminated_length": 1637.71875, "completions/min_length": 1140.0, "completions/min_terminated_length": 1140.0, "entropy": 0.05808762810193002, "epoch": 0.0432, "frac_reward_zero_std": 0.0, "grad_norm": 0.06962622702121735, "kl": 2.147458180785179, "learning_rate": 4.3501226685294365e-05, "loss": 0.0054, "num_tokens": 25906622.0, "reward": 1.5053125619888306, "reward_std": 0.14637909829616547, "rewards/rollout_reward_func/mean": 1.5053125619888306, "rewards/rollout_reward_func/std": 0.19220255315303802, "sampling/importance_sampling_ratio/max": 1.4051705598831177, "sampling/importance_sampling_ratio/mean": 1.0050350427627563, "sampling/importance_sampling_ratio/min": 0.44215327501296997, "sampling/sampling_logp_difference/max": 0.3442695140838623, "sampling/sampling_logp_difference/mean": 0.0029943459667265415, "step": 540, "step_time": 23.884721976999572 }, { "clip_ratio/high_max": 0.0028409091755747795, "clip_ratio/high_mean": 0.0014204545877873898, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014204545877873898, "completions/clipped_ratio": 0.0, "completions/max_length": 2557.0, "completions/max_terminated_length": 2557.0, "completions/mean_length": 1523.3125, "completions/mean_terminated_length": 1523.3125, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.04800809687003493, "epoch": 0.04328, "frac_reward_zero_std": 0.0, "grad_norm": 0.021456683054566383, "kl": 2.1483989357948303, "learning_rate": 4.350109634870933e-05, "loss": 0.0072, "num_tokens": 25971513.0, "reward": 1.457187533378601, "reward_std": 0.17738093435764313, "rewards/rollout_reward_func/mean": 1.457187533378601, "rewards/rollout_reward_func/std": 0.18804995715618134, "sampling/importance_sampling_ratio/max": 1.1447781324386597, "sampling/importance_sampling_ratio/mean": 0.9896172881126404, "sampling/importance_sampling_ratio/min": 0.6353939175605774, "sampling/sampling_logp_difference/max": 0.5614204406738281, "sampling/sampling_logp_difference/mean": 0.0025677443481981754, "step": 541, "step_time": 23.799264678002146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2492.0, "completions/max_terminated_length": 2492.0, "completions/mean_length": 1747.90625, "completions/mean_terminated_length": 1747.90625, "completions/min_length": 1387.0, "completions/min_terminated_length": 1387.0, "entropy": 0.026777496561408043, "epoch": 0.04336, "frac_reward_zero_std": 0.0, "grad_norm": 0.007591178175061941, "kl": 1.80949866771698, "learning_rate": 4.350096575412447e-05, "loss": 0.0044, "num_tokens": 26045170.0, "reward": 1.5049999952316284, "reward_std": 0.15999835729599, "rewards/rollout_reward_func/mean": 1.5049999952316284, "rewards/rollout_reward_func/std": 0.16529540717601776, "sampling/importance_sampling_ratio/max": 1.013851284980774, "sampling/importance_sampling_ratio/mean": 0.9974393248558044, "sampling/importance_sampling_ratio/min": 0.9416625499725342, "sampling/sampling_logp_difference/max": 0.062476158142089844, "sampling/sampling_logp_difference/mean": 0.0003856163239106536, "step": 542, "step_time": 23.56437964699944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2536.0, "completions/max_terminated_length": 2536.0, "completions/mean_length": 1872.5625, "completions/mean_terminated_length": 1872.5625, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.025515927700325847, "epoch": 0.04344, "frac_reward_zero_std": 0.0, "grad_norm": 0.0020774444565176964, "kl": 1.8493850082159042, "learning_rate": 4.3500834901541864e-05, "loss": 0.0058, "num_tokens": 26122964.0, "reward": 1.501562476158142, "reward_std": 0.1799222230911255, "rewards/rollout_reward_func/mean": 1.501562476158142, "rewards/rollout_reward_func/std": 0.20528669655323029, "sampling/importance_sampling_ratio/max": 1.0101276636123657, "sampling/importance_sampling_ratio/mean": 1.001235008239746, "sampling/importance_sampling_ratio/min": 0.9908522963523865, "sampling/sampling_logp_difference/max": 0.010425183922052383, "sampling/sampling_logp_difference/mean": 0.00029094115598127246, "step": 543, "step_time": 23.21633107299931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2555.0, "completions/max_terminated_length": 2555.0, "completions/mean_length": 1859.1875, "completions/mean_terminated_length": 1859.1875, "completions/min_length": 1379.0, "completions/min_terminated_length": 1379.0, "entropy": 0.048524091485887766, "epoch": 0.04352, "frac_reward_zero_std": 0.0, "grad_norm": 0.051885124295949936, "kl": 1.7473546117544174, "learning_rate": 4.350070379096357e-05, "loss": 0.0015, "num_tokens": 26200445.0, "reward": 1.569836139678955, "reward_std": 0.2721417546272278, "rewards/rollout_reward_func/mean": 1.569836139678955, "rewards/rollout_reward_func/std": 0.28923213481903076, "sampling/importance_sampling_ratio/max": 1.2309701442718506, "sampling/importance_sampling_ratio/mean": 0.9900065660476685, "sampling/importance_sampling_ratio/min": 0.7545739412307739, "sampling/sampling_logp_difference/max": 0.33124876022338867, "sampling/sampling_logp_difference/mean": 0.0028353589586913586, "step": 544, "step_time": 23.787770280001496 }, { "clip_ratio/high_max": 0.0021186440717428923, "clip_ratio/high_mean": 0.0010593220358714461, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010593220358714461, "completions/clipped_ratio": 0.0, "completions/max_length": 2441.0, "completions/max_terminated_length": 2441.0, "completions/mean_length": 1890.34375, "completions/mean_terminated_length": 1890.34375, "completions/min_length": 1382.0, "completions/min_terminated_length": 1382.0, "entropy": 0.07674855762161314, "epoch": 0.0436, "frac_reward_zero_std": 0.0, "grad_norm": 0.02102776989340782, "kl": 1.6838184148073196, "learning_rate": 4.350057242239167e-05, "loss": 0.0035, "num_tokens": 26278689.0, "reward": 1.5503125190734863, "reward_std": 0.16798283159732819, "rewards/rollout_reward_func/mean": 1.5503125190734863, "rewards/rollout_reward_func/std": 0.19343221187591553, "sampling/importance_sampling_ratio/max": 1.6561241149902344, "sampling/importance_sampling_ratio/mean": 1.0032631158828735, "sampling/importance_sampling_ratio/min": 0.6346138119697571, "sampling/sampling_logp_difference/max": 0.270443320274353, "sampling/sampling_logp_difference/mean": 0.0028254883363842964, "step": 545, "step_time": 23.90786105299958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2464.0, "completions/max_terminated_length": 2464.0, "completions/mean_length": 1636.0, "completions/mean_terminated_length": 1636.0, "completions/min_length": 1144.0, "completions/min_terminated_length": 1144.0, "entropy": 0.12498916941694915, "epoch": 0.04368, "frac_reward_zero_std": 0.0, "grad_norm": 0.08116038143634796, "kl": 2.174970403313637, "learning_rate": 4.350044079582825e-05, "loss": 0.0155, "num_tokens": 26347856.0, "reward": 1.5087499618530273, "reward_std": 0.17946785688400269, "rewards/rollout_reward_func/mean": 1.5087499618530273, "rewards/rollout_reward_func/std": 0.17817620933055878, "sampling/importance_sampling_ratio/max": 1.8580220937728882, "sampling/importance_sampling_ratio/mean": 1.0631723403930664, "sampling/importance_sampling_ratio/min": 0.9818664789199829, "sampling/sampling_logp_difference/max": 0.2477855682373047, "sampling/sampling_logp_difference/mean": 0.0051519302651286125, "step": 546, "step_time": 23.463999371999307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2534.0, "completions/max_terminated_length": 2534.0, "completions/mean_length": 1880.75, "completions/mean_terminated_length": 1880.75, "completions/min_length": 1379.0, "completions/min_terminated_length": 1379.0, "entropy": 0.059320672415196896, "epoch": 0.04376, "frac_reward_zero_std": 0.0, "grad_norm": 0.02384720928966999, "kl": 1.7600453197956085, "learning_rate": 4.350030891127539e-05, "loss": 0.0059, "num_tokens": 26425983.0, "reward": 1.5299999713897705, "reward_std": 0.2084425687789917, "rewards/rollout_reward_func/mean": 1.5299999713897705, "rewards/rollout_reward_func/std": 0.20900683104991913, "sampling/importance_sampling_ratio/max": 1.041379451751709, "sampling/importance_sampling_ratio/mean": 0.9855839610099792, "sampling/importance_sampling_ratio/min": 0.7354808449745178, "sampling/sampling_logp_difference/max": 0.1773284673690796, "sampling/sampling_logp_difference/mean": 0.0019149644067510962, "step": 547, "step_time": 23.231511589998263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2136.0, "completions/max_terminated_length": 2136.0, "completions/mean_length": 1688.71875, "completions/mean_terminated_length": 1688.71875, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.09257537173107266, "epoch": 0.04384, "frac_reward_zero_std": 0.0, "grad_norm": 0.033831220120191574, "kl": 2.072347030043602, "learning_rate": 4.3500176768735175e-05, "loss": -0.0003, "num_tokens": 26497553.0, "reward": 1.4912500381469727, "reward_std": 0.13328033685684204, "rewards/rollout_reward_func/mean": 1.4912500381469727, "rewards/rollout_reward_func/std": 0.15743403136730194, "sampling/importance_sampling_ratio/max": 1.2184748649597168, "sampling/importance_sampling_ratio/mean": 0.996995210647583, "sampling/importance_sampling_ratio/min": 0.7606327533721924, "sampling/sampling_logp_difference/max": 0.17357254028320312, "sampling/sampling_logp_difference/mean": 0.0031706737354397774, "step": 548, "step_time": 22.99842892599736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2115.0, "completions/max_terminated_length": 2115.0, "completions/mean_length": 1616.875, "completions/mean_terminated_length": 1616.875, "completions/min_length": 1149.0, "completions/min_terminated_length": 1149.0, "entropy": 0.04012663848698139, "epoch": 0.04392, "frac_reward_zero_std": 0.0, "grad_norm": 0.015041007660329342, "kl": 2.0443381667137146, "learning_rate": 4.350004436820971e-05, "loss": 0.0052, "num_tokens": 26565251.0, "reward": 1.5290625095367432, "reward_std": 0.17444130778312683, "rewards/rollout_reward_func/mean": 1.5290625095367432, "rewards/rollout_reward_func/std": 0.17722517251968384, "sampling/importance_sampling_ratio/max": 1.018095850944519, "sampling/importance_sampling_ratio/mean": 0.9866976141929626, "sampling/importance_sampling_ratio/min": 0.7764726281166077, "sampling/sampling_logp_difference/max": 0.2493896484375, "sampling/sampling_logp_difference/mean": 0.0012262752279639244, "step": 549, "step_time": 22.064271940002072 }, { "clip_ratio/high_max": 0.0021929824724793434, "clip_ratio/high_mean": 0.0010964912362396717, "clip_ratio/low_mean": 0.0009920635493472219, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0020885547855868936, "completions/clipped_ratio": 0.0, "completions/max_length": 2535.0, "completions/max_terminated_length": 2535.0, "completions/mean_length": 2126.21875, "completions/mean_terminated_length": 2126.21875, "completions/min_length": 1377.0, "completions/min_terminated_length": 1377.0, "entropy": 0.08209028304554522, "epoch": 0.044, "frac_reward_zero_std": 0.0, "grad_norm": 0.09578518569469452, "kl": 1.6062247157096863, "learning_rate": 4.349991170970107e-05, "loss": 0.0107, "num_tokens": 26651883.0, "reward": 1.6159374713897705, "reward_std": 0.133689284324646, "rewards/rollout_reward_func/mean": 1.6159374713897705, "rewards/rollout_reward_func/std": 0.18811215460300446, "sampling/importance_sampling_ratio/max": 1.3554415702819824, "sampling/importance_sampling_ratio/mean": 1.0038940906524658, "sampling/importance_sampling_ratio/min": 0.8776599764823914, "sampling/sampling_logp_difference/max": 0.46654248237609863, "sampling/sampling_logp_difference/mean": 0.004268597811460495, "step": 550, "step_time": 23.351991220002674 }, { "clip_ratio/high_max": 0.0018382353009656072, "clip_ratio/high_mean": 0.0009191176504828036, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009191176504828036, "completions/clipped_ratio": 0.0, "completions/max_length": 2506.0, "completions/max_terminated_length": 2506.0, "completions/mean_length": 1797.03125, "completions/mean_terminated_length": 1797.03125, "completions/min_length": 1055.0, "completions/min_terminated_length": 1055.0, "entropy": 0.09377293242141604, "epoch": 0.04408, "frac_reward_zero_std": 0.0, "grad_norm": 0.0350271537899971, "kl": 1.73772694170475, "learning_rate": 4.349977879321138e-05, "loss": 0.0148, "num_tokens": 26726875.0, "reward": 1.4996875524520874, "reward_std": 0.2109169065952301, "rewards/rollout_reward_func/mean": 1.4996875524520874, "rewards/rollout_reward_func/std": 0.26960524916648865, "sampling/importance_sampling_ratio/max": 1.0153928995132446, "sampling/importance_sampling_ratio/mean": 0.9680243134498596, "sampling/importance_sampling_ratio/min": 0.6523696184158325, "sampling/sampling_logp_difference/max": 0.20540952682495117, "sampling/sampling_logp_difference/mean": 0.003692509373649955, "step": 551, "step_time": 23.926279602999784 }, { "clip_ratio/high_max": 0.0005230125389061868, "clip_ratio/high_mean": 0.0002615062694530934, "clip_ratio/low_mean": 0.0002615062694530934, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0005230125389061868, "completions/clipped_ratio": 0.0, "completions/max_length": 2138.0, "completions/max_terminated_length": 2138.0, "completions/mean_length": 1611.71875, "completions/mean_terminated_length": 1611.71875, "completions/min_length": 1056.0, "completions/min_terminated_length": 1056.0, "entropy": 0.12064902810379863, "epoch": 0.04416, "frac_reward_zero_std": 0.0, "grad_norm": 0.050203051418066025, "kl": 1.7155736237764359, "learning_rate": 4.349964561874273e-05, "loss": 0.032, "num_tokens": 26794739.0, "reward": 1.549375057220459, "reward_std": 0.15652653574943542, "rewards/rollout_reward_func/mean": 1.549375057220459, "rewards/rollout_reward_func/std": 0.15678083896636963, "sampling/importance_sampling_ratio/max": 1.2347828149795532, "sampling/importance_sampling_ratio/mean": 0.9717563986778259, "sampling/importance_sampling_ratio/min": 0.7630941867828369, "sampling/sampling_logp_difference/max": 0.23976421356201172, "sampling/sampling_logp_difference/mean": 0.00536512304097414, "step": 552, "step_time": 25.503486852003334 }, { "clip_ratio/high_max": 0.0015432098880410194, "clip_ratio/high_mean": 0.0007716049440205097, "clip_ratio/low_mean": 0.0012212452420499176, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0019928501860704273, "completions/clipped_ratio": 0.0, "completions/max_length": 2490.0, "completions/max_terminated_length": 2490.0, "completions/mean_length": 1685.03125, "completions/mean_terminated_length": 1685.03125, "completions/min_length": 1149.0, "completions/min_terminated_length": 1149.0, "entropy": 0.14934625662863255, "epoch": 0.04424, "frac_reward_zero_std": 0.0, "grad_norm": 0.03738493472337723, "kl": 1.8819137513637543, "learning_rate": 4.349951218629723e-05, "loss": 0.0097, "num_tokens": 26865546.0, "reward": 1.5915625095367432, "reward_std": 0.12222640961408615, "rewards/rollout_reward_func/mean": 1.5915625095367432, "rewards/rollout_reward_func/std": 0.16871637105941772, "sampling/importance_sampling_ratio/max": 1.637642502784729, "sampling/importance_sampling_ratio/mean": 0.9584327936172485, "sampling/importance_sampling_ratio/min": 2.2671726217855966e-14, "sampling/sampling_logp_difference/max": 15.658135414123535, "sampling/sampling_logp_difference/mean": 0.038651809096336365, "step": 553, "step_time": 25.040444143996865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2478.0, "completions/max_terminated_length": 2478.0, "completions/mean_length": 1635.4375, "completions/mean_terminated_length": 1635.4375, "completions/min_length": 1146.0, "completions/min_terminated_length": 1146.0, "entropy": 0.1279449425637722, "epoch": 0.04432, "frac_reward_zero_std": 0.0, "grad_norm": 0.04466257244348526, "kl": 2.093495160341263, "learning_rate": 4.349937849587699e-05, "loss": -0.0014, "num_tokens": 26934551.0, "reward": 1.5671875476837158, "reward_std": 0.16849735379219055, "rewards/rollout_reward_func/mean": 1.5671875476837158, "rewards/rollout_reward_func/std": 0.17454516887664795, "sampling/importance_sampling_ratio/max": 2.3874454498291016, "sampling/importance_sampling_ratio/mean": 1.0419942140579224, "sampling/importance_sampling_ratio/min": 0.620308518409729, "sampling/sampling_logp_difference/max": 0.3373095989227295, "sampling/sampling_logp_difference/mean": 0.0071032484993338585, "step": 554, "step_time": 23.83006384699911 }, { "clip_ratio/high_max": 0.0025404448388144374, "clip_ratio/high_mean": 0.0012702224194072187, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012702224194072187, "completions/clipped_ratio": 0.0, "completions/max_length": 2499.0, "completions/max_terminated_length": 2499.0, "completions/mean_length": 1834.1875, "completions/mean_terminated_length": 1834.1875, "completions/min_length": 1198.0, "completions/min_terminated_length": 1198.0, "entropy": 0.24182669166475534, "epoch": 0.0444, "frac_reward_zero_std": 0.0, "grad_norm": 0.08301857113838196, "kl": 1.7063368409872055, "learning_rate": 4.3499244547484135e-05, "loss": -0.0094, "num_tokens": 27010591.0, "reward": 1.533750057220459, "reward_std": 0.16818106174468994, "rewards/rollout_reward_func/mean": 1.533750057220459, "rewards/rollout_reward_func/std": 0.17813999950885773, "sampling/importance_sampling_ratio/max": 1.958703637123108, "sampling/importance_sampling_ratio/mean": 1.0783041715621948, "sampling/importance_sampling_ratio/min": 0.5907301306724548, "sampling/sampling_logp_difference/max": 0.5870532989501953, "sampling/sampling_logp_difference/mean": 0.010583529248833656, "step": 555, "step_time": 25.466903193997496 }, { "clip_ratio/high_max": 0.003961054724641144, "clip_ratio/high_mean": 0.001980527362320572, "clip_ratio/low_mean": 0.0013635852374136448, "clip_ratio/low_min": 0.0005341880605556071, "clip_ratio/region_mean": 0.003344112599734217, "completions/clipped_ratio": 0.0, "completions/max_length": 2561.0, "completions/max_terminated_length": 2561.0, "completions/mean_length": 1696.9375, "completions/mean_terminated_length": 1696.9375, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.26357199251651764, "epoch": 0.04448, "frac_reward_zero_std": 0.0, "grad_norm": 0.07424619048833847, "kl": 1.5275772511959076, "learning_rate": 4.3499110341120794e-05, "loss": -0.0002, "num_tokens": 27081304.0, "reward": 1.5968749523162842, "reward_std": 0.16936732828617096, "rewards/rollout_reward_func/mean": 1.5968749523162842, "rewards/rollout_reward_func/std": 0.1869523972272873, "sampling/importance_sampling_ratio/max": 1.7808303833007812, "sampling/importance_sampling_ratio/mean": 0.9544331431388855, "sampling/importance_sampling_ratio/min": 7.190084105477496e-21, "sampling/sampling_logp_difference/max": 11.281850814819336, "sampling/sampling_logp_difference/mean": 0.04952601343393326, "step": 556, "step_time": 26.89296980400104 }, { "clip_ratio/high_max": 0.007964750519022346, "clip_ratio/high_mean": 0.0049744389252737164, "clip_ratio/low_mean": 0.002518232970032841, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007492672011721879, "completions/clipped_ratio": 0.0, "completions/max_length": 1888.0, "completions/max_terminated_length": 1888.0, "completions/mean_length": 1555.1875, "completions/mean_terminated_length": 1555.1875, "completions/min_length": 1056.0, "completions/min_terminated_length": 1056.0, "entropy": 0.30075441859662533, "epoch": 0.04456, "frac_reward_zero_std": 0.0, "grad_norm": 0.07879248261451721, "kl": 1.572910264134407, "learning_rate": 4.349897587678907e-05, "loss": 0.0299, "num_tokens": 27147306.0, "reward": 1.6509374380111694, "reward_std": 0.11498508602380753, "rewards/rollout_reward_func/mean": 1.6509374380111694, "rewards/rollout_reward_func/std": 0.14186224341392517, "sampling/importance_sampling_ratio/max": 2.590968132019043, "sampling/importance_sampling_ratio/mean": 0.9670878052711487, "sampling/importance_sampling_ratio/min": 0.5083831548690796, "sampling/sampling_logp_difference/max": 0.5345461368560791, "sampling/sampling_logp_difference/mean": 0.015501557849347591, "step": 557, "step_time": 22.959462919001453 }, { "clip_ratio/high_max": 0.0032259407453238964, "clip_ratio/high_mean": 0.0016129703726619482, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016129703726619482, "completions/clipped_ratio": 0.0, "completions/max_length": 2211.0, "completions/max_terminated_length": 2211.0, "completions/mean_length": 1728.40625, "completions/mean_terminated_length": 1728.40625, "completions/min_length": 1229.0, "completions/min_terminated_length": 1229.0, "entropy": 0.2648983057588339, "epoch": 0.04464, "frac_reward_zero_std": 0.0, "grad_norm": 0.10021505504846573, "kl": 1.675349622964859, "learning_rate": 4.349884115449111e-05, "loss": 0.0438, "num_tokens": 27219238.0, "reward": 1.6806249618530273, "reward_std": 0.1633244901895523, "rewards/rollout_reward_func/mean": 1.6806249618530273, "rewards/rollout_reward_func/std": 0.20133325457572937, "sampling/importance_sampling_ratio/max": 1.4975481033325195, "sampling/importance_sampling_ratio/mean": 1.0885347127914429, "sampling/importance_sampling_ratio/min": 0.7181669473648071, "sampling/sampling_logp_difference/max": 0.2846825122833252, "sampling/sampling_logp_difference/mean": 0.012057938612997532, "step": 558, "step_time": 24.365112999999837 }, { "clip_ratio/high_max": 0.00147058826405555, "clip_ratio/high_mean": 0.000735294132027775, "clip_ratio/low_mean": 0.001326400670222938, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002061694802250713, "completions/clipped_ratio": 0.0, "completions/max_length": 2463.0, "completions/max_terminated_length": 2463.0, "completions/mean_length": 1872.5, "completions/mean_terminated_length": 1872.5, "completions/min_length": 1215.0, "completions/min_terminated_length": 1215.0, "entropy": 0.2695632856339216, "epoch": 0.04472, "frac_reward_zero_std": 0.125, "grad_norm": 0.07155901938676834, "kl": 2.2983509302139282, "learning_rate": 4.349870617422904e-05, "loss": 0.0046, "num_tokens": 27296572.0, "reward": 1.6439062356948853, "reward_std": 0.1215234324336052, "rewards/rollout_reward_func/mean": 1.6439062356948853, "rewards/rollout_reward_func/std": 0.18774868547916412, "sampling/importance_sampling_ratio/max": 1.712899923324585, "sampling/importance_sampling_ratio/mean": 0.9620150327682495, "sampling/importance_sampling_ratio/min": 0.3851516842842102, "sampling/sampling_logp_difference/max": 0.3217787742614746, "sampling/sampling_logp_difference/mean": 0.011657848954200745, "step": 559, "step_time": 26.0191048960005 }, { "clip_ratio/high_max": 0.0017857142956927419, "clip_ratio/high_mean": 0.0017152256332337856, "clip_ratio/low_mean": 0.0006868132040835917, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024020388373173773, "completions/clipped_ratio": 0.0, "completions/max_length": 2453.0, "completions/max_terminated_length": 2453.0, "completions/mean_length": 1752.0625, "completions/mean_terminated_length": 1752.0625, "completions/min_length": 1163.0, "completions/min_terminated_length": 1163.0, "entropy": 0.3119372986257076, "epoch": 0.0448, "frac_reward_zero_std": 0.125, "grad_norm": 0.07809140533208847, "kl": 1.5841660350561142, "learning_rate": 4.349857093600499e-05, "loss": 0.0298, "num_tokens": 27369358.0, "reward": 1.729374885559082, "reward_std": 0.061200372874736786, "rewards/rollout_reward_func/mean": 1.729374885559082, "rewards/rollout_reward_func/std": 0.07237614691257477, "sampling/importance_sampling_ratio/max": 1.4498153924942017, "sampling/importance_sampling_ratio/mean": 1.0310499668121338, "sampling/importance_sampling_ratio/min": 0.44821128249168396, "sampling/sampling_logp_difference/max": 0.730445384979248, "sampling/sampling_logp_difference/mean": 0.014362530782818794, "step": 560, "step_time": 27.295389714996418 }, { "clip_ratio/high_max": 0.002273478894494474, "clip_ratio/high_mean": 0.001136739447247237, "clip_ratio/low_mean": 0.00101711560273543, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002153855049982667, "completions/clipped_ratio": 0.0, "completions/max_length": 2225.0, "completions/max_terminated_length": 2225.0, "completions/mean_length": 1692.25, "completions/mean_terminated_length": 1692.25, "completions/min_length": 1134.0, "completions/min_terminated_length": 1134.0, "entropy": 0.2988389674574137, "epoch": 0.04488, "frac_reward_zero_std": 0.0, "grad_norm": 0.08996995538473129, "kl": 1.6974306851625443, "learning_rate": 4.3498435439821125e-05, "loss": 0.0091, "num_tokens": 27439253.0, "reward": 1.6437499523162842, "reward_std": 0.13871850073337555, "rewards/rollout_reward_func/mean": 1.6437499523162842, "rewards/rollout_reward_func/std": 0.2019221931695938, "sampling/importance_sampling_ratio/max": 1.6546560525894165, "sampling/importance_sampling_ratio/mean": 0.9621283411979675, "sampling/importance_sampling_ratio/min": 0.3935607075691223, "sampling/sampling_logp_difference/max": 0.7041022777557373, "sampling/sampling_logp_difference/mean": 0.0147918201982975, "step": 561, "step_time": 25.057142325000314 }, { "clip_ratio/high_max": 0.007391635328531265, "clip_ratio/high_mean": 0.0036958176642656326, "clip_ratio/low_mean": 0.0038792883860878646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007575106050353497, "completions/clipped_ratio": 0.0, "completions/max_length": 2531.0, "completions/max_terminated_length": 2531.0, "completions/mean_length": 1957.59375, "completions/mean_terminated_length": 1957.59375, "completions/min_length": 1227.0, "completions/min_terminated_length": 1227.0, "entropy": 0.2723678592592478, "epoch": 0.04496, "frac_reward_zero_std": 0.0, "grad_norm": 0.07942330837249756, "kl": 1.7124570608139038, "learning_rate": 4.349829968567957e-05, "loss": 0.0175, "num_tokens": 27519361.0, "reward": 1.7081248760223389, "reward_std": 0.08110970258712769, "rewards/rollout_reward_func/mean": 1.7081248760223389, "rewards/rollout_reward_func/std": 0.12145455926656723, "sampling/importance_sampling_ratio/max": 1.507922649383545, "sampling/importance_sampling_ratio/mean": 1.0509501695632935, "sampling/importance_sampling_ratio/min": 0.4353257417678833, "sampling/sampling_logp_difference/max": 0.8417297601699829, "sampling/sampling_logp_difference/mean": 0.014329489320516586, "step": 562, "step_time": 25.574481809002464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2200.0, "completions/max_terminated_length": 2200.0, "completions/mean_length": 1751.625, "completions/mean_terminated_length": 1751.625, "completions/min_length": 1134.0, "completions/min_terminated_length": 1134.0, "entropy": 0.19459212571382523, "epoch": 0.04504, "frac_reward_zero_std": 0.0, "grad_norm": 1.4682766199111938, "kl": 8.051658570766449, "learning_rate": 4.3498163673582475e-05, "loss": 0.0235, "num_tokens": 27592228.0, "reward": 1.6793749332427979, "reward_std": 0.12273453176021576, "rewards/rollout_reward_func/mean": 1.6793749332427979, "rewards/rollout_reward_func/std": 0.17886626720428467, "sampling/importance_sampling_ratio/max": 1.4179431200027466, "sampling/importance_sampling_ratio/mean": 1.0281195640563965, "sampling/importance_sampling_ratio/min": 8.211151936166061e-08, "sampling/sampling_logp_difference/max": 8.609164237976074, "sampling/sampling_logp_difference/mean": 0.023136354982852936, "step": 563, "step_time": 23.9249623529995 }, { "clip_ratio/high_max": 0.004620040534064174, "clip_ratio/high_mean": 0.002310020267032087, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002310020267032087, "completions/clipped_ratio": 0.0, "completions/max_length": 2221.0, "completions/max_terminated_length": 2221.0, "completions/mean_length": 1452.40625, "completions/mean_terminated_length": 1452.40625, "completions/min_length": 1134.0, "completions/min_terminated_length": 1134.0, "entropy": 0.18447775021195412, "epoch": 0.04512, "frac_reward_zero_std": 0.0, "grad_norm": 0.04670782387256622, "kl": 2.0097319930791855, "learning_rate": 4.349802740353201e-05, "loss": 0.0077, "num_tokens": 27654153.0, "reward": 1.5893750190734863, "reward_std": 0.19048874080181122, "rewards/rollout_reward_func/mean": 1.5893750190734863, "rewards/rollout_reward_func/std": 0.22958499193191528, "sampling/importance_sampling_ratio/max": 1.4446996450424194, "sampling/importance_sampling_ratio/mean": 0.9569553136825562, "sampling/importance_sampling_ratio/min": 0.2740311920642853, "sampling/sampling_logp_difference/max": 1.2525737285614014, "sampling/sampling_logp_difference/mean": 0.010853953659534454, "step": 564, "step_time": 24.156283883998185 }, { "clip_ratio/high_max": 0.0018115942366421223, "clip_ratio/high_mean": 0.0009057971183210611, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009057971183210611, "completions/clipped_ratio": 0.0, "completions/max_length": 2185.0, "completions/max_terminated_length": 2185.0, "completions/mean_length": 1836.5625, "completions/mean_terminated_length": 1836.5625, "completions/min_length": 1497.0, "completions/min_terminated_length": 1497.0, "entropy": 0.17587346490472555, "epoch": 0.0452, "frac_reward_zero_std": 0.0, "grad_norm": 0.03648257628083229, "kl": 1.7872688621282578, "learning_rate": 4.3497890875530315e-05, "loss": 0.0217, "num_tokens": 27730026.0, "reward": 1.7540624141693115, "reward_std": 0.06052055582404137, "rewards/rollout_reward_func/mean": 1.7540624141693115, "rewards/rollout_reward_func/std": 0.07317057251930237, "sampling/importance_sampling_ratio/max": 1.4165173768997192, "sampling/importance_sampling_ratio/mean": 0.9838052988052368, "sampling/importance_sampling_ratio/min": 0.3676702082157135, "sampling/sampling_logp_difference/max": 0.3475522994995117, "sampling/sampling_logp_difference/mean": 0.008870301768183708, "step": 565, "step_time": 23.367854685997372 }, { "clip_ratio/high_max": 0.0018382353009656072, "clip_ratio/high_mean": 0.0009191176504828036, "clip_ratio/low_mean": 0.0016121031949296594, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002531220845412463, "completions/clipped_ratio": 0.0, "completions/max_length": 2474.0, "completions/max_terminated_length": 2474.0, "completions/mean_length": 1606.71875, "completions/mean_terminated_length": 1606.71875, "completions/min_length": 1057.0, "completions/min_terminated_length": 1057.0, "entropy": 0.19604859314858913, "epoch": 0.04528, "frac_reward_zero_std": 0.0, "grad_norm": 0.08504826575517654, "kl": 1.8981517255306244, "learning_rate": 4.3497754089579565e-05, "loss": 0.0126, "num_tokens": 27797798.0, "reward": 1.6740624904632568, "reward_std": 0.08170949667692184, "rewards/rollout_reward_func/mean": 1.6740624904632568, "rewards/rollout_reward_func/std": 0.12450830638408661, "sampling/importance_sampling_ratio/max": 1.2933945655822754, "sampling/importance_sampling_ratio/mean": 0.9995920658111572, "sampling/importance_sampling_ratio/min": 0.7118127346038818, "sampling/sampling_logp_difference/max": 0.38362979888916016, "sampling/sampling_logp_difference/mean": 0.010831594467163086, "step": 566, "step_time": 24.815792526000223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2203.0, "completions/max_terminated_length": 2203.0, "completions/mean_length": 1759.15625, "completions/mean_terminated_length": 1759.15625, "completions/min_length": 1134.0, "completions/min_terminated_length": 1134.0, "entropy": 0.16112570278346539, "epoch": 0.04536, "frac_reward_zero_std": 0.0, "grad_norm": 0.04531143233180046, "kl": 1.8443554043769836, "learning_rate": 4.349761704568192e-05, "loss": 0.0127, "num_tokens": 27871096.0, "reward": 1.705937385559082, "reward_std": 0.09355080872774124, "rewards/rollout_reward_func/mean": 1.705937385559082, "rewards/rollout_reward_func/std": 0.13353408873081207, "sampling/importance_sampling_ratio/max": 1.311254858970642, "sampling/importance_sampling_ratio/mean": 0.9529883861541748, "sampling/importance_sampling_ratio/min": 0.4326734244823456, "sampling/sampling_logp_difference/max": 0.685328483581543, "sampling/sampling_logp_difference/mean": 0.007187620736658573, "step": 567, "step_time": 23.475187798998377 }, { "clip_ratio/high_max": 0.0015432098880410194, "clip_ratio/high_mean": 0.0007716049440205097, "clip_ratio/low_mean": 0.002973183640278876, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037447885842993855, "completions/clipped_ratio": 0.0, "completions/max_length": 2543.0, "completions/max_terminated_length": 2543.0, "completions/mean_length": 1928.78125, "completions/mean_terminated_length": 1928.78125, "completions/min_length": 1243.0, "completions/min_terminated_length": 1243.0, "entropy": 0.15768250916153193, "epoch": 0.04544, "frac_reward_zero_std": 0.0, "grad_norm": 0.05034246668219566, "kl": 1.9411802142858505, "learning_rate": 4.3497479743839544e-05, "loss": 0.014, "num_tokens": 27950375.0, "reward": 1.7325000762939453, "reward_std": 0.08428718149662018, "rewards/rollout_reward_func/mean": 1.7325000762939453, "rewards/rollout_reward_func/std": 0.15669985115528107, "sampling/importance_sampling_ratio/max": 1.4851716756820679, "sampling/importance_sampling_ratio/mean": 0.9846786260604858, "sampling/importance_sampling_ratio/min": 0.6477238535881042, "sampling/sampling_logp_difference/max": 0.5149555206298828, "sampling/sampling_logp_difference/mean": 0.00775672122836113, "step": 568, "step_time": 25.76959964300113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0010245901066809893, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0010245901066809893, "completions/clipped_ratio": 0.0, "completions/max_length": 2475.0, "completions/max_terminated_length": 2475.0, "completions/mean_length": 1834.5, "completions/mean_terminated_length": 1834.5, "completions/min_length": 1132.0, "completions/min_terminated_length": 1132.0, "entropy": 0.12457927968353033, "epoch": 0.04552, "frac_reward_zero_std": 0.0, "grad_norm": 0.032674964517354965, "kl": 1.960961490869522, "learning_rate": 4.349734218405463e-05, "loss": 0.013, "num_tokens": 28026325.0, "reward": 1.6974999904632568, "reward_std": 0.09374083578586578, "rewards/rollout_reward_func/mean": 1.6974999904632568, "rewards/rollout_reward_func/std": 0.13462011516094208, "sampling/importance_sampling_ratio/max": 1.5195244550704956, "sampling/importance_sampling_ratio/mean": 1.040621042251587, "sampling/importance_sampling_ratio/min": 0.8543474674224854, "sampling/sampling_logp_difference/max": 0.22226619720458984, "sampling/sampling_logp_difference/mean": 0.005051853135228157, "step": 569, "step_time": 24.673591862996545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2474.0, "completions/max_terminated_length": 2474.0, "completions/mean_length": 2024.0, "completions/mean_terminated_length": 2024.0, "completions/min_length": 1494.0, "completions/min_terminated_length": 1494.0, "entropy": 0.1221923353150487, "epoch": 0.0456, "frac_reward_zero_std": 0.0, "grad_norm": 0.046898845583200455, "kl": 2.427665740251541, "learning_rate": 4.3497204366329334e-05, "loss": 0.0173, "num_tokens": 28109131.0, "reward": 1.7042410373687744, "reward_std": 0.11311163008213043, "rewards/rollout_reward_func/mean": 1.7042410373687744, "rewards/rollout_reward_func/std": 0.23355497419834137, "sampling/importance_sampling_ratio/max": 1.3069090843200684, "sampling/importance_sampling_ratio/mean": 1.032242774963379, "sampling/importance_sampling_ratio/min": 0.7072084546089172, "sampling/sampling_logp_difference/max": 0.3089423179626465, "sampling/sampling_logp_difference/mean": 0.004667968489229679, "step": 570, "step_time": 24.460678906001704 }, { "clip_ratio/high_max": 0.0024509804788976908, "clip_ratio/high_mean": 0.0012254902394488454, "clip_ratio/low_mean": 0.0021057719131931663, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033312621526420116, "completions/clipped_ratio": 0.0, "completions/max_length": 2458.0, "completions/max_terminated_length": 2458.0, "completions/mean_length": 1709.0, "completions/mean_terminated_length": 1709.0, "completions/min_length": 1132.0, "completions/min_terminated_length": 1132.0, "entropy": 0.14442884642630816, "epoch": 0.04568, "frac_reward_zero_std": 0.0, "grad_norm": 0.05546554923057556, "kl": 2.1631762236356735, "learning_rate": 4.349706629066585e-05, "loss": 0.0196, "num_tokens": 28180656.0, "reward": 1.6696875095367432, "reward_std": 0.13226525485515594, "rewards/rollout_reward_func/mean": 1.6696875095367432, "rewards/rollout_reward_func/std": 0.16948659718036652, "sampling/importance_sampling_ratio/max": 1.7860743999481201, "sampling/importance_sampling_ratio/mean": 1.0350596904754639, "sampling/importance_sampling_ratio/min": 0.6745142936706543, "sampling/sampling_logp_difference/max": 0.44863295555114746, "sampling/sampling_logp_difference/mean": 0.005865622311830521, "step": 571, "step_time": 24.554651251994073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2530.0, "completions/max_terminated_length": 2530.0, "completions/mean_length": 1928.1875, "completions/mean_terminated_length": 1928.1875, "completions/min_length": 1193.0, "completions/min_terminated_length": 1193.0, "entropy": 0.11984306573867798, "epoch": 0.04576, "frac_reward_zero_std": 0.125, "grad_norm": 0.03460849076509476, "kl": 2.588063821196556, "learning_rate": 4.349692795706638e-05, "loss": 0.0097, "num_tokens": 28259560.0, "reward": 1.6924107074737549, "reward_std": 0.08292920887470245, "rewards/rollout_reward_func/mean": 1.6924107074737549, "rewards/rollout_reward_func/std": 0.19960276782512665, "sampling/importance_sampling_ratio/max": 1.3995558023452759, "sampling/importance_sampling_ratio/mean": 0.9934592247009277, "sampling/importance_sampling_ratio/min": 0.784706711769104, "sampling/sampling_logp_difference/max": 0.3692927360534668, "sampling/sampling_logp_difference/mean": 0.004182204604148865, "step": 572, "step_time": 26.228828486997372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2519.0, "completions/max_terminated_length": 2519.0, "completions/mean_length": 1947.28125, "completions/mean_terminated_length": 1947.28125, "completions/min_length": 1163.0, "completions/min_terminated_length": 1163.0, "entropy": 0.1673612529411912, "epoch": 0.04584, "frac_reward_zero_std": 0.0, "grad_norm": 0.0639609694480896, "kl": 2.0842553079128265, "learning_rate": 4.3496789365533074e-05, "loss": 0.0058, "num_tokens": 28339694.0, "reward": 1.6907811164855957, "reward_std": 0.13326877355575562, "rewards/rollout_reward_func/mean": 1.6907811164855957, "rewards/rollout_reward_func/std": 0.22281232476234436, "sampling/importance_sampling_ratio/max": 1.3636698722839355, "sampling/importance_sampling_ratio/mean": 0.9665248394012451, "sampling/importance_sampling_ratio/min": 0.7195063233375549, "sampling/sampling_logp_difference/max": 0.503192663192749, "sampling/sampling_logp_difference/mean": 0.006860829424113035, "step": 573, "step_time": 24.904537789998358 }, { "clip_ratio/high_max": 0.0016891892300918698, "clip_ratio/high_mean": 0.0008445946150459349, "clip_ratio/low_mean": 0.0009920635493472219, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0018366581643931568, "completions/clipped_ratio": 0.0, "completions/max_length": 2539.0, "completions/max_terminated_length": 2539.0, "completions/mean_length": 1753.9375, "completions/mean_terminated_length": 1753.9375, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.17372794728726149, "epoch": 0.04592, "frac_reward_zero_std": 0.0, "grad_norm": 0.05204189941287041, "kl": 2.4710699915885925, "learning_rate": 4.349665051606816e-05, "loss": 0.0104, "num_tokens": 28412421.0, "reward": 1.6715178489685059, "reward_std": 0.14113277196884155, "rewards/rollout_reward_func/mean": 1.6715178489685059, "rewards/rollout_reward_func/std": 0.21750572323799133, "sampling/importance_sampling_ratio/max": 1.6267319917678833, "sampling/importance_sampling_ratio/mean": 0.985988199710846, "sampling/importance_sampling_ratio/min": 0.4877658784389496, "sampling/sampling_logp_difference/max": 0.3659324645996094, "sampling/sampling_logp_difference/mean": 0.008504193276166916, "step": 574, "step_time": 24.18006124999738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0012254902394488454, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012254902394488454, "completions/clipped_ratio": 0.0, "completions/max_length": 2155.0, "completions/max_terminated_length": 2155.0, "completions/mean_length": 1727.1875, "completions/mean_terminated_length": 1727.1875, "completions/min_length": 1122.0, "completions/min_terminated_length": 1122.0, "entropy": 0.17555582709610462, "epoch": 0.046, "frac_reward_zero_std": 0.0, "grad_norm": 0.1045026183128357, "kl": 2.61502942442894, "learning_rate": 4.349651140867383e-05, "loss": 0.0067, "num_tokens": 28484601.0, "reward": 1.664374828338623, "reward_std": 0.09996221959590912, "rewards/rollout_reward_func/mean": 1.664374828338623, "rewards/rollout_reward_func/std": 0.13070373237133026, "sampling/importance_sampling_ratio/max": 1.783008337020874, "sampling/importance_sampling_ratio/mean": 1.0307470560073853, "sampling/importance_sampling_ratio/min": 0.739883303642273, "sampling/sampling_logp_difference/max": 0.35597896575927734, "sampling/sampling_logp_difference/mean": 0.008632591925561428, "step": 575, "step_time": 22.640885580005488 }, { "clip_ratio/high_max": 0.0017123287543654442, "clip_ratio/high_mean": 0.0008561643771827221, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008561643771827221, "completions/clipped_ratio": 0.0, "completions/max_length": 2488.0, "completions/max_terminated_length": 2488.0, "completions/mean_length": 1782.71875, "completions/mean_terminated_length": 1782.71875, "completions/min_length": 1189.0, "completions/min_terminated_length": 1189.0, "entropy": 0.13018779922276735, "epoch": 0.04608, "frac_reward_zero_std": 0.0, "grad_norm": 0.04283088818192482, "kl": 2.3376078456640244, "learning_rate": 4.349637204335229e-05, "loss": 0.0087, "num_tokens": 28558702.0, "reward": 1.7065625190734863, "reward_std": 0.04119975119829178, "rewards/rollout_reward_func/mean": 1.7065625190734863, "rewards/rollout_reward_func/std": 0.06889025121927261, "sampling/importance_sampling_ratio/max": 1.2632999420166016, "sampling/importance_sampling_ratio/mean": 0.975411593914032, "sampling/importance_sampling_ratio/min": 0.6762810349464417, "sampling/sampling_logp_difference/max": 0.36138153076171875, "sampling/sampling_logp_difference/mean": 0.005095350090414286, "step": 576, "step_time": 25.00654940500135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0006313131307251751, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006313131307251751, "completions/clipped_ratio": 0.0, "completions/max_length": 2726.0, "completions/max_terminated_length": 2726.0, "completions/mean_length": 2026.46875, "completions/mean_terminated_length": 2026.46875, "completions/min_length": 1056.0, "completions/min_terminated_length": 1056.0, "entropy": 0.09025986213237047, "epoch": 0.04616, "frac_reward_zero_std": 0.0, "grad_norm": 0.05624719336628914, "kl": 2.0298718214035034, "learning_rate": 4.349623242010573e-05, "loss": 0.014, "num_tokens": 28640678.0, "reward": 1.7387053966522217, "reward_std": 0.09498056769371033, "rewards/rollout_reward_func/mean": 1.7387053966522217, "rewards/rollout_reward_func/std": 0.1790841966867447, "sampling/importance_sampling_ratio/max": 1.42433500289917, "sampling/importance_sampling_ratio/mean": 0.9543570876121521, "sampling/importance_sampling_ratio/min": 0.4894407093524933, "sampling/sampling_logp_difference/max": 0.7163252830505371, "sampling/sampling_logp_difference/mean": 0.0053307171911001205, "step": 577, "step_time": 26.91666969400285 }, { "clip_ratio/high_max": 0.002314814832061529, "clip_ratio/high_mean": 0.0011574074160307646, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.0, "completions/max_length": 2755.0, "completions/max_terminated_length": 2755.0, "completions/mean_length": 1893.6875, "completions/mean_terminated_length": 1893.6875, "completions/min_length": 1136.0, "completions/min_terminated_length": 1136.0, "entropy": 0.06907094363123178, "epoch": 0.04624, "frac_reward_zero_std": 0.0, "grad_norm": 0.035323191434144974, "kl": 2.0834575295448303, "learning_rate": 4.349609253893637e-05, "loss": 0.0129, "num_tokens": 28717830.0, "reward": 1.6571874618530273, "reward_std": 0.14003990590572357, "rewards/rollout_reward_func/mean": 1.6571874618530273, "rewards/rollout_reward_func/std": 0.22481869161128998, "sampling/importance_sampling_ratio/max": 1.1765869855880737, "sampling/importance_sampling_ratio/mean": 0.9561488628387451, "sampling/importance_sampling_ratio/min": 0.6296385526657104, "sampling/sampling_logp_difference/max": 0.3734598159790039, "sampling/sampling_logp_difference/mean": 0.0032570860348641872, "step": 578, "step_time": 26.644360233001862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2737.0, "completions/max_terminated_length": 2737.0, "completions/mean_length": 1871.4375, "completions/mean_terminated_length": 1871.4375, "completions/min_length": 1224.0, "completions/min_terminated_length": 1224.0, "entropy": 0.04517357982695103, "epoch": 0.04632, "frac_reward_zero_std": 0.0, "grad_norm": 0.02058584801852703, "kl": 2.2432424128055573, "learning_rate": 4.349595239984644e-05, "loss": 0.0113, "num_tokens": 28794363.0, "reward": 1.6956249475479126, "reward_std": 0.16071583330631256, "rewards/rollout_reward_func/mean": 1.6956249475479126, "rewards/rollout_reward_func/std": 0.2645472586154938, "sampling/importance_sampling_ratio/max": 1.2466130256652832, "sampling/importance_sampling_ratio/mean": 1.0209741592407227, "sampling/importance_sampling_ratio/min": 0.8310979604721069, "sampling/sampling_logp_difference/max": 0.24889421463012695, "sampling/sampling_logp_difference/mean": 0.0017067217268049717, "step": 579, "step_time": 26.72699379400001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0011574074160307646, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0011574074160307646, "completions/clipped_ratio": 0.0, "completions/max_length": 2782.0, "completions/max_terminated_length": 2782.0, "completions/mean_length": 1866.09375, "completions/mean_terminated_length": 1866.09375, "completions/min_length": 1136.0, "completions/min_terminated_length": 1136.0, "entropy": 0.04054946918040514, "epoch": 0.0464, "frac_reward_zero_std": 0.0, "grad_norm": 0.03259061276912689, "kl": 2.4593409597873688, "learning_rate": 4.349581200283814e-05, "loss": 0.0039, "num_tokens": 28870796.0, "reward": 1.7305803298950195, "reward_std": 0.11683526635169983, "rewards/rollout_reward_func/mean": 1.7305803298950195, "rewards/rollout_reward_func/std": 0.1753723919391632, "sampling/importance_sampling_ratio/max": 1.2229523658752441, "sampling/importance_sampling_ratio/mean": 1.003584861755371, "sampling/importance_sampling_ratio/min": 0.8318158388137817, "sampling/sampling_logp_difference/max": 0.3731517791748047, "sampling/sampling_logp_difference/mean": 0.0016422397457063198, "step": 580, "step_time": 26.841951472999426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2757.0, "completions/max_terminated_length": 2757.0, "completions/mean_length": 2034.1875, "completions/mean_terminated_length": 2034.1875, "completions/min_length": 1133.0, "completions/min_terminated_length": 1133.0, "entropy": 0.034419408068060875, "epoch": 0.04648, "frac_reward_zero_std": 0.0, "grad_norm": 0.02469509094953537, "kl": 2.2702308744192123, "learning_rate": 4.3495671347913706e-05, "loss": 0.016, "num_tokens": 28952829.0, "reward": 1.7448958158493042, "reward_std": 0.13726797699928284, "rewards/rollout_reward_func/mean": 1.7448958158493042, "rewards/rollout_reward_func/std": 0.21221646666526794, "sampling/importance_sampling_ratio/max": 1.3021960258483887, "sampling/importance_sampling_ratio/mean": 1.0077567100524902, "sampling/importance_sampling_ratio/min": 0.892401397228241, "sampling/sampling_logp_difference/max": 0.2616448402404785, "sampling/sampling_logp_difference/mean": 0.001170506700873375, "step": 581, "step_time": 26.44780408800034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0007183908019214869, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007183908019214869, "completions/clipped_ratio": 0.0, "completions/max_length": 2781.0, "completions/max_terminated_length": 2781.0, "completions/mean_length": 2294.96875, "completions/mean_terminated_length": 2294.96875, "completions/min_length": 1174.0, "completions/min_terminated_length": 1174.0, "entropy": 0.03519403235986829, "epoch": 0.04656, "frac_reward_zero_std": 0.0, "grad_norm": 0.027611251920461655, "kl": 1.730541616678238, "learning_rate": 4.349553043507535e-05, "loss": 0.0185, "num_tokens": 29044476.0, "reward": 1.7546875476837158, "reward_std": 0.14828094840049744, "rewards/rollout_reward_func/mean": 1.7546875476837158, "rewards/rollout_reward_func/std": 0.23850318789482117, "sampling/importance_sampling_ratio/max": 1.2102420330047607, "sampling/importance_sampling_ratio/mean": 0.9983330965042114, "sampling/importance_sampling_ratio/min": 0.8414238691329956, "sampling/sampling_logp_difference/max": 0.1989959478378296, "sampling/sampling_logp_difference/mean": 0.0012281568488106132, "step": 582, "step_time": 26.87781286000063 }, { "clip_ratio/high_max": 0.0017123287543654442, "clip_ratio/high_mean": 0.0008561643771827221, "clip_ratio/low_mean": 0.002489035134203732, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003345199511386454, "completions/clipped_ratio": 0.0, "completions/max_length": 2724.0, "completions/max_terminated_length": 2724.0, "completions/mean_length": 2054.9375, "completions/mean_terminated_length": 2054.9375, "completions/min_length": 1622.0, "completions/min_terminated_length": 1622.0, "entropy": 0.03457553149200976, "epoch": 0.04664, "frac_reward_zero_std": 0.0, "grad_norm": 0.020429516211152077, "kl": 2.3155422806739807, "learning_rate": 4.349538926432532e-05, "loss": 0.0181, "num_tokens": 29127505.0, "reward": 1.7901735305786133, "reward_std": 0.09808497130870819, "rewards/rollout_reward_func/mean": 1.7901735305786133, "rewards/rollout_reward_func/std": 0.14725284278392792, "sampling/importance_sampling_ratio/max": 1.2382577657699585, "sampling/importance_sampling_ratio/mean": 0.9966978430747986, "sampling/importance_sampling_ratio/min": 0.6990998387336731, "sampling/sampling_logp_difference/max": 0.35802626609802246, "sampling/sampling_logp_difference/mean": 0.001726797898299992, "step": 583, "step_time": 28.078023140997175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2727.0, "completions/max_terminated_length": 2727.0, "completions/mean_length": 1864.5, "completions/mean_terminated_length": 1864.5, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.03709525661543012, "epoch": 0.04672, "frac_reward_zero_std": 0.0, "grad_norm": 0.017542075365781784, "kl": 2.036255031824112, "learning_rate": 4.3495247835665845e-05, "loss": 0.0162, "num_tokens": 29203911.0, "reward": 1.7712500095367432, "reward_std": 0.07881209254264832, "rewards/rollout_reward_func/mean": 1.7712500095367432, "rewards/rollout_reward_func/std": 0.18117448687553406, "sampling/importance_sampling_ratio/max": 1.306072473526001, "sampling/importance_sampling_ratio/mean": 0.9856456518173218, "sampling/importance_sampling_ratio/min": 0.6102004051208496, "sampling/sampling_logp_difference/max": 0.23371434211730957, "sampling/sampling_logp_difference/mean": 0.0015344728017225862, "step": 584, "step_time": 26.638653275998877 }, { "clip_ratio/high_max": 0.0015243901871144772, "clip_ratio/high_mean": 0.0007621950935572386, "clip_ratio/low_mean": 0.0006377550889737904, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001399950182531029, "completions/clipped_ratio": 0.0, "completions/max_length": 2751.0, "completions/max_terminated_length": 2751.0, "completions/mean_length": 1909.28125, "completions/mean_terminated_length": 1909.28125, "completions/min_length": 1147.0, "completions/min_terminated_length": 1147.0, "entropy": 0.04707101359963417, "epoch": 0.0468, "frac_reward_zero_std": 0.125, "grad_norm": 0.032697804272174835, "kl": 1.8982254415750504, "learning_rate": 4.3495106149099164e-05, "loss": 0.0095, "num_tokens": 29281633.0, "reward": 1.7021875381469727, "reward_std": 0.11330871284008026, "rewards/rollout_reward_func/mean": 1.7021875381469727, "rewards/rollout_reward_func/std": 0.22119519114494324, "sampling/importance_sampling_ratio/max": 1.2501308917999268, "sampling/importance_sampling_ratio/mean": 0.975030779838562, "sampling/importance_sampling_ratio/min": 0.7957000732421875, "sampling/sampling_logp_difference/max": 0.24966144561767578, "sampling/sampling_logp_difference/mean": 0.002477153204381466, "step": 585, "step_time": 26.97950604900325 }, { "clip_ratio/high_max": 0.0016447368543595076, "clip_ratio/high_mean": 0.0008223684271797538, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008223684271797538, "completions/clipped_ratio": 0.0, "completions/max_length": 2709.0, "completions/max_terminated_length": 2709.0, "completions/mean_length": 2288.5, "completions/mean_terminated_length": 2288.5, "completions/min_length": 1172.0, "completions/min_terminated_length": 1172.0, "entropy": 0.040609139716252685, "epoch": 0.04688, "frac_reward_zero_std": 0.0, "grad_norm": 0.036198366433382034, "kl": 1.7162827104330063, "learning_rate": 4.349496420462751e-05, "loss": 0.0248, "num_tokens": 29373263.0, "reward": 1.7778124809265137, "reward_std": 0.09884199500083923, "rewards/rollout_reward_func/mean": 1.7778124809265137, "rewards/rollout_reward_func/std": 0.14295803010463715, "sampling/importance_sampling_ratio/max": 1.431295394897461, "sampling/importance_sampling_ratio/mean": 1.0193889141082764, "sampling/importance_sampling_ratio/min": 0.7936609983444214, "sampling/sampling_logp_difference/max": 0.37087535858154297, "sampling/sampling_logp_difference/mean": 0.0016823875484988093, "step": 586, "step_time": 27.31567250199805 }, { "clip_ratio/high_max": 0.0016025641234591603, "clip_ratio/high_mean": 0.0008012820617295802, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008012820617295802, "completions/clipped_ratio": 0.0, "completions/max_length": 2732.0, "completions/max_terminated_length": 2732.0, "completions/mean_length": 2310.46875, "completions/mean_terminated_length": 2310.46875, "completions/min_length": 1950.0, "completions/min_terminated_length": 1950.0, "entropy": 0.046074018348008394, "epoch": 0.04696, "frac_reward_zero_std": 0.125, "grad_norm": 0.03580038249492645, "kl": 1.882415622472763, "learning_rate": 4.349482200225315e-05, "loss": 0.0096, "num_tokens": 29465477.0, "reward": 1.8409374952316284, "reward_std": 0.048479966819286346, "rewards/rollout_reward_func/mean": 1.8409374952316284, "rewards/rollout_reward_func/std": 0.06586199998855591, "sampling/importance_sampling_ratio/max": 1.181662678718567, "sampling/importance_sampling_ratio/mean": 0.9641221761703491, "sampling/importance_sampling_ratio/min": 0.6988797783851624, "sampling/sampling_logp_difference/max": 0.3973988890647888, "sampling/sampling_logp_difference/mean": 0.0026738112792372704, "step": 587, "step_time": 28.4622100850047 }, { "clip_ratio/high_max": 0.0018656715983524919, "clip_ratio/high_mean": 0.0009328357991762459, "clip_ratio/low_mean": 0.0007911392604000866, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017239750595763326, "completions/clipped_ratio": 0.0, "completions/max_length": 2773.0, "completions/max_terminated_length": 2773.0, "completions/mean_length": 2159.84375, "completions/mean_terminated_length": 2159.84375, "completions/min_length": 1163.0, "completions/min_terminated_length": 1163.0, "entropy": 0.04546257574111223, "epoch": 0.04704, "frac_reward_zero_std": 0.0, "grad_norm": 0.0628182590007782, "kl": 2.2107512652873993, "learning_rate": 4.3494679541978325e-05, "loss": 0.0144, "num_tokens": 29552201.0, "reward": 1.7641518115997314, "reward_std": 0.11965693533420563, "rewards/rollout_reward_func/mean": 1.7641518115997314, "rewards/rollout_reward_func/std": 0.2252831906080246, "sampling/importance_sampling_ratio/max": 1.2966850996017456, "sampling/importance_sampling_ratio/mean": 0.9895142316818237, "sampling/importance_sampling_ratio/min": 1.2681473926523523e-10, "sampling/sampling_logp_difference/max": 17.781042098999023, "sampling/sampling_logp_difference/mean": 0.021698715165257454, "step": 588, "step_time": 26.83410318999813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2767.0, "completions/max_terminated_length": 2767.0, "completions/mean_length": 2211.21875, "completions/mean_terminated_length": 2211.21875, "completions/min_length": 1635.0, "completions/min_terminated_length": 1635.0, "entropy": 0.04133616015315056, "epoch": 0.04712, "frac_reward_zero_std": 0.0, "grad_norm": 0.013398230075836182, "kl": 1.717490330338478, "learning_rate": 4.34945368238053e-05, "loss": 0.0232, "num_tokens": 29640713.0, "reward": 1.8034374713897705, "reward_std": 0.06910195201635361, "rewards/rollout_reward_func/mean": 1.8034374713897705, "rewards/rollout_reward_func/std": 0.07128360122442245, "sampling/importance_sampling_ratio/max": 1.714211106300354, "sampling/importance_sampling_ratio/mean": 1.0176293849945068, "sampling/importance_sampling_ratio/min": 0.812361478805542, "sampling/sampling_logp_difference/max": 0.5043795108795166, "sampling/sampling_logp_difference/mean": 0.0019052018178626895, "step": 589, "step_time": 27.022660986998744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2718.0, "completions/max_terminated_length": 2718.0, "completions/mean_length": 2161.625, "completions/mean_terminated_length": 2161.625, "completions/min_length": 1619.0, "completions/min_terminated_length": 1619.0, "entropy": 0.04314515693113208, "epoch": 0.0472, "frac_reward_zero_std": 0.0, "grad_norm": 0.03526566922664642, "kl": 1.762753650546074, "learning_rate": 4.349439384773632e-05, "loss": 0.012, "num_tokens": 29727660.0, "reward": 1.794374942779541, "reward_std": 0.03787641227245331, "rewards/rollout_reward_func/mean": 1.794374942779541, "rewards/rollout_reward_func/std": 0.04976963624358177, "sampling/importance_sampling_ratio/max": 1.1536755561828613, "sampling/importance_sampling_ratio/mean": 0.9882063269615173, "sampling/importance_sampling_ratio/min": 0.6627881526947021, "sampling/sampling_logp_difference/max": 0.3727855682373047, "sampling/sampling_logp_difference/mean": 0.0020162889268249273, "step": 590, "step_time": 27.342565007998928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2732.0, "completions/max_terminated_length": 2732.0, "completions/mean_length": 2211.03125, "completions/mean_terminated_length": 2211.03125, "completions/min_length": 1635.0, "completions/min_terminated_length": 1635.0, "entropy": 0.036944866413250566, "epoch": 0.04728, "frac_reward_zero_std": 0.0, "grad_norm": 0.01662793941795826, "kl": 1.6971464306116104, "learning_rate": 4.3494250613773665e-05, "loss": 0.0238, "num_tokens": 29816641.0, "reward": 1.8040623664855957, "reward_std": 0.06364314258098602, "rewards/rollout_reward_func/mean": 1.8040623664855957, "rewards/rollout_reward_func/std": 0.06743501126766205, "sampling/importance_sampling_ratio/max": 1.1987637281417847, "sampling/importance_sampling_ratio/mean": 1.0246119499206543, "sampling/importance_sampling_ratio/min": 0.8814585208892822, "sampling/sampling_logp_difference/max": 0.1875925064086914, "sampling/sampling_logp_difference/mean": 0.0011972233187407255, "step": 591, "step_time": 28.623147771999356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2736.0, "completions/max_terminated_length": 2736.0, "completions/mean_length": 2039.03125, "completions/mean_terminated_length": 2039.03125, "completions/min_length": 1604.0, "completions/min_terminated_length": 1604.0, "entropy": 0.05247039347887039, "epoch": 0.04736, "frac_reward_zero_std": 0.0, "grad_norm": 0.05405453220009804, "kl": 1.8080898970365524, "learning_rate": 4.349410712191959e-05, "loss": 0.0234, "num_tokens": 29899066.0, "reward": 1.7871874570846558, "reward_std": 0.05080338567495346, "rewards/rollout_reward_func/mean": 1.7871874570846558, "rewards/rollout_reward_func/std": 0.06426931917667389, "sampling/importance_sampling_ratio/max": 1.5437166690826416, "sampling/importance_sampling_ratio/mean": 1.0214262008666992, "sampling/importance_sampling_ratio/min": 0.768871545791626, "sampling/sampling_logp_difference/max": 0.21503543853759766, "sampling/sampling_logp_difference/mean": 0.002357891295105219, "step": 592, "step_time": 27.27611175600032 }, { "clip_ratio/high_max": 0.002501000417396426, "clip_ratio/high_mean": 0.001250500208698213, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001250500208698213, "completions/clipped_ratio": 0.0, "completions/max_length": 2421.0, "completions/max_terminated_length": 2421.0, "completions/mean_length": 1791.34375, "completions/mean_terminated_length": 1791.34375, "completions/min_length": 1227.0, "completions/min_terminated_length": 1227.0, "entropy": 0.033155330223962665, "epoch": 0.04744, "frac_reward_zero_std": 0.0, "grad_norm": 0.04323156550526619, "kl": 1.7483617961406708, "learning_rate": 4.3493963372176376e-05, "loss": 0.0254, "num_tokens": 29972890.0, "reward": 1.6806596517562866, "reward_std": 0.23661237955093384, "rewards/rollout_reward_func/mean": 1.6806596517562866, "rewards/rollout_reward_func/std": 0.3352562189102173, "sampling/importance_sampling_ratio/max": 1.1211779117584229, "sampling/importance_sampling_ratio/mean": 0.9924331903457642, "sampling/importance_sampling_ratio/min": 0.7867768406867981, "sampling/sampling_logp_difference/max": 0.2426285743713379, "sampling/sampling_logp_difference/mean": 0.0014347245451062918, "step": 593, "step_time": 25.346692486999018 }, { "clip_ratio/high_max": 0.001923076924867928, "clip_ratio/high_mean": 0.000961538462433964, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000961538462433964, "completions/clipped_ratio": 0.0, "completions/max_length": 2788.0, "completions/max_terminated_length": 2788.0, "completions/mean_length": 2178.03125, "completions/mean_terminated_length": 2178.03125, "completions/min_length": 1176.0, "completions/min_terminated_length": 1176.0, "entropy": 0.027983971638605, "epoch": 0.04752, "frac_reward_zero_std": 0.0, "grad_norm": 0.01859484612941742, "kl": 1.7145090103149414, "learning_rate": 4.3493819364546296e-05, "loss": 0.0173, "num_tokens": 30060101.0, "reward": 1.8031249046325684, "reward_std": 0.08961233496665955, "rewards/rollout_reward_func/mean": 1.8031249046325684, "rewards/rollout_reward_func/std": 0.14200040698051453, "sampling/importance_sampling_ratio/max": 1.0476267337799072, "sampling/importance_sampling_ratio/mean": 0.9917201995849609, "sampling/importance_sampling_ratio/min": 0.7961404919624329, "sampling/sampling_logp_difference/max": 0.2436676025390625, "sampling/sampling_logp_difference/mean": 0.0009185977396555245, "step": 594, "step_time": 26.62310565700136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2778.0, "completions/max_terminated_length": 2778.0, "completions/mean_length": 1851.8125, "completions/mean_terminated_length": 1851.8125, "completions/min_length": 1135.0, "completions/min_terminated_length": 1135.0, "entropy": 0.026610327884554863, "epoch": 0.0476, "frac_reward_zero_std": 0.0, "grad_norm": 0.020761532709002495, "kl": 1.771131157875061, "learning_rate": 4.349367509903162e-05, "loss": 0.0103, "num_tokens": 30136220.0, "reward": 1.6690177917480469, "reward_std": 0.08390077948570251, "rewards/rollout_reward_func/mean": 1.6690177917480469, "rewards/rollout_reward_func/std": 0.2533598840236664, "sampling/importance_sampling_ratio/max": 1.1801831722259521, "sampling/importance_sampling_ratio/mean": 0.9941513538360596, "sampling/importance_sampling_ratio/min": 0.8678743839263916, "sampling/sampling_logp_difference/max": 0.1433095932006836, "sampling/sampling_logp_difference/mean": 0.0007920092321000993, "step": 595, "step_time": 27.82936721700389 }, { "clip_ratio/high_max": 0.003172690747305751, "clip_ratio/high_mean": 0.0015863453736528754, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015863453736528754, "completions/clipped_ratio": 0.0, "completions/max_length": 2736.0, "completions/max_terminated_length": 2736.0, "completions/mean_length": 1789.9375, "completions/mean_terminated_length": 1789.9375, "completions/min_length": 1227.0, "completions/min_terminated_length": 1227.0, "entropy": 0.03212893707677722, "epoch": 0.04768, "frac_reward_zero_std": 0.0, "grad_norm": 0.6292493939399719, "kl": 4.6428851038217545, "learning_rate": 4.349353057563466e-05, "loss": 0.011, "num_tokens": 30210165.0, "reward": 1.7271873950958252, "reward_std": 0.06906960904598236, "rewards/rollout_reward_func/mean": 1.7271873950958252, "rewards/rollout_reward_func/std": 0.19262592494487762, "sampling/importance_sampling_ratio/max": 1.2870104312896729, "sampling/importance_sampling_ratio/mean": 0.996740996837616, "sampling/importance_sampling_ratio/min": 0.656105637550354, "sampling/sampling_logp_difference/max": 0.4953126907348633, "sampling/sampling_logp_difference/mean": 0.0017212380189448595, "step": 596, "step_time": 27.023948166997798 }, { "clip_ratio/high_max": 0.0008802816737443209, "clip_ratio/high_mean": 0.00044014083687216043, "clip_ratio/low_mean": 0.0008928571478463709, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013329979847185314, "completions/clipped_ratio": 0.0, "completions/max_length": 2725.0, "completions/max_terminated_length": 2725.0, "completions/mean_length": 1900.84375, "completions/mean_terminated_length": 1900.84375, "completions/min_length": 1136.0, "completions/min_terminated_length": 1136.0, "entropy": 0.0756611367687583, "epoch": 0.04776, "frac_reward_zero_std": 0.0, "grad_norm": 0.0271714199334383, "kl": 1.9055864363908768, "learning_rate": 4.349338579435768e-05, "loss": 0.0146, "num_tokens": 30287427.0, "reward": 1.717812418937683, "reward_std": 0.1224176213145256, "rewards/rollout_reward_func/mean": 1.717812418937683, "rewards/rollout_reward_func/std": 0.2209763526916504, "sampling/importance_sampling_ratio/max": 1.194068193435669, "sampling/importance_sampling_ratio/mean": 0.9896841645240784, "sampling/importance_sampling_ratio/min": 0.8010164499282837, "sampling/sampling_logp_difference/max": 0.24674081802368164, "sampling/sampling_logp_difference/mean": 0.0028108065016567707, "step": 597, "step_time": 26.849709328000245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2426.0, "completions/max_terminated_length": 2426.0, "completions/mean_length": 1739.09375, "completions/mean_terminated_length": 1739.09375, "completions/min_length": 1132.0, "completions/min_terminated_length": 1132.0, "entropy": 0.058918820694088936, "epoch": 0.04784, "frac_reward_zero_std": 0.0, "grad_norm": 0.035104501992464066, "kl": 2.1721400171518326, "learning_rate": 4.349324075520298e-05, "loss": 0.0093, "num_tokens": 30359055.0, "reward": 1.6853125095367432, "reward_std": 0.1292807161808014, "rewards/rollout_reward_func/mean": 1.6853125095367432, "rewards/rollout_reward_func/std": 0.2450672686100006, "sampling/importance_sampling_ratio/max": 1.0688780546188354, "sampling/importance_sampling_ratio/mean": 0.9733365774154663, "sampling/importance_sampling_ratio/min": 0.6911354064941406, "sampling/sampling_logp_difference/max": 0.22610116004943848, "sampling/sampling_logp_difference/mean": 0.0021386314183473587, "step": 598, "step_time": 24.910870155998055 }, { "clip_ratio/high_max": 0.0016025641234591603, "clip_ratio/high_mean": 0.0008012820617295802, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008012820617295802, "completions/clipped_ratio": 0.0, "completions/max_length": 2747.0, "completions/max_terminated_length": 2747.0, "completions/mean_length": 2028.5625, "completions/mean_terminated_length": 2028.5625, "completions/min_length": 1250.0, "completions/min_terminated_length": 1250.0, "entropy": 0.048651745077222586, "epoch": 0.04792, "frac_reward_zero_std": 0.0, "grad_norm": 0.08052041381597519, "kl": 3.6993394941091537, "learning_rate": 4.349309545817286e-05, "loss": 0.0228, "num_tokens": 30440979.0, "reward": 1.729461669921875, "reward_std": 0.10424274206161499, "rewards/rollout_reward_func/mean": 1.729461669921875, "rewards/rollout_reward_func/std": 0.1864936500787735, "sampling/importance_sampling_ratio/max": 1.4598941802978516, "sampling/importance_sampling_ratio/mean": 1.024036169052124, "sampling/importance_sampling_ratio/min": 0.8431758880615234, "sampling/sampling_logp_difference/max": 0.27579164505004883, "sampling/sampling_logp_difference/mean": 0.0018497179262340069, "step": 599, "step_time": 27.77873468799953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2750.0, "completions/max_terminated_length": 2750.0, "completions/mean_length": 2205.0625, "completions/mean_terminated_length": 2205.0625, "completions/min_length": 1136.0, "completions/min_terminated_length": 1136.0, "entropy": 0.049673032481223345, "epoch": 0.048, "frac_reward_zero_std": 0.0, "grad_norm": 0.01149135734885931, "kl": 1.8964182287454605, "learning_rate": 4.349294990326962e-05, "loss": 0.0188, "num_tokens": 30529223.0, "reward": 1.7946875095367432, "reward_std": 0.08453357219696045, "rewards/rollout_reward_func/mean": 1.7946875095367432, "rewards/rollout_reward_func/std": 0.13538438081741333, "sampling/importance_sampling_ratio/max": 1.1313725709915161, "sampling/importance_sampling_ratio/mean": 0.9996616840362549, "sampling/importance_sampling_ratio/min": 0.7171533107757568, "sampling/sampling_logp_difference/max": 0.3570826053619385, "sampling/sampling_logp_difference/mean": 0.0013821773463860154, "step": 600, "step_time": 26.706072175998997 }, { "clip_ratio/high_max": 0.0017605633474886417, "clip_ratio/high_mean": 0.0008802816737443209, "clip_ratio/low_mean": 0.0015280739171430469, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024083555908873677, "completions/clipped_ratio": 0.0, "completions/max_length": 2783.0, "completions/max_terminated_length": 2783.0, "completions/mean_length": 2208.875, "completions/mean_terminated_length": 2208.875, "completions/min_length": 1151.0, "completions/min_terminated_length": 1151.0, "entropy": 0.06191941536962986, "epoch": 0.04808, "frac_reward_zero_std": 0.125, "grad_norm": 0.023359274491667747, "kl": 1.8146220594644547, "learning_rate": 4.349280409049556e-05, "loss": 0.02, "num_tokens": 30617779.0, "reward": 1.7047990560531616, "reward_std": 0.18327930569648743, "rewards/rollout_reward_func/mean": 1.7047990560531616, "rewards/rollout_reward_func/std": 0.2650713324546814, "sampling/importance_sampling_ratio/max": 1.222472906112671, "sampling/importance_sampling_ratio/mean": 0.9799381494522095, "sampling/importance_sampling_ratio/min": 0.7143072485923767, "sampling/sampling_logp_difference/max": 0.3477492332458496, "sampling/sampling_logp_difference/mean": 0.0023554021026939154, "step": 601, "step_time": 26.966926129998683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0016033496940508485, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016033496940508485, "completions/clipped_ratio": 0.0, "completions/max_length": 2745.0, "completions/max_terminated_length": 2745.0, "completions/mean_length": 2082.0, "completions/mean_terminated_length": 2082.0, "completions/min_length": 1226.0, "completions/min_terminated_length": 1226.0, "entropy": 0.0657735294662416, "epoch": 0.04816, "frac_reward_zero_std": 0.0, "grad_norm": 0.023633817210793495, "kl": 2.4258931279182434, "learning_rate": 4.349265801985299e-05, "loss": 0.0096, "num_tokens": 30701961.0, "reward": 1.7413020133972168, "reward_std": 0.12106546014547348, "rewards/rollout_reward_func/mean": 1.7413020133972168, "rewards/rollout_reward_func/std": 0.19209221005439758, "sampling/importance_sampling_ratio/max": 1.1653567552566528, "sampling/importance_sampling_ratio/mean": 0.9759307503700256, "sampling/importance_sampling_ratio/min": 0.7813207507133484, "sampling/sampling_logp_difference/max": 0.238816499710083, "sampling/sampling_logp_difference/mean": 0.0024415273219347, "step": 602, "step_time": 26.758621601000414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2381.0, "completions/max_terminated_length": 2381.0, "completions/mean_length": 1816.1875, "completions/mean_terminated_length": 1816.1875, "completions/min_length": 1136.0, "completions/min_terminated_length": 1136.0, "entropy": 0.08101746719330549, "epoch": 0.04824, "frac_reward_zero_std": 0.0, "grad_norm": 0.023613203316926956, "kl": 2.0875593572854996, "learning_rate": 4.3492511691344233e-05, "loss": 0.0194, "num_tokens": 30775869.0, "reward": 1.7581249475479126, "reward_std": 0.0767894834280014, "rewards/rollout_reward_func/mean": 1.7581249475479126, "rewards/rollout_reward_func/std": 0.12493708729743958, "sampling/importance_sampling_ratio/max": 1.2224376201629639, "sampling/importance_sampling_ratio/mean": 1.0026617050170898, "sampling/importance_sampling_ratio/min": 0.8489091396331787, "sampling/sampling_logp_difference/max": 0.20888543128967285, "sampling/sampling_logp_difference/mean": 0.0020153236109763384, "step": 603, "step_time": 25.977932214998873 }, { "clip_ratio/high_max": 0.003399963490664959, "clip_ratio/high_mean": 0.0016999817453324795, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016999817453324795, "completions/clipped_ratio": 0.0, "completions/max_length": 2798.0, "completions/max_terminated_length": 2798.0, "completions/mean_length": 2002.5625, "completions/mean_terminated_length": 2002.5625, "completions/min_length": 1238.0, "completions/min_terminated_length": 1238.0, "entropy": 0.07802484929561615, "epoch": 0.04832, "frac_reward_zero_std": 0.0, "grad_norm": 0.026793215423822403, "kl": 2.037396639585495, "learning_rate": 4.3492365104971595e-05, "loss": 0.0164, "num_tokens": 30856983.0, "reward": 1.7508035898208618, "reward_std": 0.13374218344688416, "rewards/rollout_reward_func/mean": 1.7508035898208618, "rewards/rollout_reward_func/std": 0.21174156665802002, "sampling/importance_sampling_ratio/max": 1.4481925964355469, "sampling/importance_sampling_ratio/mean": 1.024810791015625, "sampling/importance_sampling_ratio/min": 0.7686831951141357, "sampling/sampling_logp_difference/max": 0.3739633560180664, "sampling/sampling_logp_difference/mean": 0.0031420779414474964, "step": 604, "step_time": 27.252480779001417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0008223684271797538, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0008223684271797538, "completions/clipped_ratio": 0.0, "completions/max_length": 2748.0, "completions/max_terminated_length": 2748.0, "completions/mean_length": 2166.34375, "completions/mean_terminated_length": 2166.34375, "completions/min_length": 1162.0, "completions/min_terminated_length": 1162.0, "entropy": 0.0940632801502943, "epoch": 0.0484, "frac_reward_zero_std": 0.0, "grad_norm": 0.029155012220144272, "kl": 1.8619444668293, "learning_rate": 4.3492218260737395e-05, "loss": 0.017, "num_tokens": 30944390.0, "reward": 1.7509374618530273, "reward_std": 0.1723429262638092, "rewards/rollout_reward_func/mean": 1.7509374618530273, "rewards/rollout_reward_func/std": 0.27823591232299805, "sampling/importance_sampling_ratio/max": 1.231724739074707, "sampling/importance_sampling_ratio/mean": 1.0006358623504639, "sampling/importance_sampling_ratio/min": 0.7974903583526611, "sampling/sampling_logp_difference/max": 0.34279441833496094, "sampling/sampling_logp_difference/mean": 0.002351942704990506, "step": 605, "step_time": 26.657777281001472 }, { "clip_ratio/high_max": 0.0031665865099057555, "clip_ratio/high_mean": 0.0015832932549528778, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0015832932549528778, "completions/clipped_ratio": 0.0, "completions/max_length": 2739.0, "completions/max_terminated_length": 2739.0, "completions/mean_length": 1821.21875, "completions/mean_terminated_length": 1821.21875, "completions/min_length": 1055.0, "completions/min_terminated_length": 1055.0, "entropy": 0.11901518795639277, "epoch": 0.04848, "frac_reward_zero_std": 0.0, "grad_norm": 0.03630511090159416, "kl": 2.3871964812278748, "learning_rate": 4.349207115864397e-05, "loss": 0.0137, "num_tokens": 31019554.0, "reward": 1.6143749952316284, "reward_std": 0.2251441478729248, "rewards/rollout_reward_func/mean": 1.6143749952316284, "rewards/rollout_reward_func/std": 0.31962618231773376, "sampling/importance_sampling_ratio/max": 1.3278965950012207, "sampling/importance_sampling_ratio/mean": 0.9761210680007935, "sampling/importance_sampling_ratio/min": 0.6318883299827576, "sampling/sampling_logp_difference/max": 0.3493692874908447, "sampling/sampling_logp_difference/mean": 0.004362010397017002, "step": 606, "step_time": 26.032836345999385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2761.0, "completions/max_terminated_length": 2761.0, "completions/mean_length": 2218.5625, "completions/mean_terminated_length": 2218.5625, "completions/min_length": 1611.0, "completions/min_terminated_length": 1611.0, "entropy": 0.11652442719787359, "epoch": 0.04856, "frac_reward_zero_std": 0.0, "grad_norm": 0.030316540971398354, "kl": 1.9339845925569534, "learning_rate": 4.349192379869365e-05, "loss": 0.0106, "num_tokens": 31108016.0, "reward": 1.7866840362548828, "reward_std": 0.12198460847139359, "rewards/rollout_reward_func/mean": 1.7866840362548828, "rewards/rollout_reward_func/std": 0.2235015332698822, "sampling/importance_sampling_ratio/max": 1.2740379571914673, "sampling/importance_sampling_ratio/mean": 1.006188154220581, "sampling/importance_sampling_ratio/min": 0.8332369327545166, "sampling/sampling_logp_difference/max": 0.1260056495666504, "sampling/sampling_logp_difference/mean": 0.0035038753412663937, "step": 607, "step_time": 28.45101748199886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000735294132027775, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000735294132027775, "completions/clipped_ratio": 0.0, "completions/max_length": 2807.0, "completions/max_terminated_length": 2807.0, "completions/mean_length": 2012.96875, "completions/mean_terminated_length": 2012.96875, "completions/min_length": 1621.0, "completions/min_terminated_length": 1621.0, "entropy": 0.14706574473530054, "epoch": 0.04864, "frac_reward_zero_std": 0.0, "grad_norm": 0.05425100028514862, "kl": 2.0072427093982697, "learning_rate": 4.349177618088875e-05, "loss": 0.0383, "num_tokens": 31189439.0, "reward": 1.7446874380111694, "reward_std": 0.16152812540531158, "rewards/rollout_reward_func/mean": 1.7446874380111694, "rewards/rollout_reward_func/std": 0.3297896087169647, "sampling/importance_sampling_ratio/max": 1.6083272695541382, "sampling/importance_sampling_ratio/mean": 1.0364958047866821, "sampling/importance_sampling_ratio/min": 0.8112706542015076, "sampling/sampling_logp_difference/max": 0.23676341772079468, "sampling/sampling_logp_difference/mean": 0.0046618785709142685, "step": 608, "step_time": 28.19783598199865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2751.0, "completions/max_terminated_length": 2751.0, "completions/mean_length": 2191.28125, "completions/mean_terminated_length": 2191.28125, "completions/min_length": 1055.0, "completions/min_terminated_length": 1055.0, "entropy": 0.1258474076166749, "epoch": 0.04872, "frac_reward_zero_std": 0.0, "grad_norm": 0.027194740250706673, "kl": 1.8936708271503448, "learning_rate": 4.349162830523163e-05, "loss": 0.013, "num_tokens": 31277622.0, "reward": 1.7623263597488403, "reward_std": 0.13683225214481354, "rewards/rollout_reward_func/mean": 1.7623263597488403, "rewards/rollout_reward_func/std": 0.17009153962135315, "sampling/importance_sampling_ratio/max": 1.221761703491211, "sampling/importance_sampling_ratio/mean": 0.9983182549476624, "sampling/importance_sampling_ratio/min": 0.7909557223320007, "sampling/sampling_logp_difference/max": 0.2327871322631836, "sampling/sampling_logp_difference/mean": 0.0036012609489262104, "step": 609, "step_time": 27.025865084002362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2440.0, "completions/max_terminated_length": 2440.0, "completions/mean_length": 1724.65625, "completions/mean_terminated_length": 1724.65625, "completions/min_length": 1138.0, "completions/min_terminated_length": 1138.0, "entropy": 0.16068032756447792, "epoch": 0.0488, "frac_reward_zero_std": 0.0, "grad_norm": 0.04423225298523903, "kl": 2.3840802013874054, "learning_rate": 4.3491480171724614e-05, "loss": 0.0038, "num_tokens": 31348788.0, "reward": 1.5947222709655762, "reward_std": 0.20142440497875214, "rewards/rollout_reward_func/mean": 1.5947222709655762, "rewards/rollout_reward_func/std": 0.29986029863357544, "sampling/importance_sampling_ratio/max": 1.4229609966278076, "sampling/importance_sampling_ratio/mean": 1.053743839263916, "sampling/importance_sampling_ratio/min": 0.7823129296302795, "sampling/sampling_logp_difference/max": 0.33683228492736816, "sampling/sampling_logp_difference/mean": 0.00515399593859911, "step": 610, "step_time": 26.238558313001704 } ], "logging_steps": 1.0, "max_steps": 25000, "num_input_tokens_seen": 31348788, "num_train_epochs": 2, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }