{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.05150017243361306, "eval_steps": 500, "global_step": 896, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15703.0, "completions/max_terminated_length": 15703.0, "completions/mean_length": 8732.875, "completions/mean_terminated_length": 8732.875, "completions/min_length": 4987.0, "completions/min_terminated_length": 4987.0, "entropy": 0.3713345378637314, "epoch": 5.7477871019657434e-05, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 70583.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000192403793335, "sampling/importance_sampling_ratio/min": 1.836119345455245e-08, "sampling/sampling_logp_difference/max": 17.813026428222656, "sampling/sampling_logp_difference/mean": 0.015137896873056889, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15439.0, "completions/mean_length": 12614.125, "completions/mean_terminated_length": 10352.2001953125, "completions/min_length": 6948.0, "completions/min_terminated_length": 6948.0, "entropy": 0.36503184773027897, "epoch": 0.00011495574203931487, "frac_reward_zero_std": 0.0, "grad_norm": 0.012826107442378998, "learning_rate": 1e-05, "loss": -0.0246, "num_tokens": 172536.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999308586120605, "sampling/importance_sampling_ratio/min": 0.12457425892353058, "sampling/sampling_logp_difference/max": 2.082853317260742, "sampling/sampling_logp_difference/mean": 0.016638126224279404, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2265.0, "completions/max_terminated_length": 2265.0, "completions/mean_length": 1789.5, "completions/mean_terminated_length": 1789.5, "completions/min_length": 1057.0, "completions/min_terminated_length": 1057.0, "entropy": 0.2469680141657591, "epoch": 0.0001724336130589723, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 187676.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4334642887115479, "sampling/importance_sampling_ratio/mean": 0.9997435808181763, "sampling/importance_sampling_ratio/min": 0.7285618185997009, "sampling/sampling_logp_difference/max": 0.3600940704345703, "sampling/sampling_logp_difference/mean": 0.00893494300544262, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7182.0, "completions/max_terminated_length": 7182.0, "completions/mean_length": 2169.125, "completions/mean_terminated_length": 2169.125, "completions/min_length": 722.0, "completions/min_terminated_length": 722.0, "entropy": 0.20379706658422947, "epoch": 0.00022991148407862974, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 205845.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8169389963150024, "sampling/importance_sampling_ratio/mean": 1.0000684261322021, "sampling/importance_sampling_ratio/min": 0.6421703696250916, "sampling/sampling_logp_difference/max": 0.5971531867980957, "sampling/sampling_logp_difference/mean": 0.008074535056948662, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6306.0, "completions/max_terminated_length": 6306.0, "completions/mean_length": 2645.0, "completions/mean_terminated_length": 2645.0, "completions/min_length": 1777.0, "completions/min_terminated_length": 1777.0, "entropy": 0.25945289619266987, "epoch": 0.0002873893550982872, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 227981.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.912467122077942, "sampling/importance_sampling_ratio/mean": 1.0001256465911865, "sampling/importance_sampling_ratio/min": 0.6602612733840942, "sampling/sampling_logp_difference/max": 0.6483941078186035, "sampling/sampling_logp_difference/mean": 0.007415128871798515, "step": 5 }, { "clip_ratio/high_max": 0.00014546304009854794, "clip_ratio/high_mean": 0.00014546304009854794, "clip_ratio/low_mean": 0.00022717010870110244, "clip_ratio/low_min": 0.00022717010870110244, "clip_ratio/region_mean": 0.00037263314879965037, "completions/clipped_ratio": 0.0, "completions/max_length": 7348.0, "completions/max_terminated_length": 7348.0, "completions/mean_length": 5479.375, "completions/mean_terminated_length": 5479.375, "completions/min_length": 3462.0, "completions/min_terminated_length": 3462.0, "entropy": 0.2911783792078495, "epoch": 0.0003448672261179446, "frac_reward_zero_std": 0.0, "grad_norm": 0.010940776206552982, "learning_rate": 1e-05, "loss": -0.1459, "num_tokens": 272792.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6280268430709839, "sampling/importance_sampling_ratio/mean": 0.9998860359191895, "sampling/importance_sampling_ratio/min": 0.574209988117218, "sampling/sampling_logp_difference/max": 0.5547601580619812, "sampling/sampling_logp_difference/mean": 0.011181830428540707, "step": 6 }, { "clip_ratio/high_max": 5.0924933020723984e-05, "clip_ratio/high_mean": 5.0924933020723984e-05, "clip_ratio/low_mean": 0.00038385049265343696, "clip_ratio/low_min": 0.00038385049265343696, "clip_ratio/region_mean": 0.00043477542567416094, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 11842.0, "completions/mean_length": 9243.0, "completions/mean_terminated_length": 8222.857421875, "completions/min_length": 3075.0, "completions/min_terminated_length": 3075.0, "entropy": 0.4120573028922081, "epoch": 0.000402345097137602, "frac_reward_zero_std": 0.0, "grad_norm": 0.009335801936686039, "learning_rate": 1e-05, "loss": 0.099, "num_tokens": 348000.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8537434339523315, "sampling/importance_sampling_ratio/mean": 1.0000827312469482, "sampling/importance_sampling_ratio/min": 0.3109295070171356, "sampling/sampling_logp_difference/max": 1.1681890487670898, "sampling/sampling_logp_difference/mean": 0.0167313814163208, "step": 7 }, { "clip_ratio/high_max": 0.00019611547213571612, "clip_ratio/high_mean": 0.00019611547213571612, "clip_ratio/low_mean": 0.00048802775563672185, "clip_ratio/low_min": 0.00048802775563672185, "clip_ratio/region_mean": 0.000684143227772438, "completions/clipped_ratio": 0.0, "completions/max_length": 14617.0, "completions/max_terminated_length": 14617.0, "completions/mean_length": 9390.625, "completions/mean_terminated_length": 9390.625, "completions/min_length": 4170.0, "completions/min_terminated_length": 4170.0, "entropy": 0.45750724896788597, "epoch": 0.00045982296815725947, "frac_reward_zero_std": 0.0, "grad_norm": 0.008300665766000748, "learning_rate": 1e-05, "loss": -0.0132, "num_tokens": 424301.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000351667404175, "sampling/importance_sampling_ratio/min": 0.3896910846233368, "sampling/sampling_logp_difference/max": 0.9424009323120117, "sampling/sampling_logp_difference/mean": 0.0190004613250494, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15432.0, "completions/mean_length": 13763.0, "completions/mean_terminated_length": 11142.0, "completions/min_length": 7535.0, "completions/min_terminated_length": 7535.0, "entropy": 0.5409758687019348, "epoch": 0.0005173008391769169, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 535325.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997733235359192, "sampling/importance_sampling_ratio/min": 0.2753318250179291, "sampling/sampling_logp_difference/max": 1.289778232574463, "sampling/sampling_logp_difference/mean": 0.02159600332379341, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023255814448930323, "clip_ratio/low_min": 0.00023255814448930323, "clip_ratio/region_mean": 0.00023255814448930323, "completions/clipped_ratio": 0.0, "completions/max_length": 3956.0, "completions/max_terminated_length": 3956.0, "completions/mean_length": 2248.5, "completions/mean_terminated_length": 2248.5, "completions/min_length": 705.0, "completions/min_terminated_length": 705.0, "entropy": 0.5861556529998779, "epoch": 0.0005747787101965744, "frac_reward_zero_std": 0.0, "grad_norm": 0.029013216495513916, "learning_rate": 1e-05, "loss": -0.2064, "num_tokens": 554473.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.374398946762085, "sampling/importance_sampling_ratio/mean": 0.9997981190681458, "sampling/importance_sampling_ratio/min": 0.6573039889335632, "sampling/sampling_logp_difference/max": 0.4196087121963501, "sampling/sampling_logp_difference/mean": 0.015369054861366749, "step": 10 }, { "clip_ratio/high_max": 6.59255929349456e-05, "clip_ratio/high_mean": 6.59255929349456e-05, "clip_ratio/low_mean": 0.0005529337686311919, "clip_ratio/low_min": 0.0005529337686311919, "clip_ratio/region_mean": 0.0006188593615661375, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 14214.0, "completions/mean_length": 13965.75, "completions/mean_terminated_length": 11547.5, "completions/min_length": 8701.0, "completions/min_terminated_length": 8701.0, "entropy": 0.6653081439435482, "epoch": 0.0006322565812162317, "frac_reward_zero_std": 0.0, "grad_norm": 0.009686644189059734, "learning_rate": 1e-05, "loss": 0.0451, "num_tokens": 667743.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999228119850159, "sampling/importance_sampling_ratio/min": 0.0002580129657872021, "sampling/sampling_logp_difference/max": 8.262500762939453, "sampling/sampling_logp_difference/mean": 0.02505280077457428, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12835.0, "completions/mean_length": 10192.125, "completions/mean_terminated_length": 9307.572265625, "completions/min_length": 3177.0, "completions/min_terminated_length": 3177.0, "entropy": 0.5226390846073627, "epoch": 0.0006897344522358892, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 751376.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999798536300659, "sampling/importance_sampling_ratio/min": 0.11669129133224487, "sampling/sampling_logp_difference/max": 2.148223400115967, "sampling/sampling_logp_difference/mean": 0.020940717309713364, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13361.0, "completions/mean_length": 8044.25, "completions/mean_terminated_length": 6852.857421875, "completions/min_length": 2809.0, "completions/min_terminated_length": 2809.0, "entropy": 0.527798768132925, "epoch": 0.0007472123232555466, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 817834.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999900460243225, "sampling/importance_sampling_ratio/min": 0.31529974937438965, "sampling/sampling_logp_difference/max": 1.1542315483093262, "sampling/sampling_logp_difference/mean": 0.023021675646305084, "step": 13 }, { "clip_ratio/high_max": 8.7596352386754e-05, "clip_ratio/high_mean": 8.7596352386754e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.7596352386754e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 1512.0, "completions/max_terminated_length": 1512.0, "completions/mean_length": 984.375, "completions/mean_terminated_length": 984.375, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.36184784211218357, "epoch": 0.000804690194275204, "frac_reward_zero_std": 0.0, "grad_norm": 0.024298641830682755, "learning_rate": 1e-05, "loss": 0.1052, "num_tokens": 826485.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.2456766366958618, "sampling/importance_sampling_ratio/mean": 0.9998593926429749, "sampling/importance_sampling_ratio/min": 0.32329079508781433, "sampling/sampling_logp_difference/max": 1.1292030811309814, "sampling/sampling_logp_difference/mean": 0.012408224865794182, "step": 14 }, { "clip_ratio/high_max": 0.0001685967235971475, "clip_ratio/high_mean": 0.0001685967235971475, "clip_ratio/low_mean": 0.00014894512423779815, "clip_ratio/low_min": 0.00014894512423779815, "clip_ratio/region_mean": 0.00031754184783494566, "completions/clipped_ratio": 0.0, "completions/max_length": 14267.0, "completions/max_terminated_length": 14267.0, "completions/mean_length": 6477.0, "completions/mean_terminated_length": 6477.0, "completions/min_length": 2448.0, "completions/min_terminated_length": 2448.0, "entropy": 0.4685293845832348, "epoch": 0.0008621680652948615, "frac_reward_zero_std": 0.0, "grad_norm": 0.014940561726689339, "learning_rate": 1e-05, "loss": 0.1567, "num_tokens": 879269.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8391656875610352, "sampling/importance_sampling_ratio/mean": 1.0000338554382324, "sampling/importance_sampling_ratio/min": 0.4191240668296814, "sampling/sampling_logp_difference/max": 0.8695883750915527, "sampling/sampling_logp_difference/mean": 0.019313381984829903, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4386.0, "completions/max_terminated_length": 4386.0, "completions/mean_length": 3328.875, "completions/mean_terminated_length": 3328.875, "completions/min_length": 2100.0, "completions/min_terminated_length": 2100.0, "entropy": 0.3531157746911049, "epoch": 0.0009196459363145189, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 907412.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4276796579360962, "sampling/importance_sampling_ratio/mean": 0.9996469616889954, "sampling/importance_sampling_ratio/min": 0.6109430193901062, "sampling/sampling_logp_difference/max": 0.4927515983581543, "sampling/sampling_logp_difference/mean": 0.013146447017788887, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0005994474158796947, "clip_ratio/low_min": 0.0005994474158796947, "clip_ratio/region_mean": 0.0005994474158796947, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13680.0, "completions/mean_length": 10788.0, "completions/mean_terminated_length": 9988.572265625, "completions/min_length": 6727.0, "completions/min_terminated_length": 6727.0, "entropy": 0.3596680872142315, "epoch": 0.0009771238073341764, "frac_reward_zero_std": 0.0, "grad_norm": 0.00487625552341342, "learning_rate": 1e-05, "loss": 0.1051, "num_tokens": 995540.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.926710605621338, "sampling/importance_sampling_ratio/mean": 1.0001296997070312, "sampling/importance_sampling_ratio/min": 0.2658487856388092, "sampling/sampling_logp_difference/max": 1.3248276710510254, "sampling/sampling_logp_difference/mean": 0.01606706902384758, "step": 17 }, { "clip_ratio/high_max": 4.931870898872148e-05, "clip_ratio/high_mean": 4.931870898872148e-05, "clip_ratio/low_mean": 0.00017867315182229504, "clip_ratio/low_min": 0.00017867315182229504, "clip_ratio/region_mean": 0.00022799186081101652, "completions/clipped_ratio": 0.0, "completions/max_length": 9202.0, "completions/max_terminated_length": 9202.0, "completions/mean_length": 4511.375, "completions/mean_terminated_length": 4511.375, "completions/min_length": 1294.0, "completions/min_terminated_length": 1294.0, "entropy": 0.3361959885805845, "epoch": 0.0010346016783538338, "frac_reward_zero_std": 0.0, "grad_norm": 0.021912362426519394, "learning_rate": 1e-05, "loss": -0.2679, "num_tokens": 1033319.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000478029251099, "sampling/importance_sampling_ratio/min": 0.589799702167511, "sampling/sampling_logp_difference/max": 0.7484335899353027, "sampling/sampling_logp_difference/mean": 0.013660256750881672, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7835.0, "completions/max_terminated_length": 7835.0, "completions/mean_length": 3494.5, "completions/mean_terminated_length": 3494.5, "completions/min_length": 860.0, "completions/min_terminated_length": 860.0, "entropy": 0.26188469864428043, "epoch": 0.0010920795493734913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 1062099.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000929832458496, "sampling/importance_sampling_ratio/min": 0.5309983491897583, "sampling/sampling_logp_difference/max": 1.0871529579162598, "sampling/sampling_logp_difference/mean": 0.010413752868771553, "step": 19 }, { "clip_ratio/high_max": 1.1474206075945403e-05, "clip_ratio/high_mean": 1.1474206075945403e-05, "clip_ratio/low_mean": 0.000845233731524786, "clip_ratio/low_min": 0.000845233731524786, "clip_ratio/region_mean": 0.0008567079376007314, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15349.0, "completions/mean_length": 12602.125, "completions/mean_terminated_length": 12061.857421875, "completions/min_length": 8332.0, "completions/min_terminated_length": 8332.0, "entropy": 0.7199657186865807, "epoch": 0.0011495574203931487, "frac_reward_zero_std": 0.0, "grad_norm": 0.007488382514566183, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 1164124.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002334117889404, "sampling/importance_sampling_ratio/min": 0.02026776410639286, "sampling/sampling_logp_difference/max": 3.898723602294922, "sampling/sampling_logp_difference/mean": 0.028150716796517372, "step": 20 }, { "clip_ratio/high_max": 3.3866159355966374e-05, "clip_ratio/high_mean": 3.3866159355966374e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 3.3866159355966374e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 3691.0, "completions/max_terminated_length": 3691.0, "completions/mean_length": 2487.75, "completions/mean_terminated_length": 2487.75, "completions/min_length": 1564.0, "completions/min_terminated_length": 1564.0, "entropy": 0.30037602595984936, "epoch": 0.0012070352914128062, "frac_reward_zero_std": 0.0, "grad_norm": 0.011537784710526466, "learning_rate": 1e-05, "loss": -0.0959, "num_tokens": 1185050.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5541020631790161, "sampling/importance_sampling_ratio/mean": 1.0000855922698975, "sampling/importance_sampling_ratio/min": 0.7744051814079285, "sampling/sampling_logp_difference/max": 0.44089794158935547, "sampling/sampling_logp_difference/mean": 0.011413312517106533, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001804557723517064, "clip_ratio/low_min": 0.0001804557723517064, "clip_ratio/region_mean": 0.0001804557723517064, "completions/clipped_ratio": 0.0, "completions/max_length": 11208.0, "completions/max_terminated_length": 11208.0, "completions/mean_length": 5979.5, "completions/mean_terminated_length": 5979.5, "completions/min_length": 2086.0, "completions/min_terminated_length": 2086.0, "entropy": 0.4145108927041292, "epoch": 0.0012645131624324634, "frac_reward_zero_std": 0.0, "grad_norm": 0.01590256206691265, "learning_rate": 1e-05, "loss": 0.0982, "num_tokens": 1233846.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8413699865341187, "sampling/importance_sampling_ratio/mean": 0.9999732971191406, "sampling/importance_sampling_ratio/min": 0.41654255986213684, "sampling/sampling_logp_difference/max": 0.8757666349411011, "sampling/sampling_logp_difference/mean": 0.013827912509441376, "step": 22 }, { "clip_ratio/high_max": 1.0090409887197893e-05, "clip_ratio/high_mean": 1.0090409887197893e-05, "clip_ratio/low_mean": 7.030371489236131e-05, "clip_ratio/low_min": 7.030371489236131e-05, "clip_ratio/region_mean": 8.03941247795592e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 12388.0, "completions/max_terminated_length": 12388.0, "completions/mean_length": 4933.875, "completions/mean_terminated_length": 4933.875, "completions/min_length": 1821.0, "completions/min_terminated_length": 1821.0, "entropy": 0.27186551690101624, "epoch": 0.0013219910334521209, "frac_reward_zero_std": 0.0, "grad_norm": 0.010686766356229782, "learning_rate": 1e-05, "loss": -0.0986, "num_tokens": 1274613.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8778114318847656, "sampling/importance_sampling_ratio/mean": 1.000512957572937, "sampling/importance_sampling_ratio/min": 0.5995104312896729, "sampling/sampling_logp_difference/max": 0.6301069259643555, "sampling/sampling_logp_difference/mean": 0.011764852330088615, "step": 23 }, { "clip_ratio/high_max": 0.00021059082064311951, "clip_ratio/high_mean": 0.00021059082064311951, "clip_ratio/low_mean": 0.00010199918324360624, "clip_ratio/low_min": 0.00010199918324360624, "clip_ratio/region_mean": 0.00031259000388672575, "completions/clipped_ratio": 0.0, "completions/max_length": 15383.0, "completions/max_terminated_length": 15383.0, "completions/mean_length": 9550.625, "completions/mean_terminated_length": 9550.625, "completions/min_length": 6705.0, "completions/min_terminated_length": 6705.0, "entropy": 0.4893578588962555, "epoch": 0.0013794689044717783, "frac_reward_zero_std": 0.0, "grad_norm": 0.007482054177671671, "learning_rate": 1e-05, "loss": 0.0095, "num_tokens": 1351946.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999520778656006, "sampling/importance_sampling_ratio/min": 0.3111985921859741, "sampling/sampling_logp_difference/max": 1.1673240661621094, "sampling/sampling_logp_difference/mean": 0.019285906106233597, "step": 24 }, { "clip_ratio/high_max": 6.948304508114234e-05, "clip_ratio/high_mean": 6.948304508114234e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.948304508114234e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 1799.0, "completions/max_terminated_length": 1799.0, "completions/mean_length": 1389.0, "completions/mean_terminated_length": 1389.0, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.1900008488446474, "epoch": 0.0014369467754914358, "frac_reward_zero_std": 0.0, "grad_norm": 0.010644371621310711, "learning_rate": 1e-05, "loss": 0.0707, "num_tokens": 1364522.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2926770448684692, "sampling/importance_sampling_ratio/mean": 1.0001492500305176, "sampling/importance_sampling_ratio/min": 0.7533293962478638, "sampling/sampling_logp_difference/max": 0.2832527160644531, "sampling/sampling_logp_difference/mean": 0.0072412146255373955, "step": 25 }, { "clip_ratio/high_max": 0.00011978824113612063, "clip_ratio/high_mean": 0.00011978824113612063, "clip_ratio/low_mean": 9.366803715238348e-05, "clip_ratio/low_min": 9.366803715238348e-05, "clip_ratio/region_mean": 0.0002134562782885041, "completions/clipped_ratio": 0.0, "completions/max_length": 3192.0, "completions/max_terminated_length": 3192.0, "completions/mean_length": 2180.625, "completions/mean_terminated_length": 2180.625, "completions/min_length": 1240.0, "completions/min_terminated_length": 1240.0, "entropy": 0.37862563878297806, "epoch": 0.0014944246465110932, "frac_reward_zero_std": 0.0, "grad_norm": 0.03671795129776001, "learning_rate": 1e-05, "loss": 0.0621, "num_tokens": 1382967.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4113205671310425, "sampling/importance_sampling_ratio/mean": 0.9998127818107605, "sampling/importance_sampling_ratio/min": 0.49675899744033813, "sampling/sampling_logp_difference/max": 0.6996502876281738, "sampling/sampling_logp_difference/mean": 0.014092679135501385, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6224.0, "completions/max_terminated_length": 6224.0, "completions/mean_length": 3049.125, "completions/mean_terminated_length": 3049.125, "completions/min_length": 1252.0, "completions/min_terminated_length": 1252.0, "entropy": 1.1191394180059433, "epoch": 0.0015519025175307506, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 1409560.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6623228788375854, "sampling/importance_sampling_ratio/mean": 0.9997996091842651, "sampling/importance_sampling_ratio/min": 0.6972482204437256, "sampling/sampling_logp_difference/max": 0.5082159042358398, "sampling/sampling_logp_difference/mean": 0.023721016943454742, "step": 27 }, { "clip_ratio/high_max": 0.00017029972514137626, "clip_ratio/high_mean": 0.00017029972514137626, "clip_ratio/low_mean": 0.0003643309319159016, "clip_ratio/low_min": 0.0003643309319159016, "clip_ratio/region_mean": 0.0005346306570572779, "completions/clipped_ratio": 0.0, "completions/max_length": 11496.0, "completions/max_terminated_length": 11496.0, "completions/mean_length": 2998.0, "completions/mean_terminated_length": 2998.0, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.35346287302672863, "epoch": 0.001609380388550408, "frac_reward_zero_std": 0.0, "grad_norm": 0.026703374460339546, "learning_rate": 1e-05, "loss": 0.4945, "num_tokens": 1436712.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.8510832786560059, "sampling/importance_sampling_ratio/mean": 1.0000603199005127, "sampling/importance_sampling_ratio/min": 0.4190898835659027, "sampling/sampling_logp_difference/max": 0.8696699142456055, "sampling/sampling_logp_difference/mean": 0.016062427312135696, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7258.0, "completions/max_terminated_length": 7258.0, "completions/mean_length": 5022.75, "completions/mean_terminated_length": 5022.75, "completions/min_length": 2704.0, "completions/min_terminated_length": 2704.0, "entropy": 0.47850726172327995, "epoch": 0.0016668582595700655, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 1478150.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5747973918914795, "sampling/importance_sampling_ratio/mean": 0.9998484253883362, "sampling/importance_sampling_ratio/min": 0.2432888150215149, "sampling/sampling_logp_difference/max": 1.413506031036377, "sampling/sampling_logp_difference/mean": 0.012552469037473202, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0006041391061444301, "clip_ratio/low_min": 0.0006041391061444301, "clip_ratio/region_mean": 0.0006041391061444301, "completions/clipped_ratio": 0.0, "completions/max_length": 4163.0, "completions/max_terminated_length": 4163.0, "completions/mean_length": 2496.25, "completions/mean_terminated_length": 2496.25, "completions/min_length": 1023.0, "completions/min_terminated_length": 1023.0, "entropy": 0.4747072793543339, "epoch": 0.001724336130589723, "frac_reward_zero_std": 0.0, "grad_norm": 0.02458236925303936, "learning_rate": 1e-05, "loss": -0.0758, "num_tokens": 1499672.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4631445407867432, "sampling/importance_sampling_ratio/mean": 1.0004292726516724, "sampling/importance_sampling_ratio/min": 0.695888876914978, "sampling/sampling_logp_difference/max": 0.38058793544769287, "sampling/sampling_logp_difference/mean": 0.01702776364982128, "step": 30 }, { "clip_ratio/high_max": 0.00011294151954643894, "clip_ratio/high_mean": 0.00011294151954643894, "clip_ratio/low_mean": 0.00013817244325764477, "clip_ratio/low_min": 0.00013817244325764477, "clip_ratio/region_mean": 0.0002511139628040837, "completions/clipped_ratio": 0.0, "completions/max_length": 9644.0, "completions/max_terminated_length": 9644.0, "completions/mean_length": 4240.625, "completions/mean_terminated_length": 4240.625, "completions/min_length": 2661.0, "completions/min_terminated_length": 2661.0, "entropy": 0.40254703164100647, "epoch": 0.0017818140016093804, "frac_reward_zero_std": 0.0, "grad_norm": 0.011770940385758877, "learning_rate": 1e-05, "loss": -0.1578, "num_tokens": 1535061.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001137256622314, "sampling/importance_sampling_ratio/min": 0.5766231417655945, "sampling/sampling_logp_difference/max": 0.8971240520477295, "sampling/sampling_logp_difference/mean": 0.015235140919685364, "step": 31 }, { "clip_ratio/high_max": 3.31272094626911e-05, "clip_ratio/high_mean": 3.31272094626911e-05, "clip_ratio/low_mean": 0.00022139876818982884, "clip_ratio/low_min": 0.00022139876818982884, "clip_ratio/region_mean": 0.00025452597765251994, "completions/clipped_ratio": 0.0, "completions/max_length": 11392.0, "completions/max_terminated_length": 11392.0, "completions/mean_length": 4060.75, "completions/mean_terminated_length": 4060.75, "completions/min_length": 1180.0, "completions/min_terminated_length": 1180.0, "entropy": 0.37329025007784367, "epoch": 0.0018392918726290379, "frac_reward_zero_std": 0.0, "grad_norm": 0.02815098874270916, "learning_rate": 1e-05, "loss": 0.1951, "num_tokens": 1568635.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000230073928833, "sampling/importance_sampling_ratio/min": 0.5119144916534424, "sampling/sampling_logp_difference/max": 0.7037672996520996, "sampling/sampling_logp_difference/mean": 0.019624225795269012, "step": 32 }, { "clip_ratio/high_max": 0.00017782688155421056, "clip_ratio/high_mean": 0.00017782688155421056, "clip_ratio/low_mean": 0.0002970715577248484, "clip_ratio/low_min": 0.0002970715577248484, "clip_ratio/region_mean": 0.00047489843927905895, "completions/clipped_ratio": 0.0, "completions/max_length": 13661.0, "completions/max_terminated_length": 13661.0, "completions/mean_length": 7598.5, "completions/mean_terminated_length": 7598.5, "completions/min_length": 4159.0, "completions/min_terminated_length": 4159.0, "entropy": 0.6333885863423347, "epoch": 0.0018967697436486953, "frac_reward_zero_std": 0.0, "grad_norm": 0.013582466170191765, "learning_rate": 1e-05, "loss": 0.1817, "num_tokens": 1630687.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998713731765747, "sampling/importance_sampling_ratio/min": 0.30169931054115295, "sampling/sampling_logp_difference/max": 1.19832444190979, "sampling/sampling_logp_difference/mean": 0.02295018918812275, "step": 33 }, { "clip_ratio/high_max": 2.9680441002710722e-05, "clip_ratio/high_mean": 2.9680441002710722e-05, "clip_ratio/low_mean": 0.0005807479028590024, "clip_ratio/low_min": 0.0005807479028590024, "clip_ratio/region_mean": 0.0006104283438617131, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15896.0, "completions/mean_length": 12444.75, "completions/mean_terminated_length": 10081.2001953125, "completions/min_length": 5550.0, "completions/min_terminated_length": 5550.0, "entropy": 0.4394209682941437, "epoch": 0.0019542476146683528, "frac_reward_zero_std": 0.0, "grad_norm": 0.0108946543186903, "learning_rate": 1e-05, "loss": 0.1747, "num_tokens": 1731557.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999930202960968, "sampling/importance_sampling_ratio/min": 0.16880448162555695, "sampling/sampling_logp_difference/max": 1.7790141105651855, "sampling/sampling_logp_difference/mean": 0.01957680657505989, "step": 34 }, { "clip_ratio/high_max": 7.717753396718763e-05, "clip_ratio/high_mean": 7.717753396718763e-05, "clip_ratio/low_mean": 0.00036224109135218896, "clip_ratio/low_min": 0.00036224109135218896, "clip_ratio/region_mean": 0.0004394186253193766, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 10658.0, "completions/mean_length": 7760.375, "completions/mean_terminated_length": 6528.4287109375, "completions/min_length": 3282.0, "completions/min_terminated_length": 3282.0, "entropy": 0.3291672505438328, "epoch": 0.00201172548568801, "frac_reward_zero_std": 0.0, "grad_norm": 0.013807074166834354, "learning_rate": 1e-05, "loss": 0.2426, "num_tokens": 1794696.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.7015570402145386, "sampling/importance_sampling_ratio/mean": 0.9997872114181519, "sampling/importance_sampling_ratio/min": 0.2956359386444092, "sampling/sampling_logp_difference/max": 1.2186264991760254, "sampling/sampling_logp_difference/mean": 0.015166893601417542, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00026345426158513874, "clip_ratio/low_min": 0.00026345426158513874, "clip_ratio/region_mean": 0.00026345426158513874, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 1201.125, "completions/mean_terminated_length": 1201.125, "completions/min_length": 731.0, "completions/min_terminated_length": 731.0, "entropy": 0.273766353726387, "epoch": 0.0020692033567076677, "frac_reward_zero_std": 0.0, "grad_norm": 0.03457275405526161, "learning_rate": 1e-05, "loss": -0.0332, "num_tokens": 1805449.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.281903862953186, "sampling/importance_sampling_ratio/mean": 0.9998401999473572, "sampling/importance_sampling_ratio/min": 0.7044704556465149, "sampling/sampling_logp_difference/max": 0.350308895111084, "sampling/sampling_logp_difference/mean": 0.010058763436973095, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0006667066845693626, "clip_ratio/low_min": 0.0006667066845693626, "clip_ratio/region_mean": 0.0006667066845693626, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 11499.0, "completions/mean_length": 9040.0, "completions/mean_terminated_length": 6592.0, "completions/min_length": 2660.0, "completions/min_terminated_length": 2660.0, "entropy": 0.45479580760002136, "epoch": 0.002126681227727325, "frac_reward_zero_std": 0.0, "grad_norm": 0.012254394590854645, "learning_rate": 1e-05, "loss": 0.3725, "num_tokens": 1879545.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999937415122986, "sampling/importance_sampling_ratio/min": 0.1515624076128006, "sampling/sampling_logp_difference/max": 1.8867578506469727, "sampling/sampling_logp_difference/mean": 0.018890954554080963, "step": 37 }, { "clip_ratio/high_max": 0.00012891687038063537, "clip_ratio/high_mean": 0.00012891687038063537, "clip_ratio/low_mean": 0.0004333783217589371, "clip_ratio/low_min": 0.0004333783217589371, "clip_ratio/region_mean": 0.0005622951921395725, "completions/clipped_ratio": 0.0, "completions/max_length": 16173.0, "completions/max_terminated_length": 16173.0, "completions/mean_length": 9202.375, "completions/mean_terminated_length": 9202.375, "completions/min_length": 2678.0, "completions/min_terminated_length": 2678.0, "entropy": 0.38314999639987946, "epoch": 0.0021841590987469826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01417507790029049, "learning_rate": 1e-05, "loss": 0.0678, "num_tokens": 1954084.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000090599060059, "sampling/importance_sampling_ratio/min": 0.2041807472705841, "sampling/sampling_logp_difference/max": 1.588749647140503, "sampling/sampling_logp_difference/mean": 0.01605384796857834, "step": 38 }, { "clip_ratio/high_max": 0.0001823197799240006, "clip_ratio/high_mean": 0.0001823197799240006, "clip_ratio/low_mean": 0.0003852306108456105, "clip_ratio/low_min": 0.0003852306108456105, "clip_ratio/region_mean": 0.0005675503907696111, "completions/clipped_ratio": 0.0, "completions/max_length": 14687.0, "completions/max_terminated_length": 14687.0, "completions/mean_length": 8147.0, "completions/mean_terminated_length": 8147.0, "completions/min_length": 916.0, "completions/min_terminated_length": 916.0, "entropy": 0.46798358857631683, "epoch": 0.00224163696976664, "frac_reward_zero_std": 0.0, "grad_norm": 0.025270069018006325, "learning_rate": 1e-05, "loss": -0.3309, "num_tokens": 2020420.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.9406111240386963, "sampling/importance_sampling_ratio/mean": 1.0002000331878662, "sampling/importance_sampling_ratio/min": 0.36602357029914856, "sampling/sampling_logp_difference/max": 1.0050575733184814, "sampling/sampling_logp_difference/mean": 0.020551156252622604, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.865824540611356e-05, "clip_ratio/low_min": 9.865824540611356e-05, "clip_ratio/region_mean": 9.865824540611356e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 1972.0, "completions/max_terminated_length": 1972.0, "completions/mean_length": 1393.875, "completions/mean_terminated_length": 1393.875, "completions/min_length": 918.0, "completions/min_terminated_length": 918.0, "entropy": 0.3229250833392143, "epoch": 0.0022991148407862975, "frac_reward_zero_std": 0.0, "grad_norm": 0.025179797783493996, "learning_rate": 1e-05, "loss": 0.0512, "num_tokens": 2032915.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4177850484848022, "sampling/importance_sampling_ratio/mean": 1.0002374649047852, "sampling/importance_sampling_ratio/min": 0.7231569290161133, "sampling/sampling_logp_difference/max": 0.34909582138061523, "sampling/sampling_logp_difference/mean": 0.01290525309741497, "step": 40 }, { "clip_ratio/high_max": 0.00015636347234249115, "clip_ratio/high_mean": 0.00015636347234249115, "clip_ratio/low_mean": 0.00015422578144352883, "clip_ratio/low_min": 0.00015422578144352883, "clip_ratio/region_mean": 0.00031058925378602, "completions/clipped_ratio": 0.0, "completions/max_length": 7163.0, "completions/max_terminated_length": 7163.0, "completions/mean_length": 5693.625, "completions/mean_terminated_length": 5693.625, "completions/min_length": 3660.0, "completions/min_terminated_length": 3660.0, "entropy": 0.6593504920601845, "epoch": 0.0023565927118059547, "frac_reward_zero_std": 0.0, "grad_norm": 0.009168462827801704, "learning_rate": 1e-05, "loss": 0.0491, "num_tokens": 2079400.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6397300958633423, "sampling/importance_sampling_ratio/mean": 1.0000221729278564, "sampling/importance_sampling_ratio/min": 0.5670493841171265, "sampling/sampling_logp_difference/max": 0.5673089027404785, "sampling/sampling_logp_difference/mean": 0.023446809500455856, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3320.0, "completions/max_terminated_length": 3320.0, "completions/mean_length": 2165.625, "completions/mean_terminated_length": 2165.625, "completions/min_length": 1036.0, "completions/min_terminated_length": 1036.0, "entropy": 0.4291406534612179, "epoch": 0.0024140705828256124, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2097741.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4754000902175903, "sampling/importance_sampling_ratio/mean": 1.0001964569091797, "sampling/importance_sampling_ratio/min": 0.6778994202613831, "sampling/sampling_logp_difference/max": 0.38892924785614014, "sampling/sampling_logp_difference/mean": 0.01713942177593708, "step": 42 }, { "clip_ratio/high_max": 6.959911115700379e-05, "clip_ratio/high_mean": 6.959911115700379e-05, "clip_ratio/low_mean": 0.0003096781365456991, "clip_ratio/low_min": 0.0003096781365456991, "clip_ratio/region_mean": 0.0003792772477027029, "completions/clipped_ratio": 0.0, "completions/max_length": 2918.0, "completions/max_terminated_length": 2918.0, "completions/mean_length": 2122.125, "completions/mean_terminated_length": 2122.125, "completions/min_length": 1466.0, "completions/min_terminated_length": 1466.0, "entropy": 0.2825094908475876, "epoch": 0.0024715484538452696, "frac_reward_zero_std": 0.0, "grad_norm": 0.029945719987154007, "learning_rate": 1e-05, "loss": 0.1292, "num_tokens": 2115766.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4234094619750977, "sampling/importance_sampling_ratio/mean": 0.9999959468841553, "sampling/importance_sampling_ratio/min": 0.7060609459877014, "sampling/sampling_logp_difference/max": 0.3530550003051758, "sampling/sampling_logp_difference/mean": 0.011036572977900505, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15865.0, "completions/mean_length": 12600.375, "completions/mean_terminated_length": 12059.857421875, "completions/min_length": 5038.0, "completions/min_terminated_length": 5038.0, "entropy": 0.29098295606672764, "epoch": 0.002529026324864927, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2217897.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997394680976868, "sampling/importance_sampling_ratio/min": 0.047701530158519745, "sampling/sampling_logp_difference/max": 3.0427918434143066, "sampling/sampling_logp_difference/mean": 0.012049276381731033, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5978.0, "completions/max_terminated_length": 5978.0, "completions/mean_length": 5043.375, "completions/mean_terminated_length": 5043.375, "completions/min_length": 3933.0, "completions/min_terminated_length": 3933.0, "entropy": 0.4449828788638115, "epoch": 0.0025865041958845845, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2259348.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.769199252128601, "sampling/importance_sampling_ratio/mean": 0.9998717904090881, "sampling/importance_sampling_ratio/min": 0.5246632695198059, "sampling/sampling_logp_difference/max": 0.6449985504150391, "sampling/sampling_logp_difference/mean": 0.017409298568964005, "step": 45 }, { "clip_ratio/high_max": 4.307374183554202e-05, "clip_ratio/high_mean": 4.307374183554202e-05, "clip_ratio/low_mean": 0.0003977762389695272, "clip_ratio/low_min": 0.0003977762389695272, "clip_ratio/region_mean": 0.0004408499808050692, "completions/clipped_ratio": 0.0, "completions/max_length": 2902.0, "completions/max_terminated_length": 2902.0, "completions/mean_length": 1450.25, "completions/mean_terminated_length": 1450.25, "completions/min_length": 557.0, "completions/min_terminated_length": 557.0, "entropy": 0.2576371170580387, "epoch": 0.0026439820669042417, "frac_reward_zero_std": 0.0, "grad_norm": 0.037199344485998154, "learning_rate": 1e-05, "loss": -0.1005, "num_tokens": 2271630.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.4901281595230103, "sampling/importance_sampling_ratio/mean": 0.9999934434890747, "sampling/importance_sampling_ratio/min": 0.6777300834655762, "sampling/sampling_logp_difference/max": 0.3988621234893799, "sampling/sampling_logp_difference/mean": 0.011057121679186821, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010794473200803623, "clip_ratio/low_min": 0.00010794473200803623, "clip_ratio/region_mean": 0.00010794473200803623, "completions/clipped_ratio": 0.0, "completions/max_length": 2922.0, "completions/max_terminated_length": 2922.0, "completions/mean_length": 1511.75, "completions/mean_terminated_length": 1511.75, "completions/min_length": 729.0, "completions/min_terminated_length": 729.0, "entropy": 0.5865042172372341, "epoch": 0.0027014599379238994, "frac_reward_zero_std": 0.0, "grad_norm": 0.02290233224630356, "learning_rate": 1e-05, "loss": -0.1436, "num_tokens": 2284788.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.3138504028320312, "sampling/importance_sampling_ratio/mean": 1.0002350807189941, "sampling/importance_sampling_ratio/min": 0.6800664067268372, "sampling/sampling_logp_difference/max": 0.38556480407714844, "sampling/sampling_logp_difference/mean": 0.016870463266968727, "step": 47 }, { "clip_ratio/high_max": 3.955070496886037e-05, "clip_ratio/high_mean": 3.955070496886037e-05, "clip_ratio/low_mean": 2.347930785617791e-05, "clip_ratio/low_min": 2.347930785617791e-05, "clip_ratio/region_mean": 6.303001282503828e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10707.0, "completions/max_terminated_length": 10707.0, "completions/mean_length": 6775.0, "completions/mean_terminated_length": 6775.0, "completions/min_length": 1813.0, "completions/min_terminated_length": 1813.0, "entropy": 0.6008276157081127, "epoch": 0.0027589378089435566, "frac_reward_zero_std": 0.0, "grad_norm": 0.005901714321225882, "learning_rate": 1e-05, "loss": 0.3085, "num_tokens": 2340020.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998616576194763, "sampling/importance_sampling_ratio/min": 0.5181363224983215, "sampling/sampling_logp_difference/max": 0.7540040016174316, "sampling/sampling_logp_difference/mean": 0.018575357273221016, "step": 48 }, { "clip_ratio/high_max": 7.636024292878574e-05, "clip_ratio/high_mean": 7.636024292878574e-05, "clip_ratio/low_mean": 0.00025442260812269524, "clip_ratio/low_min": 0.00025442260812269524, "clip_ratio/region_mean": 0.000330782851051481, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14986.0, "completions/mean_length": 11027.875, "completions/mean_terminated_length": 9242.5, "completions/min_length": 2475.0, "completions/min_terminated_length": 2475.0, "entropy": 0.45021577924489975, "epoch": 0.0028164156799632143, "frac_reward_zero_std": 0.0, "grad_norm": 0.021097583696246147, "learning_rate": 1e-05, "loss": 0.0651, "num_tokens": 2429131.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999630451202393, "sampling/importance_sampling_ratio/min": 0.4296703040599823, "sampling/sampling_logp_difference/max": 0.8447370529174805, "sampling/sampling_logp_difference/mean": 0.020308000966906548, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10777.0, "completions/max_terminated_length": 10777.0, "completions/mean_length": 8693.625, "completions/mean_terminated_length": 8693.625, "completions/min_length": 5111.0, "completions/min_terminated_length": 5111.0, "entropy": 0.6992930024862289, "epoch": 0.0028738935509828715, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2500248.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998568892478943, "sampling/importance_sampling_ratio/min": 0.37074294686317444, "sampling/sampling_logp_difference/max": 0.9922463893890381, "sampling/sampling_logp_difference/mean": 0.027893485501408577, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013150973245501518, "clip_ratio/low_min": 0.00013150973245501518, "clip_ratio/region_mean": 0.00013150973245501518, "completions/clipped_ratio": 0.0, "completions/max_length": 3448.0, "completions/max_terminated_length": 3448.0, "completions/mean_length": 2144.5, "completions/mean_terminated_length": 2144.5, "completions/min_length": 1325.0, "completions/min_terminated_length": 1325.0, "entropy": 0.20928144082427025, "epoch": 0.002931371422002529, "frac_reward_zero_std": 0.0, "grad_norm": 0.05555763095617294, "learning_rate": 1e-05, "loss": -0.0964, "num_tokens": 2518124.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4257904291152954, "sampling/importance_sampling_ratio/mean": 0.9999476671218872, "sampling/importance_sampling_ratio/min": 0.6867164969444275, "sampling/sampling_logp_difference/max": 0.37583374977111816, "sampling/sampling_logp_difference/mean": 0.00927009154111147, "step": 51 }, { "clip_ratio/high_max": 6.471852248068899e-05, "clip_ratio/high_mean": 6.471852248068899e-05, "clip_ratio/low_mean": 6.35001269984059e-05, "clip_ratio/low_min": 6.35001269984059e-05, "clip_ratio/region_mean": 0.0001282186494790949, "completions/clipped_ratio": 0.0, "completions/max_length": 9808.0, "completions/max_terminated_length": 9808.0, "completions/mean_length": 6485.875, "completions/mean_terminated_length": 6485.875, "completions/min_length": 4964.0, "completions/min_terminated_length": 4964.0, "entropy": 0.26626385375857353, "epoch": 0.0029888492930221864, "frac_reward_zero_std": 0.0, "grad_norm": 0.05183282867074013, "learning_rate": 1e-05, "loss": 0.0755, "num_tokens": 2571451.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000094175338745, "sampling/importance_sampling_ratio/min": 0.46055662631988525, "sampling/sampling_logp_difference/max": 0.9583044052124023, "sampling/sampling_logp_difference/mean": 0.011562701314687729, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5567.0, "completions/max_terminated_length": 5567.0, "completions/mean_length": 4548.25, "completions/mean_terminated_length": 4548.25, "completions/min_length": 2948.0, "completions/min_terminated_length": 2948.0, "entropy": 0.3451690226793289, "epoch": 0.003046327164041844, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2608957.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4250214099884033, "sampling/importance_sampling_ratio/mean": 1.0001447200775146, "sampling/importance_sampling_ratio/min": 0.6228136420249939, "sampling/sampling_logp_difference/max": 0.4735078811645508, "sampling/sampling_logp_difference/mean": 0.013225145637989044, "step": 53 }, { "clip_ratio/high_max": 0.0002130183820554521, "clip_ratio/high_mean": 0.0002130183820554521, "clip_ratio/low_mean": 0.000633551215287298, "clip_ratio/low_min": 0.000633551215287298, "clip_ratio/region_mean": 0.0008465695973427501, "completions/clipped_ratio": 0.0, "completions/max_length": 8079.0, "completions/max_terminated_length": 8079.0, "completions/mean_length": 5219.375, "completions/mean_terminated_length": 5219.375, "completions/min_length": 2274.0, "completions/min_terminated_length": 2274.0, "entropy": 0.681399904191494, "epoch": 0.0031038050350615013, "frac_reward_zero_std": 0.0, "grad_norm": 0.038597557693719864, "learning_rate": 1e-05, "loss": -0.2008, "num_tokens": 2653176.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000501871109009, "sampling/importance_sampling_ratio/min": 0.5542822480201721, "sampling/sampling_logp_difference/max": 0.8316373825073242, "sampling/sampling_logp_difference/mean": 0.02502310648560524, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13798.0, "completions/mean_length": 12660.875, "completions/mean_terminated_length": 11419.833984375, "completions/min_length": 9486.0, "completions/min_terminated_length": 9486.0, "entropy": 0.6816116347908974, "epoch": 0.003161282906081159, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2755687.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000107288360596, "sampling/importance_sampling_ratio/min": 0.04938606917858124, "sampling/sampling_logp_difference/max": 3.008086919784546, "sampling/sampling_logp_difference/mean": 0.026097681373357773, "step": 55 }, { "clip_ratio/high_max": 0.00016612962645012885, "clip_ratio/high_mean": 0.00016612962645012885, "clip_ratio/low_mean": 0.00011158721463289112, "clip_ratio/low_min": 0.00011158721463289112, "clip_ratio/region_mean": 0.00027771684108301997, "completions/clipped_ratio": 0.0, "completions/max_length": 6250.0, "completions/max_terminated_length": 6250.0, "completions/mean_length": 5129.125, "completions/mean_terminated_length": 5129.125, "completions/min_length": 3532.0, "completions/min_terminated_length": 3532.0, "entropy": 0.3505010213702917, "epoch": 0.003218760777100816, "frac_reward_zero_std": 0.0, "grad_norm": 0.009208712726831436, "learning_rate": 1e-05, "loss": 0.0324, "num_tokens": 2797536.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4510549306869507, "sampling/importance_sampling_ratio/mean": 0.9997953176498413, "sampling/importance_sampling_ratio/min": 0.4308338165283203, "sampling/sampling_logp_difference/max": 0.8420329093933105, "sampling/sampling_logp_difference/mean": 0.012967368587851524, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12147.0, "completions/max_terminated_length": 12147.0, "completions/mean_length": 7532.125, "completions/mean_terminated_length": 7532.125, "completions/min_length": 3595.0, "completions/min_terminated_length": 3595.0, "entropy": 0.545828327536583, "epoch": 0.0032762386481204734, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2859137.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998393058776855, "sampling/importance_sampling_ratio/min": 0.3288186490535736, "sampling/sampling_logp_difference/max": 1.2061529159545898, "sampling/sampling_logp_difference/mean": 0.02107314020395279, "step": 57 }, { "clip_ratio/high_max": 5.484861685545184e-05, "clip_ratio/high_mean": 5.484861685545184e-05, "clip_ratio/low_mean": 8.833922038320452e-05, "clip_ratio/low_min": 8.833922038320452e-05, "clip_ratio/region_mean": 0.00014318783723865636, "completions/clipped_ratio": 0.0, "completions/max_length": 2403.0, "completions/max_terminated_length": 2403.0, "completions/mean_length": 1772.375, "completions/mean_terminated_length": 1772.375, "completions/min_length": 1263.0, "completions/min_terminated_length": 1263.0, "entropy": 0.2984078824520111, "epoch": 0.003333716519140131, "frac_reward_zero_std": 0.0, "grad_norm": 0.01238580048084259, "learning_rate": 1e-05, "loss": -0.0716, "num_tokens": 2874292.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3563395738601685, "sampling/importance_sampling_ratio/mean": 1.0003012418746948, "sampling/importance_sampling_ratio/min": 0.6631757616996765, "sampling/sampling_logp_difference/max": 0.4107152223587036, "sampling/sampling_logp_difference/mean": 0.011635399423539639, "step": 58 }, { "clip_ratio/high_max": 9.920635056914762e-05, "clip_ratio/high_mean": 9.920635056914762e-05, "clip_ratio/low_mean": 0.00020814879826502874, "clip_ratio/low_min": 0.00020814879826502874, "clip_ratio/region_mean": 0.00030735514883417636, "completions/clipped_ratio": 0.0, "completions/max_length": 1637.0, "completions/max_terminated_length": 1637.0, "completions/mean_length": 1393.875, "completions/mean_terminated_length": 1393.875, "completions/min_length": 1012.0, "completions/min_terminated_length": 1012.0, "entropy": 0.24145445227622986, "epoch": 0.0033911943901597883, "frac_reward_zero_std": 0.0, "grad_norm": 0.021861892193555832, "learning_rate": 1e-05, "loss": -0.0578, "num_tokens": 2887627.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4735182523727417, "sampling/importance_sampling_ratio/mean": 1.0001130104064941, "sampling/importance_sampling_ratio/min": 0.631110429763794, "sampling/sampling_logp_difference/max": 0.46027445793151855, "sampling/sampling_logp_difference/mean": 0.009926659055054188, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6529.0, "completions/max_terminated_length": 6529.0, "completions/mean_length": 4031.625, "completions/mean_terminated_length": 4031.625, "completions/min_length": 1792.0, "completions/min_terminated_length": 1792.0, "entropy": 0.2522127367556095, "epoch": 0.003448672261179446, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 2920792.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5025330781936646, "sampling/importance_sampling_ratio/mean": 1.0001198053359985, "sampling/importance_sampling_ratio/min": 0.6018195152282715, "sampling/sampling_logp_difference/max": 0.5077977180480957, "sampling/sampling_logp_difference/mean": 0.010761220008134842, "step": 60 }, { "clip_ratio/high_max": 0.00024247844339697622, "clip_ratio/high_mean": 0.00024247844339697622, "clip_ratio/low_mean": 4.752851600642316e-05, "clip_ratio/low_min": 4.752851600642316e-05, "clip_ratio/region_mean": 0.0002900069594033994, "completions/clipped_ratio": 0.0, "completions/max_length": 3546.0, "completions/max_terminated_length": 3546.0, "completions/mean_length": 2080.25, "completions/mean_terminated_length": 2080.25, "completions/min_length": 967.0, "completions/min_terminated_length": 967.0, "entropy": 0.2911796625703573, "epoch": 0.003506150132199103, "frac_reward_zero_std": 0.0, "grad_norm": 0.023584146052598953, "learning_rate": 1e-05, "loss": -0.0732, "num_tokens": 2938794.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4307900667190552, "sampling/importance_sampling_ratio/mean": 0.9999948740005493, "sampling/importance_sampling_ratio/min": 0.69541996717453, "sampling/sampling_logp_difference/max": 0.3632392883300781, "sampling/sampling_logp_difference/mean": 0.012206352315843105, "step": 61 }, { "clip_ratio/high_max": 1.4455880773311947e-05, "clip_ratio/high_mean": 1.4455880773311947e-05, "clip_ratio/low_mean": 0.000513993112690514, "clip_ratio/low_min": 0.000513993112690514, "clip_ratio/region_mean": 0.0005284489934638259, "completions/clipped_ratio": 0.0, "completions/max_length": 8647.0, "completions/max_terminated_length": 8647.0, "completions/mean_length": 4010.375, "completions/mean_terminated_length": 4010.375, "completions/min_length": 740.0, "completions/min_terminated_length": 740.0, "entropy": 0.4742611162364483, "epoch": 0.003563628003218761, "frac_reward_zero_std": 0.0, "grad_norm": 0.03585368022322655, "learning_rate": 1e-05, "loss": -0.0353, "num_tokens": 2975653.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6222310066223145, "sampling/importance_sampling_ratio/mean": 1.0001109838485718, "sampling/importance_sampling_ratio/min": 0.608767569065094, "sampling/sampling_logp_difference/max": 0.4963188171386719, "sampling/sampling_logp_difference/mean": 0.020225893706083298, "step": 62 }, { "clip_ratio/high_max": 0.00017478327390563209, "clip_ratio/high_mean": 0.00017478327390563209, "clip_ratio/low_mean": 0.00015009606431704015, "clip_ratio/low_min": 0.00015009606431704015, "clip_ratio/region_mean": 0.00032487933822267223, "completions/clipped_ratio": 0.0, "completions/max_length": 8328.0, "completions/max_terminated_length": 8328.0, "completions/mean_length": 6091.625, "completions/mean_terminated_length": 6091.625, "completions/min_length": 3429.0, "completions/min_terminated_length": 3429.0, "entropy": 0.4262738786637783, "epoch": 0.003621105874238418, "frac_reward_zero_std": 0.0, "grad_norm": 0.0088336281478405, "learning_rate": 1e-05, "loss": 0.1299, "num_tokens": 3025186.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6910793781280518, "sampling/importance_sampling_ratio/mean": 1.0002009868621826, "sampling/importance_sampling_ratio/min": 0.6178554892539978, "sampling/sampling_logp_difference/max": 0.525367021560669, "sampling/sampling_logp_difference/mean": 0.016626553609967232, "step": 63 }, { "clip_ratio/high_max": 8.472792978864163e-05, "clip_ratio/high_mean": 8.472792978864163e-05, "clip_ratio/low_mean": 0.0005569919776462484, "clip_ratio/low_min": 0.0005569919776462484, "clip_ratio/region_mean": 0.00064171990743489, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12677.0, "completions/mean_length": 9709.75, "completions/mean_terminated_length": 8756.2861328125, "completions/min_length": 4627.0, "completions/min_terminated_length": 4627.0, "entropy": 0.5099417231976986, "epoch": 0.0036785837452580758, "frac_reward_zero_std": 0.0, "grad_norm": 0.025481535121798515, "learning_rate": 1e-05, "loss": 0.057, "num_tokens": 3104288.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999805212020874, "sampling/importance_sampling_ratio/min": 0.3367921710014343, "sampling/sampling_logp_difference/max": 1.3402471542358398, "sampling/sampling_logp_difference/mean": 0.021615201607346535, "step": 64 }, { "clip_ratio/high_max": 6.347670478135115e-05, "clip_ratio/high_mean": 6.347670478135115e-05, "clip_ratio/low_mean": 0.0005828946596011519, "clip_ratio/low_min": 0.0005828946596011519, "clip_ratio/region_mean": 0.0006463713643825031, "completions/clipped_ratio": 0.0, "completions/max_length": 14769.0, "completions/max_terminated_length": 14769.0, "completions/mean_length": 11125.625, "completions/mean_terminated_length": 11125.625, "completions/min_length": 8889.0, "completions/min_terminated_length": 8889.0, "entropy": 0.5693316161632538, "epoch": 0.003736061616277733, "frac_reward_zero_std": 0.0, "grad_norm": 0.013858629390597343, "learning_rate": 1e-05, "loss": 0.1261, "num_tokens": 3194197.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999197721481323, "sampling/importance_sampling_ratio/min": 0.20770741999149323, "sampling/sampling_logp_difference/max": 1.571624755859375, "sampling/sampling_logp_difference/mean": 0.022273730486631393, "step": 65 }, { "clip_ratio/high_max": 4.7023926526890136e-05, "clip_ratio/high_mean": 4.7023926526890136e-05, "clip_ratio/low_mean": 0.0003786109300563112, "clip_ratio/low_min": 0.0003786109300563112, "clip_ratio/region_mean": 0.0004256348565832013, "completions/clipped_ratio": 0.0, "completions/max_length": 7961.0, "completions/max_terminated_length": 7961.0, "completions/mean_length": 5469.125, "completions/mean_terminated_length": 5469.125, "completions/min_length": 3050.0, "completions/min_terminated_length": 3050.0, "entropy": 0.37077387794852257, "epoch": 0.0037935394872973907, "frac_reward_zero_std": 0.0, "grad_norm": 0.014726302586495876, "learning_rate": 1e-05, "loss": -0.0125, "num_tokens": 3239414.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4613500833511353, "sampling/importance_sampling_ratio/mean": 1.00039803981781, "sampling/importance_sampling_ratio/min": 0.6130610108375549, "sampling/sampling_logp_difference/max": 0.48929083347320557, "sampling/sampling_logp_difference/mean": 0.015297925099730492, "step": 66 }, { "clip_ratio/high_max": 0.00014758230099687353, "clip_ratio/high_mean": 0.00014758230099687353, "clip_ratio/low_mean": 0.00034344276355113834, "clip_ratio/low_min": 0.00034344276355113834, "clip_ratio/region_mean": 0.0004910250645480119, "completions/clipped_ratio": 0.0, "completions/max_length": 15818.0, "completions/max_terminated_length": 15818.0, "completions/mean_length": 4518.875, "completions/mean_terminated_length": 4518.875, "completions/min_length": 710.0, "completions/min_terminated_length": 710.0, "entropy": 0.46321990713477135, "epoch": 0.003851017358317048, "frac_reward_zero_std": 0.0, "grad_norm": 0.052609484642744064, "learning_rate": 1e-05, "loss": 0.1724, "num_tokens": 3276549.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999647736549377, "sampling/importance_sampling_ratio/min": 0.5366171598434448, "sampling/sampling_logp_difference/max": 0.7253210544586182, "sampling/sampling_logp_difference/mean": 0.019297108054161072, "step": 67 }, { "clip_ratio/high_max": 5.84329609409906e-05, "clip_ratio/high_mean": 5.84329609409906e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.84329609409906e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6515.0, "completions/max_terminated_length": 6515.0, "completions/mean_length": 3167.125, "completions/mean_terminated_length": 3167.125, "completions/min_length": 930.0, "completions/min_terminated_length": 930.0, "entropy": 0.3499685563147068, "epoch": 0.0039084952293367056, "frac_reward_zero_std": 0.0, "grad_norm": 0.008895262144505978, "learning_rate": 1e-05, "loss": -0.2496, "num_tokens": 3303014.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.476549744606018, "sampling/importance_sampling_ratio/mean": 1.000215768814087, "sampling/importance_sampling_ratio/min": 0.6207373738288879, "sampling/sampling_logp_difference/max": 0.47684717178344727, "sampling/sampling_logp_difference/mean": 0.015071067027747631, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 1.4886269127600826e-05, "clip_ratio/low_min": 1.4886269127600826e-05, "clip_ratio/region_mean": 1.4886269127600826e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 8397.0, "completions/max_terminated_length": 8397.0, "completions/mean_length": 4625.375, "completions/mean_terminated_length": 4625.375, "completions/min_length": 1610.0, "completions/min_terminated_length": 1610.0, "entropy": 0.5824310332536697, "epoch": 0.003965973100356363, "frac_reward_zero_std": 0.0, "grad_norm": 0.1078847348690033, "learning_rate": 1e-05, "loss": 0.2885, "num_tokens": 3341649.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6486999988555908, "sampling/importance_sampling_ratio/mean": 0.9998449683189392, "sampling/importance_sampling_ratio/min": 0.6175811886787415, "sampling/sampling_logp_difference/max": 0.4999871253967285, "sampling/sampling_logp_difference/mean": 0.017474355176091194, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3218.0, "completions/max_terminated_length": 3218.0, "completions/mean_length": 1611.0, "completions/mean_terminated_length": 1611.0, "completions/min_length": 498.0, "completions/min_terminated_length": 498.0, "entropy": 0.44486694410443306, "epoch": 0.00402345097137602, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3355417.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2839782238006592, "sampling/importance_sampling_ratio/mean": 1.0003026723861694, "sampling/importance_sampling_ratio/min": 0.6977416276931763, "sampling/sampling_logp_difference/max": 0.3599064350128174, "sampling/sampling_logp_difference/mean": 0.016955537721514702, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11105.0, "completions/max_terminated_length": 11105.0, "completions/mean_length": 6453.75, "completions/mean_terminated_length": 6453.75, "completions/min_length": 2829.0, "completions/min_terminated_length": 2829.0, "entropy": 0.5774021446704865, "epoch": 0.004080928842395678, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3408039.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9741193056106567, "sampling/importance_sampling_ratio/mean": 1.0002410411834717, "sampling/importance_sampling_ratio/min": 0.5421406626701355, "sampling/sampling_logp_difference/max": 0.6801223754882812, "sampling/sampling_logp_difference/mean": 0.017538031563162804, "step": 71 }, { "clip_ratio/high_max": 7.407679731841199e-05, "clip_ratio/high_mean": 7.407679731841199e-05, "clip_ratio/low_mean": 0.0003438445564825088, "clip_ratio/low_min": 0.0003438445564825088, "clip_ratio/region_mean": 0.00041792135380092077, "completions/clipped_ratio": 0.0, "completions/max_length": 9270.0, "completions/max_terminated_length": 9270.0, "completions/mean_length": 5584.875, "completions/mean_terminated_length": 5584.875, "completions/min_length": 498.0, "completions/min_terminated_length": 498.0, "entropy": 0.44911012426018715, "epoch": 0.004138406713415335, "frac_reward_zero_std": 0.0, "grad_norm": 0.016307076439261436, "learning_rate": 1e-05, "loss": 0.2551, "num_tokens": 3454102.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8314783573150635, "sampling/importance_sampling_ratio/mean": 1.0001745223999023, "sampling/importance_sampling_ratio/min": 0.47552651166915894, "sampling/sampling_logp_difference/max": 0.7433326244354248, "sampling/sampling_logp_difference/mean": 0.01905350759625435, "step": 72 }, { "clip_ratio/high_max": 0.00010738677883637138, "clip_ratio/high_mean": 0.00010738677883637138, "clip_ratio/low_mean": 0.0003271719397162087, "clip_ratio/low_min": 0.0003271719397162087, "clip_ratio/region_mean": 0.0004345587185525801, "completions/clipped_ratio": 0.0, "completions/max_length": 11830.0, "completions/max_terminated_length": 11830.0, "completions/mean_length": 6688.75, "completions/mean_terminated_length": 6688.75, "completions/min_length": 2061.0, "completions/min_terminated_length": 2061.0, "entropy": 0.7740826979279518, "epoch": 0.004195884584434992, "frac_reward_zero_std": 0.0, "grad_norm": 0.029672987759113312, "learning_rate": 1e-05, "loss": 0.2613, "num_tokens": 3508732.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8884772062301636, "sampling/importance_sampling_ratio/mean": 1.000227689743042, "sampling/importance_sampling_ratio/min": 0.4901808798313141, "sampling/sampling_logp_difference/max": 0.7129808664321899, "sampling/sampling_logp_difference/mean": 0.02653191238641739, "step": 73 }, { "clip_ratio/high_max": 6.572279562533367e-05, "clip_ratio/high_mean": 6.572279562533367e-05, "clip_ratio/low_mean": 0.0006653135860688053, "clip_ratio/low_min": 0.0006653135860688053, "clip_ratio/region_mean": 0.000731036381694139, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14942.0, "completions/mean_length": 11513.625, "completions/mean_terminated_length": 10817.857421875, "completions/min_length": 6219.0, "completions/min_terminated_length": 6219.0, "entropy": 0.6061702743172646, "epoch": 0.00425336245545465, "frac_reward_zero_std": 0.0, "grad_norm": 0.023116735741496086, "learning_rate": 1e-05, "loss": 0.1467, "num_tokens": 3601689.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999862790107727, "sampling/importance_sampling_ratio/min": 0.0003409076016396284, "sampling/sampling_logp_difference/max": 7.983899116516113, "sampling/sampling_logp_difference/mean": 0.022893456742167473, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3982.0, "completions/max_terminated_length": 3982.0, "completions/mean_length": 2700.875, "completions/mean_terminated_length": 2700.875, "completions/min_length": 659.0, "completions/min_terminated_length": 659.0, "entropy": 0.1945251878350973, "epoch": 0.0043108403264743075, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3624448.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4087806940078735, "sampling/importance_sampling_ratio/mean": 0.9997677803039551, "sampling/importance_sampling_ratio/min": 0.44225573539733887, "sampling/sampling_logp_difference/max": 0.8158669471740723, "sampling/sampling_logp_difference/mean": 0.007831585593521595, "step": 75 }, { "clip_ratio/high_max": 2.9169259505579248e-05, "clip_ratio/high_mean": 2.9169259505579248e-05, "clip_ratio/low_mean": 0.0010002476192312315, "clip_ratio/low_min": 0.0010002476192312315, "clip_ratio/region_mean": 0.0010294168787368108, "completions/clipped_ratio": 0.0, "completions/max_length": 12856.0, "completions/max_terminated_length": 12856.0, "completions/mean_length": 9036.75, "completions/mean_terminated_length": 9036.75, "completions/min_length": 3947.0, "completions/min_terminated_length": 3947.0, "entropy": 0.584886621683836, "epoch": 0.004368318197493965, "frac_reward_zero_std": 0.0, "grad_norm": 0.007610360626131296, "learning_rate": 1e-05, "loss": -0.1496, "num_tokens": 3698494.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000782012939453, "sampling/importance_sampling_ratio/min": 0.4218885898590088, "sampling/sampling_logp_difference/max": 0.8630140423774719, "sampling/sampling_logp_difference/mean": 0.02367301844060421, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2698145736940205e-05, "clip_ratio/low_min": 5.2698145736940205e-05, "clip_ratio/region_mean": 5.2698145736940205e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 2372.0, "completions/max_terminated_length": 2372.0, "completions/mean_length": 1625.625, "completions/mean_terminated_length": 1625.625, "completions/min_length": 835.0, "completions/min_terminated_length": 835.0, "entropy": 0.7885649353265762, "epoch": 0.004425796068513622, "frac_reward_zero_std": 0.0, "grad_norm": 0.0428813137114048, "learning_rate": 1e-05, "loss": -0.0962, "num_tokens": 3712811.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.2535712718963623, "sampling/importance_sampling_ratio/mean": 0.9997212886810303, "sampling/importance_sampling_ratio/min": 0.6434804797172546, "sampling/sampling_logp_difference/max": 0.44086360931396484, "sampling/sampling_logp_difference/mean": 0.020571956411004066, "step": 77 }, { "clip_ratio/high_max": 3.8287769712042063e-05, "clip_ratio/high_mean": 3.8287769712042063e-05, "clip_ratio/low_mean": 0.0008040851753321476, "clip_ratio/low_min": 0.0008040851753321476, "clip_ratio/region_mean": 0.0008423729450441897, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13059.0, "completions/mean_length": 11664.875, "completions/mean_terminated_length": 10091.833984375, "completions/min_length": 6610.0, "completions/min_terminated_length": 6610.0, "entropy": 0.5955601409077644, "epoch": 0.00448327393953328, "frac_reward_zero_std": 0.0, "grad_norm": 0.009304461069405079, "learning_rate": 1e-05, "loss": -0.0425, "num_tokens": 3807858.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000051736831665, "sampling/importance_sampling_ratio/min": 0.40974199771881104, "sampling/sampling_logp_difference/max": 0.8922276496887207, "sampling/sampling_logp_difference/mean": 0.02575019933283329, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8903.0, "completions/max_terminated_length": 8903.0, "completions/mean_length": 4943.375, "completions/mean_terminated_length": 4943.375, "completions/min_length": 2102.0, "completions/min_terminated_length": 2102.0, "entropy": 0.3662714660167694, "epoch": 0.004540751810552937, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3849109.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7069648504257202, "sampling/importance_sampling_ratio/mean": 1.0002577304840088, "sampling/importance_sampling_ratio/min": 0.42224836349487305, "sampling/sampling_logp_difference/max": 0.8621616363525391, "sampling/sampling_logp_difference/mean": 0.014560350216925144, "step": 79 }, { "clip_ratio/high_max": 4.0380606151302345e-05, "clip_ratio/high_mean": 4.0380606151302345e-05, "clip_ratio/low_mean": 0.00026428982164361514, "clip_ratio/low_min": 0.00026428982164361514, "clip_ratio/region_mean": 0.0003046704277949175, "completions/clipped_ratio": 0.0, "completions/max_length": 8607.0, "completions/max_terminated_length": 8607.0, "completions/mean_length": 5977.625, "completions/mean_terminated_length": 5977.625, "completions/min_length": 3968.0, "completions/min_terminated_length": 3968.0, "entropy": 0.48602984473109245, "epoch": 0.004598229681572595, "frac_reward_zero_std": 0.0, "grad_norm": 0.02175828441977501, "learning_rate": 1e-05, "loss": 0.047, "num_tokens": 3898130.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6929270029067993, "sampling/importance_sampling_ratio/mean": 0.999931812286377, "sampling/importance_sampling_ratio/min": 0.5149385333061218, "sampling/sampling_logp_difference/max": 0.6637077331542969, "sampling/sampling_logp_difference/mean": 0.015433047898113728, "step": 80 }, { "clip_ratio/high_max": 3.3530042856000364e-05, "clip_ratio/high_mean": 3.3530042856000364e-05, "clip_ratio/low_mean": 0.00010602205293253064, "clip_ratio/low_min": 0.00010602205293253064, "clip_ratio/region_mean": 0.000139552095788531, "completions/clipped_ratio": 0.0, "completions/max_length": 3728.0, "completions/max_terminated_length": 3728.0, "completions/mean_length": 2809.5, "completions/mean_terminated_length": 2809.5, "completions/min_length": 2034.0, "completions/min_terminated_length": 2034.0, "entropy": 0.289394024759531, "epoch": 0.004655707552592252, "frac_reward_zero_std": 0.0, "grad_norm": 0.011299347504973412, "learning_rate": 1e-05, "loss": -0.0571, "num_tokens": 3921782.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3607003688812256, "sampling/importance_sampling_ratio/mean": 1.000383973121643, "sampling/importance_sampling_ratio/min": 0.5568434596061707, "sampling/sampling_logp_difference/max": 0.5854711532592773, "sampling/sampling_logp_difference/mean": 0.010291689075529575, "step": 81 }, { "clip_ratio/high_max": 0.00010765574734250549, "clip_ratio/high_mean": 0.00010765574734250549, "clip_ratio/low_mean": 0.0003638860216597095, "clip_ratio/low_min": 0.0003638860216597095, "clip_ratio/region_mean": 0.000471541769002215, "completions/clipped_ratio": 0.0, "completions/max_length": 5725.0, "completions/max_terminated_length": 5725.0, "completions/mean_length": 4045.5, "completions/mean_terminated_length": 4045.5, "completions/min_length": 2295.0, "completions/min_terminated_length": 2295.0, "entropy": 0.5646822117269039, "epoch": 0.004713185423611909, "frac_reward_zero_std": 0.0, "grad_norm": 0.023033710196614265, "learning_rate": 1e-05, "loss": -0.0664, "num_tokens": 3954906.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.473975658416748, "sampling/importance_sampling_ratio/mean": 0.9994459748268127, "sampling/importance_sampling_ratio/min": 0.667750358581543, "sampling/sampling_logp_difference/max": 0.40384089946746826, "sampling/sampling_logp_difference/mean": 0.02020481415092945, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9426.0, "completions/max_terminated_length": 9426.0, "completions/mean_length": 5180.5, "completions/mean_terminated_length": 5180.5, "completions/min_length": 3161.0, "completions/min_terminated_length": 3161.0, "entropy": 0.5163888968527317, "epoch": 0.004770663294631567, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3997582.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5550614595413208, "sampling/importance_sampling_ratio/mean": 0.9998461604118347, "sampling/importance_sampling_ratio/min": 0.41633352637290955, "sampling/sampling_logp_difference/max": 0.8762686252593994, "sampling/sampling_logp_difference/mean": 0.01981530524790287, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4797.0, "completions/max_terminated_length": 4797.0, "completions/mean_length": 3375.0, "completions/mean_terminated_length": 3375.0, "completions/min_length": 1937.0, "completions/min_terminated_length": 1937.0, "entropy": 0.23298371024429798, "epoch": 0.004828141165651225, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 4025438.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.409183382987976, "sampling/importance_sampling_ratio/mean": 1.0002565383911133, "sampling/importance_sampling_ratio/min": 0.670382022857666, "sampling/sampling_logp_difference/max": 0.39990758895874023, "sampling/sampling_logp_difference/mean": 0.008978866040706635, "step": 84 }, { "clip_ratio/high_max": 6.28058460279135e-05, "clip_ratio/high_mean": 6.28058460279135e-05, "clip_ratio/low_mean": 0.0004359620616014581, "clip_ratio/low_min": 0.0004359620616014581, "clip_ratio/region_mean": 0.0004987679076293716, "completions/clipped_ratio": 0.0, "completions/max_length": 13690.0, "completions/max_terminated_length": 13690.0, "completions/mean_length": 9661.75, "completions/mean_terminated_length": 9661.75, "completions/min_length": 7069.0, "completions/min_terminated_length": 7069.0, "entropy": 0.41262468323111534, "epoch": 0.0048856190366708815, "frac_reward_zero_std": 0.0, "grad_norm": 0.013142373412847519, "learning_rate": 1e-05, "loss": -0.0268, "num_tokens": 4103852.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000039339065552, "sampling/importance_sampling_ratio/min": 0.28515318036079407, "sampling/sampling_logp_difference/max": 1.2547287940979004, "sampling/sampling_logp_difference/mean": 0.01729038543999195, "step": 85 }, { "clip_ratio/high_max": 0.00018562689729151316, "clip_ratio/high_mean": 0.00018562689729151316, "clip_ratio/low_mean": 8.496465306961909e-05, "clip_ratio/low_min": 8.496465306961909e-05, "clip_ratio/region_mean": 0.00027059155036113225, "completions/clipped_ratio": 0.0, "completions/max_length": 14712.0, "completions/max_terminated_length": 14712.0, "completions/mean_length": 10643.125, "completions/mean_terminated_length": 10643.125, "completions/min_length": 7521.0, "completions/min_terminated_length": 7521.0, "entropy": 0.4584597982466221, "epoch": 0.004943096907690539, "frac_reward_zero_std": 0.0, "grad_norm": 0.005199153907597065, "learning_rate": 1e-05, "loss": 0.1354, "num_tokens": 4189781.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998418092727661, "sampling/importance_sampling_ratio/min": 0.19236509501934052, "sampling/sampling_logp_difference/max": 1.6483601331710815, "sampling/sampling_logp_difference/mean": 0.017366668209433556, "step": 86 }, { "clip_ratio/high_max": 4.416961019160226e-05, "clip_ratio/high_mean": 4.416961019160226e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.416961019160226e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 2962.0, "completions/max_terminated_length": 2962.0, "completions/mean_length": 1943.625, "completions/mean_terminated_length": 1943.625, "completions/min_length": 1028.0, "completions/min_terminated_length": 1028.0, "entropy": 0.25489529222249985, "epoch": 0.005000574778710197, "frac_reward_zero_std": 0.0, "grad_norm": 0.07604192942380905, "learning_rate": 1e-05, "loss": -0.0092, "num_tokens": 4206050.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.397850751876831, "sampling/importance_sampling_ratio/mean": 0.9999982118606567, "sampling/importance_sampling_ratio/min": 0.6308754086494446, "sampling/sampling_logp_difference/max": 0.4606468677520752, "sampling/sampling_logp_difference/mean": 0.010172372683882713, "step": 87 }, { "clip_ratio/high_max": 0.00013542795204557478, "clip_ratio/high_mean": 0.00013542795204557478, "clip_ratio/low_mean": 0.0006206220132298768, "clip_ratio/low_min": 0.0006206220132298768, "clip_ratio/region_mean": 0.0007560499652754515, "completions/clipped_ratio": 0.0, "completions/max_length": 1202.0, "completions/max_terminated_length": 1202.0, "completions/mean_length": 763.5, "completions/mean_terminated_length": 763.5, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.33439876325428486, "epoch": 0.005058052649729854, "frac_reward_zero_std": 0.0, "grad_norm": 0.060591693967580795, "learning_rate": 1e-05, "loss": -0.0851, "num_tokens": 4215230.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4514975547790527, "sampling/importance_sampling_ratio/mean": 1.0007790327072144, "sampling/importance_sampling_ratio/min": 0.6699815988540649, "sampling/sampling_logp_difference/max": 0.40050506591796875, "sampling/sampling_logp_difference/mean": 0.014445881359279156, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1787.0, "completions/max_terminated_length": 1787.0, "completions/mean_length": 1103.875, "completions/mean_terminated_length": 1103.875, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.508196271955967, "epoch": 0.005115530520749511, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 4225269.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.282193660736084, "sampling/importance_sampling_ratio/mean": 1.000566005706787, "sampling/importance_sampling_ratio/min": 0.6705912947654724, "sampling/sampling_logp_difference/max": 0.3995954990386963, "sampling/sampling_logp_difference/mean": 0.01484319195151329, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000270635835477151, "clip_ratio/low_min": 0.000270635835477151, "clip_ratio/region_mean": 0.000270635835477151, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 3458.0, "completions/mean_length": 4003.5, "completions/mean_terminated_length": 2234.857177734375, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.4536377377808094, "epoch": 0.005173008391769169, "frac_reward_zero_std": 0.0, "grad_norm": 0.08329286426305771, "learning_rate": 1e-05, "loss": 0.4597, "num_tokens": 4258313.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.8575853109359741, "sampling/importance_sampling_ratio/mean": 1.0002970695495605, "sampling/importance_sampling_ratio/min": 0.2592966556549072, "sampling/sampling_logp_difference/max": 1.3497824668884277, "sampling/sampling_logp_difference/mean": 0.020333070307970047, "step": 90 }, { "clip_ratio/high_max": 6.797360038035549e-05, "clip_ratio/high_mean": 6.797360038035549e-05, "clip_ratio/low_mean": 0.00015368337153631728, "clip_ratio/low_min": 0.00015368337153631728, "clip_ratio/region_mean": 0.00022165697191667277, "completions/clipped_ratio": 0.0, "completions/max_length": 10391.0, "completions/max_terminated_length": 10391.0, "completions/mean_length": 5971.75, "completions/mean_terminated_length": 5971.75, "completions/min_length": 2969.0, "completions/min_terminated_length": 2969.0, "entropy": 0.4571814499795437, "epoch": 0.005230486262788827, "frac_reward_zero_std": 0.0, "grad_norm": 0.022652029991149902, "learning_rate": 1e-05, "loss": 0.1659, "num_tokens": 4306711.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6500544548034668, "sampling/importance_sampling_ratio/mean": 1.0000139474868774, "sampling/importance_sampling_ratio/min": 0.39687731862068176, "sampling/sampling_logp_difference/max": 0.9241280555725098, "sampling/sampling_logp_difference/mean": 0.018285445868968964, "step": 91 }, { "clip_ratio/high_max": 0.00013248470713733695, "clip_ratio/high_mean": 0.00013248470713733695, "clip_ratio/low_mean": 0.00011607853230088949, "clip_ratio/low_min": 0.00011607853230088949, "clip_ratio/region_mean": 0.00024856323943822645, "completions/clipped_ratio": 0.0, "completions/max_length": 7538.0, "completions/max_terminated_length": 7538.0, "completions/mean_length": 4436.125, "completions/mean_terminated_length": 4436.125, "completions/min_length": 2237.0, "completions/min_terminated_length": 2237.0, "entropy": 0.26149668358266354, "epoch": 0.005287964133808483, "frac_reward_zero_std": 0.0, "grad_norm": 0.008685613982379436, "learning_rate": 1e-05, "loss": 0.2472, "num_tokens": 4343240.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5667108297348022, "sampling/importance_sampling_ratio/mean": 1.0001031160354614, "sampling/importance_sampling_ratio/min": 0.6066851615905762, "sampling/sampling_logp_difference/max": 0.49974536895751953, "sampling/sampling_logp_difference/mean": 0.010060268454253674, "step": 92 }, { "clip_ratio/high_max": 1.6438716556876898e-05, "clip_ratio/high_mean": 1.6438716556876898e-05, "clip_ratio/low_mean": 8.397716010222211e-05, "clip_ratio/low_min": 8.397716010222211e-05, "clip_ratio/region_mean": 0.00010041587665909901, "completions/clipped_ratio": 0.0, "completions/max_length": 9654.0, "completions/max_terminated_length": 9654.0, "completions/mean_length": 4659.125, "completions/mean_terminated_length": 4659.125, "completions/min_length": 1628.0, "completions/min_terminated_length": 1628.0, "entropy": 0.43837379664182663, "epoch": 0.005345442004828141, "frac_reward_zero_std": 0.0, "grad_norm": 0.050454385578632355, "learning_rate": 1e-05, "loss": -0.5178, "num_tokens": 4382209.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998786449432373, "sampling/importance_sampling_ratio/min": 0.43582969903945923, "sampling/sampling_logp_difference/max": 0.8305037021636963, "sampling/sampling_logp_difference/mean": 0.014977103099226952, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2592.0, "completions/max_terminated_length": 2592.0, "completions/mean_length": 1523.75, "completions/mean_terminated_length": 1523.75, "completions/min_length": 947.0, "completions/min_terminated_length": 947.0, "entropy": 0.263191694393754, "epoch": 0.005402919875847799, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 4395311.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5041006803512573, "sampling/importance_sampling_ratio/mean": 0.9999500513076782, "sampling/importance_sampling_ratio/min": 0.6959434151649475, "sampling/sampling_logp_difference/max": 0.4081951379776001, "sampling/sampling_logp_difference/mean": 0.010398940183222294, "step": 94 }, { "clip_ratio/high_max": 7.324352191062644e-05, "clip_ratio/high_mean": 7.324352191062644e-05, "clip_ratio/low_mean": 0.00010513036249903962, "clip_ratio/low_min": 0.00010513036249903962, "clip_ratio/region_mean": 0.00017837388440966606, "completions/clipped_ratio": 0.0, "completions/max_length": 3657.0, "completions/max_terminated_length": 3657.0, "completions/mean_length": 2862.875, "completions/mean_terminated_length": 2862.875, "completions/min_length": 1961.0, "completions/min_terminated_length": 1961.0, "entropy": 0.2586438972502947, "epoch": 0.005460397746867456, "frac_reward_zero_std": 0.0, "grad_norm": 0.0077639296650886536, "learning_rate": 1e-05, "loss": -0.0599, "num_tokens": 4419494.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.560463547706604, "sampling/importance_sampling_ratio/mean": 1.0001232624053955, "sampling/importance_sampling_ratio/min": 0.6484719514846802, "sampling/sampling_logp_difference/max": 0.44498300552368164, "sampling/sampling_logp_difference/mean": 0.010486197657883167, "step": 95 }, { "clip_ratio/high_max": 0.0001295057481911499, "clip_ratio/high_mean": 0.0001295057481911499, "clip_ratio/low_mean": 0.0005240441350906622, "clip_ratio/low_min": 0.0005240441350906622, "clip_ratio/region_mean": 0.0006535498832818121, "completions/clipped_ratio": 0.0, "completions/max_length": 11747.0, "completions/max_terminated_length": 11747.0, "completions/mean_length": 7692.625, "completions/mean_terminated_length": 7692.625, "completions/min_length": 4612.0, "completions/min_terminated_length": 4612.0, "entropy": 0.4370035044848919, "epoch": 0.005517875617887113, "frac_reward_zero_std": 0.0, "grad_norm": 0.01865573227405548, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 4481763.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.9858016967773438, "sampling/importance_sampling_ratio/mean": 0.9999425411224365, "sampling/importance_sampling_ratio/min": 0.360934317111969, "sampling/sampling_logp_difference/max": 1.0190593004226685, "sampling/sampling_logp_difference/mean": 0.016827711835503578, "step": 96 }, { "clip_ratio/high_max": 1.5083865946508013e-05, "clip_ratio/high_mean": 1.5083865946508013e-05, "clip_ratio/low_mean": 0.00090244751481805, "clip_ratio/low_min": 0.00090244751481805, "clip_ratio/region_mean": 0.000917531380764558, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 12921.0, "completions/mean_length": 12423.625, "completions/mean_terminated_length": 10047.400390625, "completions/min_length": 7634.0, "completions/min_terminated_length": 7634.0, "entropy": 0.6236320100724697, "epoch": 0.005575353488906771, "frac_reward_zero_std": 0.0, "grad_norm": 0.006804484408348799, "learning_rate": 1e-05, "loss": 0.118, "num_tokens": 4582400.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997172355651855, "sampling/importance_sampling_ratio/min": 0.3413894474506378, "sampling/sampling_logp_difference/max": 1.0747313499450684, "sampling/sampling_logp_difference/mean": 0.025754448026418686, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 15624.0, "completions/mean_length": 13475.5, "completions/mean_terminated_length": 12506.0, "completions/min_length": 4824.0, "completions/min_terminated_length": 4824.0, "entropy": 0.4783651791512966, "epoch": 0.0056328313599264285, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 4691060.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999643564224243, "sampling/importance_sampling_ratio/min": 0.30113595724105835, "sampling/sampling_logp_difference/max": 1.213273525238037, "sampling/sampling_logp_difference/mean": 0.020278790965676308, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 2.6158835680689663e-05, "clip_ratio/low_min": 2.6158835680689663e-05, "clip_ratio/region_mean": 2.6158835680689663e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 9557.0, "completions/max_terminated_length": 9557.0, "completions/mean_length": 2165.875, "completions/mean_terminated_length": 2165.875, "completions/min_length": 794.0, "completions/min_terminated_length": 794.0, "entropy": 0.29633556492626667, "epoch": 0.005690309230946085, "frac_reward_zero_std": 0.0, "grad_norm": 0.013613307848572731, "learning_rate": 1e-05, "loss": 0.784, "num_tokens": 4709435.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7876187562942505, "sampling/importance_sampling_ratio/mean": 0.999664306640625, "sampling/importance_sampling_ratio/min": 0.6190581917762756, "sampling/sampling_logp_difference/max": 0.5808844566345215, "sampling/sampling_logp_difference/mean": 0.013285611756145954, "step": 99 }, { "clip_ratio/high_max": 9.427890472579747e-05, "clip_ratio/high_mean": 9.427890472579747e-05, "clip_ratio/low_mean": 0.0008690841023053508, "clip_ratio/low_min": 0.0008690841023053508, "clip_ratio/region_mean": 0.0009633630070311483, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14278.0, "completions/mean_length": 12535.125, "completions/mean_terminated_length": 11985.2861328125, "completions/min_length": 6118.0, "completions/min_terminated_length": 6118.0, "entropy": 0.5694211684167385, "epoch": 0.005747787101965743, "frac_reward_zero_std": 0.0, "grad_norm": 0.006562459748238325, "learning_rate": 1e-05, "loss": 0.0231, "num_tokens": 4810828.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003129243850708, "sampling/importance_sampling_ratio/min": 0.2958182096481323, "sampling/sampling_logp_difference/max": 1.2180101871490479, "sampling/sampling_logp_difference/mean": 0.025095155462622643, "step": 100 }, { "clip_ratio/high_max": 0.00021666069733328186, "clip_ratio/high_mean": 0.00021666069733328186, "clip_ratio/low_mean": 0.00034195535408798605, "clip_ratio/low_min": 0.00034195535408798605, "clip_ratio/region_mean": 0.0005586160514212679, "completions/clipped_ratio": 0.0, "completions/max_length": 5100.0, "completions/max_terminated_length": 5100.0, "completions/mean_length": 3674.875, "completions/mean_terminated_length": 3674.875, "completions/min_length": 609.0, "completions/min_terminated_length": 609.0, "entropy": 0.6064490713179111, "epoch": 0.005805264972985401, "frac_reward_zero_std": 0.0, "grad_norm": 0.04777848348021507, "learning_rate": 1e-05, "loss": 0.0129, "num_tokens": 4841243.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4659724235534668, "sampling/importance_sampling_ratio/mean": 1.0000799894332886, "sampling/importance_sampling_ratio/min": 0.6710418462753296, "sampling/sampling_logp_difference/max": 0.39892375469207764, "sampling/sampling_logp_difference/mean": 0.023423591628670692, "step": 101 }, { "clip_ratio/high_max": 0.00020406878411449725, "clip_ratio/high_mean": 0.00020406878411449725, "clip_ratio/low_mean": 0.00021950393420411274, "clip_ratio/low_min": 0.00021950393420411274, "clip_ratio/region_mean": 0.00042357271831861, "completions/clipped_ratio": 0.0, "completions/max_length": 9886.0, "completions/max_terminated_length": 9886.0, "completions/mean_length": 5910.5, "completions/mean_terminated_length": 5910.5, "completions/min_length": 842.0, "completions/min_terminated_length": 842.0, "entropy": 0.5742521323263645, "epoch": 0.005862742844005058, "frac_reward_zero_std": 0.0, "grad_norm": 0.02844126522541046, "learning_rate": 1e-05, "loss": -0.2702, "num_tokens": 4889535.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6540199518203735, "sampling/importance_sampling_ratio/mean": 0.9999129176139832, "sampling/importance_sampling_ratio/min": 0.4713459610939026, "sampling/sampling_logp_difference/max": 0.7521629333496094, "sampling/sampling_logp_difference/mean": 0.023292189463973045, "step": 102 }, { "clip_ratio/high_max": 7.457700303348247e-05, "clip_ratio/high_mean": 7.457700303348247e-05, "clip_ratio/low_mean": 0.0003259427976445295, "clip_ratio/low_min": 0.0003259427976445295, "clip_ratio/region_mean": 0.00040051980067801196, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14820.0, "completions/mean_length": 11594.75, "completions/mean_terminated_length": 10910.572265625, "completions/min_length": 6098.0, "completions/min_terminated_length": 6098.0, "entropy": 0.3187015615403652, "epoch": 0.005920220715024715, "frac_reward_zero_std": 0.0, "grad_norm": 0.014056019484996796, "learning_rate": 1e-05, "loss": 0.1221, "num_tokens": 4983725.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999797344207764, "sampling/importance_sampling_ratio/min": 0.0509355291724205, "sampling/sampling_logp_difference/max": 2.9771945476531982, "sampling/sampling_logp_difference/mean": 0.015568562783300877, "step": 103 }, { "clip_ratio/high_max": 7.59027407184476e-05, "clip_ratio/high_mean": 7.59027407184476e-05, "clip_ratio/low_mean": 0.00043579248449532315, "clip_ratio/low_min": 0.00043579248449532315, "clip_ratio/region_mean": 0.0005116952252137708, "completions/clipped_ratio": 0.0, "completions/max_length": 10378.0, "completions/max_terminated_length": 10378.0, "completions/mean_length": 6279.375, "completions/mean_terminated_length": 6279.375, "completions/min_length": 3401.0, "completions/min_terminated_length": 3401.0, "entropy": 0.6004678979516029, "epoch": 0.005977698586044373, "frac_reward_zero_std": 0.0, "grad_norm": 0.037188686430454254, "learning_rate": 1e-05, "loss": 0.1846, "num_tokens": 5035672.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001723766326904, "sampling/importance_sampling_ratio/min": 0.39308810234069824, "sampling/sampling_logp_difference/max": 1.280113697052002, "sampling/sampling_logp_difference/mean": 0.02408575639128685, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020284376842027996, "clip_ratio/low_min": 0.00020284376842027996, "clip_ratio/region_mean": 0.00020284376842027996, "completions/clipped_ratio": 0.0, "completions/max_length": 12887.0, "completions/max_terminated_length": 12887.0, "completions/mean_length": 8273.75, "completions/mean_terminated_length": 8273.75, "completions/min_length": 3123.0, "completions/min_terminated_length": 3123.0, "entropy": 0.8106596171855927, "epoch": 0.0060351764570640305, "frac_reward_zero_std": 0.0, "grad_norm": 0.019368216395378113, "learning_rate": 1e-05, "loss": 0.3, "num_tokens": 5102726.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003316402435303, "sampling/importance_sampling_ratio/min": 0.37379640340805054, "sampling/sampling_logp_difference/max": 0.984044075012207, "sampling/sampling_logp_difference/mean": 0.02187422849237919, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4021.0, "completions/max_terminated_length": 4021.0, "completions/mean_length": 2854.375, "completions/mean_terminated_length": 2854.375, "completions/min_length": 2024.0, "completions/min_terminated_length": 2024.0, "entropy": 0.2449374794960022, "epoch": 0.006092654328083688, "frac_reward_zero_std": 0.0, "grad_norm": 0.008184988051652908, "learning_rate": 1e-05, "loss": -0.086, "num_tokens": 5126889.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3411176204681396, "sampling/importance_sampling_ratio/mean": 0.9995095729827881, "sampling/importance_sampling_ratio/min": 0.7121059894561768, "sampling/sampling_logp_difference/max": 0.33952856063842773, "sampling/sampling_logp_difference/mean": 0.010186538100242615, "step": 106 }, { "clip_ratio/high_max": 0.00023311110089707654, "clip_ratio/high_mean": 0.00023311110089707654, "clip_ratio/low_mean": 0.0005337549955584109, "clip_ratio/low_min": 0.0005337549955584109, "clip_ratio/region_mean": 0.0007668660964554874, "completions/clipped_ratio": 0.0, "completions/max_length": 7455.0, "completions/max_terminated_length": 7455.0, "completions/mean_length": 5222.0, "completions/mean_terminated_length": 5222.0, "completions/min_length": 2643.0, "completions/min_terminated_length": 2643.0, "entropy": 0.7336124926805496, "epoch": 0.006150132199103345, "frac_reward_zero_std": 0.0, "grad_norm": 0.036571402102708817, "learning_rate": 1e-05, "loss": 0.0096, "num_tokens": 5170073.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6328125, "sampling/importance_sampling_ratio/mean": 0.9998865127563477, "sampling/importance_sampling_ratio/min": 0.4239436388015747, "sampling/sampling_logp_difference/max": 0.8581547737121582, "sampling/sampling_logp_difference/mean": 0.027363304048776627, "step": 107 }, { "clip_ratio/high_max": 5.095334017823916e-05, "clip_ratio/high_mean": 5.095334017823916e-05, "clip_ratio/low_mean": 0.00020405950272106566, "clip_ratio/low_min": 0.00020405950272106566, "clip_ratio/region_mean": 0.0002550128428993048, "completions/clipped_ratio": 0.0, "completions/max_length": 7622.0, "completions/max_terminated_length": 7622.0, "completions/mean_length": 5145.75, "completions/mean_terminated_length": 5145.75, "completions/min_length": 3266.0, "completions/min_terminated_length": 3266.0, "entropy": 0.4340439885854721, "epoch": 0.006207610070123003, "frac_reward_zero_std": 0.0, "grad_norm": 0.04015190154314041, "learning_rate": 1e-05, "loss": 0.0621, "num_tokens": 5212927.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.7912691831588745, "sampling/importance_sampling_ratio/mean": 0.999870240688324, "sampling/importance_sampling_ratio/min": 0.33116602897644043, "sampling/sampling_logp_difference/max": 1.105135440826416, "sampling/sampling_logp_difference/mean": 0.014029542915523052, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010089762690768111, "clip_ratio/low_min": 0.00010089762690768111, "clip_ratio/region_mean": 0.00010089762690768111, "completions/clipped_ratio": 0.0, "completions/max_length": 9632.0, "completions/max_terminated_length": 9632.0, "completions/mean_length": 5762.5, "completions/mean_terminated_length": 5762.5, "completions/min_length": 3287.0, "completions/min_terminated_length": 3287.0, "entropy": 0.39456985145807266, "epoch": 0.00626508794114266, "frac_reward_zero_std": 0.0, "grad_norm": 0.009497498162090778, "learning_rate": 1e-05, "loss": 0.1078, "num_tokens": 5260299.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001640319824219, "sampling/importance_sampling_ratio/min": 0.3906859755516052, "sampling/sampling_logp_difference/max": 1.2457599639892578, "sampling/sampling_logp_difference/mean": 0.013365420512855053, "step": 109 }, { "clip_ratio/high_max": 6.160670454846695e-05, "clip_ratio/high_mean": 6.160670454846695e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.160670454846695e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 2574.0, "completions/max_terminated_length": 2574.0, "completions/mean_length": 1961.125, "completions/mean_terminated_length": 1961.125, "completions/min_length": 1186.0, "completions/min_terminated_length": 1186.0, "entropy": 0.20434438437223434, "epoch": 0.006322565812162318, "frac_reward_zero_std": 0.0, "grad_norm": 0.054287102073431015, "learning_rate": 1e-05, "loss": -0.1395, "num_tokens": 5277180.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9964170455932617, "sampling/importance_sampling_ratio/mean": 0.9997534155845642, "sampling/importance_sampling_ratio/min": 0.6984014511108398, "sampling/sampling_logp_difference/max": 0.6913540363311768, "sampling/sampling_logp_difference/mean": 0.008898675441741943, "step": 110 }, { "clip_ratio/high_max": 0.00012034870269417297, "clip_ratio/high_mean": 0.00012034870269417297, "clip_ratio/low_mean": 5.5915901612024754e-05, "clip_ratio/low_min": 5.5915901612024754e-05, "clip_ratio/region_mean": 0.00017626460430619773, "completions/clipped_ratio": 0.0, "completions/max_length": 9815.0, "completions/max_terminated_length": 9815.0, "completions/mean_length": 7040.375, "completions/mean_terminated_length": 7040.375, "completions/min_length": 4229.0, "completions/min_terminated_length": 4229.0, "entropy": 0.2732951082289219, "epoch": 0.006380043683181975, "frac_reward_zero_std": 0.0, "grad_norm": 0.006093955133110285, "learning_rate": 1e-05, "loss": -0.1292, "num_tokens": 5334319.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8080084323883057, "sampling/importance_sampling_ratio/mean": 1.0001444816589355, "sampling/importance_sampling_ratio/min": 0.40781381726264954, "sampling/sampling_logp_difference/max": 0.896944522857666, "sampling/sampling_logp_difference/mean": 0.011279252357780933, "step": 111 }, { "clip_ratio/high_max": 3.80923374905251e-05, "clip_ratio/high_mean": 3.80923374905251e-05, "clip_ratio/low_mean": 0.0008192299137590453, "clip_ratio/low_min": 0.0008192299137590453, "clip_ratio/region_mean": 0.0008573222512495704, "completions/clipped_ratio": 0.0, "completions/max_length": 7425.0, "completions/max_terminated_length": 7425.0, "completions/mean_length": 5289.25, "completions/mean_terminated_length": 5289.25, "completions/min_length": 3101.0, "completions/min_terminated_length": 3101.0, "entropy": 0.5866907760500908, "epoch": 0.006437521554201632, "frac_reward_zero_std": 0.0, "grad_norm": 0.02391251176595688, "learning_rate": 1e-05, "loss": -0.0641, "num_tokens": 5377417.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.7714844942092896, "sampling/importance_sampling_ratio/mean": 1.0000824928283691, "sampling/importance_sampling_ratio/min": 0.5712507367134094, "sampling/sampling_logp_difference/max": 0.5718178749084473, "sampling/sampling_logp_difference/mean": 0.023795749992132187, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7667.0, "completions/max_terminated_length": 7667.0, "completions/mean_length": 4092.25, "completions/mean_terminated_length": 4092.25, "completions/min_length": 2210.0, "completions/min_terminated_length": 2210.0, "entropy": 0.26948732510209084, "epoch": 0.00649499942522129, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 5411419.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5129711627960205, "sampling/importance_sampling_ratio/mean": 1.0002878904342651, "sampling/importance_sampling_ratio/min": 0.6073850393295288, "sampling/sampling_logp_difference/max": 0.4985923767089844, "sampling/sampling_logp_difference/mean": 0.011067938059568405, "step": 113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10053.0, "completions/max_terminated_length": 10053.0, "completions/mean_length": 6838.0, "completions/mean_terminated_length": 6838.0, "completions/min_length": 4493.0, "completions/min_terminated_length": 4493.0, "entropy": 0.36648314259946346, "epoch": 0.006552477296240947, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 5467379.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999701976776123, "sampling/importance_sampling_ratio/min": 0.34160375595092773, "sampling/sampling_logp_difference/max": 1.074103832244873, "sampling/sampling_logp_difference/mean": 0.01511444803327322, "step": 114 }, { "clip_ratio/high_max": 0.0002783380914479494, "clip_ratio/high_mean": 0.0002783380914479494, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0002783380914479494, "completions/clipped_ratio": 0.0, "completions/max_length": 2412.0, "completions/max_terminated_length": 2412.0, "completions/mean_length": 1630.5, "completions/mean_terminated_length": 1630.5, "completions/min_length": 910.0, "completions/min_terminated_length": 910.0, "entropy": 0.3034412022680044, "epoch": 0.0066099551672606045, "frac_reward_zero_std": 0.0, "grad_norm": 0.01175200566649437, "learning_rate": 1e-05, "loss": -0.0583, "num_tokens": 5481591.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3706064224243164, "sampling/importance_sampling_ratio/mean": 0.9998339414596558, "sampling/importance_sampling_ratio/min": 0.2978171408176422, "sampling/sampling_logp_difference/max": 1.211275577545166, "sampling/sampling_logp_difference/mean": 0.011887015774846077, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022903785429662094, "clip_ratio/low_min": 0.00022903785429662094, "clip_ratio/region_mean": 0.00022903785429662094, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.0, "completions/max_terminated_length": 1171.0, "completions/mean_length": 1044.375, "completions/mean_terminated_length": 1044.375, "completions/min_length": 936.0, "completions/min_terminated_length": 936.0, "entropy": 0.3192705698311329, "epoch": 0.006667433038280262, "frac_reward_zero_std": 0.0, "grad_norm": 0.05706014484167099, "learning_rate": 1e-05, "loss": -0.0136, "num_tokens": 5491114.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.4043923616409302, "sampling/importance_sampling_ratio/mean": 1.0009198188781738, "sampling/importance_sampling_ratio/min": 0.6967563033103943, "sampling/sampling_logp_difference/max": 0.36131954193115234, "sampling/sampling_logp_difference/mean": 0.012880105525255203, "step": 116 }, { "clip_ratio/high_max": 2.0398172637214884e-05, "clip_ratio/high_mean": 2.0398172637214884e-05, "clip_ratio/low_mean": 0.000778431014623493, "clip_ratio/low_min": 0.000778431014623493, "clip_ratio/region_mean": 0.0007988291872607078, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15170.0, "completions/mean_length": 9836.875, "completions/mean_terminated_length": 8901.572265625, "completions/min_length": 5382.0, "completions/min_terminated_length": 5382.0, "entropy": 0.5065955854952335, "epoch": 0.00672491090929992, "frac_reward_zero_std": 0.0, "grad_norm": 0.0058839512057602406, "learning_rate": 1e-05, "loss": 0.1335, "num_tokens": 5570745.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003448724746704, "sampling/importance_sampling_ratio/min": 0.30787843465805054, "sampling/sampling_logp_difference/max": 2.0774405002593994, "sampling/sampling_logp_difference/mean": 0.0221853144466877, "step": 117 }, { "clip_ratio/high_max": 6.92520770826377e-05, "clip_ratio/high_mean": 6.92520770826377e-05, "clip_ratio/low_mean": 6.154603761387989e-05, "clip_ratio/low_min": 6.154603761387989e-05, "clip_ratio/region_mean": 0.0001307981146965176, "completions/clipped_ratio": 0.0, "completions/max_length": 2781.0, "completions/max_terminated_length": 2781.0, "completions/mean_length": 1831.625, "completions/mean_terminated_length": 1831.625, "completions/min_length": 1006.0, "completions/min_terminated_length": 1006.0, "entropy": 0.57236672565341, "epoch": 0.006782388780319577, "frac_reward_zero_std": 0.0, "grad_norm": 0.015471331775188446, "learning_rate": 1e-05, "loss": 0.0389, "num_tokens": 5586390.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.295737385749817, "sampling/importance_sampling_ratio/mean": 1.0000808238983154, "sampling/importance_sampling_ratio/min": 0.7419808506965637, "sampling/sampling_logp_difference/max": 0.2984318733215332, "sampling/sampling_logp_difference/mean": 0.015503806062042713, "step": 118 }, { "clip_ratio/high_max": 0.00015082918798725586, "clip_ratio/high_mean": 0.00015082918798725586, "clip_ratio/low_mean": 0.00022748402261640877, "clip_ratio/low_min": 0.00022748402261640877, "clip_ratio/region_mean": 0.00037831321060366463, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 16313.0, "completions/mean_length": 9942.75, "completions/mean_terminated_length": 9022.572265625, "completions/min_length": 2978.0, "completions/min_terminated_length": 2978.0, "entropy": 0.474183764308691, "epoch": 0.006839866651339234, "frac_reward_zero_std": 0.0, "grad_norm": 0.004840504843741655, "learning_rate": 1e-05, "loss": 0.2649, "num_tokens": 5666788.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997504949569702, "sampling/importance_sampling_ratio/min": 0.3404845893383026, "sampling/sampling_logp_difference/max": 1.077385425567627, "sampling/sampling_logp_difference/mean": 0.020352305844426155, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025774889945751056, "clip_ratio/low_min": 0.00025774889945751056, "clip_ratio/region_mean": 0.00025774889945751056, "completions/clipped_ratio": 0.0, "completions/max_length": 1997.0, "completions/max_terminated_length": 1997.0, "completions/mean_length": 1550.375, "completions/mean_terminated_length": 1550.375, "completions/min_length": 978.0, "completions/min_terminated_length": 978.0, "entropy": 0.33403414487838745, "epoch": 0.006897344522358892, "frac_reward_zero_std": 0.0, "grad_norm": 0.04866841435432434, "learning_rate": 1e-05, "loss": 0.0581, "num_tokens": 5680391.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.364777684211731, "sampling/importance_sampling_ratio/mean": 0.9999312162399292, "sampling/importance_sampling_ratio/min": 0.6394995450973511, "sampling/sampling_logp_difference/max": 0.4470694065093994, "sampling/sampling_logp_difference/mean": 0.013162706978619099, "step": 120 }, { "clip_ratio/high_max": 6.664296961389482e-05, "clip_ratio/high_mean": 6.664296961389482e-05, "clip_ratio/low_mean": 0.0003867654777423013, "clip_ratio/low_min": 0.0003867654777423013, "clip_ratio/region_mean": 0.0004534084473561961, "completions/clipped_ratio": 0.0, "completions/max_length": 15640.0, "completions/max_terminated_length": 15640.0, "completions/mean_length": 7863.375, "completions/mean_terminated_length": 7863.375, "completions/min_length": 2762.0, "completions/min_terminated_length": 2762.0, "entropy": 0.43769025802612305, "epoch": 0.00695482239337855, "frac_reward_zero_std": 0.0, "grad_norm": 0.03385671600699425, "learning_rate": 1e-05, "loss": 0.1263, "num_tokens": 5744842.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997232556343079, "sampling/importance_sampling_ratio/min": 0.1058628186583519, "sampling/sampling_logp_difference/max": 2.2456111907958984, "sampling/sampling_logp_difference/mean": 0.020388441160321236, "step": 121 }, { "clip_ratio/high_max": 8.19703327579191e-05, "clip_ratio/high_mean": 8.19703327579191e-05, "clip_ratio/low_mean": 0.00010615710925776511, "clip_ratio/low_min": 0.00010615710925776511, "clip_ratio/region_mean": 0.0001881274420156842, "completions/clipped_ratio": 0.0, "completions/max_length": 8899.0, "completions/max_terminated_length": 8899.0, "completions/mean_length": 5869.375, "completions/mean_terminated_length": 5869.375, "completions/min_length": 3585.0, "completions/min_terminated_length": 3585.0, "entropy": 0.3783104941248894, "epoch": 0.007012300264398206, "frac_reward_zero_std": 0.0, "grad_norm": 0.0070150443352758884, "learning_rate": 1e-05, "loss": -0.0695, "num_tokens": 5793069.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4635522365570068, "sampling/importance_sampling_ratio/mean": 1.0000921487808228, "sampling/importance_sampling_ratio/min": 0.5364006161689758, "sampling/sampling_logp_difference/max": 0.6228740215301514, "sampling/sampling_logp_difference/mean": 0.014711553230881691, "step": 122 }, { "clip_ratio/high_max": 1.9290124328108504e-05, "clip_ratio/high_mean": 1.9290124328108504e-05, "clip_ratio/low_mean": 0.0003349564867676236, "clip_ratio/low_min": 0.0003349564867676236, "clip_ratio/region_mean": 0.0003542466110957321, "completions/clipped_ratio": 0.0, "completions/max_length": 9364.0, "completions/max_terminated_length": 9364.0, "completions/mean_length": 4458.625, "completions/mean_terminated_length": 4458.625, "completions/min_length": 1577.0, "completions/min_terminated_length": 1577.0, "entropy": 0.6041946038603783, "epoch": 0.007069778135417864, "frac_reward_zero_std": 0.0, "grad_norm": 0.045138485729694366, "learning_rate": 1e-05, "loss": 0.1476, "num_tokens": 5830658.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4898751974105835, "sampling/importance_sampling_ratio/mean": 1.0004459619522095, "sampling/importance_sampling_ratio/min": 0.5976069569587708, "sampling/sampling_logp_difference/max": 0.5148220062255859, "sampling/sampling_logp_difference/mean": 0.019251661375164986, "step": 123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002357012745051179, "clip_ratio/low_min": 0.0002357012745051179, "clip_ratio/region_mean": 0.0002357012745051179, "completions/clipped_ratio": 0.0, "completions/max_length": 3192.0, "completions/max_terminated_length": 3192.0, "completions/mean_length": 1549.375, "completions/mean_terminated_length": 1549.375, "completions/min_length": 946.0, "completions/min_terminated_length": 946.0, "entropy": 0.3446086347103119, "epoch": 0.007127256006437522, "frac_reward_zero_std": 0.0, "grad_norm": 0.025366822257637978, "learning_rate": 1e-05, "loss": 0.3489, "num_tokens": 5843965.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5565142631530762, "sampling/importance_sampling_ratio/mean": 0.9995758533477783, "sampling/importance_sampling_ratio/min": 0.6307206749916077, "sampling/sampling_logp_difference/max": 0.4608922004699707, "sampling/sampling_logp_difference/mean": 0.015049204230308533, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4179.0, "completions/max_terminated_length": 4179.0, "completions/mean_length": 3381.5, "completions/mean_terminated_length": 3381.5, "completions/min_length": 2415.0, "completions/min_terminated_length": 2415.0, "entropy": 0.23659361898899078, "epoch": 0.007184733877457179, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 5872265.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3590935468673706, "sampling/importance_sampling_ratio/mean": 1.0000966787338257, "sampling/importance_sampling_ratio/min": 0.6720110774040222, "sampling/sampling_logp_difference/max": 0.39748048782348633, "sampling/sampling_logp_difference/mean": 0.009295837953686714, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1474.0, "completions/max_terminated_length": 1474.0, "completions/mean_length": 1049.625, "completions/mean_terminated_length": 1049.625, "completions/min_length": 845.0, "completions/min_terminated_length": 845.0, "entropy": 0.30449257232248783, "epoch": 0.007242211748476836, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 5881782.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.395222783088684, "sampling/importance_sampling_ratio/mean": 1.0002799034118652, "sampling/importance_sampling_ratio/min": 0.6292243599891663, "sampling/sampling_logp_difference/max": 0.46326732635498047, "sampling/sampling_logp_difference/mean": 0.01346125639975071, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0004151017274125479, "clip_ratio/low_min": 0.0004151017274125479, "clip_ratio/region_mean": 0.0004151017274125479, "completions/clipped_ratio": 0.0, "completions/max_length": 9615.0, "completions/max_terminated_length": 9615.0, "completions/mean_length": 2985.625, "completions/mean_terminated_length": 2985.625, "completions/min_length": 1412.0, "completions/min_terminated_length": 1412.0, "entropy": 0.5704441592097282, "epoch": 0.007299689619496494, "frac_reward_zero_std": 0.0, "grad_norm": 0.020893597975373268, "learning_rate": 1e-05, "loss": 0.175, "num_tokens": 5906651.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8009577989578247, "sampling/importance_sampling_ratio/mean": 1.0002970695495605, "sampling/importance_sampling_ratio/min": 0.5241647958755493, "sampling/sampling_logp_difference/max": 0.645949125289917, "sampling/sampling_logp_difference/mean": 0.023233845829963684, "step": 127 }, { "clip_ratio/high_max": 0.00020371542268549092, "clip_ratio/high_mean": 0.00020371542268549092, "clip_ratio/low_mean": 0.000363931103493087, "clip_ratio/low_min": 0.000363931103493087, "clip_ratio/region_mean": 0.0005676465261785779, "completions/clipped_ratio": 0.0, "completions/max_length": 14664.0, "completions/max_terminated_length": 14664.0, "completions/mean_length": 11802.625, "completions/mean_terminated_length": 11802.625, "completions/min_length": 7176.0, "completions/min_terminated_length": 7176.0, "entropy": 0.6763871014118195, "epoch": 0.0073571674905161515, "frac_reward_zero_std": 0.0, "grad_norm": 0.009250272996723652, "learning_rate": 1e-05, "loss": 0.1183, "num_tokens": 6002128.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000865459442139, "sampling/importance_sampling_ratio/min": 0.2995777130126953, "sampling/sampling_logp_difference/max": 1.2053813934326172, "sampling/sampling_logp_difference/mean": 0.02760196290910244, "step": 128 }, { "clip_ratio/high_max": 0.00012694765064225066, "clip_ratio/high_mean": 0.00012694765064225066, "clip_ratio/low_mean": 0.00015172972052823752, "clip_ratio/low_min": 0.00015172972052823752, "clip_ratio/region_mean": 0.0002786773711704882, "completions/clipped_ratio": 0.0, "completions/max_length": 10329.0, "completions/max_terminated_length": 10329.0, "completions/mean_length": 5570.75, "completions/mean_terminated_length": 5570.75, "completions/min_length": 3491.0, "completions/min_terminated_length": 3491.0, "entropy": 0.45906075090169907, "epoch": 0.007414645361535808, "frac_reward_zero_std": 0.0, "grad_norm": 0.008599936030805111, "learning_rate": 1e-05, "loss": -0.0399, "num_tokens": 6047822.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8152507543563843, "sampling/importance_sampling_ratio/mean": 0.9997912049293518, "sampling/importance_sampling_ratio/min": 0.4613805413246155, "sampling/sampling_logp_difference/max": 0.7735320925712585, "sampling/sampling_logp_difference/mean": 0.017998823896050453, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 16384.0, "completions/max_terminated_length": 14605.0, "completions/mean_length": 15545.875, "completions/mean_terminated_length": 13031.5, "completions/min_length": 11458.0, "completions/min_terminated_length": 11458.0, "entropy": 0.19472824409604073, "epoch": 0.007472123232555466, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6173981.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001029968261719, "sampling/importance_sampling_ratio/min": 0.310546338558197, "sampling/sampling_logp_difference/max": 1.1694221496582031, "sampling/sampling_logp_difference/mean": 0.00943661853671074, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9302.0, "completions/max_terminated_length": 9302.0, "completions/mean_length": 5712.5, "completions/mean_terminated_length": 5712.5, "completions/min_length": 3497.0, "completions/min_terminated_length": 3497.0, "entropy": 0.6666118577122688, "epoch": 0.007529601103575124, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6220553.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000160932540894, "sampling/importance_sampling_ratio/min": 0.27957940101623535, "sampling/sampling_logp_difference/max": 1.2744688987731934, "sampling/sampling_logp_difference/mean": 0.025103742256760597, "step": 131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2163.0, "completions/max_terminated_length": 2163.0, "completions/mean_length": 1661.625, "completions/mean_terminated_length": 1661.625, "completions/min_length": 1090.0, "completions/min_terminated_length": 1090.0, "entropy": 0.15260465070605278, "epoch": 0.007587078974594781, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6234718.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.404661774635315, "sampling/importance_sampling_ratio/mean": 1.0002719163894653, "sampling/importance_sampling_ratio/min": 0.6761270761489868, "sampling/sampling_logp_difference/max": 0.39137423038482666, "sampling/sampling_logp_difference/mean": 0.006605846807360649, "step": 132 }, { "clip_ratio/high_max": 5.024858546676114e-05, "clip_ratio/high_mean": 5.024858546676114e-05, "clip_ratio/low_mean": 0.0008835435146465898, "clip_ratio/low_min": 0.0008835435146465898, "clip_ratio/region_mean": 0.0009337921001133509, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13484.0, "completions/mean_length": 7938.125, "completions/mean_terminated_length": 6731.57177734375, "completions/min_length": 3555.0, "completions/min_terminated_length": 3555.0, "entropy": 0.3728845566511154, "epoch": 0.007644556845614438, "frac_reward_zero_std": 0.0, "grad_norm": 0.007273501250892878, "learning_rate": 1e-05, "loss": 0.1902, "num_tokens": 6299279.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000244140625, "sampling/importance_sampling_ratio/min": 0.44445252418518066, "sampling/sampling_logp_difference/max": 0.9838747978210449, "sampling/sampling_logp_difference/mean": 0.016618642956018448, "step": 133 }, { "clip_ratio/high_max": 6.319514795904979e-05, "clip_ratio/high_mean": 6.319514795904979e-05, "clip_ratio/low_mean": 5.792400406789966e-05, "clip_ratio/low_min": 5.792400406789966e-05, "clip_ratio/region_mean": 0.00012111915202694945, "completions/clipped_ratio": 0.0, "completions/max_length": 2867.0, "completions/max_terminated_length": 2867.0, "completions/mean_length": 2057.25, "completions/mean_terminated_length": 2057.25, "completions/min_length": 1652.0, "completions/min_terminated_length": 1652.0, "entropy": 0.28987678699195385, "epoch": 0.007702034716634096, "frac_reward_zero_std": 0.0, "grad_norm": 0.01631118729710579, "learning_rate": 1e-05, "loss": -0.0321, "num_tokens": 6316705.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4888286590576172, "sampling/importance_sampling_ratio/mean": 1.000054955482483, "sampling/importance_sampling_ratio/min": 0.6797762513160706, "sampling/sampling_logp_difference/max": 0.39798974990844727, "sampling/sampling_logp_difference/mean": 0.011149604804813862, "step": 134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7552.0, "completions/max_terminated_length": 7552.0, "completions/mean_length": 3074.625, "completions/mean_terminated_length": 3074.625, "completions/min_length": 1472.0, "completions/min_terminated_length": 1472.0, "entropy": 0.19418606348335743, "epoch": 0.0077595125876537534, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6342766.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5750576257705688, "sampling/importance_sampling_ratio/mean": 1.000051498413086, "sampling/importance_sampling_ratio/min": 0.6158427596092224, "sampling/sampling_logp_difference/max": 0.48476362228393555, "sampling/sampling_logp_difference/mean": 0.00793286319822073, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4646.0, "completions/max_terminated_length": 4646.0, "completions/mean_length": 3114.375, "completions/mean_terminated_length": 3114.375, "completions/min_length": 1475.0, "completions/min_terminated_length": 1475.0, "entropy": 0.5646222084760666, "epoch": 0.007816990458673411, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6368873.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4438854455947876, "sampling/importance_sampling_ratio/mean": 0.9999485611915588, "sampling/importance_sampling_ratio/min": 0.6187267303466797, "sampling/sampling_logp_difference/max": 0.48009157180786133, "sampling/sampling_logp_difference/mean": 0.020424988120794296, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003590989581425674, "clip_ratio/low_min": 0.0003590989581425674, "clip_ratio/region_mean": 0.0003590989581425674, "completions/clipped_ratio": 0.0, "completions/max_length": 1278.0, "completions/max_terminated_length": 1278.0, "completions/mean_length": 970.625, "completions/mean_terminated_length": 970.625, "completions/min_length": 694.0, "completions/min_terminated_length": 694.0, "entropy": 0.23127288930118084, "epoch": 0.007874468329693069, "frac_reward_zero_std": 0.0, "grad_norm": 0.017832156270742416, "learning_rate": 1e-05, "loss": -0.0371, "num_tokens": 6377558.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.275546669960022, "sampling/importance_sampling_ratio/mean": 1.0002614259719849, "sampling/importance_sampling_ratio/min": 0.6965088248252869, "sampling/sampling_logp_difference/max": 0.36167478561401367, "sampling/sampling_logp_difference/mean": 0.009490367025136948, "step": 137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013839952953276224, "clip_ratio/low_min": 0.00013839952953276224, "clip_ratio/region_mean": 0.00013839952953276224, "completions/clipped_ratio": 0.0, "completions/max_length": 6932.0, "completions/max_terminated_length": 6932.0, "completions/mean_length": 4874.625, "completions/mean_terminated_length": 4874.625, "completions/min_length": 2478.0, "completions/min_terminated_length": 2478.0, "entropy": 0.6169449612498283, "epoch": 0.007931946200712726, "frac_reward_zero_std": 0.0, "grad_norm": 0.0270911306142807, "learning_rate": 1e-05, "loss": -0.0552, "num_tokens": 6417859.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001020431518555, "sampling/importance_sampling_ratio/min": 0.5970737338066101, "sampling/sampling_logp_difference/max": 0.7154791355133057, "sampling/sampling_logp_difference/mean": 0.015228682197630405, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2991.0, "completions/max_terminated_length": 2991.0, "completions/mean_length": 2041.875, "completions/mean_terminated_length": 2041.875, "completions/min_length": 1428.0, "completions/min_terminated_length": 1428.0, "entropy": 0.180243456736207, "epoch": 0.007989424071732382, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6435058.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3439971208572388, "sampling/importance_sampling_ratio/mean": 1.0000247955322266, "sampling/importance_sampling_ratio/min": 0.6104732751846313, "sampling/sampling_logp_difference/max": 0.49352073669433594, "sampling/sampling_logp_difference/mean": 0.007695259992033243, "step": 139 }, { "clip_ratio/high_max": 7.995203668542672e-05, "clip_ratio/high_mean": 7.995203668542672e-05, "clip_ratio/low_mean": 0.00029105868452461436, "clip_ratio/low_min": 0.00029105868452461436, "clip_ratio/region_mean": 0.0003710107212100411, "completions/clipped_ratio": 0.0, "completions/max_length": 5747.0, "completions/max_terminated_length": 5747.0, "completions/mean_length": 3175.125, "completions/mean_terminated_length": 3175.125, "completions/min_length": 1304.0, "completions/min_terminated_length": 1304.0, "entropy": 0.3940267525613308, "epoch": 0.00804690194275204, "frac_reward_zero_std": 0.0, "grad_norm": 0.019426902756094933, "learning_rate": 1e-05, "loss": -0.1836, "num_tokens": 6461451.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.800508737564087, "sampling/importance_sampling_ratio/mean": 0.9997274875640869, "sampling/importance_sampling_ratio/min": 0.4268714189529419, "sampling/sampling_logp_difference/max": 0.851272463798523, "sampling/sampling_logp_difference/mean": 0.016020389273762703, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3173.0, "completions/max_terminated_length": 3173.0, "completions/mean_length": 2345.75, "completions/mean_terminated_length": 2345.75, "completions/min_length": 1536.0, "completions/min_terminated_length": 1536.0, "entropy": 0.19869183097034693, "epoch": 0.008104379813771698, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6481257.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.636795163154602, "sampling/importance_sampling_ratio/mean": 1.000019907951355, "sampling/importance_sampling_ratio/min": 0.6332414746284485, "sampling/sampling_logp_difference/max": 0.4927401542663574, "sampling/sampling_logp_difference/mean": 0.008244763128459454, "step": 141 }, { "clip_ratio/high_max": 0.00017878098151413724, "clip_ratio/high_mean": 0.00017878098151413724, "clip_ratio/low_mean": 0.0002086660679196939, "clip_ratio/low_min": 0.0002086660679196939, "clip_ratio/region_mean": 0.0003874470494338311, "completions/clipped_ratio": 0.0, "completions/max_length": 7785.0, "completions/max_terminated_length": 7785.0, "completions/mean_length": 5947.625, "completions/mean_terminated_length": 5947.625, "completions/min_length": 3582.0, "completions/min_terminated_length": 3582.0, "entropy": 0.29591018334031105, "epoch": 0.008161857684791355, "frac_reward_zero_std": 0.0, "grad_norm": 0.0072945160791277885, "learning_rate": 1e-05, "loss": 0.0789, "num_tokens": 6529750.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997889995574951, "sampling/importance_sampling_ratio/min": 0.374642550945282, "sampling/sampling_logp_difference/max": 0.9817829132080078, "sampling/sampling_logp_difference/mean": 0.013533972203731537, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1034.0, "completions/max_terminated_length": 1034.0, "completions/mean_length": 813.625, "completions/mean_terminated_length": 813.625, "completions/min_length": 532.0, "completions/min_terminated_length": 532.0, "entropy": 0.557020515203476, "epoch": 0.008219335555811013, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6538291.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2654404640197754, "sampling/importance_sampling_ratio/mean": 1.0006256103515625, "sampling/importance_sampling_ratio/min": 0.7822061777114868, "sampling/sampling_logp_difference/max": 0.2456369400024414, "sampling/sampling_logp_difference/mean": 0.015856806188821793, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3080.0, "completions/max_terminated_length": 3080.0, "completions/mean_length": 1541.0, "completions/mean_terminated_length": 1541.0, "completions/min_length": 913.0, "completions/min_terminated_length": 913.0, "entropy": 0.2369413785636425, "epoch": 0.00827681342683067, "frac_reward_zero_std": 0.0, "grad_norm": 0.009322436526417732, "learning_rate": 1e-05, "loss": 0.3527, "num_tokens": 6551547.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3686444759368896, "sampling/importance_sampling_ratio/mean": 1.000044822692871, "sampling/importance_sampling_ratio/min": 0.7421360015869141, "sampling/sampling_logp_difference/max": 0.31382083892822266, "sampling/sampling_logp_difference/mean": 0.009175000712275505, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6132.0, "completions/max_terminated_length": 6132.0, "completions/mean_length": 3661.75, "completions/mean_terminated_length": 3661.75, "completions/min_length": 1555.0, "completions/min_terminated_length": 1555.0, "entropy": 0.41266872733831406, "epoch": 0.008334291297850328, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6581657.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6183322668075562, "sampling/importance_sampling_ratio/mean": 1.000028133392334, "sampling/importance_sampling_ratio/min": 0.648280143737793, "sampling/sampling_logp_difference/max": 0.4813961982727051, "sampling/sampling_logp_difference/mean": 0.01620025560259819, "step": 145 }, { "clip_ratio/high_max": 0.00013272334399516694, "clip_ratio/high_mean": 0.00013272334399516694, "clip_ratio/low_mean": 0.00012308468103583436, "clip_ratio/low_min": 0.00012308468103583436, "clip_ratio/region_mean": 0.0002558080250310013, "completions/clipped_ratio": 0.0, "completions/max_length": 14048.0, "completions/max_terminated_length": 14048.0, "completions/mean_length": 10919.5, "completions/mean_terminated_length": 10919.5, "completions/min_length": 8052.0, "completions/min_terminated_length": 8052.0, "entropy": 0.46391235664486885, "epoch": 0.008391769168869984, "frac_reward_zero_std": 0.0, "grad_norm": 0.011609341949224472, "learning_rate": 1e-05, "loss": -0.0312, "num_tokens": 6670045.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998831748962402, "sampling/importance_sampling_ratio/min": 0.267840713262558, "sampling/sampling_logp_difference/max": 1.3173627853393555, "sampling/sampling_logp_difference/mean": 0.019570058211684227, "step": 146 }, { "clip_ratio/high_max": 0.0002113405971613247, "clip_ratio/high_mean": 0.0002113405971613247, "clip_ratio/low_mean": 0.00013204225979279727, "clip_ratio/low_min": 0.00013204225979279727, "clip_ratio/region_mean": 0.00034338285695412196, "completions/clipped_ratio": 0.0, "completions/max_length": 3221.0, "completions/max_terminated_length": 3221.0, "completions/mean_length": 2172.5, "completions/mean_terminated_length": 2172.5, "completions/min_length": 894.0, "completions/min_terminated_length": 894.0, "entropy": 0.3832019381225109, "epoch": 0.008449247039889642, "frac_reward_zero_std": 0.0, "grad_norm": 0.009099716320633888, "learning_rate": 1e-05, "loss": 0.1086, "num_tokens": 6688649.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4237631559371948, "sampling/importance_sampling_ratio/mean": 0.9996580481529236, "sampling/importance_sampling_ratio/min": 0.6517035961151123, "sampling/sampling_logp_difference/max": 0.4281654357910156, "sampling/sampling_logp_difference/mean": 0.01533388253301382, "step": 147 }, { "clip_ratio/high_max": 3.787980313063599e-05, "clip_ratio/high_mean": 3.787980313063599e-05, "clip_ratio/low_mean": 8.50484793772921e-05, "clip_ratio/low_min": 8.50484793772921e-05, "clip_ratio/region_mean": 0.0001229282825079281, "completions/clipped_ratio": 0.0, "completions/max_length": 11758.0, "completions/max_terminated_length": 11758.0, "completions/mean_length": 5206.0, "completions/mean_terminated_length": 5206.0, "completions/min_length": 2350.0, "completions/min_terminated_length": 2350.0, "entropy": 0.44523949921131134, "epoch": 0.0085067249109093, "frac_reward_zero_std": 0.0, "grad_norm": 0.00463527487590909, "learning_rate": 1e-05, "loss": 0.4445, "num_tokens": 6731249.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4365485906600952, "sampling/importance_sampling_ratio/mean": 0.9995553493499756, "sampling/importance_sampling_ratio/min": 0.6018043160438538, "sampling/sampling_logp_difference/max": 0.5078229904174805, "sampling/sampling_logp_difference/mean": 0.013541629537940025, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1831.0, "completions/max_terminated_length": 1831.0, "completions/mean_length": 1088.25, "completions/mean_terminated_length": 1088.25, "completions/min_length": 860.0, "completions/min_terminated_length": 860.0, "entropy": 0.24122720398008823, "epoch": 0.008564202781928957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6741003.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3806281089782715, "sampling/importance_sampling_ratio/mean": 1.0003173351287842, "sampling/importance_sampling_ratio/min": 0.6448020339012146, "sampling/sampling_logp_difference/max": 0.4388120174407959, "sampling/sampling_logp_difference/mean": 0.008040092885494232, "step": 149 }, { "clip_ratio/high_max": 0.00015799710672581568, "clip_ratio/high_mean": 0.00015799710672581568, "clip_ratio/low_mean": 2.6057952709379606e-05, "clip_ratio/low_min": 2.6057952709379606e-05, "clip_ratio/region_mean": 0.0001840550594351953, "completions/clipped_ratio": 0.0, "completions/max_length": 4797.0, "completions/max_terminated_length": 4797.0, "completions/mean_length": 2548.0, "completions/mean_terminated_length": 2548.0, "completions/min_length": 1179.0, "completions/min_terminated_length": 1179.0, "entropy": 0.4162476062774658, "epoch": 0.008621680652948615, "frac_reward_zero_std": 0.0, "grad_norm": 0.010240177623927593, "learning_rate": 1e-05, "loss": 0.3125, "num_tokens": 6762675.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4358079433441162, "sampling/importance_sampling_ratio/mean": 1.000313401222229, "sampling/importance_sampling_ratio/min": 0.6893447041511536, "sampling/sampling_logp_difference/max": 0.37201380729675293, "sampling/sampling_logp_difference/mean": 0.013260569423437119, "step": 150 }, { "clip_ratio/high_max": 6.30233535048319e-05, "clip_ratio/high_mean": 6.30233535048319e-05, "clip_ratio/low_mean": 0.00042105267857550643, "clip_ratio/low_min": 0.00042105267857550643, "clip_ratio/region_mean": 0.00048407603208033834, "completions/clipped_ratio": 0.0, "completions/max_length": 12958.0, "completions/max_terminated_length": 12958.0, "completions/mean_length": 9031.875, "completions/mean_terminated_length": 9031.875, "completions/min_length": 4371.0, "completions/min_terminated_length": 4371.0, "entropy": 0.42483629286289215, "epoch": 0.008679158523968273, "frac_reward_zero_std": 0.0, "grad_norm": 0.029668988659977913, "learning_rate": 1e-05, "loss": 0.067, "num_tokens": 6836258.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.7135207653045654, "sampling/importance_sampling_ratio/mean": 1.0001245737075806, "sampling/importance_sampling_ratio/min": 0.335127055644989, "sampling/sampling_logp_difference/max": 1.093245506286621, "sampling/sampling_logp_difference/mean": 0.019100628793239594, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1469.0, "completions/max_terminated_length": 1469.0, "completions/mean_length": 910.0, "completions/mean_terminated_length": 910.0, "completions/min_length": 554.0, "completions/min_terminated_length": 554.0, "entropy": 0.301215760409832, "epoch": 0.00873663639498793, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6844242.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.296616792678833, "sampling/importance_sampling_ratio/mean": 0.9998509287834167, "sampling/importance_sampling_ratio/min": 0.695597767829895, "sampling/sampling_logp_difference/max": 0.36298370361328125, "sampling/sampling_logp_difference/mean": 0.011990005150437355, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1782.0, "completions/max_terminated_length": 1782.0, "completions/mean_length": 1431.0, "completions/mean_terminated_length": 1431.0, "completions/min_length": 814.0, "completions/min_terminated_length": 814.0, "entropy": 0.2171723898500204, "epoch": 0.008794114266007588, "frac_reward_zero_std": 0.0, "grad_norm": 0.01422662939876318, "learning_rate": 1e-05, "loss": -0.0469, "num_tokens": 6856698.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3222421407699585, "sampling/importance_sampling_ratio/mean": 0.9996227622032166, "sampling/importance_sampling_ratio/min": 0.6277742385864258, "sampling/sampling_logp_difference/max": 0.4655747413635254, "sampling/sampling_logp_difference/mean": 0.009093833155930042, "step": 153 }, { "clip_ratio/high_max": 0.00020458265498746186, "clip_ratio/high_mean": 0.00020458265498746186, "clip_ratio/low_mean": 0.000125376129290089, "clip_ratio/low_min": 0.000125376129290089, "clip_ratio/region_mean": 0.0003299587842775509, "completions/clipped_ratio": 0.0, "completions/max_length": 1637.0, "completions/max_terminated_length": 1637.0, "completions/mean_length": 1273.375, "completions/mean_terminated_length": 1273.375, "completions/min_length": 997.0, "completions/min_terminated_length": 997.0, "entropy": 0.21633286029100418, "epoch": 0.008851592137027244, "frac_reward_zero_std": 0.0, "grad_norm": 0.059154924005270004, "learning_rate": 1e-05, "loss": -0.0553, "num_tokens": 6867837.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5729005336761475, "sampling/importance_sampling_ratio/mean": 0.9998074769973755, "sampling/importance_sampling_ratio/min": 0.6064621210098267, "sampling/sampling_logp_difference/max": 0.5001130104064941, "sampling/sampling_logp_difference/mean": 0.008797754533588886, "step": 154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2549.0, "completions/max_terminated_length": 2549.0, "completions/mean_length": 1737.375, "completions/mean_terminated_length": 1737.375, "completions/min_length": 976.0, "completions/min_terminated_length": 976.0, "entropy": 0.21732072532176971, "epoch": 0.008909070008046902, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6883088.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2848201990127563, "sampling/importance_sampling_ratio/mean": 1.0001574754714966, "sampling/importance_sampling_ratio/min": 0.49255454540252686, "sampling/sampling_logp_difference/max": 0.7081500887870789, "sampling/sampling_logp_difference/mean": 0.008940266445279121, "step": 155 }, { "clip_ratio/high_max": 4.288164564059116e-05, "clip_ratio/high_mean": 4.288164564059116e-05, "clip_ratio/low_mean": 9.36329597607255e-05, "clip_ratio/low_min": 9.36329597607255e-05, "clip_ratio/region_mean": 0.00013651460540131666, "completions/clipped_ratio": 0.0, "completions/max_length": 2915.0, "completions/max_terminated_length": 2915.0, "completions/mean_length": 1733.0, "completions/mean_terminated_length": 1733.0, "completions/min_length": 743.0, "completions/min_terminated_length": 743.0, "entropy": 0.2394898571074009, "epoch": 0.00896654787906656, "frac_reward_zero_std": 0.0, "grad_norm": 0.06945914030075073, "learning_rate": 1e-05, "loss": -0.0806, "num_tokens": 6898088.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3556156158447266, "sampling/importance_sampling_ratio/mean": 1.0001763105392456, "sampling/importance_sampling_ratio/min": 0.6855784058570862, "sampling/sampling_logp_difference/max": 0.37749242782592773, "sampling/sampling_logp_difference/mean": 0.010261802934110165, "step": 156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6742.0, "completions/max_terminated_length": 6742.0, "completions/mean_length": 3985.5, "completions/mean_terminated_length": 3985.5, "completions/min_length": 1301.0, "completions/min_terminated_length": 1301.0, "entropy": 0.3295823559165001, "epoch": 0.009024025750086217, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6931212.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3633723258972168, "sampling/importance_sampling_ratio/mean": 0.9998742341995239, "sampling/importance_sampling_ratio/min": 0.6257620453834534, "sampling/sampling_logp_difference/max": 0.4687851071357727, "sampling/sampling_logp_difference/mean": 0.014020584523677826, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6554.0, "completions/max_terminated_length": 6554.0, "completions/mean_length": 3004.375, "completions/mean_terminated_length": 3004.375, "completions/min_length": 1153.0, "completions/min_terminated_length": 1153.0, "entropy": 0.2464207522571087, "epoch": 0.009081503621105875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6955927.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4352158308029175, "sampling/importance_sampling_ratio/mean": 0.9999135732650757, "sampling/importance_sampling_ratio/min": 0.6133259534835815, "sampling/sampling_logp_difference/max": 0.488858699798584, "sampling/sampling_logp_difference/mean": 0.01061803288757801, "step": 158 }, { "clip_ratio/high_max": 3.569319142116001e-05, "clip_ratio/high_mean": 3.569319142116001e-05, "clip_ratio/low_mean": 0.00049591064453125, "clip_ratio/low_min": 0.00049591064453125, "clip_ratio/region_mean": 0.00053160383595241, "completions/clipped_ratio": 0.75, "completions/max_length": 16384.0, "completions/max_terminated_length": 14928.0, "completions/mean_length": 15631.875, "completions/mean_terminated_length": 13375.5, "completions/min_length": 11823.0, "completions/min_terminated_length": 11823.0, "entropy": 0.3680622801184654, "epoch": 0.009138981492125532, "frac_reward_zero_std": 0.0, "grad_norm": 0.017388461157679558, "learning_rate": 1e-05, "loss": 0.0778, "num_tokens": 7081950.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999274611473083, "sampling/importance_sampling_ratio/min": 0.18218164145946503, "sampling/sampling_logp_difference/max": 1.7027510404586792, "sampling/sampling_logp_difference/mean": 0.01629018411040306, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4961.0, "completions/max_terminated_length": 4961.0, "completions/mean_length": 4246.625, "completions/mean_terminated_length": 4246.625, "completions/min_length": 3206.0, "completions/min_terminated_length": 3206.0, "entropy": 0.3007093593478203, "epoch": 0.00919645936314519, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7116963.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4973028898239136, "sampling/importance_sampling_ratio/mean": 1.000224232673645, "sampling/importance_sampling_ratio/min": 0.6020798683166504, "sampling/sampling_logp_difference/max": 0.5073652267456055, "sampling/sampling_logp_difference/mean": 0.011745051480829716, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14000.0, "completions/max_terminated_length": 14000.0, "completions/mean_length": 6351.75, "completions/mean_terminated_length": 6351.75, "completions/min_length": 1756.0, "completions/min_terminated_length": 1756.0, "entropy": 0.5105179361999035, "epoch": 0.009253937234164846, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7168593.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000451803207397, "sampling/importance_sampling_ratio/min": 0.07039714604616165, "sampling/sampling_logp_difference/max": 2.6536026000976562, "sampling/sampling_logp_difference/mean": 0.021646125242114067, "step": 161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8577.0, "completions/max_terminated_length": 8577.0, "completions/mean_length": 4887.25, "completions/mean_terminated_length": 4887.25, "completions/min_length": 2752.0, "completions/min_terminated_length": 2752.0, "entropy": 0.43511849269270897, "epoch": 0.009311415105184503, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7208419.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4992823600769043, "sampling/importance_sampling_ratio/mean": 0.9998119473457336, "sampling/importance_sampling_ratio/min": 0.5442285537719727, "sampling/sampling_logp_difference/max": 0.6083860397338867, "sampling/sampling_logp_difference/mean": 0.016671858727931976, "step": 162 }, { "clip_ratio/high_max": 0.00022961249851505272, "clip_ratio/high_mean": 0.00022961249851505272, "clip_ratio/low_mean": 0.0002116431569447741, "clip_ratio/low_min": 0.0002116431569447741, "clip_ratio/region_mean": 0.0004412556554598268, "completions/clipped_ratio": 0.0, "completions/max_length": 12969.0, "completions/max_terminated_length": 12969.0, "completions/mean_length": 9163.5, "completions/mean_terminated_length": 9163.5, "completions/min_length": 4514.0, "completions/min_terminated_length": 4514.0, "entropy": 0.49257949367165565, "epoch": 0.009368892976204161, "frac_reward_zero_std": 0.0, "grad_norm": 0.006566404365003109, "learning_rate": 1e-05, "loss": 0.0194, "num_tokens": 7282679.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998836517333984, "sampling/importance_sampling_ratio/min": 0.3657211363315582, "sampling/sampling_logp_difference/max": 1.0058841705322266, "sampling/sampling_logp_difference/mean": 0.020654143765568733, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 16245.0, "completions/mean_length": 11264.25, "completions/mean_terminated_length": 10532.857421875, "completions/min_length": 6847.0, "completions/min_terminated_length": 6847.0, "entropy": 0.7053465247154236, "epoch": 0.009426370847223819, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7374009.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000031590461731, "sampling/importance_sampling_ratio/min": 0.302106112241745, "sampling/sampling_logp_difference/max": 1.3456833362579346, "sampling/sampling_logp_difference/mean": 0.02689768746495247, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1544.0, "completions/max_terminated_length": 1544.0, "completions/mean_length": 1000.25, "completions/mean_terminated_length": 1000.25, "completions/min_length": 640.0, "completions/min_terminated_length": 640.0, "entropy": 0.6362258717417717, "epoch": 0.009483848718243476, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7382827.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2800841331481934, "sampling/importance_sampling_ratio/mean": 1.00029456615448, "sampling/importance_sampling_ratio/min": 0.7599013447761536, "sampling/sampling_logp_difference/max": 0.274566650390625, "sampling/sampling_logp_difference/mean": 0.017346898093819618, "step": 165 }, { "clip_ratio/high_max": 0.00012542009426397271, "clip_ratio/high_mean": 0.00012542009426397271, "clip_ratio/low_mean": 0.00020808458793908358, "clip_ratio/low_min": 0.00020808458793908358, "clip_ratio/region_mean": 0.0003335046822030563, "completions/clipped_ratio": 0.0, "completions/max_length": 6953.0, "completions/max_terminated_length": 6953.0, "completions/mean_length": 3977.5, "completions/mean_terminated_length": 3977.5, "completions/min_length": 2419.0, "completions/min_terminated_length": 2419.0, "entropy": 0.2975945267826319, "epoch": 0.009541326589263134, "frac_reward_zero_std": 0.0, "grad_norm": 0.0099942646920681, "learning_rate": 1e-05, "loss": 0.1789, "num_tokens": 7416031.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8197044134140015, "sampling/importance_sampling_ratio/mean": 1.0002262592315674, "sampling/importance_sampling_ratio/min": 0.6547252535820007, "sampling/sampling_logp_difference/max": 0.5986740589141846, "sampling/sampling_logp_difference/mean": 0.01162185799330473, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14021.0, "completions/mean_length": 7496.625, "completions/mean_terminated_length": 4534.1669921875, "completions/min_length": 1406.0, "completions/min_terminated_length": 1406.0, "entropy": 0.27918735332787037, "epoch": 0.009598804460282792, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7477228.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998542666435242, "sampling/importance_sampling_ratio/min": 0.4236017167568207, "sampling/sampling_logp_difference/max": 0.8589615821838379, "sampling/sampling_logp_difference/mean": 0.014311072416603565, "step": 167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1618.0, "completions/max_terminated_length": 1618.0, "completions/mean_length": 661.25, "completions/mean_terminated_length": 661.25, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.20153271220624447, "epoch": 0.00965628233130245, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7483510.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3847815990447998, "sampling/importance_sampling_ratio/mean": 0.9990352392196655, "sampling/importance_sampling_ratio/min": 0.6852436065673828, "sampling/sampling_logp_difference/max": 0.3779808282852173, "sampling/sampling_logp_difference/mean": 0.009825999848544598, "step": 168 }, { "clip_ratio/high_max": 6.28524212515913e-05, "clip_ratio/high_mean": 6.28524212515913e-05, "clip_ratio/low_mean": 0.0007833648342057131, "clip_ratio/low_min": 0.0007833648342057131, "clip_ratio/region_mean": 0.0008462172554573044, "completions/clipped_ratio": 0.0, "completions/max_length": 6571.0, "completions/max_terminated_length": 6571.0, "completions/mean_length": 4029.25, "completions/mean_terminated_length": 4029.25, "completions/min_length": 950.0, "completions/min_terminated_length": 950.0, "entropy": 0.3966398164629936, "epoch": 0.009713760202322105, "frac_reward_zero_std": 0.0, "grad_norm": 0.03468601778149605, "learning_rate": 1e-05, "loss": 0.0808, "num_tokens": 7517736.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4819731712341309, "sampling/importance_sampling_ratio/mean": 0.9999225735664368, "sampling/importance_sampling_ratio/min": 0.4870922863483429, "sampling/sampling_logp_difference/max": 0.719301700592041, "sampling/sampling_logp_difference/mean": 0.01768554002046585, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2629.0, "completions/max_terminated_length": 2629.0, "completions/mean_length": 2078.125, "completions/mean_terminated_length": 2078.125, "completions/min_length": 1474.0, "completions/min_terminated_length": 1474.0, "entropy": 0.25761128030717373, "epoch": 0.009771238073341763, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7536521.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3825703859329224, "sampling/importance_sampling_ratio/mean": 0.9997692108154297, "sampling/importance_sampling_ratio/min": 0.6662405729293823, "sampling/sampling_logp_difference/max": 0.4061044454574585, "sampling/sampling_logp_difference/mean": 0.01018275786191225, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4610.0, "completions/max_terminated_length": 4610.0, "completions/mean_length": 2941.875, "completions/mean_terminated_length": 2941.875, "completions/min_length": 1798.0, "completions/min_terminated_length": 1798.0, "entropy": 0.7922124117612839, "epoch": 0.00982871594436142, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7561912.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.655548334121704, "sampling/importance_sampling_ratio/mean": 1.000400424003601, "sampling/importance_sampling_ratio/min": 0.6684586405754089, "sampling/sampling_logp_difference/max": 0.5041322708129883, "sampling/sampling_logp_difference/mean": 0.019063184037804604, "step": 171 }, { "clip_ratio/high_max": 6.214732457010541e-05, "clip_ratio/high_mean": 6.214732457010541e-05, "clip_ratio/low_mean": 0.00022542186343343928, "clip_ratio/low_min": 0.00022542186343343928, "clip_ratio/region_mean": 0.0002875691880035447, "completions/clipped_ratio": 0.0, "completions/max_length": 11616.0, "completions/max_terminated_length": 11616.0, "completions/mean_length": 6968.0, "completions/mean_terminated_length": 6968.0, "completions/min_length": 4674.0, "completions/min_terminated_length": 4674.0, "entropy": 0.3691304251551628, "epoch": 0.009886193815381078, "frac_reward_zero_std": 0.0, "grad_norm": 0.043332166969776154, "learning_rate": 1e-05, "loss": 0.2118, "num_tokens": 7618896.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8682059049606323, "sampling/importance_sampling_ratio/mean": 1.000428318977356, "sampling/importance_sampling_ratio/min": 0.4910013973712921, "sampling/sampling_logp_difference/max": 0.7113083600997925, "sampling/sampling_logp_difference/mean": 0.015282669104635715, "step": 172 }, { "clip_ratio/high_max": 0.00016217795018746983, "clip_ratio/high_mean": 0.00016217795018746983, "clip_ratio/low_mean": 0.00034929594403365627, "clip_ratio/low_min": 0.00034929594403365627, "clip_ratio/region_mean": 0.0005114738942211261, "completions/clipped_ratio": 0.0, "completions/max_length": 15481.0, "completions/max_terminated_length": 15481.0, "completions/mean_length": 9216.875, "completions/mean_terminated_length": 9216.875, "completions/min_length": 3655.0, "completions/min_terminated_length": 3655.0, "entropy": 0.6708216592669487, "epoch": 0.009943671686400736, "frac_reward_zero_std": 0.0, "grad_norm": 0.030088379979133606, "learning_rate": 1e-05, "loss": 0.1355, "num_tokens": 7693511.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.8613359928131104, "sampling/importance_sampling_ratio/mean": 0.999386191368103, "sampling/importance_sampling_ratio/min": 0.4131677448749542, "sampling/sampling_logp_difference/max": 0.8839015960693359, "sampling/sampling_logp_difference/mean": 0.027095327153801918, "step": 173 }, { "clip_ratio/high_max": 5.058680835645646e-05, "clip_ratio/high_mean": 5.058680835645646e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.058680835645646e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 4366.0, "completions/max_terminated_length": 4366.0, "completions/mean_length": 1801.875, "completions/mean_terminated_length": 1801.875, "completions/min_length": 610.0, "completions/min_terminated_length": 610.0, "entropy": 0.24556374922394753, "epoch": 0.010001149557420394, "frac_reward_zero_std": 0.0, "grad_norm": 0.011647317558526993, "learning_rate": 1e-05, "loss": -0.1987, "num_tokens": 7709054.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6970995664596558, "sampling/importance_sampling_ratio/mean": 0.9994653463363647, "sampling/importance_sampling_ratio/min": 0.6818316578865051, "sampling/sampling_logp_difference/max": 0.5289206504821777, "sampling/sampling_logp_difference/mean": 0.010972586460411549, "step": 174 }, { "clip_ratio/high_max": 0.00013646288425661623, "clip_ratio/high_mean": 0.00013646288425661623, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00013646288425661623, "completions/clipped_ratio": 0.0, "completions/max_length": 1411.0, "completions/max_terminated_length": 1411.0, "completions/mean_length": 850.875, "completions/mean_terminated_length": 850.875, "completions/min_length": 528.0, "completions/min_terminated_length": 528.0, "entropy": 0.28919071331620216, "epoch": 0.010058627428440051, "frac_reward_zero_std": 0.0, "grad_norm": 0.10567350685596466, "learning_rate": 1e-05, "loss": -0.0169, "num_tokens": 7716869.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002267360687256, "sampling/importance_sampling_ratio/min": 0.6456519365310669, "sampling/sampling_logp_difference/max": 0.8261299133300781, "sampling/sampling_logp_difference/mean": 0.012845244258642197, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5941.0, "completions/max_terminated_length": 5941.0, "completions/mean_length": 3116.375, "completions/mean_terminated_length": 3116.375, "completions/min_length": 1754.0, "completions/min_terminated_length": 1754.0, "entropy": 0.264639051631093, "epoch": 0.010116105299459707, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7742856.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3892130851745605, "sampling/importance_sampling_ratio/mean": 1.0001221895217896, "sampling/importance_sampling_ratio/min": 0.5892245769500732, "sampling/sampling_logp_difference/max": 0.5289478302001953, "sampling/sampling_logp_difference/mean": 0.010956798680126667, "step": 176 }, { "clip_ratio/high_max": 7.992327300598845e-05, "clip_ratio/high_mean": 7.992327300598845e-05, "clip_ratio/low_mean": 0.00014164306048769504, "clip_ratio/low_min": 0.00014164306048769504, "clip_ratio/region_mean": 0.0002215663334936835, "completions/clipped_ratio": 0.0, "completions/max_length": 1765.0, "completions/max_terminated_length": 1765.0, "completions/mean_length": 1319.75, "completions/mean_terminated_length": 1319.75, "completions/min_length": 698.0, "completions/min_terminated_length": 698.0, "entropy": 0.24970881827175617, "epoch": 0.010173583170479365, "frac_reward_zero_std": 0.0, "grad_norm": 0.016001326963305473, "learning_rate": 1e-05, "loss": 0.1191, "num_tokens": 7754102.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.398911952972412, "sampling/importance_sampling_ratio/mean": 1.0002716779708862, "sampling/importance_sampling_ratio/min": 0.6763787269592285, "sampling/sampling_logp_difference/max": 0.3910020589828491, "sampling/sampling_logp_difference/mean": 0.010832153260707855, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10050.0, "completions/max_terminated_length": 10050.0, "completions/mean_length": 4924.625, "completions/mean_terminated_length": 4924.625, "completions/min_length": 1384.0, "completions/min_terminated_length": 1384.0, "entropy": 0.39915452152490616, "epoch": 0.010231061041499023, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7795035.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003193616867065, "sampling/importance_sampling_ratio/min": 0.4675396680831909, "sampling/sampling_logp_difference/max": 1.1266989707946777, "sampling/sampling_logp_difference/mean": 0.01402293797582388, "step": 178 }, { "clip_ratio/high_max": 0.00014332833416119684, "clip_ratio/high_mean": 0.00014332833416119684, "clip_ratio/low_mean": 0.0005741491113440134, "clip_ratio/low_min": 0.0005741491113440134, "clip_ratio/region_mean": 0.0007174774455052102, "completions/clipped_ratio": 0.0, "completions/max_length": 6375.0, "completions/max_terminated_length": 6375.0, "completions/mean_length": 3390.625, "completions/mean_terminated_length": 3390.625, "completions/min_length": 559.0, "completions/min_terminated_length": 559.0, "entropy": 0.6463589556515217, "epoch": 0.01028853891251868, "frac_reward_zero_std": 0.0, "grad_norm": 0.024347297847270966, "learning_rate": 1e-05, "loss": -0.2196, "num_tokens": 7826600.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6255055665969849, "sampling/importance_sampling_ratio/mean": 0.9999240040779114, "sampling/importance_sampling_ratio/min": 0.30804312229156494, "sampling/sampling_logp_difference/max": 1.1775155067443848, "sampling/sampling_logp_difference/mean": 0.026130210608243942, "step": 179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5437.0, "completions/max_terminated_length": 5437.0, "completions/mean_length": 2589.5, "completions/mean_terminated_length": 2589.5, "completions/min_length": 1455.0, "completions/min_terminated_length": 1455.0, "entropy": 0.4448091350495815, "epoch": 0.010346016783538338, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7848316.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5598030090332031, "sampling/importance_sampling_ratio/mean": 1.0002201795578003, "sampling/importance_sampling_ratio/min": 0.6510143280029297, "sampling/sampling_logp_difference/max": 0.44455957412719727, "sampling/sampling_logp_difference/mean": 0.01733734831213951, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15770.0, "completions/max_terminated_length": 15770.0, "completions/mean_length": 11755.25, "completions/mean_terminated_length": 11755.25, "completions/min_length": 5009.0, "completions/min_terminated_length": 5009.0, "entropy": 0.524201687425375, "epoch": 0.010403494654557996, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7943358.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999759793281555, "sampling/importance_sampling_ratio/min": 0.17881280183792114, "sampling/sampling_logp_difference/max": 1.7214158773422241, "sampling/sampling_logp_difference/mean": 0.022415822371840477, "step": 181 }, { "clip_ratio/high_max": 0.0002902467967942357, "clip_ratio/high_mean": 0.0002902467967942357, "clip_ratio/low_mean": 0.00016747052723076195, "clip_ratio/low_min": 0.00016747052723076195, "clip_ratio/region_mean": 0.00045771732402499765, "completions/clipped_ratio": 0.0, "completions/max_length": 11196.0, "completions/max_terminated_length": 11196.0, "completions/mean_length": 3628.25, "completions/mean_terminated_length": 3628.25, "completions/min_length": 1508.0, "completions/min_terminated_length": 1508.0, "entropy": 0.6214040480554104, "epoch": 0.010460972525577653, "frac_reward_zero_std": 0.0, "grad_norm": 0.17145849764347076, "learning_rate": 1e-05, "loss": 0.7378, "num_tokens": 7973424.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6428632736206055, "sampling/importance_sampling_ratio/mean": 0.9999727606773376, "sampling/importance_sampling_ratio/min": 0.31693235039711, "sampling/sampling_logp_difference/max": 1.1490669250488281, "sampling/sampling_logp_difference/mean": 0.02465110458433628, "step": 182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3473.0, "completions/max_terminated_length": 3473.0, "completions/mean_length": 2117.75, "completions/mean_terminated_length": 2117.75, "completions/min_length": 1016.0, "completions/min_terminated_length": 1016.0, "entropy": 0.42756710946559906, "epoch": 0.01051845039659731, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7991398.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4277199506759644, "sampling/importance_sampling_ratio/mean": 0.9998186230659485, "sampling/importance_sampling_ratio/min": 0.6188682317733765, "sampling/sampling_logp_difference/max": 0.47986292839050293, "sampling/sampling_logp_difference/mean": 0.016678232699632645, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9563.0, "completions/max_terminated_length": 9563.0, "completions/mean_length": 5603.75, "completions/mean_terminated_length": 5603.75, "completions/min_length": 2811.0, "completions/min_terminated_length": 2811.0, "entropy": 0.2840829510241747, "epoch": 0.010575928267616967, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8037348.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8708668947219849, "sampling/importance_sampling_ratio/mean": 1.0001227855682373, "sampling/importance_sampling_ratio/min": 0.36112356185913086, "sampling/sampling_logp_difference/max": 1.0185351371765137, "sampling/sampling_logp_difference/mean": 0.012249253690242767, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0004523384304775391, "clip_ratio/low_min": 0.0004523384304775391, "clip_ratio/region_mean": 0.0004523384304775391, "completions/clipped_ratio": 0.0, "completions/max_length": 11455.0, "completions/max_terminated_length": 11455.0, "completions/mean_length": 7349.875, "completions/mean_terminated_length": 7349.875, "completions/min_length": 3444.0, "completions/min_terminated_length": 3444.0, "entropy": 0.34330933913588524, "epoch": 0.010633406138636625, "frac_reward_zero_std": 0.0, "grad_norm": 0.00419212132692337, "learning_rate": 1e-05, "loss": -0.0421, "num_tokens": 8096979.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999867677688599, "sampling/importance_sampling_ratio/min": 0.5320678353309631, "sampling/sampling_logp_difference/max": 0.8504753112792969, "sampling/sampling_logp_difference/mean": 0.014090518467128277, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3496.0, "completions/max_terminated_length": 3496.0, "completions/mean_length": 2670.375, "completions/mean_terminated_length": 2670.375, "completions/min_length": 2176.0, "completions/min_terminated_length": 2176.0, "entropy": 0.25105168111622334, "epoch": 0.010690884009656282, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8119302.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3162728548049927, "sampling/importance_sampling_ratio/mean": 1.0000258684158325, "sampling/importance_sampling_ratio/min": 0.5002357959747314, "sampling/sampling_logp_difference/max": 0.692675769329071, "sampling/sampling_logp_difference/mean": 0.00980860274285078, "step": 186 }, { "clip_ratio/high_max": 4.545454430626705e-05, "clip_ratio/high_mean": 4.545454430626705e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.545454430626705e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 3995.0, "completions/max_terminated_length": 3995.0, "completions/mean_length": 2801.0, "completions/mean_terminated_length": 2801.0, "completions/min_length": 1778.0, "completions/min_terminated_length": 1778.0, "entropy": 0.34686275385320187, "epoch": 0.01074836188067594, "frac_reward_zero_std": 0.0, "grad_norm": 0.07881250977516174, "learning_rate": 1e-05, "loss": -0.0175, "num_tokens": 8142862.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.474563479423523, "sampling/importance_sampling_ratio/mean": 0.9998308420181274, "sampling/importance_sampling_ratio/min": 0.6655871272087097, "sampling/sampling_logp_difference/max": 0.407085657119751, "sampling/sampling_logp_difference/mean": 0.013679606840014458, "step": 187 }, { "clip_ratio/high_max": 9.280240919906646e-05, "clip_ratio/high_mean": 9.280240919906646e-05, "clip_ratio/low_mean": 0.0005474619974847883, "clip_ratio/low_min": 0.0005474619974847883, "clip_ratio/region_mean": 0.0006402644066838548, "completions/clipped_ratio": 0.0, "completions/max_length": 13911.0, "completions/max_terminated_length": 13911.0, "completions/mean_length": 6992.75, "completions/mean_terminated_length": 6992.75, "completions/min_length": 3146.0, "completions/min_terminated_length": 3146.0, "entropy": 0.439095851033926, "epoch": 0.010805839751695598, "frac_reward_zero_std": 0.0, "grad_norm": 0.014007357880473137, "learning_rate": 1e-05, "loss": 0.3076, "num_tokens": 8199732.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.884044885635376, "sampling/importance_sampling_ratio/mean": 1.000054955482483, "sampling/importance_sampling_ratio/min": 0.2771124541759491, "sampling/sampling_logp_difference/max": 1.2833318710327148, "sampling/sampling_logp_difference/mean": 0.01880401186645031, "step": 188 }, { "clip_ratio/high_max": 0.00017074525840143906, "clip_ratio/high_mean": 0.00017074525840143906, "clip_ratio/low_mean": 0.00018244860257254913, "clip_ratio/low_min": 0.00018244860257254913, "clip_ratio/region_mean": 0.0003531938609739882, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13772.0, "completions/mean_length": 10991.0, "completions/mean_terminated_length": 10220.572265625, "completions/min_length": 5741.0, "completions/min_terminated_length": 5741.0, "entropy": 0.5909396111965179, "epoch": 0.010863317622715255, "frac_reward_zero_std": 0.0, "grad_norm": 0.006729756481945515, "learning_rate": 1e-05, "loss": 0.2001, "num_tokens": 8288572.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003269910812378, "sampling/importance_sampling_ratio/min": 0.35161498188972473, "sampling/sampling_logp_difference/max": 1.0452184677124023, "sampling/sampling_logp_difference/mean": 0.024106120690703392, "step": 189 }, { "clip_ratio/high_max": 4.512167470238637e-05, "clip_ratio/high_mean": 4.512167470238637e-05, "clip_ratio/low_mean": 1.870417509053368e-05, "clip_ratio/low_min": 1.870417509053368e-05, "clip_ratio/region_mean": 6.382584979292005e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10334.0, "completions/max_terminated_length": 10334.0, "completions/mean_length": 7156.25, "completions/mean_terminated_length": 7156.25, "completions/min_length": 1146.0, "completions/min_terminated_length": 1146.0, "entropy": 0.5183168314397335, "epoch": 0.010920795493734913, "frac_reward_zero_std": 0.0, "grad_norm": 0.027234913781285286, "learning_rate": 1e-05, "loss": -0.2447, "num_tokens": 8347710.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000864267349243, "sampling/importance_sampling_ratio/min": 0.42131614685058594, "sampling/sampling_logp_difference/max": 0.99310302734375, "sampling/sampling_logp_difference/mean": 0.015133515931665897, "step": 190 }, { "clip_ratio/high_max": 0.0001190261282317806, "clip_ratio/high_mean": 0.0001190261282317806, "clip_ratio/low_mean": 0.00020377377950353548, "clip_ratio/low_min": 0.00020377377950353548, "clip_ratio/region_mean": 0.0003227999077353161, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 11128.0, "completions/mean_length": 8664.375, "completions/mean_terminated_length": 7561.57177734375, "completions/min_length": 4658.0, "completions/min_terminated_length": 4658.0, "entropy": 0.46362604573369026, "epoch": 0.010978273364754569, "frac_reward_zero_std": 0.0, "grad_norm": 0.013075378723442554, "learning_rate": 1e-05, "loss": 0.1808, "num_tokens": 8418145.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000049114227295, "sampling/importance_sampling_ratio/min": 0.30332186818122864, "sampling/sampling_logp_difference/max": 1.1929607391357422, "sampling/sampling_logp_difference/mean": 0.019484898075461388, "step": 191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5233.0, "completions/max_terminated_length": 5233.0, "completions/mean_length": 2935.5, "completions/mean_terminated_length": 2935.5, "completions/min_length": 2037.0, "completions/min_terminated_length": 2037.0, "entropy": 0.20832776837050915, "epoch": 0.011035751235774226, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8442229.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3631702661514282, "sampling/importance_sampling_ratio/mean": 0.9995654225349426, "sampling/importance_sampling_ratio/min": 0.647485077381134, "sampling/sampling_logp_difference/max": 0.434659481048584, "sampling/sampling_logp_difference/mean": 0.008618641644716263, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0011076702794525772, "clip_ratio/low_min": 0.0011076702794525772, "clip_ratio/region_mean": 0.0011076702794525772, "completions/clipped_ratio": 0.0, "completions/max_length": 12091.0, "completions/max_terminated_length": 12091.0, "completions/mean_length": 9176.5, "completions/mean_terminated_length": 9176.5, "completions/min_length": 7268.0, "completions/min_terminated_length": 7268.0, "entropy": 0.8082810416817665, "epoch": 0.011093229106793884, "frac_reward_zero_std": 0.0, "grad_norm": 0.009606908075511456, "learning_rate": 1e-05, "loss": 0.0418, "num_tokens": 8516945.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999254941940308, "sampling/importance_sampling_ratio/min": 0.24914436042308807, "sampling/sampling_logp_difference/max": 1.3897228240966797, "sampling/sampling_logp_difference/mean": 0.03110113926231861, "step": 193 }, { "clip_ratio/high_max": 0.0001240125511685619, "clip_ratio/high_mean": 0.0001240125511685619, "clip_ratio/low_mean": 0.0002643555781105533, "clip_ratio/low_min": 0.0002643555781105533, "clip_ratio/region_mean": 0.00038836812927911524, "completions/clipped_ratio": 0.0, "completions/max_length": 9104.0, "completions/max_terminated_length": 9104.0, "completions/mean_length": 6047.625, "completions/mean_terminated_length": 6047.625, "completions/min_length": 2831.0, "completions/min_terminated_length": 2831.0, "entropy": 0.7152147218585014, "epoch": 0.011150706977813542, "frac_reward_zero_std": 0.0, "grad_norm": 0.01703065074980259, "learning_rate": 1e-05, "loss": 0.0503, "num_tokens": 8566526.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8028453588485718, "sampling/importance_sampling_ratio/mean": 1.000258207321167, "sampling/importance_sampling_ratio/min": 0.4833535850048065, "sampling/sampling_logp_difference/max": 0.7270069122314453, "sampling/sampling_logp_difference/mean": 0.02330091968178749, "step": 194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14530.0, "completions/mean_length": 14316.125, "completions/mean_terminated_length": 13626.833984375, "completions/min_length": 12473.0, "completions/min_terminated_length": 12473.0, "entropy": 0.5538299642503262, "epoch": 0.0112081848488332, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8684231.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998871684074402, "sampling/importance_sampling_ratio/min": 0.3882474899291992, "sampling/sampling_logp_difference/max": 1.178421974182129, "sampling/sampling_logp_difference/mean": 0.021088674664497375, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9680.0, "completions/max_terminated_length": 9680.0, "completions/mean_length": 3682.5, "completions/mean_terminated_length": 3682.5, "completions/min_length": 1100.0, "completions/min_terminated_length": 1100.0, "entropy": 0.3728352524340153, "epoch": 0.011265662719852857, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8714371.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004141330718994, "sampling/importance_sampling_ratio/min": 0.5050770044326782, "sampling/sampling_logp_difference/max": 0.8590295314788818, "sampling/sampling_logp_difference/mean": 0.017004601657390594, "step": 196 }, { "clip_ratio/high_max": 8.552064900868572e-05, "clip_ratio/high_mean": 8.552064900868572e-05, "clip_ratio/low_mean": 0.0002187505378969945, "clip_ratio/low_min": 0.0002187505378969945, "clip_ratio/region_mean": 0.0003042711869056802, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15636.0, "completions/mean_length": 12618.375, "completions/mean_terminated_length": 12080.4287109375, "completions/min_length": 7212.0, "completions/min_terminated_length": 7212.0, "entropy": 0.558051310479641, "epoch": 0.011323140590872515, "frac_reward_zero_std": 0.0, "grad_norm": 0.006278068292886019, "learning_rate": 1e-05, "loss": 0.1062, "num_tokens": 8816014.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999608993530273, "sampling/importance_sampling_ratio/min": 0.06352178007364273, "sampling/sampling_logp_difference/max": 2.7563724517822266, "sampling/sampling_logp_difference/mean": 0.020380595698952675, "step": 197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0008949943621701095, "clip_ratio/low_min": 0.0008949943621701095, "clip_ratio/region_mean": 0.0008949943621701095, "completions/clipped_ratio": 0.0, "completions/max_length": 9994.0, "completions/max_terminated_length": 9994.0, "completions/mean_length": 4796.375, "completions/mean_terminated_length": 4796.375, "completions/min_length": 941.0, "completions/min_terminated_length": 941.0, "entropy": 0.5178602710366249, "epoch": 0.01138061846189217, "frac_reward_zero_std": 0.0, "grad_norm": 0.014132932759821415, "learning_rate": 1e-05, "loss": -0.0059, "num_tokens": 8855321.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000486373901367, "sampling/importance_sampling_ratio/min": 0.32621052861213684, "sampling/sampling_logp_difference/max": 1.1202123165130615, "sampling/sampling_logp_difference/mean": 0.02108672820031643, "step": 198 }, { "clip_ratio/high_max": 0.00011427201388869435, "clip_ratio/high_mean": 0.00011427201388869435, "clip_ratio/low_mean": 9.918212890625e-05, "clip_ratio/low_min": 9.918212890625e-05, "clip_ratio/region_mean": 0.00021345414279494435, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13635.0, "completions/mean_length": 8562.75, "completions/mean_terminated_length": 7445.4287109375, "completions/min_length": 1630.0, "completions/min_terminated_length": 1630.0, "entropy": 0.47923652827739716, "epoch": 0.011438096332911828, "frac_reward_zero_std": 0.0, "grad_norm": 0.009647444821894169, "learning_rate": 1e-05, "loss": 0.3229, "num_tokens": 8925015.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001194477081299, "sampling/importance_sampling_ratio/min": 0.4713723063468933, "sampling/sampling_logp_difference/max": 0.9788780212402344, "sampling/sampling_logp_difference/mean": 0.022269317880272865, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8952.0, "completions/max_terminated_length": 8952.0, "completions/mean_length": 5280.125, "completions/mean_terminated_length": 5280.125, "completions/min_length": 1581.0, "completions/min_terminated_length": 1581.0, "entropy": 0.6958541795611382, "epoch": 0.011495574203931486, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 8968592.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8806509971618652, "sampling/importance_sampling_ratio/mean": 0.9999502897262573, "sampling/importance_sampling_ratio/min": 0.4117402136325836, "sampling/sampling_logp_difference/max": 0.8873627185821533, "sampling/sampling_logp_difference/mean": 0.020331639796495438, "step": 200 }, { "clip_ratio/high_max": 9.95652808342129e-05, "clip_ratio/high_mean": 9.95652808342129e-05, "clip_ratio/low_mean": 8.650519157527015e-05, "clip_ratio/low_min": 8.650519157527015e-05, "clip_ratio/region_mean": 0.00018607047240948305, "completions/clipped_ratio": 0.0, "completions/max_length": 13644.0, "completions/max_terminated_length": 13644.0, "completions/mean_length": 7140.375, "completions/mean_terminated_length": 7140.375, "completions/min_length": 4784.0, "completions/min_terminated_length": 4784.0, "entropy": 0.5029181391000748, "epoch": 0.011553052074951144, "frac_reward_zero_std": 0.0, "grad_norm": 0.005729257129132748, "learning_rate": 1e-05, "loss": -0.0672, "num_tokens": 9026771.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.832338809967041, "sampling/importance_sampling_ratio/mean": 1.0003610849380493, "sampling/importance_sampling_ratio/min": 0.3870709538459778, "sampling/sampling_logp_difference/max": 0.9491472244262695, "sampling/sampling_logp_difference/mean": 0.01959267072379589, "step": 201 }, { "clip_ratio/high_max": 0.00011046524923585821, "clip_ratio/high_mean": 0.00011046524923585821, "clip_ratio/low_mean": 0.00018594362700241618, "clip_ratio/low_min": 0.00018594362700241618, "clip_ratio/region_mean": 0.0002964088762382744, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14093.0, "completions/mean_length": 10794.625, "completions/mean_terminated_length": 9996.1435546875, "completions/min_length": 6482.0, "completions/min_terminated_length": 6482.0, "entropy": 0.6566261723637581, "epoch": 0.011610529945970801, "frac_reward_zero_std": 0.0, "grad_norm": 0.03386719897389412, "learning_rate": 1e-05, "loss": 0.1077, "num_tokens": 9114312.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997841715812683, "sampling/importance_sampling_ratio/min": 0.36285603046417236, "sampling/sampling_logp_difference/max": 1.013749122619629, "sampling/sampling_logp_difference/mean": 0.02372993715107441, "step": 202 }, { "clip_ratio/high_max": 0.00010559167640167288, "clip_ratio/high_mean": 0.00010559167640167288, "clip_ratio/low_mean": 6.103515625e-05, "clip_ratio/low_min": 6.103515625e-05, "clip_ratio/region_mean": 0.00016662683265167288, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15414.0, "completions/mean_length": 11427.875, "completions/mean_terminated_length": 10719.857421875, "completions/min_length": 8661.0, "completions/min_terminated_length": 8661.0, "entropy": 0.3644193671643734, "epoch": 0.011668007816990459, "frac_reward_zero_std": 0.0, "grad_norm": 0.004561516456305981, "learning_rate": 1e-05, "loss": 0.1532, "num_tokens": 9206919.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999821782112122, "sampling/importance_sampling_ratio/min": 0.19193419814109802, "sampling/sampling_logp_difference/max": 1.6506026983261108, "sampling/sampling_logp_difference/mean": 0.01567949913442135, "step": 203 }, { "clip_ratio/high_max": 6.510417006211355e-05, "clip_ratio/high_mean": 6.510417006211355e-05, "clip_ratio/low_mean": 4.36300178989768e-05, "clip_ratio/low_min": 4.36300178989768e-05, "clip_ratio/region_mean": 0.00010873418796109036, "completions/clipped_ratio": 0.0, "completions/max_length": 3913.0, "completions/max_terminated_length": 3913.0, "completions/mean_length": 2745.625, "completions/mean_terminated_length": 2745.625, "completions/min_length": 1920.0, "completions/min_terminated_length": 1920.0, "entropy": 0.29109179601073265, "epoch": 0.011725485688010117, "frac_reward_zero_std": 0.0, "grad_norm": 0.008285376243293285, "learning_rate": 1e-05, "loss": 0.0152, "num_tokens": 9230540.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6389014720916748, "sampling/importance_sampling_ratio/mean": 1.0000051259994507, "sampling/importance_sampling_ratio/min": 0.6845753192901611, "sampling/sampling_logp_difference/max": 0.49402618408203125, "sampling/sampling_logp_difference/mean": 0.012220971286296844, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4695.0, "completions/max_terminated_length": 4695.0, "completions/mean_length": 2600.625, "completions/mean_terminated_length": 2600.625, "completions/min_length": 1549.0, "completions/min_terminated_length": 1549.0, "entropy": 0.34315982833504677, "epoch": 0.011782963559029774, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9252385.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3601510524749756, "sampling/importance_sampling_ratio/mean": 0.9998400211334229, "sampling/importance_sampling_ratio/min": 0.6423700451850891, "sampling/sampling_logp_difference/max": 0.44259071350097656, "sampling/sampling_logp_difference/mean": 0.013423820957541466, "step": 205 }, { "clip_ratio/high_max": 8.21611683932133e-06, "clip_ratio/high_mean": 8.21611683932133e-06, "clip_ratio/low_mean": 0.0006934340599400457, "clip_ratio/low_min": 0.0006934340599400457, "clip_ratio/region_mean": 0.000701650176779367, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15214.0, "completions/mean_length": 13387.5, "completions/mean_terminated_length": 10391.0, "completions/min_length": 4242.0, "completions/min_terminated_length": 4242.0, "entropy": 0.43935373052954674, "epoch": 0.01184044143004943, "frac_reward_zero_std": 0.0, "grad_norm": 0.005486832931637764, "learning_rate": 1e-05, "loss": -0.0484, "num_tokens": 9361069.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000028133392334, "sampling/importance_sampling_ratio/min": 0.007944649085402489, "sampling/sampling_logp_difference/max": 4.835256576538086, "sampling/sampling_logp_difference/mean": 0.017225248739123344, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1860.0, "completions/max_terminated_length": 1860.0, "completions/mean_length": 1221.25, "completions/mean_terminated_length": 1221.25, "completions/min_length": 875.0, "completions/min_terminated_length": 875.0, "entropy": 0.2764971349388361, "epoch": 0.011897919301069088, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9372015.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2674356698989868, "sampling/importance_sampling_ratio/mean": 0.9998180270195007, "sampling/importance_sampling_ratio/min": 0.6043380498886108, "sampling/sampling_logp_difference/max": 0.5036215782165527, "sampling/sampling_logp_difference/mean": 0.012156876735389233, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 16378.0, "completions/mean_length": 12198.0, "completions/mean_terminated_length": 10802.6669921875, "completions/min_length": 5198.0, "completions/min_terminated_length": 5198.0, "entropy": 0.8942966908216476, "epoch": 0.011955397172088746, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9471783.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000622272491455, "sampling/importance_sampling_ratio/min": 0.3456362187862396, "sampling/sampling_logp_difference/max": 1.062368392944336, "sampling/sampling_logp_difference/mean": 0.030848519876599312, "step": 208 }, { "clip_ratio/high_max": 2.7043628506362438e-05, "clip_ratio/high_mean": 2.7043628506362438e-05, "clip_ratio/low_mean": 0.00016821823646751, "clip_ratio/low_min": 0.00016821823646751, "clip_ratio/region_mean": 0.00019526186497387243, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14125.0, "completions/mean_length": 10032.875, "completions/mean_terminated_length": 9125.572265625, "completions/min_length": 3332.0, "completions/min_terminated_length": 3332.0, "entropy": 0.8033781796693802, "epoch": 0.012012875043108403, "frac_reward_zero_std": 0.0, "grad_norm": 0.011191771365702152, "learning_rate": 1e-05, "loss": 0.2732, "num_tokens": 9553038.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999014735221863, "sampling/importance_sampling_ratio/min": 0.019850613549351692, "sampling/sampling_logp_difference/max": 3.919520378112793, "sampling/sampling_logp_difference/mean": 0.023044193163514137, "step": 209 }, { "clip_ratio/high_max": 3.402286165510304e-05, "clip_ratio/high_mean": 3.402286165510304e-05, "clip_ratio/low_mean": 4.693954178947024e-05, "clip_ratio/low_min": 4.693954178947024e-05, "clip_ratio/region_mean": 8.096240344457328e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 3674.0, "completions/max_terminated_length": 3674.0, "completions/mean_length": 2677.625, "completions/mean_terminated_length": 2677.625, "completions/min_length": 1793.0, "completions/min_terminated_length": 1793.0, "entropy": 0.2406750489026308, "epoch": 0.012070352914128061, "frac_reward_zero_std": 0.0, "grad_norm": 0.009171653538942337, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 9576307.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3651903867721558, "sampling/importance_sampling_ratio/mean": 0.9999209046363831, "sampling/importance_sampling_ratio/min": 0.6137738823890686, "sampling/sampling_logp_difference/max": 0.488128662109375, "sampling/sampling_logp_difference/mean": 0.010427516885101795, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0007341744203586131, "clip_ratio/low_min": 0.0007341744203586131, "clip_ratio/region_mean": 0.0007341744203586131, "completions/clipped_ratio": 0.0, "completions/max_length": 3958.0, "completions/max_terminated_length": 3958.0, "completions/mean_length": 1602.75, "completions/mean_terminated_length": 1602.75, "completions/min_length": 652.0, "completions/min_terminated_length": 652.0, "entropy": 0.4816881865262985, "epoch": 0.012127830785147719, "frac_reward_zero_std": 0.0, "grad_norm": 0.05415642634034157, "learning_rate": 1e-05, "loss": -0.534, "num_tokens": 9590377.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.3418947458267212, "sampling/importance_sampling_ratio/mean": 1.0001122951507568, "sampling/importance_sampling_ratio/min": 0.6219221949577332, "sampling/sampling_logp_difference/max": 0.47494029998779297, "sampling/sampling_logp_difference/mean": 0.021240253001451492, "step": 211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6170.0, "completions/max_terminated_length": 6170.0, "completions/mean_length": 3812.25, "completions/mean_terminated_length": 3812.25, "completions/min_length": 1433.0, "completions/min_terminated_length": 1433.0, "entropy": 0.5724687688052654, "epoch": 0.012185308656167376, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9621771.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4833757877349854, "sampling/importance_sampling_ratio/mean": 1.0000051259994507, "sampling/importance_sampling_ratio/min": 0.6280277967453003, "sampling/sampling_logp_difference/max": 0.46517086029052734, "sampling/sampling_logp_difference/mean": 0.015964122489094734, "step": 212 }, { "clip_ratio/high_max": 5.3949072025716305e-05, "clip_ratio/high_mean": 5.3949072025716305e-05, "clip_ratio/low_mean": 0.0003672853927128017, "clip_ratio/low_min": 0.0003672853927128017, "clip_ratio/region_mean": 0.000421234464738518, "completions/clipped_ratio": 0.0, "completions/max_length": 2317.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 1286.375, "completions/mean_terminated_length": 1286.375, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.616693627089262, "epoch": 0.012242786527187032, "frac_reward_zero_std": 0.0, "grad_norm": 0.11791826784610748, "learning_rate": 1e-05, "loss": -0.5173, "num_tokens": 9632854.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4110692739486694, "sampling/importance_sampling_ratio/mean": 1.0003750324249268, "sampling/importance_sampling_ratio/min": 0.623399555683136, "sampling/sampling_logp_difference/max": 0.4725675582885742, "sampling/sampling_logp_difference/mean": 0.025106515735387802, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2425.0, "completions/max_terminated_length": 2425.0, "completions/mean_length": 2065.875, "completions/mean_terminated_length": 2065.875, "completions/min_length": 1775.0, "completions/min_terminated_length": 1775.0, "entropy": 0.27775036357343197, "epoch": 0.01230026439820669, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9650789.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3647032976150513, "sampling/importance_sampling_ratio/mean": 0.9997227787971497, "sampling/importance_sampling_ratio/min": 0.6230998635292053, "sampling/sampling_logp_difference/max": 0.47304844856262207, "sampling/sampling_logp_difference/mean": 0.012043694034218788, "step": 214 }, { "clip_ratio/high_max": 7.259000994963571e-05, "clip_ratio/high_mean": 7.259000994963571e-05, "clip_ratio/low_mean": 0.0008480450778733939, "clip_ratio/low_min": 0.0008480450778733939, "clip_ratio/region_mean": 0.0009206350878230296, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14367.0, "completions/mean_length": 11202.75, "completions/mean_terminated_length": 9475.6669921875, "completions/min_length": 6518.0, "completions/min_terminated_length": 6518.0, "entropy": 0.49712123721838, "epoch": 0.012357742269226347, "frac_reward_zero_std": 0.0, "grad_norm": 0.010083692148327827, "learning_rate": 1e-05, "loss": 0.0807, "num_tokens": 9742363.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002573728561401, "sampling/importance_sampling_ratio/min": 0.1632498800754547, "sampling/sampling_logp_difference/max": 1.8124732971191406, "sampling/sampling_logp_difference/mean": 0.020661983639001846, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14677.0, "completions/mean_length": 12185.375, "completions/mean_terminated_length": 10785.833984375, "completions/min_length": 3584.0, "completions/min_terminated_length": 3584.0, "entropy": 0.6123535353690386, "epoch": 0.012415220140246005, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9840670.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999887228012085, "sampling/importance_sampling_ratio/min": 0.3155209720134735, "sampling/sampling_logp_difference/max": 1.1535301208496094, "sampling/sampling_logp_difference/mean": 0.025949751958251, "step": 216 }, { "clip_ratio/high_max": 0.00018627778990776278, "clip_ratio/high_mean": 0.00018627778990776278, "clip_ratio/low_mean": 0.000316828147333581, "clip_ratio/low_min": 0.000316828147333581, "clip_ratio/region_mean": 0.0005031059372413438, "completions/clipped_ratio": 0.0, "completions/max_length": 11685.0, "completions/max_terminated_length": 11685.0, "completions/mean_length": 6222.0, "completions/mean_terminated_length": 6222.0, "completions/min_length": 1375.0, "completions/min_terminated_length": 1375.0, "entropy": 0.8033790811896324, "epoch": 0.012472698011265663, "frac_reward_zero_std": 0.0, "grad_norm": 0.013911571353673935, "learning_rate": 1e-05, "loss": 0.2239, "num_tokens": 9891734.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997472167015076, "sampling/importance_sampling_ratio/min": 0.3719465434551239, "sampling/sampling_logp_difference/max": 0.9890050888061523, "sampling/sampling_logp_difference/mean": 0.031158994883298874, "step": 217 }, { "clip_ratio/high_max": 0.0001009012121357955, "clip_ratio/high_mean": 0.0001009012121357955, "clip_ratio/low_mean": 0.00027443770159152336, "clip_ratio/low_min": 0.00027443770159152336, "clip_ratio/region_mean": 0.00037533891372731887, "completions/clipped_ratio": 0.0, "completions/max_length": 15150.0, "completions/max_terminated_length": 15150.0, "completions/mean_length": 11302.75, "completions/mean_terminated_length": 11302.75, "completions/min_length": 7471.0, "completions/min_terminated_length": 7471.0, "entropy": 0.4500351585447788, "epoch": 0.01253017588228532, "frac_reward_zero_std": 0.0, "grad_norm": 0.00955692958086729, "learning_rate": 1e-05, "loss": 0.1302, "num_tokens": 9983764.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999442100524902, "sampling/importance_sampling_ratio/min": 0.005013084504753351, "sampling/sampling_logp_difference/max": 5.295703887939453, "sampling/sampling_logp_difference/mean": 0.019716009497642517, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6111.0, "completions/max_terminated_length": 6111.0, "completions/mean_length": 4018.125, "completions/mean_terminated_length": 4018.125, "completions/min_length": 3109.0, "completions/min_terminated_length": 3109.0, "entropy": 0.5289942175149918, "epoch": 0.012587653753304978, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10017245.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4753402471542358, "sampling/importance_sampling_ratio/mean": 0.9994773268699646, "sampling/importance_sampling_ratio/min": 0.5852012038230896, "sampling/sampling_logp_difference/max": 0.535799503326416, "sampling/sampling_logp_difference/mean": 0.01781732775270939, "step": 219 }, { "clip_ratio/high_max": 0.00010511302025406621, "clip_ratio/high_mean": 0.00010511302025406621, "clip_ratio/low_mean": 0.0004118060605833307, "clip_ratio/low_min": 0.0004118060605833307, "clip_ratio/region_mean": 0.0005169190808373969, "completions/clipped_ratio": 0.0, "completions/max_length": 15955.0, "completions/max_terminated_length": 15955.0, "completions/mean_length": 9534.75, "completions/mean_terminated_length": 9534.75, "completions/min_length": 4445.0, "completions/min_terminated_length": 4445.0, "entropy": 0.6315775439143181, "epoch": 0.012645131624324636, "frac_reward_zero_std": 0.0, "grad_norm": 0.03455370292067528, "learning_rate": 1e-05, "loss": 0.4175, "num_tokens": 10094955.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998697638511658, "sampling/importance_sampling_ratio/min": 0.07237821817398071, "sampling/sampling_logp_difference/max": 2.625849962234497, "sampling/sampling_logp_difference/mean": 0.026303892955183983, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12028.0, "completions/max_terminated_length": 12028.0, "completions/mean_length": 7468.625, "completions/mean_terminated_length": 7468.625, "completions/min_length": 3702.0, "completions/min_terminated_length": 3702.0, "entropy": 0.4179515987634659, "epoch": 0.012702609495344292, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10155752.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7167187929153442, "sampling/importance_sampling_ratio/mean": 1.0000470876693726, "sampling/importance_sampling_ratio/min": 0.38231536746025085, "sampling/sampling_logp_difference/max": 0.9615094661712646, "sampling/sampling_logp_difference/mean": 0.01747564598917961, "step": 221 }, { "clip_ratio/high_max": 3.70562520402018e-05, "clip_ratio/high_mean": 3.70562520402018e-05, "clip_ratio/low_mean": 0.0004943426047248067, "clip_ratio/low_min": 0.0004943426047248067, "clip_ratio/region_mean": 0.0005313988567650085, "completions/clipped_ratio": 0.625, "completions/max_length": 16384.0, "completions/max_terminated_length": 13493.0, "completions/mean_length": 14093.375, "completions/mean_terminated_length": 10275.6669921875, "completions/min_length": 5608.0, "completions/min_terminated_length": 5608.0, "entropy": 0.40119199454784393, "epoch": 0.01276008736636395, "frac_reward_zero_std": 0.0, "grad_norm": 0.008288045413792133, "learning_rate": 1e-05, "loss": 0.174, "num_tokens": 10269739.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998534917831421, "sampling/importance_sampling_ratio/min": 0.025376958772540092, "sampling/sampling_logp_difference/max": 3.6739137172698975, "sampling/sampling_logp_difference/mean": 0.018384510651230812, "step": 222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3782.0, "completions/max_terminated_length": 3782.0, "completions/mean_length": 2814.125, "completions/mean_terminated_length": 2814.125, "completions/min_length": 2102.0, "completions/min_terminated_length": 2102.0, "entropy": 0.2692510113120079, "epoch": 0.012817565237383607, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10292940.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4655894041061401, "sampling/importance_sampling_ratio/mean": 0.9999130368232727, "sampling/importance_sampling_ratio/min": 0.6186597943305969, "sampling/sampling_logp_difference/max": 0.48019981384277344, "sampling/sampling_logp_difference/mean": 0.010910087265074253, "step": 223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1424.0, "completions/max_terminated_length": 1424.0, "completions/mean_length": 909.25, "completions/mean_terminated_length": 909.25, "completions/min_length": 502.0, "completions/min_terminated_length": 502.0, "entropy": 0.32026074081659317, "epoch": 0.012875043108403265, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10300974.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5538291931152344, "sampling/importance_sampling_ratio/mean": 1.0001140832901, "sampling/importance_sampling_ratio/min": 0.6256819367408752, "sampling/sampling_logp_difference/max": 0.46891307830810547, "sampling/sampling_logp_difference/mean": 0.014822527766227722, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10535.0, "completions/max_terminated_length": 10535.0, "completions/mean_length": 3011.75, "completions/mean_terminated_length": 3011.75, "completions/min_length": 1347.0, "completions/min_terminated_length": 1347.0, "entropy": 0.39564137905836105, "epoch": 0.012932520979422922, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10326124.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5386968851089478, "sampling/importance_sampling_ratio/mean": 1.00032377243042, "sampling/importance_sampling_ratio/min": 0.6109485626220703, "sampling/sampling_logp_difference/max": 0.49274253845214844, "sampling/sampling_logp_difference/mean": 0.019274462014436722, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7352.0, "completions/max_terminated_length": 7352.0, "completions/mean_length": 3362.625, "completions/mean_terminated_length": 3362.625, "completions/min_length": 1670.0, "completions/min_terminated_length": 1670.0, "entropy": 0.4048736020922661, "epoch": 0.01298999885044258, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10353761.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4652092456817627, "sampling/importance_sampling_ratio/mean": 0.9997941255569458, "sampling/importance_sampling_ratio/min": 0.5652009844779968, "sampling/sampling_logp_difference/max": 0.5705738067626953, "sampling/sampling_logp_difference/mean": 0.017642401158809662, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5755.0, "completions/max_terminated_length": 5755.0, "completions/mean_length": 2942.5, "completions/mean_terminated_length": 2942.5, "completions/min_length": 883.0, "completions/min_terminated_length": 883.0, "entropy": 0.8773008808493614, "epoch": 0.013047476721462238, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10378221.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.842469573020935, "sampling/importance_sampling_ratio/mean": 1.000606656074524, "sampling/importance_sampling_ratio/min": 0.616563081741333, "sampling/sampling_logp_difference/max": 0.6111068725585938, "sampling/sampling_logp_difference/mean": 0.025237642228603363, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10473.0, "completions/max_terminated_length": 10473.0, "completions/mean_length": 3174.875, "completions/mean_terminated_length": 3174.875, "completions/min_length": 566.0, "completions/min_terminated_length": 566.0, "entropy": 0.2972034942358732, "epoch": 0.013104954592481894, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10404676.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4848393201828003, "sampling/importance_sampling_ratio/mean": 1.0002814531326294, "sampling/importance_sampling_ratio/min": 0.6210651397705078, "sampling/sampling_logp_difference/max": 0.4763193130493164, "sampling/sampling_logp_difference/mean": 0.012838711962103844, "step": 228 }, { "clip_ratio/high_max": 0.00014502731937682256, "clip_ratio/high_mean": 0.00014502731937682256, "clip_ratio/low_mean": 4.468275074032135e-05, "clip_ratio/low_min": 4.468275074032135e-05, "clip_ratio/region_mean": 0.00018971007011714391, "completions/clipped_ratio": 0.0, "completions/max_length": 5595.0, "completions/max_terminated_length": 5595.0, "completions/mean_length": 4151.0, "completions/mean_terminated_length": 4151.0, "completions/min_length": 3123.0, "completions/min_terminated_length": 3123.0, "entropy": 0.5627840459346771, "epoch": 0.013162432463501551, "frac_reward_zero_std": 0.0, "grad_norm": 0.011595550924539566, "learning_rate": 1e-05, "loss": 0.1234, "num_tokens": 10440100.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.7026751041412354, "sampling/importance_sampling_ratio/mean": 1.0003058910369873, "sampling/importance_sampling_ratio/min": 0.5480839014053345, "sampling/sampling_logp_difference/max": 0.6013269424438477, "sampling/sampling_logp_difference/mean": 0.020583635196089745, "step": 229 }, { "clip_ratio/high_max": 0.00010347706665925216, "clip_ratio/high_mean": 0.00010347706665925216, "clip_ratio/low_mean": 8.392333984375e-05, "clip_ratio/low_min": 8.392333984375e-05, "clip_ratio/region_mean": 0.00018740040650300216, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12809.0, "completions/mean_length": 10427.125, "completions/mean_terminated_length": 9576.1435546875, "completions/min_length": 6924.0, "completions/min_terminated_length": 6924.0, "entropy": 0.4362257868051529, "epoch": 0.013219910334521209, "frac_reward_zero_std": 0.0, "grad_norm": 0.005803901236504316, "learning_rate": 1e-05, "loss": 0.202, "num_tokens": 10525341.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000170111656189, "sampling/importance_sampling_ratio/min": 0.26937708258628845, "sampling/sampling_logp_difference/max": 1.311643123626709, "sampling/sampling_logp_difference/mean": 0.01951228268444538, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3820.0, "completions/max_terminated_length": 3820.0, "completions/mean_length": 2411.375, "completions/mean_terminated_length": 2411.375, "completions/min_length": 1320.0, "completions/min_terminated_length": 1320.0, "entropy": 0.3224306609481573, "epoch": 0.013277388205540867, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10545624.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5646144151687622, "sampling/importance_sampling_ratio/mean": 0.9997329115867615, "sampling/importance_sampling_ratio/min": 0.6744040250778198, "sampling/sampling_logp_difference/max": 0.44763946533203125, "sampling/sampling_logp_difference/mean": 0.013830232433974743, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0005034416935814079, "clip_ratio/low_min": 0.0005034416935814079, "clip_ratio/region_mean": 0.0005034416935814079, "completions/clipped_ratio": 0.0, "completions/max_length": 8818.0, "completions/max_terminated_length": 8818.0, "completions/mean_length": 5082.75, "completions/mean_terminated_length": 5082.75, "completions/min_length": 1564.0, "completions/min_terminated_length": 1564.0, "entropy": 0.5696996338665485, "epoch": 0.013334866076560524, "frac_reward_zero_std": 0.0, "grad_norm": 0.006232178304344416, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 10587430.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.537739634513855, "sampling/importance_sampling_ratio/mean": 1.0001429319381714, "sampling/importance_sampling_ratio/min": 0.4392375946044922, "sampling/sampling_logp_difference/max": 0.8227148056030273, "sampling/sampling_logp_difference/mean": 0.02268536016345024, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7178.0, "completions/max_terminated_length": 7178.0, "completions/mean_length": 3350.5, "completions/mean_terminated_length": 3350.5, "completions/min_length": 1589.0, "completions/min_terminated_length": 1589.0, "entropy": 0.41542892530560493, "epoch": 0.013392343947580182, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10615994.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5274951457977295, "sampling/importance_sampling_ratio/mean": 1.0001320838928223, "sampling/importance_sampling_ratio/min": 0.5196384191513062, "sampling/sampling_logp_difference/max": 0.6546220779418945, "sampling/sampling_logp_difference/mean": 0.01948987878859043, "step": 233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9857.0, "completions/max_terminated_length": 9857.0, "completions/mean_length": 5487.5, "completions/mean_terminated_length": 5487.5, "completions/min_length": 3320.0, "completions/min_terminated_length": 3320.0, "entropy": 0.38110455498099327, "epoch": 0.01344982181859984, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10660734.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.81279456615448, "sampling/importance_sampling_ratio/mean": 1.00006103515625, "sampling/importance_sampling_ratio/min": 0.581904947757721, "sampling/sampling_logp_difference/max": 0.5948696136474609, "sampling/sampling_logp_difference/mean": 0.015486331656575203, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12979.0, "completions/max_terminated_length": 12979.0, "completions/mean_length": 5788.625, "completions/mean_terminated_length": 5788.625, "completions/min_length": 2136.0, "completions/min_terminated_length": 2136.0, "entropy": 0.4809899814426899, "epoch": 0.013507299689619497, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10708523.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5933127403259277, "sampling/importance_sampling_ratio/mean": 0.9999656677246094, "sampling/importance_sampling_ratio/min": 0.2556094825267792, "sampling/sampling_logp_difference/max": 1.3641045093536377, "sampling/sampling_logp_difference/mean": 0.017151974141597748, "step": 235 }, { "clip_ratio/high_max": 2.4659042537678033e-05, "clip_ratio/high_mean": 2.4659042537678033e-05, "clip_ratio/low_mean": 0.0005239465244812891, "clip_ratio/low_min": 0.0005239465244812891, "clip_ratio/region_mean": 0.0005486055670189671, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 14968.0, "completions/mean_length": 12166.25, "completions/mean_terminated_length": 7948.5, "completions/min_length": 4181.0, "completions/min_terminated_length": 4181.0, "entropy": 0.4801964499056339, "epoch": 0.013564777560639153, "frac_reward_zero_std": 0.0, "grad_norm": 0.005800375249236822, "learning_rate": 1e-05, "loss": 0.1765, "num_tokens": 10807061.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998948574066162, "sampling/importance_sampling_ratio/min": 6.112750128295374e-08, "sampling/sampling_logp_difference/max": 16.61030387878418, "sampling/sampling_logp_difference/mean": 0.020769845694303513, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 16148.0, "completions/mean_length": 13540.125, "completions/mean_terminated_length": 12592.1669921875, "completions/min_length": 9726.0, "completions/min_terminated_length": 9726.0, "entropy": 0.9904661029577255, "epoch": 0.013622255431658811, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10916926.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997573494911194, "sampling/importance_sampling_ratio/min": 8.059122774284333e-05, "sampling/sampling_logp_difference/max": 9.42612075805664, "sampling/sampling_logp_difference/mean": 0.032993484288454056, "step": 237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4654.0, "completions/max_terminated_length": 4654.0, "completions/mean_length": 2031.25, "completions/mean_terminated_length": 2031.25, "completions/min_length": 983.0, "completions/min_terminated_length": 983.0, "entropy": 0.23056718334555626, "epoch": 0.013679733302678469, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10934704.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5085855722427368, "sampling/importance_sampling_ratio/mean": 0.9995875358581543, "sampling/importance_sampling_ratio/min": 0.6115107536315918, "sampling/sampling_logp_difference/max": 0.4918227195739746, "sampling/sampling_logp_difference/mean": 0.01120684016495943, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.768800423946232e-05, "clip_ratio/low_min": 7.768800423946232e-05, "clip_ratio/region_mean": 7.768800423946232e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6228.0, "completions/max_terminated_length": 6228.0, "completions/mean_length": 2806.25, "completions/mean_terminated_length": 2806.25, "completions/min_length": 998.0, "completions/min_terminated_length": 998.0, "entropy": 1.0021002143621445, "epoch": 0.013737211173698126, "frac_reward_zero_std": 0.0, "grad_norm": 0.009687749668955803, "learning_rate": 1e-05, "loss": -0.1695, "num_tokens": 10958122.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4409617185592651, "sampling/importance_sampling_ratio/mean": 0.9998266100883484, "sampling/importance_sampling_ratio/min": 0.2617310881614685, "sampling/sampling_logp_difference/max": 1.340437650680542, "sampling/sampling_logp_difference/mean": 0.024621441960334778, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12936.0, "completions/max_terminated_length": 12936.0, "completions/mean_length": 8497.25, "completions/mean_terminated_length": 8497.25, "completions/min_length": 4639.0, "completions/min_terminated_length": 4639.0, "entropy": 0.5409430265426636, "epoch": 0.013794689044717784, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11026940.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9139245748519897, "sampling/importance_sampling_ratio/mean": 0.9997201561927795, "sampling/importance_sampling_ratio/min": 0.35531097650527954, "sampling/sampling_logp_difference/max": 1.034761905670166, "sampling/sampling_logp_difference/mean": 0.020615391433238983, "step": 240 }, { "clip_ratio/high_max": 2.386103369644843e-05, "clip_ratio/high_mean": 2.386103369644843e-05, "clip_ratio/low_mean": 0.0006688979556201957, "clip_ratio/low_min": 0.0006688979556201957, "clip_ratio/region_mean": 0.0006927589893166441, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15716.0, "completions/mean_length": 11776.375, "completions/mean_terminated_length": 11118.1435546875, "completions/min_length": 7469.0, "completions/min_terminated_length": 7469.0, "entropy": 0.6553877890110016, "epoch": 0.013852166915737442, "frac_reward_zero_std": 0.0, "grad_norm": 0.005486266687512398, "learning_rate": 1e-05, "loss": -0.1183, "num_tokens": 11122135.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000144243240356, "sampling/importance_sampling_ratio/min": 0.010656909085810184, "sampling/sampling_logp_difference/max": 4.541546821594238, "sampling/sampling_logp_difference/mean": 0.024141818284988403, "step": 241 }, { "clip_ratio/high_max": 1.9644821804831736e-05, "clip_ratio/high_mean": 1.9644821804831736e-05, "clip_ratio/low_mean": 0.00018991712477145484, "clip_ratio/low_min": 0.00018991712477145484, "clip_ratio/region_mean": 0.00020956194657628657, "completions/clipped_ratio": 0.0, "completions/max_length": 10198.0, "completions/max_terminated_length": 10198.0, "completions/mean_length": 8131.625, "completions/mean_terminated_length": 8131.625, "completions/min_length": 6354.0, "completions/min_terminated_length": 6354.0, "entropy": 0.890590064227581, "epoch": 0.0139096447867571, "frac_reward_zero_std": 0.0, "grad_norm": 0.013450238853693008, "learning_rate": 1e-05, "loss": -0.0102, "num_tokens": 11188132.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.781151533126831, "sampling/importance_sampling_ratio/mean": 0.9999437928199768, "sampling/importance_sampling_ratio/min": 0.45966771245002747, "sampling/sampling_logp_difference/max": 0.7772514820098877, "sampling/sampling_logp_difference/mean": 0.02461942844092846, "step": 242 }, { "clip_ratio/high_max": 6.480041338363662e-05, "clip_ratio/high_mean": 6.480041338363662e-05, "clip_ratio/low_mean": 6.854949606349692e-05, "clip_ratio/low_min": 6.854949606349692e-05, "clip_ratio/region_mean": 0.00013334990944713354, "completions/clipped_ratio": 0.0, "completions/max_length": 3913.0, "completions/max_terminated_length": 3913.0, "completions/mean_length": 1979.125, "completions/mean_terminated_length": 1979.125, "completions/min_length": 748.0, "completions/min_terminated_length": 748.0, "entropy": 0.3730858415365219, "epoch": 0.013967122657776755, "frac_reward_zero_std": 0.0, "grad_norm": 0.07335606217384338, "learning_rate": 1e-05, "loss": 0.0606, "num_tokens": 11205037.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.7882685661315918, "sampling/importance_sampling_ratio/mean": 0.9999749660491943, "sampling/importance_sampling_ratio/min": 0.5497262477874756, "sampling/sampling_logp_difference/max": 0.598334789276123, "sampling/sampling_logp_difference/mean": 0.016838887706398964, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1756.0, "completions/max_terminated_length": 1756.0, "completions/mean_length": 1302.75, "completions/mean_terminated_length": 1302.75, "completions/min_length": 857.0, "completions/min_terminated_length": 857.0, "entropy": 0.23300896771252155, "epoch": 0.014024600528796413, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11216075.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5093194246292114, "sampling/importance_sampling_ratio/mean": 1.0003427267074585, "sampling/importance_sampling_ratio/min": 0.6418024301528931, "sampling/sampling_logp_difference/max": 0.44347476959228516, "sampling/sampling_logp_difference/mean": 0.009614585898816586, "step": 244 }, { "clip_ratio/high_max": 2.7015345040126704e-05, "clip_ratio/high_mean": 2.7015345040126704e-05, "clip_ratio/low_mean": 0.00046870691221556626, "clip_ratio/low_min": 0.00046870691221556626, "clip_ratio/region_mean": 0.000495722257255693, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13101.0, "completions/mean_length": 9661.0, "completions/mean_terminated_length": 8700.572265625, "completions/min_length": 4408.0, "completions/min_terminated_length": 4408.0, "entropy": 0.30384486727416515, "epoch": 0.01408207839981607, "frac_reward_zero_std": 0.0, "grad_norm": 0.008967713452875614, "learning_rate": 1e-05, "loss": 0.1583, "num_tokens": 11294371.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998798966407776, "sampling/importance_sampling_ratio/min": 0.1724100410938263, "sampling/sampling_logp_difference/max": 1.7578797340393066, "sampling/sampling_logp_difference/mean": 0.014898917637765408, "step": 245 }, { "clip_ratio/high_max": 0.00010216360897175036, "clip_ratio/high_mean": 0.00010216360897175036, "clip_ratio/low_mean": 0.0009872470836853608, "clip_ratio/low_min": 0.0009872470836853608, "clip_ratio/region_mean": 0.0010894106926571112, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13869.0, "completions/mean_length": 11561.375, "completions/mean_terminated_length": 9953.833984375, "completions/min_length": 3559.0, "completions/min_terminated_length": 3559.0, "entropy": 0.6489362716674805, "epoch": 0.014139556270835728, "frac_reward_zero_std": 0.0, "grad_norm": 0.009956092573702335, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 11387998.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000132441520691, "sampling/importance_sampling_ratio/min": 0.02731258235871792, "sampling/sampling_logp_difference/max": 3.600407838821411, "sampling/sampling_logp_difference/mean": 0.02649668976664543, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7619.0, "completions/max_terminated_length": 7619.0, "completions/mean_length": 3206.375, "completions/mean_terminated_length": 3206.375, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 1.464602880179882, "epoch": 0.014197034141855386, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11415401.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4338622093200684, "sampling/importance_sampling_ratio/mean": 0.9999920725822449, "sampling/importance_sampling_ratio/min": 0.44867074489593506, "sampling/sampling_logp_difference/max": 0.8014659881591797, "sampling/sampling_logp_difference/mean": 0.03130270913243294, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6551.0, "completions/max_terminated_length": 6551.0, "completions/mean_length": 4104.125, "completions/mean_terminated_length": 4104.125, "completions/min_length": 1975.0, "completions/min_terminated_length": 1975.0, "entropy": 0.3689926713705063, "epoch": 0.014254512012875043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11449394.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5026180744171143, "sampling/importance_sampling_ratio/mean": 0.9996576309204102, "sampling/importance_sampling_ratio/min": 0.5714125633239746, "sampling/sampling_logp_difference/max": 0.5596437454223633, "sampling/sampling_logp_difference/mean": 0.01587335765361786, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15847.0, "completions/max_terminated_length": 15847.0, "completions/mean_length": 11492.625, "completions/mean_terminated_length": 11492.625, "completions/min_length": 8562.0, "completions/min_terminated_length": 8562.0, "entropy": 0.5737185478210449, "epoch": 0.014311989883894701, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11542447.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000476837158203, "sampling/importance_sampling_ratio/min": 0.14115479588508606, "sampling/sampling_logp_difference/max": 1.9578981399536133, "sampling/sampling_logp_difference/mean": 0.02276143990457058, "step": 249 }, { "clip_ratio/high_max": 0.00013715856039198115, "clip_ratio/high_mean": 0.00013715856039198115, "clip_ratio/low_mean": 0.0001709281132207252, "clip_ratio/low_min": 0.0001709281132207252, "clip_ratio/region_mean": 0.00030808667361270636, "completions/clipped_ratio": 0.0, "completions/max_length": 7131.0, "completions/max_terminated_length": 7131.0, "completions/mean_length": 4421.625, "completions/mean_terminated_length": 4421.625, "completions/min_length": 1999.0, "completions/min_terminated_length": 1999.0, "entropy": 0.2619555275887251, "epoch": 0.014369467754914359, "frac_reward_zero_std": 0.0, "grad_norm": 0.011488448828458786, "learning_rate": 1e-05, "loss": 0.2443, "num_tokens": 11580052.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7280057668685913, "sampling/importance_sampling_ratio/mean": 1.0001354217529297, "sampling/importance_sampling_ratio/min": 0.6068691611289978, "sampling/sampling_logp_difference/max": 0.5469679832458496, "sampling/sampling_logp_difference/mean": 0.012207957915961742, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2934.0, "completions/max_terminated_length": 2934.0, "completions/mean_length": 1419.75, "completions/mean_terminated_length": 1419.75, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "entropy": 0.28696313686668873, "epoch": 0.014426945625934015, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11592610.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.443624496459961, "sampling/importance_sampling_ratio/mean": 0.9992883801460266, "sampling/importance_sampling_ratio/min": 0.6237903237342834, "sampling/sampling_logp_difference/max": 0.4719409942626953, "sampling/sampling_logp_difference/mean": 0.01366155780851841, "step": 251 }, { "clip_ratio/high_max": 0.00017145523452199996, "clip_ratio/high_mean": 0.00017145523452199996, "clip_ratio/low_mean": 0.00010699764243327081, "clip_ratio/low_min": 0.00010699764243327081, "clip_ratio/region_mean": 0.00027845287695527077, "completions/clipped_ratio": 0.0, "completions/max_length": 13382.0, "completions/max_terminated_length": 13382.0, "completions/mean_length": 8514.75, "completions/mean_terminated_length": 8514.75, "completions/min_length": 5461.0, "completions/min_terminated_length": 5461.0, "entropy": 0.5025256536900997, "epoch": 0.014484423496953672, "frac_reward_zero_std": 0.0, "grad_norm": 0.005746009759604931, "learning_rate": 1e-05, "loss": 0.0348, "num_tokens": 11661440.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000219702720642, "sampling/importance_sampling_ratio/min": 0.5012786388397217, "sampling/sampling_logp_difference/max": 0.9241889715194702, "sampling/sampling_logp_difference/mean": 0.020174650475382805, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.6195942054037e-05, "clip_ratio/low_min": 6.6195942054037e-05, "clip_ratio/region_mean": 6.6195942054037e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 5665.0, "completions/max_terminated_length": 5665.0, "completions/mean_length": 3291.5, "completions/mean_terminated_length": 3291.5, "completions/min_length": 1059.0, "completions/min_terminated_length": 1059.0, "entropy": 0.43801668658852577, "epoch": 0.01454190136797333, "frac_reward_zero_std": 0.0, "grad_norm": 0.005613123066723347, "learning_rate": 1e-05, "loss": 0.2547, "num_tokens": 11688764.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.419911503791809, "sampling/importance_sampling_ratio/mean": 1.0000346899032593, "sampling/importance_sampling_ratio/min": 0.6072993278503418, "sampling/sampling_logp_difference/max": 0.4987335205078125, "sampling/sampling_logp_difference/mean": 0.014720753766596317, "step": 253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2179.0, "completions/max_terminated_length": 2179.0, "completions/mean_length": 884.375, "completions/mean_terminated_length": 884.375, "completions/min_length": 461.0, "completions/min_terminated_length": 461.0, "entropy": 0.28187943436205387, "epoch": 0.014599379238992988, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11696519.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.696502923965454, "sampling/importance_sampling_ratio/mean": 0.999819278717041, "sampling/importance_sampling_ratio/min": 0.5635024309158325, "sampling/sampling_logp_difference/max": 0.5735836029052734, "sampling/sampling_logp_difference/mean": 0.01351790502667427, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3781.0, "completions/max_terminated_length": 3781.0, "completions/mean_length": 2290.625, "completions/mean_terminated_length": 2290.625, "completions/min_length": 843.0, "completions/min_terminated_length": 843.0, "entropy": 0.3055574521422386, "epoch": 0.014656857110012645, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11716028.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3616334199905396, "sampling/importance_sampling_ratio/mean": 1.0000905990600586, "sampling/importance_sampling_ratio/min": 0.5984509587287903, "sampling/sampling_logp_difference/max": 0.513410747051239, "sampling/sampling_logp_difference/mean": 0.013125410303473473, "step": 255 }, { "clip_ratio/high_max": 5.335124114935752e-05, "clip_ratio/high_mean": 5.335124114935752e-05, "clip_ratio/low_mean": 0.00042540972390270326, "clip_ratio/low_min": 0.00042540972390270326, "clip_ratio/region_mean": 0.0004787609650520608, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15349.0, "completions/mean_length": 12992.0, "completions/mean_terminated_length": 12507.4287109375, "completions/min_length": 8727.0, "completions/min_terminated_length": 8727.0, "entropy": 0.6972725093364716, "epoch": 0.014714334981032303, "frac_reward_zero_std": 0.0, "grad_norm": 0.009135198779404163, "learning_rate": 1e-05, "loss": 0.0787, "num_tokens": 11820764.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000097393989563, "sampling/importance_sampling_ratio/min": 0.16634340584278107, "sampling/sampling_logp_difference/max": 1.793700933456421, "sampling/sampling_logp_difference/mean": 0.02293052151799202, "step": 256 }, { "clip_ratio/high_max": 4.2088422560482286e-05, "clip_ratio/high_mean": 4.2088422560482286e-05, "clip_ratio/low_mean": 8.430281741311774e-05, "clip_ratio/low_min": 8.430281741311774e-05, "clip_ratio/region_mean": 0.00012639123997360002, "completions/clipped_ratio": 0.0, "completions/max_length": 10320.0, "completions/max_terminated_length": 10320.0, "completions/mean_length": 5065.625, "completions/mean_terminated_length": 5065.625, "completions/min_length": 3496.0, "completions/min_terminated_length": 3496.0, "entropy": 0.26933291368186474, "epoch": 0.01477181285205196, "frac_reward_zero_std": 0.0, "grad_norm": 0.05237021669745445, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 11862289.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5922070741653442, "sampling/importance_sampling_ratio/mean": 0.9999058246612549, "sampling/importance_sampling_ratio/min": 0.5469368100166321, "sampling/sampling_logp_difference/max": 0.6034219861030579, "sampling/sampling_logp_difference/mean": 0.012624706141650677, "step": 257 }, { "clip_ratio/high_max": 0.00012257248636160512, "clip_ratio/high_mean": 0.00012257248636160512, "clip_ratio/low_mean": 0.0001102778987842612, "clip_ratio/low_min": 0.0001102778987842612, "clip_ratio/region_mean": 0.0002328503851458663, "completions/clipped_ratio": 0.0, "completions/max_length": 6737.0, "completions/max_terminated_length": 6737.0, "completions/mean_length": 4876.625, "completions/mean_terminated_length": 4876.625, "completions/min_length": 3497.0, "completions/min_terminated_length": 3497.0, "entropy": 0.345124926418066, "epoch": 0.014829290723071617, "frac_reward_zero_std": 0.0, "grad_norm": 0.05227786675095558, "learning_rate": 1e-05, "loss": -0.0249, "num_tokens": 11904054.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001052618026733, "sampling/importance_sampling_ratio/min": 0.5758959054946899, "sampling/sampling_logp_difference/max": 0.7921288013458252, "sampling/sampling_logp_difference/mean": 0.014587976969778538, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3329.0, "completions/max_terminated_length": 3329.0, "completions/mean_length": 2546.125, "completions/mean_terminated_length": 2546.125, "completions/min_length": 1418.0, "completions/min_terminated_length": 1418.0, "entropy": 0.25126997753977776, "epoch": 0.014886768594091274, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11925727.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4259095191955566, "sampling/importance_sampling_ratio/mean": 1.0000795125961304, "sampling/importance_sampling_ratio/min": 0.6376045346260071, "sampling/sampling_logp_difference/max": 0.45003700256347656, "sampling/sampling_logp_difference/mean": 0.01043980848044157, "step": 259 }, { "clip_ratio/high_max": 2.7790129024651833e-05, "clip_ratio/high_mean": 2.7790129024651833e-05, "clip_ratio/low_mean": 0.0009038363714353181, "clip_ratio/low_min": 0.0009038363714353181, "clip_ratio/region_mean": 0.00093162650045997, "completions/clipped_ratio": 0.0, "completions/max_length": 15221.0, "completions/max_terminated_length": 15221.0, "completions/mean_length": 10797.625, "completions/mean_terminated_length": 10797.625, "completions/min_length": 6488.0, "completions/min_terminated_length": 6488.0, "entropy": 0.872970461845398, "epoch": 0.014944246465110932, "frac_reward_zero_std": 0.0, "grad_norm": 0.006912244018167257, "learning_rate": 1e-05, "loss": 0.0593, "num_tokens": 12013652.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002055168151855, "sampling/importance_sampling_ratio/min": 0.05752217769622803, "sampling/sampling_logp_difference/max": 2.8555846214294434, "sampling/sampling_logp_difference/mean": 0.03274575620889664, "step": 260 }, { "clip_ratio/high_max": 2.3955539290909655e-05, "clip_ratio/high_mean": 2.3955539290909655e-05, "clip_ratio/low_mean": 6.0837119235657156e-05, "clip_ratio/low_min": 6.0837119235657156e-05, "clip_ratio/region_mean": 8.479265852656681e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10804.0, "completions/max_terminated_length": 10804.0, "completions/mean_length": 6303.625, "completions/mean_terminated_length": 6303.625, "completions/min_length": 4371.0, "completions/min_terminated_length": 4371.0, "entropy": 1.3393941968679428, "epoch": 0.01500172433613059, "frac_reward_zero_std": 0.0, "grad_norm": 0.0125952810049057, "learning_rate": 1e-05, "loss": -0.071, "num_tokens": 12064993.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4859604835510254, "sampling/importance_sampling_ratio/mean": 0.9997767806053162, "sampling/importance_sampling_ratio/min": 0.3659899830818176, "sampling/sampling_logp_difference/max": 1.0051493644714355, "sampling/sampling_logp_difference/mean": 0.028279611840844154, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14001.0, "completions/max_terminated_length": 14001.0, "completions/mean_length": 9267.75, "completions/mean_terminated_length": 9267.75, "completions/min_length": 4333.0, "completions/min_terminated_length": 4333.0, "entropy": 0.7848407402634621, "epoch": 0.015059202207150247, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12139951.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998934864997864, "sampling/importance_sampling_ratio/min": 0.31592896580696106, "sampling/sampling_logp_difference/max": 1.486649990081787, "sampling/sampling_logp_difference/mean": 0.020874252542853355, "step": 262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8796.0, "completions/max_terminated_length": 8796.0, "completions/mean_length": 5854.0, "completions/mean_terminated_length": 5854.0, "completions/min_length": 3834.0, "completions/min_terminated_length": 3834.0, "entropy": 0.6252594366669655, "epoch": 0.015116680078169905, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12187543.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9118043184280396, "sampling/importance_sampling_ratio/mean": 0.9999527335166931, "sampling/importance_sampling_ratio/min": 0.6088029742240906, "sampling/sampling_logp_difference/max": 0.6480474472045898, "sampling/sampling_logp_difference/mean": 0.022176310420036316, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3143.0, "completions/max_terminated_length": 3143.0, "completions/mean_length": 1706.125, "completions/mean_terminated_length": 1706.125, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.3878425918519497, "epoch": 0.015174157949189563, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12201816.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.500133752822876, "sampling/importance_sampling_ratio/mean": 1.0005122423171997, "sampling/importance_sampling_ratio/min": 0.6614216566085815, "sampling/sampling_logp_difference/max": 0.4133636951446533, "sampling/sampling_logp_difference/mean": 0.016923129558563232, "step": 264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10087.0, "completions/max_terminated_length": 10087.0, "completions/mean_length": 5456.75, "completions/mean_terminated_length": 5456.75, "completions/min_length": 3458.0, "completions/min_terminated_length": 3458.0, "entropy": 0.4956372305750847, "epoch": 0.01523163582020922, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12246822.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7300132513046265, "sampling/importance_sampling_ratio/mean": 1.0002621412277222, "sampling/importance_sampling_ratio/min": 0.6069631576538086, "sampling/sampling_logp_difference/max": 0.5481290817260742, "sampling/sampling_logp_difference/mean": 0.01882501319050789, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9249.0, "completions/max_terminated_length": 9249.0, "completions/mean_length": 5672.125, "completions/mean_terminated_length": 5672.125, "completions/min_length": 4063.0, "completions/min_terminated_length": 4063.0, "entropy": 0.38724199309945107, "epoch": 0.015289113691228876, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12293935.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7353522777557373, "sampling/importance_sampling_ratio/mean": 0.9997246265411377, "sampling/importance_sampling_ratio/min": 0.4818136394023895, "sampling/sampling_logp_difference/max": 0.7301979064941406, "sampling/sampling_logp_difference/mean": 0.01586085744202137, "step": 266 }, { "clip_ratio/high_max": 5.242153383733239e-05, "clip_ratio/high_mean": 5.242153383733239e-05, "clip_ratio/low_mean": 0.00019502839313645381, "clip_ratio/low_min": 0.00019502839313645381, "clip_ratio/region_mean": 0.0002474499269737862, "completions/clipped_ratio": 0.0, "completions/max_length": 7770.0, "completions/max_terminated_length": 7770.0, "completions/mean_length": 5646.875, "completions/mean_terminated_length": 5646.875, "completions/min_length": 4140.0, "completions/min_terminated_length": 4140.0, "entropy": 0.7165233716368675, "epoch": 0.015346591562248534, "frac_reward_zero_std": 0.0, "grad_norm": 0.021525582298636436, "learning_rate": 1e-05, "loss": -0.1338, "num_tokens": 12340294.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999697804450989, "sampling/importance_sampling_ratio/min": 0.38163527846336365, "sampling/sampling_logp_difference/max": 1.1706171035766602, "sampling/sampling_logp_difference/mean": 0.01964937150478363, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6600.0, "completions/max_terminated_length": 6600.0, "completions/mean_length": 4055.5, "completions/mean_terminated_length": 4055.5, "completions/min_length": 1793.0, "completions/min_terminated_length": 1793.0, "entropy": 0.49006812646985054, "epoch": 0.015404069433268192, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12374034.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4653664827346802, "sampling/importance_sampling_ratio/mean": 0.9998965263366699, "sampling/importance_sampling_ratio/min": 0.5483196377754211, "sampling/sampling_logp_difference/max": 0.6008968353271484, "sampling/sampling_logp_difference/mean": 0.019602404907345772, "step": 268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7732.0, "completions/max_terminated_length": 7732.0, "completions/mean_length": 4636.875, "completions/mean_terminated_length": 4636.875, "completions/min_length": 1654.0, "completions/min_terminated_length": 1654.0, "entropy": 0.5612859316170216, "epoch": 0.01546154730428785, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12412681.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.99969083070755, "sampling/importance_sampling_ratio/min": 0.3210018277168274, "sampling/sampling_logp_difference/max": 1.1363084316253662, "sampling/sampling_logp_difference/mean": 0.016598373651504517, "step": 269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11061.0, "completions/max_terminated_length": 11061.0, "completions/mean_length": 4946.125, "completions/mean_terminated_length": 4946.125, "completions/min_length": 1497.0, "completions/min_terminated_length": 1497.0, "entropy": 0.8044924661517143, "epoch": 0.015519025175307507, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12453498.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999637007713318, "sampling/importance_sampling_ratio/min": 2.09754239222093e-06, "sampling/sampling_logp_difference/max": 13.07474422454834, "sampling/sampling_logp_difference/mean": 0.022526336833834648, "step": 270 }, { "clip_ratio/high_max": 1.0969723916787189e-05, "clip_ratio/high_mean": 1.0969723916787189e-05, "clip_ratio/low_mean": 0.0007923787270556204, "clip_ratio/low_min": 0.0007923787270556204, "clip_ratio/region_mean": 0.0008033484509724076, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 15828.0, "completions/mean_length": 14050.25, "completions/mean_terminated_length": 13272.333984375, "completions/min_length": 9970.0, "completions/min_terminated_length": 9970.0, "entropy": 0.861279807984829, "epoch": 0.015576503046327165, "frac_reward_zero_std": 0.0, "grad_norm": 0.008018149062991142, "learning_rate": 1e-05, "loss": 0.0669, "num_tokens": 12567996.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999126195907593, "sampling/importance_sampling_ratio/min": 0.10851297527551651, "sampling/sampling_logp_difference/max": 2.2208855152130127, "sampling/sampling_logp_difference/mean": 0.02991190180182457, "step": 271 }, { "clip_ratio/high_max": 0.0002770495702861808, "clip_ratio/high_mean": 0.0002770495702861808, "clip_ratio/low_mean": 5.847953070770018e-05, "clip_ratio/low_min": 5.847953070770018e-05, "clip_ratio/region_mean": 0.000335529100993881, "completions/clipped_ratio": 0.0, "completions/max_length": 10993.0, "completions/max_terminated_length": 10993.0, "completions/mean_length": 4628.25, "completions/mean_terminated_length": 4628.25, "completions/min_length": 2676.0, "completions/min_terminated_length": 2676.0, "entropy": 0.5769103765487671, "epoch": 0.015633980917346822, "frac_reward_zero_std": 0.0, "grad_norm": 0.010209813714027405, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 12606310.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0006463527679443, "sampling/importance_sampling_ratio/min": 0.41931721568107605, "sampling/sampling_logp_difference/max": 0.869127631187439, "sampling/sampling_logp_difference/mean": 0.0236323531717062, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5765.0, "completions/max_terminated_length": 5765.0, "completions/mean_length": 5065.125, "completions/mean_terminated_length": 5065.125, "completions/min_length": 4460.0, "completions/min_terminated_length": 4460.0, "entropy": 0.36210478469729424, "epoch": 0.015691458788366478, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12647839.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999702572822571, "sampling/importance_sampling_ratio/min": 0.605582058429718, "sampling/sampling_logp_difference/max": 0.7214326858520508, "sampling/sampling_logp_difference/mean": 0.014195986092090607, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6531243899371475e-05, "clip_ratio/low_min": 5.6531243899371475e-05, "clip_ratio/region_mean": 5.6531243899371475e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 13267.0, "completions/max_terminated_length": 13267.0, "completions/mean_length": 4615.625, "completions/mean_terminated_length": 4615.625, "completions/min_length": 2352.0, "completions/min_terminated_length": 2352.0, "entropy": 0.33000186271965504, "epoch": 0.015748936659386138, "frac_reward_zero_std": 0.0, "grad_norm": 0.12380556017160416, "learning_rate": 1e-05, "loss": 0.6623, "num_tokens": 12685988.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000193476676941, "sampling/importance_sampling_ratio/min": 0.541982889175415, "sampling/sampling_logp_difference/max": 1.0451645851135254, "sampling/sampling_logp_difference/mean": 0.01526904758065939, "step": 274 }, { "clip_ratio/high_max": 3.398894114070572e-05, "clip_ratio/high_mean": 3.398894114070572e-05, "clip_ratio/low_mean": 0.0008203986872103997, "clip_ratio/low_min": 0.0008203986872103997, "clip_ratio/region_mean": 0.0008543876283511054, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 16029.0, "completions/mean_length": 13501.0, "completions/mean_terminated_length": 11771.2001953125, "completions/min_length": 7672.0, "completions/min_terminated_length": 7672.0, "entropy": 0.6330981440842152, "epoch": 0.015806414530405793, "frac_reward_zero_std": 0.0, "grad_norm": 0.011000245809555054, "learning_rate": 1e-05, "loss": 0.1659, "num_tokens": 12795020.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998465180397034, "sampling/importance_sampling_ratio/min": 1.6777634073150693e-06, "sampling/sampling_logp_difference/max": 13.298048973083496, "sampling/sampling_logp_difference/mean": 0.02542637661099434, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5439.0, "completions/max_terminated_length": 5439.0, "completions/mean_length": 2809.625, "completions/mean_terminated_length": 2809.625, "completions/min_length": 1897.0, "completions/min_terminated_length": 1897.0, "entropy": 0.2729002833366394, "epoch": 0.015863892401425453, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12818761.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4460861682891846, "sampling/importance_sampling_ratio/mean": 1.000095009803772, "sampling/importance_sampling_ratio/min": 0.47927430272102356, "sampling/sampling_logp_difference/max": 0.7354822158813477, "sampling/sampling_logp_difference/mean": 0.012092736549675465, "step": 276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5087.0, "completions/max_terminated_length": 5087.0, "completions/mean_length": 3614.5, "completions/mean_terminated_length": 3614.5, "completions/min_length": 3021.0, "completions/min_terminated_length": 3021.0, "entropy": 0.47840742766857147, "epoch": 0.01592137027244511, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12848629.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.450257658958435, "sampling/importance_sampling_ratio/mean": 0.9998756051063538, "sampling/importance_sampling_ratio/min": 0.6375133395195007, "sampling/sampling_logp_difference/max": 0.4501800537109375, "sampling/sampling_logp_difference/mean": 0.013919536024332047, "step": 277 }, { "clip_ratio/high_max": 3.3866159355966374e-05, "clip_ratio/high_mean": 3.3866159355966374e-05, "clip_ratio/low_mean": 0.0006137364034657367, "clip_ratio/low_min": 0.0006137364034657367, "clip_ratio/region_mean": 0.000647602562821703, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 11073.0, "completions/mean_length": 10755.125, "completions/mean_terminated_length": 7377.80029296875, "completions/min_length": 2662.0, "completions/min_terminated_length": 2662.0, "entropy": 0.5293156765401363, "epoch": 0.015978848143464765, "frac_reward_zero_std": 0.0, "grad_norm": 0.01830984279513359, "learning_rate": 1e-05, "loss": 0.3361, "num_tokens": 12936262.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003126859664917, "sampling/importance_sampling_ratio/min": 0.20826758444309235, "sampling/sampling_logp_difference/max": 1.8035707473754883, "sampling/sampling_logp_difference/mean": 0.023199284449219704, "step": 278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 2.079520891129505e-05, "clip_ratio/low_min": 2.079520891129505e-05, "clip_ratio/region_mean": 2.079520891129505e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6011.0, "completions/max_terminated_length": 6011.0, "completions/mean_length": 4201.625, "completions/mean_terminated_length": 4201.625, "completions/min_length": 3057.0, "completions/min_terminated_length": 3057.0, "entropy": 0.1944421762600541, "epoch": 0.016036326014484424, "frac_reward_zero_std": 0.0, "grad_norm": 0.005797348450869322, "learning_rate": 1e-05, "loss": 0.1522, "num_tokens": 12970995.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.550799012184143, "sampling/importance_sampling_ratio/mean": 1.0001014471054077, "sampling/importance_sampling_ratio/min": 0.5867496132850647, "sampling/sampling_logp_difference/max": 0.5331571102142334, "sampling/sampling_logp_difference/mean": 0.00934829842299223, "step": 279 }, { "clip_ratio/high_max": 0.0001857445422501769, "clip_ratio/high_mean": 0.0001857445422501769, "clip_ratio/low_mean": 5.3636558732250705e-05, "clip_ratio/low_min": 5.3636558732250705e-05, "clip_ratio/region_mean": 0.0002393811009824276, "completions/clipped_ratio": 0.0, "completions/max_length": 9322.0, "completions/max_terminated_length": 9322.0, "completions/mean_length": 5055.5, "completions/mean_terminated_length": 5055.5, "completions/min_length": 2883.0, "completions/min_terminated_length": 2883.0, "entropy": 0.5680248625576496, "epoch": 0.01609380388550408, "frac_reward_zero_std": 0.0, "grad_norm": 0.0072536226361989975, "learning_rate": 1e-05, "loss": 0.2987, "num_tokens": 13014351.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4945365190505981, "sampling/importance_sampling_ratio/mean": 0.9998281002044678, "sampling/importance_sampling_ratio/min": 0.558586061000824, "sampling/sampling_logp_difference/max": 0.5823465585708618, "sampling/sampling_logp_difference/mean": 0.02077416516840458, "step": 280 }, { "clip_ratio/high_max": 4.7874134907033294e-05, "clip_ratio/high_mean": 4.7874134907033294e-05, "clip_ratio/low_mean": 0.0006891895754961297, "clip_ratio/low_min": 0.0006891895754961297, "clip_ratio/region_mean": 0.000737063710403163, "completions/clipped_ratio": 0.0, "completions/max_length": 9291.0, "completions/max_terminated_length": 9291.0, "completions/mean_length": 6910.375, "completions/mean_terminated_length": 6910.375, "completions/min_length": 1861.0, "completions/min_terminated_length": 1861.0, "entropy": 0.587644200772047, "epoch": 0.01615128175652374, "frac_reward_zero_std": 0.0, "grad_norm": 0.019552532583475113, "learning_rate": 1e-05, "loss": -0.1338, "num_tokens": 13070618.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999970197677612, "sampling/importance_sampling_ratio/min": 0.4577460289001465, "sampling/sampling_logp_difference/max": 0.7814407348632812, "sampling/sampling_logp_difference/mean": 0.02277231030166149, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 1.6823687474243343e-05, "clip_ratio/low_min": 1.6823687474243343e-05, "clip_ratio/region_mean": 1.6823687474243343e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 7430.0, "completions/max_terminated_length": 7430.0, "completions/mean_length": 3620.375, "completions/mean_terminated_length": 3620.375, "completions/min_length": 1958.0, "completions/min_terminated_length": 1958.0, "entropy": 0.41035582683980465, "epoch": 0.016208759627543395, "frac_reward_zero_std": 0.0, "grad_norm": 0.005621267948299646, "learning_rate": 1e-05, "loss": 0.3721, "num_tokens": 13100749.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6037373542785645, "sampling/importance_sampling_ratio/mean": 1.0002422332763672, "sampling/importance_sampling_ratio/min": 0.6739776134490967, "sampling/sampling_logp_difference/max": 0.4723367691040039, "sampling/sampling_logp_difference/mean": 0.012744064442813396, "step": 282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12909.0, "completions/max_terminated_length": 12909.0, "completions/mean_length": 7139.75, "completions/mean_terminated_length": 7139.75, "completions/min_length": 3028.0, "completions/min_terminated_length": 3028.0, "entropy": 0.3640165962278843, "epoch": 0.016266237498563055, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13159011.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8976774215698242, "sampling/importance_sampling_ratio/mean": 0.9998587369918823, "sampling/importance_sampling_ratio/min": 0.5414568185806274, "sampling/sampling_logp_difference/max": 0.6406307220458984, "sampling/sampling_logp_difference/mean": 0.015155038796365261, "step": 283 }, { "clip_ratio/high_max": 8.563771370972972e-05, "clip_ratio/high_mean": 8.563771370972972e-05, "clip_ratio/low_mean": 0.00022893887216923758, "clip_ratio/low_min": 0.00022893887216923758, "clip_ratio/region_mean": 0.0003145765858789673, "completions/clipped_ratio": 0.0, "completions/max_length": 11161.0, "completions/max_terminated_length": 11161.0, "completions/mean_length": 7993.25, "completions/mean_terminated_length": 7993.25, "completions/min_length": 4849.0, "completions/min_terminated_length": 4849.0, "entropy": 0.6444938853383064, "epoch": 0.01632371536958271, "frac_reward_zero_std": 0.0, "grad_norm": 0.045867763459682465, "learning_rate": 1e-05, "loss": 0.1413, "num_tokens": 13223757.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999954879283905, "sampling/importance_sampling_ratio/min": 0.10870083421468735, "sampling/sampling_logp_difference/max": 2.219155788421631, "sampling/sampling_logp_difference/mean": 0.024017199873924255, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4588.0, "completions/max_terminated_length": 4588.0, "completions/mean_length": 2995.125, "completions/mean_terminated_length": 2995.125, "completions/min_length": 1353.0, "completions/min_terminated_length": 1353.0, "entropy": 0.38860574923455715, "epoch": 0.016381193240602367, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13248526.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996554255485535, "sampling/importance_sampling_ratio/min": 0.5708932876586914, "sampling/sampling_logp_difference/max": 0.7145235538482666, "sampling/sampling_logp_difference/mean": 0.016480296850204468, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 15923.0, "completions/mean_length": 14015.375, "completions/mean_terminated_length": 13225.833984375, "completions/min_length": 11374.0, "completions/min_terminated_length": 11374.0, "entropy": 0.4670153819024563, "epoch": 0.016438671111622026, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13361801.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002036094665527, "sampling/importance_sampling_ratio/min": 0.39538297057151794, "sampling/sampling_logp_difference/max": 0.9279004335403442, "sampling/sampling_logp_difference/mean": 0.018330631777644157, "step": 286 }, { "clip_ratio/high_max": 6.584697439393494e-05, "clip_ratio/high_mean": 6.584697439393494e-05, "clip_ratio/low_mean": 4.254305531503633e-05, "clip_ratio/low_min": 4.254305531503633e-05, "clip_ratio/region_mean": 0.00010839002970897127, "completions/clipped_ratio": 0.0, "completions/max_length": 14691.0, "completions/max_terminated_length": 14691.0, "completions/mean_length": 8030.25, "completions/mean_terminated_length": 8030.25, "completions/min_length": 2975.0, "completions/min_terminated_length": 2975.0, "entropy": 0.6689124628901482, "epoch": 0.016496148982641682, "frac_reward_zero_std": 0.0, "grad_norm": 0.005533882882446051, "learning_rate": 1e-05, "loss": 0.2931, "num_tokens": 13427291.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000213384628296, "sampling/importance_sampling_ratio/min": 0.22132240235805511, "sampling/sampling_logp_difference/max": 1.5081348419189453, "sampling/sampling_logp_difference/mean": 0.020459897816181183, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001402131310896948, "clip_ratio/low_min": 0.0001402131310896948, "clip_ratio/region_mean": 0.0001402131310896948, "completions/clipped_ratio": 0.0, "completions/max_length": 3510.0, "completions/max_terminated_length": 3510.0, "completions/mean_length": 1911.125, "completions/mean_terminated_length": 1911.125, "completions/min_length": 1122.0, "completions/min_terminated_length": 1122.0, "entropy": 0.40920743718743324, "epoch": 0.01655362685366134, "frac_reward_zero_std": 0.0, "grad_norm": 0.01969943195581436, "learning_rate": 1e-05, "loss": -0.0788, "num_tokens": 13443700.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.2755016088485718, "sampling/importance_sampling_ratio/mean": 1.0000691413879395, "sampling/importance_sampling_ratio/min": 0.6171529293060303, "sampling/sampling_logp_difference/max": 0.4826383590698242, "sampling/sampling_logp_difference/mean": 0.013039803132414818, "step": 288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3207.0, "completions/max_terminated_length": 3207.0, "completions/mean_length": 2260.0, "completions/mean_terminated_length": 2260.0, "completions/min_length": 1123.0, "completions/min_terminated_length": 1123.0, "entropy": 0.24728919006884098, "epoch": 0.016611104724680997, "frac_reward_zero_std": 0.0, "grad_norm": 0.008269250392913818, "learning_rate": 1e-05, "loss": 0.0429, "num_tokens": 13463084.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.536453127861023, "sampling/importance_sampling_ratio/mean": 1.0002949237823486, "sampling/importance_sampling_ratio/min": 0.6286458969116211, "sampling/sampling_logp_difference/max": 0.4641871452331543, "sampling/sampling_logp_difference/mean": 0.008462048135697842, "step": 289 }, { "clip_ratio/high_max": 8.435608469881117e-05, "clip_ratio/high_mean": 8.435608469881117e-05, "clip_ratio/low_mean": 0.0005368705315049738, "clip_ratio/low_min": 0.0005368705315049738, "clip_ratio/region_mean": 0.0006212266162037849, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 12413.0, "completions/mean_length": 10500.5, "completions/mean_terminated_length": 8539.333984375, "completions/min_length": 5737.0, "completions/min_terminated_length": 5737.0, "entropy": 0.39540280401706696, "epoch": 0.016668582595700657, "frac_reward_zero_std": 0.0, "grad_norm": 0.014503099955618382, "learning_rate": 1e-05, "loss": 0.106, "num_tokens": 13548144.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002013444900513, "sampling/importance_sampling_ratio/min": 0.4272690713405609, "sampling/sampling_logp_difference/max": 0.9638075828552246, "sampling/sampling_logp_difference/mean": 0.017945846542716026, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3095.0, "completions/max_terminated_length": 3095.0, "completions/mean_length": 2623.5, "completions/mean_terminated_length": 2623.5, "completions/min_length": 2247.0, "completions/min_terminated_length": 2247.0, "entropy": 0.19449901394546032, "epoch": 0.016726060466720313, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13570308.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.380873441696167, "sampling/importance_sampling_ratio/mean": 0.9999493956565857, "sampling/importance_sampling_ratio/min": 0.6107633113861084, "sampling/sampling_logp_difference/max": 0.4930458068847656, "sampling/sampling_logp_difference/mean": 0.008187439292669296, "step": 291 }, { "clip_ratio/high_max": 0.00012352219346212223, "clip_ratio/high_mean": 0.00012352219346212223, "clip_ratio/low_mean": 0.00023838504421291873, "clip_ratio/low_min": 0.00023838504421291873, "clip_ratio/region_mean": 0.00036190723767504096, "completions/clipped_ratio": 0.0, "completions/max_length": 11632.0, "completions/max_terminated_length": 11632.0, "completions/mean_length": 5898.125, "completions/mean_terminated_length": 5898.125, "completions/min_length": 2433.0, "completions/min_terminated_length": 2433.0, "entropy": 0.3482863698154688, "epoch": 0.01678353833773997, "frac_reward_zero_std": 0.0, "grad_norm": 0.05515585094690323, "learning_rate": 1e-05, "loss": 0.5142, "num_tokens": 13619077.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999215602874756, "sampling/importance_sampling_ratio/min": 0.47605910897254944, "sampling/sampling_logp_difference/max": 0.742213249206543, "sampling/sampling_logp_difference/mean": 0.016893424093723297, "step": 292 }, { "clip_ratio/high_max": 9.655231951910537e-05, "clip_ratio/high_mean": 9.655231951910537e-05, "clip_ratio/low_mean": 0.00034231648896820843, "clip_ratio/low_min": 0.00034231648896820843, "clip_ratio/region_mean": 0.0004388688084873138, "completions/clipped_ratio": 0.0, "completions/max_length": 9685.0, "completions/max_terminated_length": 9685.0, "completions/mean_length": 4638.375, "completions/mean_terminated_length": 4638.375, "completions/min_length": 1932.0, "completions/min_terminated_length": 1932.0, "entropy": 0.5883205719292164, "epoch": 0.016841016208759628, "frac_reward_zero_std": 0.0, "grad_norm": 0.028066974133253098, "learning_rate": 1e-05, "loss": 0.1608, "num_tokens": 13657504.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.80884850025177, "sampling/importance_sampling_ratio/mean": 0.9993683099746704, "sampling/importance_sampling_ratio/min": 0.388579785823822, "sampling/sampling_logp_difference/max": 0.9452568292617798, "sampling/sampling_logp_difference/mean": 0.021590445190668106, "step": 293 }, { "clip_ratio/high_max": 1.4988009752414655e-05, "clip_ratio/high_mean": 1.4988009752414655e-05, "clip_ratio/low_mean": 0.0007542960302089341, "clip_ratio/low_min": 0.0007542960302089341, "clip_ratio/region_mean": 0.0007692840399613488, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14972.0, "completions/mean_length": 12955.125, "completions/mean_terminated_length": 11812.1669921875, "completions/min_length": 8340.0, "completions/min_terminated_length": 8340.0, "entropy": 0.6050230599939823, "epoch": 0.016898494079779284, "frac_reward_zero_std": 0.0, "grad_norm": 0.005949932616204023, "learning_rate": 1e-05, "loss": 0.1258, "num_tokens": 13762497.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999903678894043, "sampling/importance_sampling_ratio/min": 0.02129349298775196, "sampling/sampling_logp_difference/max": 3.849353790283203, "sampling/sampling_logp_difference/mean": 0.022619644179940224, "step": 294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10462.0, "completions/max_terminated_length": 10462.0, "completions/mean_length": 7412.0, "completions/mean_terminated_length": 7412.0, "completions/min_length": 5436.0, "completions/min_terminated_length": 5436.0, "entropy": 0.6282271333038807, "epoch": 0.016955971950798943, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13822553.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999313354492188, "sampling/importance_sampling_ratio/min": 0.3860255777835846, "sampling/sampling_logp_difference/max": 0.9518516063690186, "sampling/sampling_logp_difference/mean": 0.02337348833680153, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7661.0, "completions/max_terminated_length": 7661.0, "completions/mean_length": 5066.875, "completions/mean_terminated_length": 5066.875, "completions/min_length": 2909.0, "completions/min_terminated_length": 2909.0, "entropy": 0.41942763701081276, "epoch": 0.0170134498218186, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13864168.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6161707639694214, "sampling/importance_sampling_ratio/mean": 1.0000927448272705, "sampling/importance_sampling_ratio/min": 0.6308397650718689, "sampling/sampling_logp_difference/max": 0.4800596237182617, "sampling/sampling_logp_difference/mean": 0.016679463908076286, "step": 296 }, { "clip_ratio/high_max": 2.8115160603192635e-05, "clip_ratio/high_mean": 2.8115160603192635e-05, "clip_ratio/low_mean": 0.00034843457979150116, "clip_ratio/low_min": 0.00034843457979150116, "clip_ratio/region_mean": 0.0003765497403946938, "completions/clipped_ratio": 0.0, "completions/max_length": 5227.0, "completions/max_terminated_length": 5227.0, "completions/mean_length": 3872.875, "completions/mean_terminated_length": 3872.875, "completions/min_length": 2402.0, "completions/min_terminated_length": 2402.0, "entropy": 0.456345247104764, "epoch": 0.01707092769283826, "frac_reward_zero_std": 0.0, "grad_norm": 0.04224399849772453, "learning_rate": 1e-05, "loss": 0.1888, "num_tokens": 13896335.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.3437304496765137, "sampling/importance_sampling_ratio/mean": 0.9995068311691284, "sampling/importance_sampling_ratio/min": 0.5417106747627258, "sampling/sampling_logp_difference/max": 0.6130232810974121, "sampling/sampling_logp_difference/mean": 0.016502318903803825, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15812.0, "completions/mean_length": 14368.375, "completions/mean_terminated_length": 12352.75, "completions/min_length": 6324.0, "completions/min_terminated_length": 6324.0, "entropy": 0.5045627877116203, "epoch": 0.017128405563857915, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14012450.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000554323196411, "sampling/importance_sampling_ratio/min": 0.2854660153388977, "sampling/sampling_logp_difference/max": 1.2536323070526123, "sampling/sampling_logp_difference/mean": 0.023292748257517815, "step": 298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1333.0, "completions/max_terminated_length": 1333.0, "completions/mean_length": 992.875, "completions/mean_terminated_length": 992.875, "completions/min_length": 658.0, "completions/min_terminated_length": 658.0, "entropy": 0.31343047320842743, "epoch": 0.01718588343487757, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14021705.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7160178422927856, "sampling/importance_sampling_ratio/mean": 0.999776303768158, "sampling/importance_sampling_ratio/min": 0.6452099680900574, "sampling/sampling_logp_difference/max": 0.5400063991546631, "sampling/sampling_logp_difference/mean": 0.014022443443536758, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3130.0, "completions/max_terminated_length": 3130.0, "completions/mean_length": 2598.25, "completions/mean_terminated_length": 2598.25, "completions/min_length": 2000.0, "completions/min_terminated_length": 2000.0, "entropy": 0.3396691009402275, "epoch": 0.01724336130589723, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14043851.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8454477787017822, "sampling/importance_sampling_ratio/mean": 1.0004243850708008, "sampling/importance_sampling_ratio/min": 0.676132321357727, "sampling/sampling_logp_difference/max": 0.6127219200134277, "sampling/sampling_logp_difference/mean": 0.01019161008298397, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6920.0, "completions/max_terminated_length": 6920.0, "completions/mean_length": 4282.625, "completions/mean_terminated_length": 4282.625, "completions/min_length": 1837.0, "completions/min_terminated_length": 1837.0, "entropy": 1.0960607826709747, "epoch": 0.017300839176916886, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14079664.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.480582356452942, "sampling/importance_sampling_ratio/mean": 1.000103235244751, "sampling/importance_sampling_ratio/min": 0.6124796271324158, "sampling/sampling_logp_difference/max": 0.49023962020874023, "sampling/sampling_logp_difference/mean": 0.026110351085662842, "step": 301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2898.0, "completions/max_terminated_length": 2898.0, "completions/mean_length": 2414.375, "completions/mean_terminated_length": 2414.375, "completions/min_length": 1927.0, "completions/min_terminated_length": 1927.0, "entropy": 0.23792114667594433, "epoch": 0.017358317047936545, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14099739.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2692124843597412, "sampling/importance_sampling_ratio/mean": 0.9998177886009216, "sampling/importance_sampling_ratio/min": 0.6417900323867798, "sampling/sampling_logp_difference/max": 0.4434940814971924, "sampling/sampling_logp_difference/mean": 0.009595055133104324, "step": 302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15818.0, "completions/max_terminated_length": 15818.0, "completions/mean_length": 8399.25, "completions/mean_terminated_length": 8399.25, "completions/min_length": 3962.0, "completions/min_terminated_length": 3962.0, "entropy": 0.38004388101398945, "epoch": 0.0174157949189562, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14168077.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998432397842407, "sampling/importance_sampling_ratio/min": 0.4143325090408325, "sampling/sampling_logp_difference/max": 0.8810864686965942, "sampling/sampling_logp_difference/mean": 0.01728796400129795, "step": 303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 13862.0, "completions/mean_length": 14596.875, "completions/mean_terminated_length": 12809.75, "completions/min_length": 12360.0, "completions/min_terminated_length": 12360.0, "entropy": 0.520825169980526, "epoch": 0.01747327278997586, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14286556.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998247027397156, "sampling/importance_sampling_ratio/min": 0.0023807960096746683, "sampling/sampling_logp_difference/max": 6.04032039642334, "sampling/sampling_logp_difference/mean": 0.02044062502682209, "step": 304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3011.0, "completions/max_terminated_length": 3011.0, "completions/mean_length": 1954.125, "completions/mean_terminated_length": 1954.125, "completions/min_length": 1091.0, "completions/min_terminated_length": 1091.0, "entropy": 1.3233292549848557, "epoch": 0.017530750660995516, "frac_reward_zero_std": 0.0, "grad_norm": 0.05288930609822273, "learning_rate": 1e-05, "loss": -0.1429, "num_tokens": 14303885.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.3491226434707642, "sampling/importance_sampling_ratio/mean": 1.0000087022781372, "sampling/importance_sampling_ratio/min": 0.6237119436264038, "sampling/sampling_logp_difference/max": 0.47206664085388184, "sampling/sampling_logp_difference/mean": 0.02923406846821308, "step": 305 }, { "clip_ratio/high_max": 9.506751302978955e-05, "clip_ratio/high_mean": 9.506751302978955e-05, "clip_ratio/low_mean": 0.00028230962925590575, "clip_ratio/low_min": 0.00028230962925590575, "clip_ratio/region_mean": 0.0003773771422856953, "completions/clipped_ratio": 0.0, "completions/max_length": 14650.0, "completions/max_terminated_length": 14650.0, "completions/mean_length": 6245.0, "completions/mean_terminated_length": 6245.0, "completions/min_length": 971.0, "completions/min_terminated_length": 971.0, "entropy": 0.5631169006228447, "epoch": 0.017588228532015176, "frac_reward_zero_std": 0.0, "grad_norm": 0.03470282256603241, "learning_rate": 1e-05, "loss": -0.0158, "num_tokens": 14355293.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000054121017456, "sampling/importance_sampling_ratio/min": 0.4764721989631653, "sampling/sampling_logp_difference/max": 0.8508948087692261, "sampling/sampling_logp_difference/mean": 0.02263343334197998, "step": 306 }, { "clip_ratio/high_max": 6.634154306084383e-05, "clip_ratio/high_mean": 6.634154306084383e-05, "clip_ratio/low_mean": 8.58442799653858e-05, "clip_ratio/low_min": 8.58442799653858e-05, "clip_ratio/region_mean": 0.00015218582302622963, "completions/clipped_ratio": 0.0, "completions/max_length": 11649.0, "completions/max_terminated_length": 11649.0, "completions/mean_length": 6025.0, "completions/mean_terminated_length": 6025.0, "completions/min_length": 1995.0, "completions/min_terminated_length": 1995.0, "entropy": 0.3133881501853466, "epoch": 0.017645706403034832, "frac_reward_zero_std": 0.0, "grad_norm": 0.005668353755027056, "learning_rate": 1e-05, "loss": 0.3299, "num_tokens": 14404573.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.944168210029602, "sampling/importance_sampling_ratio/mean": 0.9998335242271423, "sampling/importance_sampling_ratio/min": 0.5526868104934692, "sampling/sampling_logp_difference/max": 0.6648342609405518, "sampling/sampling_logp_difference/mean": 0.01425552275031805, "step": 307 }, { "clip_ratio/high_max": 4.29996543971356e-05, "clip_ratio/high_mean": 4.29996543971356e-05, "clip_ratio/low_mean": 9.245562250725925e-05, "clip_ratio/low_min": 9.245562250725925e-05, "clip_ratio/region_mean": 0.00013545527690439485, "completions/clipped_ratio": 0.0, "completions/max_length": 2980.0, "completions/max_terminated_length": 2980.0, "completions/mean_length": 1745.75, "completions/mean_terminated_length": 1745.75, "completions/min_length": 930.0, "completions/min_terminated_length": 930.0, "entropy": 0.30218998342752457, "epoch": 0.017703184274054488, "frac_reward_zero_std": 0.0, "grad_norm": 0.01708727888762951, "learning_rate": 1e-05, "loss": -0.0797, "num_tokens": 14419387.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6625542640686035, "sampling/importance_sampling_ratio/mean": 1.0002760887145996, "sampling/importance_sampling_ratio/min": 0.628564715385437, "sampling/sampling_logp_difference/max": 0.5083551406860352, "sampling/sampling_logp_difference/mean": 0.013656601309776306, "step": 308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11742.0, "completions/max_terminated_length": 11742.0, "completions/mean_length": 8171.25, "completions/mean_terminated_length": 8171.25, "completions/min_length": 4699.0, "completions/min_terminated_length": 4699.0, "entropy": 0.5202376432716846, "epoch": 0.017760662145074147, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14485589.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8411750793457031, "sampling/importance_sampling_ratio/mean": 1.0003048181533813, "sampling/importance_sampling_ratio/min": 0.4910643994808197, "sampling/sampling_logp_difference/max": 0.7111799716949463, "sampling/sampling_logp_difference/mean": 0.020186688750982285, "step": 309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3891.0, "completions/max_terminated_length": 3891.0, "completions/mean_length": 1940.25, "completions/mean_terminated_length": 1940.25, "completions/min_length": 642.0, "completions/min_terminated_length": 642.0, "entropy": 0.31665168702602386, "epoch": 0.017818140016093803, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14501911.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4268609285354614, "sampling/importance_sampling_ratio/mean": 0.9999961256980896, "sampling/importance_sampling_ratio/min": 0.6249384880065918, "sampling/sampling_logp_difference/max": 0.47010207176208496, "sampling/sampling_logp_difference/mean": 0.013832678087055683, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6001.0, "completions/max_terminated_length": 6001.0, "completions/mean_length": 2945.5, "completions/mean_terminated_length": 2945.5, "completions/min_length": 1819.0, "completions/min_terminated_length": 1819.0, "entropy": 0.20620127767324448, "epoch": 0.017875617887113462, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14526491.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3218950033187866, "sampling/importance_sampling_ratio/mean": 0.9999508857727051, "sampling/importance_sampling_ratio/min": 0.6079448461532593, "sampling/sampling_logp_difference/max": 0.49767112731933594, "sampling/sampling_logp_difference/mean": 0.010383360087871552, "step": 311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3221.0, "completions/max_terminated_length": 3221.0, "completions/mean_length": 1896.25, "completions/mean_terminated_length": 1896.25, "completions/min_length": 1382.0, "completions/min_terminated_length": 1382.0, "entropy": 0.3582219146192074, "epoch": 0.01793309575813312, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14542309.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5014512538909912, "sampling/importance_sampling_ratio/mean": 1.0002200603485107, "sampling/importance_sampling_ratio/min": 0.647032618522644, "sampling/sampling_logp_difference/max": 0.43535852432250977, "sampling/sampling_logp_difference/mean": 0.015400128439068794, "step": 312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6912.0, "completions/max_terminated_length": 6912.0, "completions/mean_length": 4067.125, "completions/mean_terminated_length": 4067.125, "completions/min_length": 685.0, "completions/min_terminated_length": 685.0, "entropy": 0.517782075330615, "epoch": 0.017990573629152778, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14575734.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.418588399887085, "sampling/importance_sampling_ratio/mean": 1.0005604028701782, "sampling/importance_sampling_ratio/min": 0.5757678151130676, "sampling/sampling_logp_difference/max": 0.5520508289337158, "sampling/sampling_logp_difference/mean": 0.022016707807779312, "step": 313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5348.0, "completions/max_terminated_length": 5348.0, "completions/mean_length": 3690.625, "completions/mean_terminated_length": 3690.625, "completions/min_length": 2375.0, "completions/min_terminated_length": 2375.0, "entropy": 0.289380444213748, "epoch": 0.018048051500172434, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14606091.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4627519845962524, "sampling/importance_sampling_ratio/mean": 0.9998953938484192, "sampling/importance_sampling_ratio/min": 0.6064385771751404, "sampling/sampling_logp_difference/max": 0.5001518726348877, "sampling/sampling_logp_difference/mean": 0.01178512629121542, "step": 314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10975.0, "completions/max_terminated_length": 10975.0, "completions/mean_length": 4978.25, "completions/mean_terminated_length": 4978.25, "completions/min_length": 2371.0, "completions/min_terminated_length": 2371.0, "entropy": 0.5141785815358162, "epoch": 0.01810552937119209, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14647109.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5613679885864258, "sampling/importance_sampling_ratio/mean": 0.9998388290405273, "sampling/importance_sampling_ratio/min": 0.4340689778327942, "sampling/sampling_logp_difference/max": 0.8345518112182617, "sampling/sampling_logp_difference/mean": 0.021983593702316284, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018217236720374785, "clip_ratio/low_min": 0.00018217236720374785, "clip_ratio/region_mean": 0.00018217236720374785, "completions/clipped_ratio": 0.0, "completions/max_length": 7029.0, "completions/max_terminated_length": 7029.0, "completions/mean_length": 5591.375, "completions/mean_terminated_length": 5591.375, "completions/min_length": 2976.0, "completions/min_terminated_length": 2976.0, "entropy": 0.7261018678545952, "epoch": 0.01816300724221175, "frac_reward_zero_std": 0.0, "grad_norm": 0.009648680686950684, "learning_rate": 1e-05, "loss": 0.1141, "num_tokens": 14693080.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7077374458312988, "sampling/importance_sampling_ratio/mean": 0.9998070001602173, "sampling/importance_sampling_ratio/min": 0.5865379571914673, "sampling/sampling_logp_difference/max": 0.5351693630218506, "sampling/sampling_logp_difference/mean": 0.020792080089449883, "step": 316 }, { "clip_ratio/high_max": 1.1924067621293943e-05, "clip_ratio/high_mean": 1.1924067621293943e-05, "clip_ratio/low_mean": 0.00020796335957129486, "clip_ratio/low_min": 0.00020796335957129486, "clip_ratio/region_mean": 0.0002198874271925888, "completions/clipped_ratio": 0.0, "completions/max_length": 10483.0, "completions/max_terminated_length": 10483.0, "completions/mean_length": 3633.75, "completions/mean_terminated_length": 3633.75, "completions/min_length": 1952.0, "completions/min_terminated_length": 1952.0, "entropy": 0.3380902595818043, "epoch": 0.018220485113231405, "frac_reward_zero_std": 0.0, "grad_norm": 0.007336276583373547, "learning_rate": 1e-05, "loss": -0.6667, "num_tokens": 14723190.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004078149795532, "sampling/importance_sampling_ratio/min": 0.5002537369728088, "sampling/sampling_logp_difference/max": 0.7221651077270508, "sampling/sampling_logp_difference/mean": 0.013298140838742256, "step": 317 }, { "clip_ratio/high_max": 3.0288345442386344e-05, "clip_ratio/high_mean": 3.0288345442386344e-05, "clip_ratio/low_mean": 0.0007701888825977221, "clip_ratio/low_min": 0.0007701888825977221, "clip_ratio/region_mean": 0.0008004772280401085, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15778.0, "completions/mean_length": 15147.375, "completions/mean_terminated_length": 13910.75, "completions/min_length": 12328.0, "completions/min_terminated_length": 12328.0, "entropy": 0.6852492205798626, "epoch": 0.018277962984251064, "frac_reward_zero_std": 0.0, "grad_norm": 0.006635353900492191, "learning_rate": 1e-05, "loss": 0.0648, "num_tokens": 14845417.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998446106910706, "sampling/importance_sampling_ratio/min": 0.031520646065473557, "sampling/sampling_logp_difference/max": 3.4571125507354736, "sampling/sampling_logp_difference/mean": 0.02578580379486084, "step": 318 }, { "clip_ratio/high_max": 0.00013082451914669946, "clip_ratio/high_mean": 0.00013082451914669946, "clip_ratio/low_mean": 0.00016318538109771907, "clip_ratio/low_min": 0.00016318538109771907, "clip_ratio/region_mean": 0.00029400990024441853, "completions/clipped_ratio": 0.0, "completions/max_length": 3081.0, "completions/max_terminated_length": 3081.0, "completions/mean_length": 1920.375, "completions/mean_terminated_length": 1920.375, "completions/min_length": 766.0, "completions/min_terminated_length": 766.0, "entropy": 0.4040065184235573, "epoch": 0.01833544085527072, "frac_reward_zero_std": 0.0, "grad_norm": 0.07099581509828568, "learning_rate": 1e-05, "loss": -0.2124, "num_tokens": 14861540.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.299908995628357, "sampling/importance_sampling_ratio/mean": 1.000104546546936, "sampling/importance_sampling_ratio/min": 0.6175916194915771, "sampling/sampling_logp_difference/max": 0.48192787170410156, "sampling/sampling_logp_difference/mean": 0.01673056185245514, "step": 319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002039020328084007, "clip_ratio/low_min": 0.0002039020328084007, "clip_ratio/region_mean": 0.0002039020328084007, "completions/clipped_ratio": 0.0, "completions/max_length": 10435.0, "completions/max_terminated_length": 10435.0, "completions/mean_length": 7574.375, "completions/mean_terminated_length": 7574.375, "completions/min_length": 4115.0, "completions/min_terminated_length": 4115.0, "entropy": 0.6065731458365917, "epoch": 0.01839291872629038, "frac_reward_zero_std": 0.0, "grad_norm": 0.014942534267902374, "learning_rate": 1e-05, "loss": 0.0561, "num_tokens": 14923455.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6245927810668945, "sampling/importance_sampling_ratio/mean": 0.9999759793281555, "sampling/importance_sampling_ratio/min": 0.19265475869178772, "sampling/sampling_logp_difference/max": 1.6468554735183716, "sampling/sampling_logp_difference/mean": 0.018679451197385788, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7008.0, "completions/max_terminated_length": 7008.0, "completions/mean_length": 4057.75, "completions/mean_terminated_length": 4057.75, "completions/min_length": 1055.0, "completions/min_terminated_length": 1055.0, "entropy": 0.3605603091418743, "epoch": 0.018450396597310036, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14956845.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001881122589111, "sampling/importance_sampling_ratio/min": 0.5888962149620056, "sampling/sampling_logp_difference/max": 0.8162369728088379, "sampling/sampling_logp_difference/mean": 0.017093200236558914, "step": 321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14493.0, "completions/mean_length": 12585.875, "completions/mean_terminated_length": 11319.833984375, "completions/min_length": 5750.0, "completions/min_terminated_length": 5750.0, "entropy": 0.8011277616024017, "epoch": 0.01850787446832969, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15058364.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999358057975769, "sampling/importance_sampling_ratio/min": 0.00016261611017398536, "sampling/sampling_logp_difference/max": 8.72411823272705, "sampling/sampling_logp_difference/mean": 0.026181455701589584, "step": 322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8791.0, "completions/max_terminated_length": 8791.0, "completions/mean_length": 4230.875, "completions/mean_terminated_length": 4230.875, "completions/min_length": 1668.0, "completions/min_terminated_length": 1668.0, "entropy": 0.38680815510451794, "epoch": 0.01856535233934935, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15093627.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7237930297851562, "sampling/importance_sampling_ratio/mean": 1.0000313520431519, "sampling/importance_sampling_ratio/min": 0.5658870339393616, "sampling/sampling_logp_difference/max": 0.5693607330322266, "sampling/sampling_logp_difference/mean": 0.01748107559978962, "step": 323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5690.0, "completions/max_terminated_length": 5690.0, "completions/mean_length": 2776.875, "completions/mean_terminated_length": 2776.875, "completions/min_length": 1956.0, "completions/min_terminated_length": 1956.0, "entropy": 0.7585382387042046, "epoch": 0.018622830210369007, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15117330.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8585883378982544, "sampling/importance_sampling_ratio/mean": 1.0003856420516968, "sampling/importance_sampling_ratio/min": 0.6560837626457214, "sampling/sampling_logp_difference/max": 0.6198172569274902, "sampling/sampling_logp_difference/mean": 0.021034816280007362, "step": 324 }, { "clip_ratio/high_max": 8.878271182766184e-05, "clip_ratio/high_mean": 8.878271182766184e-05, "clip_ratio/low_mean": 9.278733341488987e-05, "clip_ratio/low_min": 9.278733341488987e-05, "clip_ratio/region_mean": 0.00018157004524255171, "completions/clipped_ratio": 0.0, "completions/max_length": 8083.0, "completions/max_terminated_length": 8083.0, "completions/mean_length": 4144.125, "completions/mean_terminated_length": 4144.125, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.5541177727282047, "epoch": 0.018680308081388666, "frac_reward_zero_std": 0.0, "grad_norm": 0.07104233652353287, "learning_rate": 1e-05, "loss": 0.0185, "num_tokens": 15152315.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7055503129959106, "sampling/importance_sampling_ratio/mean": 1.00011146068573, "sampling/importance_sampling_ratio/min": 0.3620549738407135, "sampling/sampling_logp_difference/max": 1.0159592628479004, "sampling/sampling_logp_difference/mean": 0.02261045016348362, "step": 325 }, { "clip_ratio/high_max": 4.524165706243366e-05, "clip_ratio/high_mean": 4.524165706243366e-05, "clip_ratio/low_mean": 0.000441106480138842, "clip_ratio/low_min": 0.000441106480138842, "clip_ratio/region_mean": 0.0004863481372012757, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15512.0, "completions/mean_length": 12479.875, "completions/mean_terminated_length": 11922.1435546875, "completions/min_length": 9749.0, "completions/min_terminated_length": 9749.0, "entropy": 0.7386438101530075, "epoch": 0.018737785952408322, "frac_reward_zero_std": 0.0, "grad_norm": 0.016033122316002846, "learning_rate": 1e-05, "loss": -0.0465, "num_tokens": 15253610.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000121593475342, "sampling/importance_sampling_ratio/min": 0.37671196460723877, "sampling/sampling_logp_difference/max": 0.9762744903564453, "sampling/sampling_logp_difference/mean": 0.02541348896920681, "step": 326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4390.0, "completions/max_terminated_length": 4390.0, "completions/mean_length": 1495.875, "completions/mean_terminated_length": 1495.875, "completions/min_length": 564.0, "completions/min_terminated_length": 564.0, "entropy": 0.3768142517656088, "epoch": 0.01879526382342798, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15267001.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2828130722045898, "sampling/importance_sampling_ratio/mean": 1.0002108812332153, "sampling/importance_sampling_ratio/min": 0.6261738538742065, "sampling/sampling_logp_difference/max": 0.4681272506713867, "sampling/sampling_logp_difference/mean": 0.018461311236023903, "step": 327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1976.0, "completions/max_terminated_length": 1976.0, "completions/mean_length": 1566.5, "completions/mean_terminated_length": 1566.5, "completions/min_length": 837.0, "completions/min_terminated_length": 837.0, "entropy": 0.5904304720461369, "epoch": 0.018852741694447638, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15280733.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3126988410949707, "sampling/importance_sampling_ratio/mean": 0.9998319745063782, "sampling/importance_sampling_ratio/min": 0.6127909421920776, "sampling/sampling_logp_difference/max": 0.48973143100738525, "sampling/sampling_logp_difference/mean": 0.01677585020661354, "step": 328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2742.0, "completions/max_terminated_length": 2742.0, "completions/mean_length": 2204.0, "completions/mean_terminated_length": 2204.0, "completions/min_length": 1336.0, "completions/min_terminated_length": 1336.0, "entropy": 0.3187308609485626, "epoch": 0.018910219565467293, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15299893.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4487329721450806, "sampling/importance_sampling_ratio/mean": 1.000268578529358, "sampling/importance_sampling_ratio/min": 0.6670248508453369, "sampling/sampling_logp_difference/max": 0.40492796897888184, "sampling/sampling_logp_difference/mean": 0.01043415255844593, "step": 329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7615.0, "completions/max_terminated_length": 7615.0, "completions/mean_length": 4871.875, "completions/mean_terminated_length": 4871.875, "completions/min_length": 3167.0, "completions/min_terminated_length": 3167.0, "entropy": 0.39124204590916634, "epoch": 0.018967697436486953, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15340076.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4911690950393677, "sampling/importance_sampling_ratio/mean": 0.9998763203620911, "sampling/importance_sampling_ratio/min": 0.4422745108604431, "sampling/sampling_logp_difference/max": 0.8158245086669922, "sampling/sampling_logp_difference/mean": 0.01586175337433815, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 16384.0, "completions/max_terminated_length": 16316.0, "completions/mean_length": 16359.25, "completions/mean_terminated_length": 16285.0, "completions/min_length": 16254.0, "completions/min_terminated_length": 16254.0, "entropy": 0.6022672429680824, "epoch": 0.01902517530750661, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15472374.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999624490737915, "sampling/importance_sampling_ratio/min": 0.27834710478782654, "sampling/sampling_logp_difference/max": 1.278886318206787, "sampling/sampling_logp_difference/mean": 0.02268480323255062, "step": 331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3686.0, "completions/max_terminated_length": 3686.0, "completions/mean_length": 3110.5, "completions/mean_terminated_length": 3110.5, "completions/min_length": 2322.0, "completions/min_terminated_length": 2322.0, "entropy": 0.4437192752957344, "epoch": 0.019082653178526268, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15498058.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3044779300689697, "sampling/importance_sampling_ratio/mean": 1.000067114830017, "sampling/importance_sampling_ratio/min": 0.6156363487243652, "sampling/sampling_logp_difference/max": 0.48509883880615234, "sampling/sampling_logp_difference/mean": 0.015801699832081795, "step": 332 }, { "clip_ratio/high_max": 1.7615557226235978e-05, "clip_ratio/high_mean": 1.7615557226235978e-05, "clip_ratio/low_mean": 0.0006183970363053959, "clip_ratio/low_min": 0.0006183970363053959, "clip_ratio/region_mean": 0.0006360125935316319, "completions/clipped_ratio": 0.0, "completions/max_length": 10678.0, "completions/max_terminated_length": 10678.0, "completions/mean_length": 7056.25, "completions/mean_terminated_length": 7056.25, "completions/min_length": 3294.0, "completions/min_terminated_length": 3294.0, "entropy": 0.7451484203338623, "epoch": 0.019140131049545924, "frac_reward_zero_std": 0.0, "grad_norm": 0.010370392352342606, "learning_rate": 1e-05, "loss": -0.0022, "num_tokens": 15555412.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997419118881226, "sampling/importance_sampling_ratio/min": 0.4893128573894501, "sampling/sampling_logp_difference/max": 0.7159855365753174, "sampling/sampling_logp_difference/mean": 0.026656731963157654, "step": 333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.742178811691701e-05, "clip_ratio/low_min": 6.742178811691701e-05, "clip_ratio/region_mean": 6.742178811691701e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 4934.0, "completions/max_terminated_length": 4934.0, "completions/mean_length": 3034.125, "completions/mean_terminated_length": 3034.125, "completions/min_length": 1231.0, "completions/min_terminated_length": 1231.0, "entropy": 0.5302278809249401, "epoch": 0.019197608920565584, "frac_reward_zero_std": 0.0, "grad_norm": 0.06457998603582382, "learning_rate": 1e-05, "loss": -0.1002, "num_tokens": 15580781.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.501657485961914, "sampling/importance_sampling_ratio/mean": 1.0004233121871948, "sampling/importance_sampling_ratio/min": 0.6905239820480347, "sampling/sampling_logp_difference/max": 0.4065694808959961, "sampling/sampling_logp_difference/mean": 0.01687503792345524, "step": 334 }, { "clip_ratio/high_max": 3.468056456767954e-05, "clip_ratio/high_mean": 3.468056456767954e-05, "clip_ratio/low_mean": 0.0002981360230478458, "clip_ratio/low_min": 0.0002981360230478458, "clip_ratio/region_mean": 0.00033281658761552535, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15099.0, "completions/mean_length": 10131.625, "completions/mean_terminated_length": 9238.4287109375, "completions/min_length": 4174.0, "completions/min_terminated_length": 4174.0, "entropy": 0.4654873348772526, "epoch": 0.01925508679158524, "frac_reward_zero_std": 0.0, "grad_norm": 0.010863181203603745, "learning_rate": 1e-05, "loss": 0.1874, "num_tokens": 15662874.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999262094497681, "sampling/importance_sampling_ratio/min": 0.42790281772613525, "sampling/sampling_logp_difference/max": 0.8488591909408569, "sampling/sampling_logp_difference/mean": 0.021146049723029137, "step": 335 }, { "clip_ratio/high_max": 4.132231333642267e-05, "clip_ratio/high_mean": 4.132231333642267e-05, "clip_ratio/low_mean": 3.9316419133683667e-05, "clip_ratio/low_min": 3.9316419133683667e-05, "clip_ratio/region_mean": 8.063873247010633e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 9538.0, "completions/max_terminated_length": 9538.0, "completions/mean_length": 5125.25, "completions/mean_terminated_length": 5125.25, "completions/min_length": 1507.0, "completions/min_terminated_length": 1507.0, "entropy": 0.7648974284529686, "epoch": 0.0193125646626049, "frac_reward_zero_std": 0.0, "grad_norm": 0.0052408394403755665, "learning_rate": 1e-05, "loss": 0.3051, "num_tokens": 15704684.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6716387271881104, "sampling/importance_sampling_ratio/mean": 1.000206470489502, "sampling/importance_sampling_ratio/min": 0.5740965008735657, "sampling/sampling_logp_difference/max": 0.5549578666687012, "sampling/sampling_logp_difference/mean": 0.021035529673099518, "step": 336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0007146268180804327, "clip_ratio/low_min": 0.0007146268180804327, "clip_ratio/region_mean": 0.0007146268180804327, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 13494.0, "completions/mean_length": 13404.625, "completions/mean_terminated_length": 11617.0, "completions/min_length": 9977.0, "completions/min_terminated_length": 9977.0, "entropy": 0.43265021592378616, "epoch": 0.019370042533624555, "frac_reward_zero_std": 0.0, "grad_norm": 0.00584243331104517, "learning_rate": 1e-05, "loss": 0.0768, "num_tokens": 15813009.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999116063117981, "sampling/importance_sampling_ratio/min": 0.002006195019930601, "sampling/sampling_logp_difference/max": 6.211515426635742, "sampling/sampling_logp_difference/mean": 0.018601318821310997, "step": 337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13395.0, "completions/max_terminated_length": 13395.0, "completions/mean_length": 9461.375, "completions/mean_terminated_length": 9461.375, "completions/min_length": 6637.0, "completions/min_terminated_length": 6637.0, "entropy": 0.8805797770619392, "epoch": 0.01942752040464421, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15890124.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001945495605469, "sampling/importance_sampling_ratio/min": 0.3986915946006775, "sampling/sampling_logp_difference/max": 0.9195671081542969, "sampling/sampling_logp_difference/mean": 0.02378488890826702, "step": 338 }, { "clip_ratio/high_max": 0.00014442485553445294, "clip_ratio/high_mean": 0.00014442485553445294, "clip_ratio/low_mean": 0.00019609275477705523, "clip_ratio/low_min": 0.00019609275477705523, "clip_ratio/region_mean": 0.0003405176103115082, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 10936.0, "completions/mean_length": 9337.75, "completions/mean_terminated_length": 8331.1435546875, "completions/min_length": 6615.0, "completions/min_terminated_length": 6615.0, "entropy": 0.6978324055671692, "epoch": 0.01948499827566387, "frac_reward_zero_std": 0.0, "grad_norm": 0.01227813120931387, "learning_rate": 1e-05, "loss": 0.1759, "num_tokens": 15966842.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8344930410385132, "sampling/importance_sampling_ratio/mean": 1.0003163814544678, "sampling/importance_sampling_ratio/min": 0.4923921227455139, "sampling/sampling_logp_difference/max": 0.7084798812866211, "sampling/sampling_logp_difference/mean": 0.02756013721227646, "step": 339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6524.0, "completions/max_terminated_length": 6524.0, "completions/mean_length": 4721.375, "completions/mean_terminated_length": 4721.375, "completions/min_length": 3315.0, "completions/min_terminated_length": 3315.0, "entropy": 0.32055202685296535, "epoch": 0.019542476146683526, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16005621.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3077168464660645, "sampling/importance_sampling_ratio/mean": 1.000072717666626, "sampling/importance_sampling_ratio/min": 0.4791582226753235, "sampling/sampling_logp_difference/max": 0.7357244491577148, "sampling/sampling_logp_difference/mean": 0.0142374187707901, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3782.0, "completions/max_terminated_length": 3782.0, "completions/mean_length": 2638.75, "completions/mean_terminated_length": 2638.75, "completions/min_length": 1449.0, "completions/min_terminated_length": 1449.0, "entropy": 0.32627006992697716, "epoch": 0.019599954017703185, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16028171.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6159237623214722, "sampling/importance_sampling_ratio/mean": 0.9995152354240417, "sampling/importance_sampling_ratio/min": 0.6188905239105225, "sampling/sampling_logp_difference/max": 0.4799067974090576, "sampling/sampling_logp_difference/mean": 0.015043683350086212, "step": 341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00030060322387726046, "clip_ratio/low_min": 0.00030060322387726046, "clip_ratio/region_mean": 0.00030060322387726046, "completions/clipped_ratio": 0.0, "completions/max_length": 11077.0, "completions/max_terminated_length": 11077.0, "completions/mean_length": 4994.375, "completions/mean_terminated_length": 4994.375, "completions/min_length": 1190.0, "completions/min_terminated_length": 1190.0, "entropy": 1.0550551414489746, "epoch": 0.01965743188872284, "frac_reward_zero_std": 0.0, "grad_norm": 0.017271967604756355, "learning_rate": 1e-05, "loss": 0.2909, "num_tokens": 16069574.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6729991436004639, "sampling/importance_sampling_ratio/mean": 1.0001438856124878, "sampling/importance_sampling_ratio/min": 0.4280719459056854, "sampling/sampling_logp_difference/max": 0.8484640121459961, "sampling/sampling_logp_difference/mean": 0.026202231645584106, "step": 342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14414.0, "completions/mean_length": 12635.25, "completions/mean_terminated_length": 12099.71484375, "completions/min_length": 8481.0, "completions/min_terminated_length": 8481.0, "entropy": 0.6609185971319675, "epoch": 0.0197149097597425, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16171688.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999833881855011, "sampling/importance_sampling_ratio/min": 0.34503352642059326, "sampling/sampling_logp_difference/max": 1.0641136169433594, "sampling/sampling_logp_difference/mean": 0.0253476370126009, "step": 343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6682.0, "completions/max_terminated_length": 6682.0, "completions/mean_length": 3645.875, "completions/mean_terminated_length": 3645.875, "completions/min_length": 1887.0, "completions/min_terminated_length": 1887.0, "entropy": 0.39544339664280415, "epoch": 0.019772387630762157, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16201879.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9815831184387207, "sampling/importance_sampling_ratio/mean": 1.0000919103622437, "sampling/importance_sampling_ratio/min": 0.5613101124763489, "sampling/sampling_logp_difference/max": 0.6838960647583008, "sampling/sampling_logp_difference/mean": 0.016911331564188004, "step": 344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 14822.0, "completions/mean_length": 14651.375, "completions/mean_terminated_length": 12918.75, "completions/min_length": 11623.0, "completions/min_terminated_length": 11623.0, "entropy": 0.42336586117744446, "epoch": 0.019829865501781813, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16320738.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000054955482483, "sampling/importance_sampling_ratio/min": 0.1662822961807251, "sampling/sampling_logp_difference/max": 1.7940683364868164, "sampling/sampling_logp_difference/mean": 0.019420776516199112, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11950.0, "completions/max_terminated_length": 11950.0, "completions/mean_length": 6997.125, "completions/mean_terminated_length": 6997.125, "completions/min_length": 4513.0, "completions/min_terminated_length": 4513.0, "entropy": 0.2710230275988579, "epoch": 0.019887343372801472, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16377835.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7474852800369263, "sampling/importance_sampling_ratio/mean": 1.0000462532043457, "sampling/importance_sampling_ratio/min": 0.4111810028553009, "sampling/sampling_logp_difference/max": 0.8887218236923218, "sampling/sampling_logp_difference/mean": 0.012591187842190266, "step": 346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5051.0, "completions/max_terminated_length": 5051.0, "completions/mean_length": 2853.625, "completions/mean_terminated_length": 2853.625, "completions/min_length": 1698.0, "completions/min_terminated_length": 1698.0, "entropy": 0.4040024057030678, "epoch": 0.019944821243821128, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16401672.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.735011339187622, "sampling/importance_sampling_ratio/mean": 0.9998360872268677, "sampling/importance_sampling_ratio/min": 0.49195700883865356, "sampling/sampling_logp_difference/max": 0.7093639373779297, "sampling/sampling_logp_difference/mean": 0.016577383503317833, "step": 347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2932.0, "completions/max_terminated_length": 2932.0, "completions/mean_length": 1817.0, "completions/mean_terminated_length": 1817.0, "completions/min_length": 759.0, "completions/min_terminated_length": 759.0, "entropy": 0.3871239461004734, "epoch": 0.020002299114840787, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16417128.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5381892919540405, "sampling/importance_sampling_ratio/mean": 1.0006383657455444, "sampling/importance_sampling_ratio/min": 0.623935341835022, "sampling/sampling_logp_difference/max": 0.4717085361480713, "sampling/sampling_logp_difference/mean": 0.016869783401489258, "step": 348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5132.0, "completions/max_terminated_length": 5132.0, "completions/mean_length": 3433.0, "completions/mean_terminated_length": 3433.0, "completions/min_length": 2218.0, "completions/min_terminated_length": 2218.0, "entropy": 0.6093649119138718, "epoch": 0.020059776985860443, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16445944.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5278382301330566, "sampling/importance_sampling_ratio/mean": 0.9998639822006226, "sampling/importance_sampling_ratio/min": 0.5801088809967041, "sampling/sampling_logp_difference/max": 0.5445394515991211, "sampling/sampling_logp_difference/mean": 0.023235652595758438, "step": 349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13434.0, "completions/max_terminated_length": 13434.0, "completions/mean_length": 9765.5, "completions/mean_terminated_length": 9765.5, "completions/min_length": 5986.0, "completions/min_terminated_length": 5986.0, "entropy": 0.9009614214301109, "epoch": 0.020117254856880103, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16524996.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995908737182617, "sampling/importance_sampling_ratio/min": 0.3160809278488159, "sampling/sampling_logp_difference/max": 1.1751902103424072, "sampling/sampling_logp_difference/mean": 0.02548110857605934, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9958.0, "completions/max_terminated_length": 9958.0, "completions/mean_length": 5576.125, "completions/mean_terminated_length": 5576.125, "completions/min_length": 1937.0, "completions/min_terminated_length": 1937.0, "entropy": 1.234039157629013, "epoch": 0.02017473272789976, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16571245.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6989067792892456, "sampling/importance_sampling_ratio/mean": 0.9999986886978149, "sampling/importance_sampling_ratio/min": 0.5405990481376648, "sampling/sampling_logp_difference/max": 0.6150774955749512, "sampling/sampling_logp_difference/mean": 0.030534079298377037, "step": 351 }, { "clip_ratio/high_max": 0.00012208950647618622, "clip_ratio/high_mean": 0.00012208950647618622, "clip_ratio/low_mean": 6.016847328282893e-05, "clip_ratio/low_min": 6.016847328282893e-05, "clip_ratio/region_mean": 0.00018225797975901514, "completions/clipped_ratio": 0.0, "completions/max_length": 5135.0, "completions/max_terminated_length": 5135.0, "completions/mean_length": 3478.125, "completions/mean_terminated_length": 3478.125, "completions/min_length": 2563.0, "completions/min_terminated_length": 2563.0, "entropy": 0.3595081530511379, "epoch": 0.020232210598919415, "frac_reward_zero_std": 0.0, "grad_norm": 0.009329551830887794, "learning_rate": 1e-05, "loss": 0.0687, "num_tokens": 16600134.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5045642852783203, "sampling/importance_sampling_ratio/mean": 1.0004687309265137, "sampling/importance_sampling_ratio/min": 0.5318603515625, "sampling/sampling_logp_difference/max": 0.6313743591308594, "sampling/sampling_logp_difference/mean": 0.014808500185608864, "step": 352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010229132749373093, "clip_ratio/low_min": 0.00010229132749373093, "clip_ratio/region_mean": 0.00010229132749373093, "completions/clipped_ratio": 0.0, "completions/max_length": 2319.0, "completions/max_terminated_length": 2319.0, "completions/mean_length": 1595.875, "completions/mean_terminated_length": 1595.875, "completions/min_length": 865.0, "completions/min_terminated_length": 865.0, "entropy": 0.35693978890776634, "epoch": 0.020289688469939074, "frac_reward_zero_std": 0.0, "grad_norm": 0.024179993197321892, "learning_rate": 1e-05, "loss": -0.1834, "num_tokens": 16613805.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4377808570861816, "sampling/importance_sampling_ratio/mean": 1.0000147819519043, "sampling/importance_sampling_ratio/min": 0.57708340883255, "sampling/sampling_logp_difference/max": 0.5497684478759766, "sampling/sampling_logp_difference/mean": 0.015388967469334602, "step": 353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 16384.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 16384.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 16384.0, "completions/min_terminated_length": 0.0, "entropy": 0.66748046875, "epoch": 0.02034716634095873, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16746693.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000501871109009, "sampling/importance_sampling_ratio/min": 0.17032411694526672, "sampling/sampling_logp_difference/max": 1.7700520753860474, "sampling/sampling_logp_difference/mean": 0.02706432342529297, "step": 354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3290.0, "completions/max_terminated_length": 3290.0, "completions/mean_length": 1433.75, "completions/mean_terminated_length": 1433.75, "completions/min_length": 506.0, "completions/min_terminated_length": 506.0, "entropy": 0.4565188344568014, "epoch": 0.02040464421197839, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16759131.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3809913396835327, "sampling/importance_sampling_ratio/mean": 0.999972939491272, "sampling/importance_sampling_ratio/min": 0.61844402551651, "sampling/sampling_logp_difference/max": 0.480548620223999, "sampling/sampling_logp_difference/mean": 0.021480221301317215, "step": 355 }, { "clip_ratio/high_max": 3.810596899711527e-05, "clip_ratio/high_mean": 3.810596899711527e-05, "clip_ratio/low_mean": 0.00023517667432315648, "clip_ratio/low_min": 0.00023517667432315648, "clip_ratio/region_mean": 0.00027328264332027175, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 11950.0, "completions/mean_length": 10748.875, "completions/mean_terminated_length": 8870.5, "completions/min_length": 5878.0, "completions/min_terminated_length": 5878.0, "entropy": 0.3735651969909668, "epoch": 0.020462122082998045, "frac_reward_zero_std": 0.0, "grad_norm": 0.02715384028851986, "learning_rate": 1e-05, "loss": 0.2014, "num_tokens": 16846426.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001177787780762, "sampling/importance_sampling_ratio/min": 0.3348197340965271, "sampling/sampling_logp_difference/max": 1.094162940979004, "sampling/sampling_logp_difference/mean": 0.015553958714008331, "step": 356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12477.0, "completions/max_terminated_length": 12477.0, "completions/mean_length": 7907.25, "completions/mean_terminated_length": 7907.25, "completions/min_length": 5728.0, "completions/min_terminated_length": 5728.0, "entropy": 0.6937508285045624, "epoch": 0.020519599954017705, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16910732.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001392364501953, "sampling/importance_sampling_ratio/min": 0.4836191236972809, "sampling/sampling_logp_difference/max": 0.8179230690002441, "sampling/sampling_logp_difference/mean": 0.027213335037231445, "step": 357 }, { "clip_ratio/high_max": 9.627788131183479e-05, "clip_ratio/high_mean": 9.627788131183479e-05, "clip_ratio/low_mean": 0.0003370559716131538, "clip_ratio/low_min": 0.0003370559716131538, "clip_ratio/region_mean": 0.0004333338529249886, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13925.0, "completions/mean_length": 11331.5, "completions/mean_terminated_length": 9647.333984375, "completions/min_length": 5608.0, "completions/min_terminated_length": 5608.0, "entropy": 0.3750923369079828, "epoch": 0.02057707782503736, "frac_reward_zero_std": 0.0, "grad_norm": 0.023595118895173073, "learning_rate": 1e-05, "loss": 0.2705, "num_tokens": 17002504.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999678909778595, "sampling/importance_sampling_ratio/min": 0.310112327337265, "sampling/sampling_logp_difference/max": 1.170820713043213, "sampling/sampling_logp_difference/mean": 0.017307985574007034, "step": 358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9512.0, "completions/max_terminated_length": 9512.0, "completions/mean_length": 4008.875, "completions/mean_terminated_length": 4008.875, "completions/min_length": 1897.0, "completions/min_terminated_length": 1897.0, "entropy": 0.603480126708746, "epoch": 0.020634555696057016, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17035511.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7548553943634033, "sampling/importance_sampling_ratio/mean": 0.9997280240058899, "sampling/importance_sampling_ratio/min": 0.48254984617233276, "sampling/sampling_logp_difference/max": 0.7286710739135742, "sampling/sampling_logp_difference/mean": 0.022598395124077797, "step": 359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15712.0, "completions/max_terminated_length": 15712.0, "completions/mean_length": 12256.25, "completions/mean_terminated_length": 12256.25, "completions/min_length": 7515.0, "completions/min_terminated_length": 7515.0, "entropy": 0.9838720113039017, "epoch": 0.020692033567076676, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17135345.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.00010347366333, "sampling/importance_sampling_ratio/min": 0.2992424964904785, "sampling/sampling_logp_difference/max": 1.2065010070800781, "sampling/sampling_logp_difference/mean": 0.0314478874206543, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3067.0, "completions/max_terminated_length": 3067.0, "completions/mean_length": 2019.5, "completions/mean_terminated_length": 2019.5, "completions/min_length": 865.0, "completions/min_terminated_length": 865.0, "entropy": 0.4162302501499653, "epoch": 0.020749511438096332, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17152701.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4723769426345825, "sampling/importance_sampling_ratio/mean": 0.9998703598976135, "sampling/importance_sampling_ratio/min": 0.6035754084587097, "sampling/sampling_logp_difference/max": 0.5048842430114746, "sampling/sampling_logp_difference/mean": 0.018714936450123787, "step": 361 }, { "clip_ratio/high_max": 5.953218715148978e-05, "clip_ratio/high_mean": 5.953218715148978e-05, "clip_ratio/low_mean": 0.0005276044685160741, "clip_ratio/low_min": 0.0005276044685160741, "clip_ratio/region_mean": 0.0005871366556675639, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15048.0, "completions/mean_length": 12956.375, "completions/mean_terminated_length": 9528.75, "completions/min_length": 6243.0, "completions/min_terminated_length": 6243.0, "entropy": 0.44832662865519524, "epoch": 0.02080698930911599, "frac_reward_zero_std": 0.0, "grad_norm": 0.01465114951133728, "learning_rate": 1e-05, "loss": 0.2945, "num_tokens": 17258872.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999077916145325, "sampling/importance_sampling_ratio/min": 0.3361400067806244, "sampling/sampling_logp_difference/max": 1.154249668121338, "sampling/sampling_logp_difference/mean": 0.01987970992922783, "step": 362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10306.0, "completions/max_terminated_length": 10306.0, "completions/mean_length": 4658.25, "completions/mean_terminated_length": 4658.25, "completions/min_length": 1282.0, "completions/min_terminated_length": 1282.0, "entropy": 0.6721976362168789, "epoch": 0.020864467180135647, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17297490.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7022991180419922, "sampling/importance_sampling_ratio/mean": 1.0000230073928833, "sampling/importance_sampling_ratio/min": 0.0958167016506195, "sampling/sampling_logp_difference/max": 2.34531831741333, "sampling/sampling_logp_difference/mean": 0.02582489140331745, "step": 363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6522.0, "completions/max_terminated_length": 6522.0, "completions/mean_length": 2912.0, "completions/mean_terminated_length": 2912.0, "completions/min_length": 2044.0, "completions/min_terminated_length": 2044.0, "entropy": 0.22699221409857273, "epoch": 0.020921945051155307, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17321722.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6960196495056152, "sampling/importance_sampling_ratio/mean": 1.0000361204147339, "sampling/importance_sampling_ratio/min": 0.4738141596317291, "sampling/sampling_logp_difference/max": 0.7469401359558105, "sampling/sampling_logp_difference/mean": 0.010224299505352974, "step": 364 }, { "clip_ratio/high_max": 7.319757969526108e-05, "clip_ratio/high_mean": 7.319757969526108e-05, "clip_ratio/low_mean": 0.00011551292845979333, "clip_ratio/low_min": 0.00011551292845979333, "clip_ratio/region_mean": 0.0001887105081550544, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 8141.0, "completions/mean_length": 7507.125, "completions/mean_terminated_length": 6239.00048828125, "completions/min_length": 3957.0, "completions/min_terminated_length": 3957.0, "entropy": 0.3593865130096674, "epoch": 0.020979422922174962, "frac_reward_zero_std": 0.0, "grad_norm": 0.010079027153551579, "learning_rate": 1e-05, "loss": 0.1919, "num_tokens": 17383611.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6150532960891724, "sampling/importance_sampling_ratio/mean": 0.9999769926071167, "sampling/importance_sampling_ratio/min": 0.15059253573417664, "sampling/sampling_logp_difference/max": 1.8931775093078613, "sampling/sampling_logp_difference/mean": 0.016810156404972076, "step": 365 }, { "clip_ratio/high_max": 7.529172580689192e-05, "clip_ratio/high_mean": 7.529172580689192e-05, "clip_ratio/low_mean": 0.0001947502387338318, "clip_ratio/low_min": 0.0001947502387338318, "clip_ratio/region_mean": 0.0002700419645407237, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12793.0, "completions/mean_length": 10771.375, "completions/mean_terminated_length": 9969.572265625, "completions/min_length": 6930.0, "completions/min_terminated_length": 6930.0, "entropy": 0.7980213016271591, "epoch": 0.02103690079319462, "frac_reward_zero_std": 0.0, "grad_norm": 0.010477994568645954, "learning_rate": 1e-05, "loss": 0.1912, "num_tokens": 17470710.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9356058835983276, "sampling/importance_sampling_ratio/mean": 1.0001732110977173, "sampling/importance_sampling_ratio/min": 0.43703436851501465, "sampling/sampling_logp_difference/max": 0.8277435302734375, "sampling/sampling_logp_difference/mean": 0.025370774790644646, "step": 366 }, { "clip_ratio/high_max": 6.0065296565881e-05, "clip_ratio/high_mean": 6.0065296565881e-05, "clip_ratio/low_mean": 0.0005909008614253253, "clip_ratio/low_min": 0.0005909008614253253, "clip_ratio/region_mean": 0.0006509661579912063, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15605.0, "completions/mean_length": 9916.125, "completions/mean_terminated_length": 8992.1435546875, "completions/min_length": 4208.0, "completions/min_terminated_length": 4208.0, "entropy": 0.45364048704504967, "epoch": 0.021094378664214278, "frac_reward_zero_std": 0.0, "grad_norm": 0.008153869770467281, "learning_rate": 1e-05, "loss": -0.0246, "num_tokens": 17551735.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997556209564209, "sampling/importance_sampling_ratio/min": 0.268157958984375, "sampling/sampling_logp_difference/max": 1.3161790370941162, "sampling/sampling_logp_difference/mean": 0.019931312650442123, "step": 367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4067.0, "completions/max_terminated_length": 4067.0, "completions/mean_length": 2725.0, "completions/mean_terminated_length": 2725.0, "completions/min_length": 1576.0, "completions/min_terminated_length": 1576.0, "entropy": 0.5497967749834061, "epoch": 0.021151856535233934, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17574487.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6456069946289062, "sampling/importance_sampling_ratio/mean": 1.00068199634552, "sampling/importance_sampling_ratio/min": 0.609940230846405, "sampling/sampling_logp_difference/max": 0.4981093406677246, "sampling/sampling_logp_difference/mean": 0.021459870040416718, "step": 368 }, { "clip_ratio/high_max": 9.761245200934354e-05, "clip_ratio/high_mean": 9.761245200934354e-05, "clip_ratio/low_mean": 0.00023482833057641983, "clip_ratio/low_min": 0.00023482833057641983, "clip_ratio/region_mean": 0.00033244078258576337, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15704.0, "completions/mean_length": 11187.5, "completions/mean_terminated_length": 10445.1435546875, "completions/min_length": 4421.0, "completions/min_terminated_length": 4421.0, "entropy": 0.37645208090543747, "epoch": 0.021209334406253593, "frac_reward_zero_std": 0.0, "grad_norm": 0.03603425621986389, "learning_rate": 1e-05, "loss": 0.2343, "num_tokens": 17665315.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999097585678101, "sampling/importance_sampling_ratio/min": 0.16250598430633545, "sampling/sampling_logp_difference/max": 1.8170404434204102, "sampling/sampling_logp_difference/mean": 0.018346307799220085, "step": 369 }, { "clip_ratio/high_max": 1.8873623048420995e-05, "clip_ratio/high_mean": 1.8873623048420995e-05, "clip_ratio/low_mean": 0.00012164050713181496, "clip_ratio/low_min": 0.00012164050713181496, "clip_ratio/region_mean": 0.00014051413018023595, "completions/clipped_ratio": 0.0, "completions/max_length": 12125.0, "completions/max_terminated_length": 12125.0, "completions/mean_length": 5782.25, "completions/mean_terminated_length": 5782.25, "completions/min_length": 1970.0, "completions/min_terminated_length": 1970.0, "entropy": 0.6423214338719845, "epoch": 0.02126681227727325, "frac_reward_zero_std": 0.0, "grad_norm": 0.02861444652080536, "learning_rate": 1e-05, "loss": -0.0707, "num_tokens": 17712965.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8723466396331787, "sampling/importance_sampling_ratio/mean": 0.9999862909317017, "sampling/importance_sampling_ratio/min": 0.4887623190879822, "sampling/sampling_logp_difference/max": 0.715878963470459, "sampling/sampling_logp_difference/mean": 0.018169758841395378, "step": 370 }, { "clip_ratio/high_max": 0.0002698010503081605, "clip_ratio/high_mean": 0.0002698010503081605, "clip_ratio/low_mean": 0.00021440846467157826, "clip_ratio/low_min": 0.00021440846467157826, "clip_ratio/region_mean": 0.00048420951497973874, "completions/clipped_ratio": 0.0, "completions/max_length": 2882.0, "completions/max_terminated_length": 2882.0, "completions/mean_length": 2059.5, "completions/mean_terminated_length": 2059.5, "completions/min_length": 864.0, "completions/min_terminated_length": 864.0, "entropy": 0.5068723000586033, "epoch": 0.02132429014829291, "frac_reward_zero_std": 0.0, "grad_norm": 0.08042344450950623, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 17730601.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5238127708435059, "sampling/importance_sampling_ratio/mean": 0.9996336698532104, "sampling/importance_sampling_ratio/min": 0.5397042036056519, "sampling/sampling_logp_difference/max": 0.6167340278625488, "sampling/sampling_logp_difference/mean": 0.021951928734779358, "step": 371 }, { "clip_ratio/high_max": 1.83769479917828e-05, "clip_ratio/high_mean": 1.83769479917828e-05, "clip_ratio/low_mean": 0.00040829650970408693, "clip_ratio/low_min": 0.00040829650970408693, "clip_ratio/region_mean": 0.00042667345769586973, "completions/clipped_ratio": 0.0, "completions/max_length": 9130.0, "completions/max_terminated_length": 9130.0, "completions/mean_length": 6032.375, "completions/mean_terminated_length": 6032.375, "completions/min_length": 3842.0, "completions/min_terminated_length": 3842.0, "entropy": 0.3433102909475565, "epoch": 0.021381768019312564, "frac_reward_zero_std": 0.0, "grad_norm": 0.022114230319857597, "learning_rate": 1e-05, "loss": 0.1585, "num_tokens": 17780132.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.5154314041137695, "sampling/importance_sampling_ratio/mean": 1.0000022649765015, "sampling/importance_sampling_ratio/min": 0.5724499821662903, "sampling/sampling_logp_difference/max": 0.5578298568725586, "sampling/sampling_logp_difference/mean": 0.014348246157169342, "step": 372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.632597018731758e-05, "clip_ratio/low_min": 8.632597018731758e-05, "clip_ratio/region_mean": 8.632597018731758e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 3596.0, "completions/max_terminated_length": 3596.0, "completions/mean_length": 2008.5, "completions/mean_terminated_length": 2008.5, "completions/min_length": 964.0, "completions/min_terminated_length": 964.0, "entropy": 0.3825047891587019, "epoch": 0.021439245890332224, "frac_reward_zero_std": 0.0, "grad_norm": 0.024160167202353477, "learning_rate": 1e-05, "loss": -0.2159, "num_tokens": 17797496.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4042750597000122, "sampling/importance_sampling_ratio/mean": 1.0002102851867676, "sampling/importance_sampling_ratio/min": 0.7001754641532898, "sampling/sampling_logp_difference/max": 0.35642433166503906, "sampling/sampling_logp_difference/mean": 0.013086330145597458, "step": 373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10115.0, "completions/max_terminated_length": 10115.0, "completions/mean_length": 5735.125, "completions/mean_terminated_length": 5735.125, "completions/min_length": 3126.0, "completions/min_terminated_length": 3126.0, "entropy": 0.9846834391355515, "epoch": 0.02149672376135188, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17844697.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001513957977295, "sampling/importance_sampling_ratio/min": 0.41287025809288025, "sampling/sampling_logp_difference/max": 0.8846219182014465, "sampling/sampling_logp_difference/mean": 0.02507157251238823, "step": 374 }, { "clip_ratio/high_max": 0.00012527878880064236, "clip_ratio/high_mean": 0.00012527878880064236, "clip_ratio/low_mean": 0.00017507003212813288, "clip_ratio/low_min": 0.00017507003212813288, "clip_ratio/region_mean": 0.00030034882092877524, "completions/clipped_ratio": 0.0, "completions/max_length": 10439.0, "completions/max_terminated_length": 10439.0, "completions/mean_length": 6156.875, "completions/mean_terminated_length": 6156.875, "completions/min_length": 2142.0, "completions/min_terminated_length": 2142.0, "entropy": 0.846487246453762, "epoch": 0.021554201632371536, "frac_reward_zero_std": 0.0, "grad_norm": 0.01263112761080265, "learning_rate": 1e-05, "loss": -0.2307, "num_tokens": 17895160.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6920719146728516, "sampling/importance_sampling_ratio/mean": 0.9999474287033081, "sampling/importance_sampling_ratio/min": 0.2458421289920807, "sampling/sampling_logp_difference/max": 1.4030656814575195, "sampling/sampling_logp_difference/mean": 0.028860243037343025, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7747.0, "completions/max_terminated_length": 7747.0, "completions/mean_length": 4877.375, "completions/mean_terminated_length": 4877.375, "completions/min_length": 2300.0, "completions/min_terminated_length": 2300.0, "entropy": 0.3635020200163126, "epoch": 0.021611679503391195, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17935579.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6226881742477417, "sampling/importance_sampling_ratio/mean": 0.9995774626731873, "sampling/importance_sampling_ratio/min": 0.5516060590744019, "sampling/sampling_logp_difference/max": 0.5949211120605469, "sampling/sampling_logp_difference/mean": 0.01612044870853424, "step": 376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013702968863071874, "clip_ratio/low_min": 0.00013702968863071874, "clip_ratio/region_mean": 0.00013702968863071874, "completions/clipped_ratio": 0.0, "completions/max_length": 6776.0, "completions/max_terminated_length": 6776.0, "completions/mean_length": 3584.125, "completions/mean_terminated_length": 3584.125, "completions/min_length": 1255.0, "completions/min_terminated_length": 1255.0, "entropy": 1.4472458511590958, "epoch": 0.02166915737441085, "frac_reward_zero_std": 0.0, "grad_norm": 0.09286519885063171, "learning_rate": 1e-05, "loss": -0.1914, "num_tokens": 17965828.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4383729696273804, "sampling/importance_sampling_ratio/mean": 0.9994940161705017, "sampling/importance_sampling_ratio/min": 0.530176043510437, "sampling/sampling_logp_difference/max": 0.6345462799072266, "sampling/sampling_logp_difference/mean": 0.03094877488911152, "step": 377 }, { "clip_ratio/high_max": 8.652422911836766e-05, "clip_ratio/high_mean": 8.652422911836766e-05, "clip_ratio/low_mean": 8.914233876566868e-05, "clip_ratio/low_min": 8.914233876566868e-05, "clip_ratio/region_mean": 0.00017566656788403634, "completions/clipped_ratio": 0.0, "completions/max_length": 8465.0, "completions/max_terminated_length": 8465.0, "completions/mean_length": 4396.375, "completions/mean_terminated_length": 4396.375, "completions/min_length": 2541.0, "completions/min_terminated_length": 2541.0, "entropy": 0.845998540520668, "epoch": 0.02172663524543051, "frac_reward_zero_std": 0.0, "grad_norm": 0.07467586547136307, "learning_rate": 1e-05, "loss": 0.2373, "num_tokens": 18002471.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000317931175232, "sampling/importance_sampling_ratio/min": 0.4164700210094452, "sampling/sampling_logp_difference/max": 0.9122521877288818, "sampling/sampling_logp_difference/mean": 0.022327328100800514, "step": 378 }, { "clip_ratio/high_max": 7.681223723920994e-05, "clip_ratio/high_mean": 7.681223723920994e-05, "clip_ratio/low_mean": 0.0002786066252156161, "clip_ratio/low_min": 0.0002786066252156161, "clip_ratio/region_mean": 0.000355418862454826, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12453.0, "completions/mean_length": 10504.0, "completions/mean_terminated_length": 9664.0, "completions/min_length": 6857.0, "completions/min_terminated_length": 6857.0, "entropy": 0.43550832010805607, "epoch": 0.021784113116450166, "frac_reward_zero_std": 0.0, "grad_norm": 0.007709996774792671, "learning_rate": 1e-05, "loss": 0.2212, "num_tokens": 18087655.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000183343887329, "sampling/importance_sampling_ratio/min": 0.5077204704284668, "sampling/sampling_logp_difference/max": 0.8771915435791016, "sampling/sampling_logp_difference/mean": 0.018277566879987717, "step": 379 }, { "clip_ratio/high_max": 4.323030952946283e-05, "clip_ratio/high_mean": 4.323030952946283e-05, "clip_ratio/low_mean": 0.0002932506122306222, "clip_ratio/low_min": 0.0002932506122306222, "clip_ratio/region_mean": 0.00033648092176008504, "completions/clipped_ratio": 0.0, "completions/max_length": 9631.0, "completions/max_terminated_length": 9631.0, "completions/mean_length": 7189.0, "completions/mean_terminated_length": 7189.0, "completions/min_length": 4571.0, "completions/min_terminated_length": 4571.0, "entropy": 0.8732515946030617, "epoch": 0.021841590987469826, "frac_reward_zero_std": 0.0, "grad_norm": 0.021911537274718285, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 18146159.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6794453859329224, "sampling/importance_sampling_ratio/mean": 0.9997799396514893, "sampling/importance_sampling_ratio/min": 0.287708044052124, "sampling/sampling_logp_difference/max": 1.2458090782165527, "sampling/sampling_logp_difference/mean": 0.02427300438284874, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8625.0, "completions/max_terminated_length": 8625.0, "completions/mean_length": 5537.0, "completions/mean_terminated_length": 5537.0, "completions/min_length": 3167.0, "completions/min_terminated_length": 3167.0, "entropy": 0.8039069324731827, "epoch": 0.02189906885848948, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18191703.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998453855514526, "sampling/importance_sampling_ratio/min": 0.5594327449798584, "sampling/sampling_logp_difference/max": 0.7885527610778809, "sampling/sampling_logp_difference/mean": 0.030826354399323463, "step": 381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2796.0, "completions/max_terminated_length": 2796.0, "completions/mean_length": 2149.625, "completions/mean_terminated_length": 2149.625, "completions/min_length": 1581.0, "completions/min_terminated_length": 1581.0, "entropy": 0.14457772858440876, "epoch": 0.021956546729509138, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18210036.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2777059078216553, "sampling/importance_sampling_ratio/mean": 0.9998260140419006, "sampling/importance_sampling_ratio/min": 0.6714066863059998, "sampling/sampling_logp_difference/max": 0.3983802795410156, "sampling/sampling_logp_difference/mean": 0.006943520158529282, "step": 382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10632.0, "completions/max_terminated_length": 10632.0, "completions/mean_length": 9324.0, "completions/mean_terminated_length": 9324.0, "completions/min_length": 7091.0, "completions/min_terminated_length": 7091.0, "entropy": 1.1208415552973747, "epoch": 0.022014024600528797, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18285828.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999837875366211, "sampling/importance_sampling_ratio/min": 0.19099198281764984, "sampling/sampling_logp_difference/max": 1.6555237770080566, "sampling/sampling_logp_difference/mean": 0.034007880836725235, "step": 383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5809.0, "completions/max_terminated_length": 5809.0, "completions/mean_length": 3529.25, "completions/mean_terminated_length": 3529.25, "completions/min_length": 2132.0, "completions/min_terminated_length": 2132.0, "entropy": 0.365608349442482, "epoch": 0.022071502471548453, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18314958.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7880229949951172, "sampling/importance_sampling_ratio/mean": 0.9999485015869141, "sampling/importance_sampling_ratio/min": 0.6207014322280884, "sampling/sampling_logp_difference/max": 0.5811104774475098, "sampling/sampling_logp_difference/mean": 0.012056337669491768, "step": 384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000892564799869433, "clip_ratio/low_min": 0.000892564799869433, "clip_ratio/region_mean": 0.000892564799869433, "completions/clipped_ratio": 0.0, "completions/max_length": 9224.0, "completions/max_terminated_length": 9224.0, "completions/mean_length": 5925.625, "completions/mean_terminated_length": 5925.625, "completions/min_length": 2486.0, "completions/min_terminated_length": 2486.0, "entropy": 0.4147707037627697, "epoch": 0.022128980342568112, "frac_reward_zero_std": 0.0, "grad_norm": 0.013555883429944515, "learning_rate": 1e-05, "loss": -0.0184, "num_tokens": 18363507.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6463449001312256, "sampling/importance_sampling_ratio/mean": 1.0002241134643555, "sampling/importance_sampling_ratio/min": 0.19371020793914795, "sampling/sampling_logp_difference/max": 1.6413919925689697, "sampling/sampling_logp_difference/mean": 0.017880231142044067, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2853.0, "completions/max_terminated_length": 2853.0, "completions/mean_length": 2532.375, "completions/mean_terminated_length": 2532.375, "completions/min_length": 2126.0, "completions/min_terminated_length": 2126.0, "entropy": 0.22236023470759392, "epoch": 0.022186458213587768, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18384830.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.325555443763733, "sampling/importance_sampling_ratio/mean": 1.000349760055542, "sampling/importance_sampling_ratio/min": 0.6102064251899719, "sampling/sampling_logp_difference/max": 0.4939579963684082, "sampling/sampling_logp_difference/mean": 0.009665180929005146, "step": 386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.0, "completions/max_terminated_length": 820.0, "completions/mean_length": 607.375, "completions/mean_terminated_length": 607.375, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.30915217101573944, "epoch": 0.022243936084607428, "frac_reward_zero_std": 0.0, "grad_norm": 0.10359089821577072, "learning_rate": 1e-05, "loss": -0.0848, "num_tokens": 18390657.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.315879464149475, "sampling/importance_sampling_ratio/mean": 0.9994267225265503, "sampling/importance_sampling_ratio/min": 0.6805714964866638, "sampling/sampling_logp_difference/max": 0.38482236862182617, "sampling/sampling_logp_difference/mean": 0.013176491484045982, "step": 387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15622.0, "completions/mean_length": 14695.375, "completions/mean_terminated_length": 14454.1435546875, "completions/min_length": 13439.0, "completions/min_terminated_length": 13439.0, "entropy": 0.43820831179618835, "epoch": 0.022301413955627084, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18509676.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999627470970154, "sampling/importance_sampling_ratio/min": 0.06175379827618599, "sampling/sampling_logp_difference/max": 2.784599781036377, "sampling/sampling_logp_difference/mean": 0.019379397854208946, "step": 388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8079.0, "completions/max_terminated_length": 8079.0, "completions/mean_length": 5298.125, "completions/mean_terminated_length": 5298.125, "completions/min_length": 3780.0, "completions/min_terminated_length": 3780.0, "entropy": 0.19917121902108192, "epoch": 0.02235889182664674, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18552909.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7135921716690063, "sampling/importance_sampling_ratio/mean": 1.0001752376556396, "sampling/importance_sampling_ratio/min": 0.6549367904663086, "sampling/sampling_logp_difference/max": 0.5385918617248535, "sampling/sampling_logp_difference/mean": 0.008984100073575974, "step": 389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5017.0, "completions/max_terminated_length": 5017.0, "completions/mean_length": 3152.875, "completions/mean_terminated_length": 3152.875, "completions/min_length": 1652.0, "completions/min_terminated_length": 1652.0, "entropy": 0.5698305554687977, "epoch": 0.0224163696976664, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18578932.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4063564538955688, "sampling/importance_sampling_ratio/mean": 0.9999165534973145, "sampling/importance_sampling_ratio/min": 0.5449109673500061, "sampling/sampling_logp_difference/max": 0.6071329116821289, "sampling/sampling_logp_difference/mean": 0.022845176979899406, "step": 390 }, { "clip_ratio/high_max": 6.760654105164576e-05, "clip_ratio/high_mean": 6.760654105164576e-05, "clip_ratio/low_mean": 0.00014923234266461805, "clip_ratio/low_min": 0.00014923234266461805, "clip_ratio/region_mean": 0.0002168388837162638, "completions/clipped_ratio": 0.0, "completions/max_length": 7397.0, "completions/max_terminated_length": 7397.0, "completions/mean_length": 4748.125, "completions/mean_terminated_length": 4748.125, "completions/min_length": 1977.0, "completions/min_terminated_length": 1977.0, "entropy": 0.4516385570168495, "epoch": 0.022473847568686055, "frac_reward_zero_std": 0.0, "grad_norm": 0.009269390255212784, "learning_rate": 1e-05, "loss": -0.0142, "num_tokens": 18621637.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9333773851394653, "sampling/importance_sampling_ratio/mean": 1.0000908374786377, "sampling/importance_sampling_ratio/min": 0.19427525997161865, "sampling/sampling_logp_difference/max": 1.638479232788086, "sampling/sampling_logp_difference/mean": 0.019084813073277473, "step": 391 }, { "clip_ratio/high_max": 8.494792746205349e-05, "clip_ratio/high_mean": 8.494792746205349e-05, "clip_ratio/low_mean": 0.00021836556697962806, "clip_ratio/low_min": 0.00021836556697962806, "clip_ratio/region_mean": 0.00030331349444168154, "completions/clipped_ratio": 0.0, "completions/max_length": 14989.0, "completions/max_terminated_length": 14989.0, "completions/mean_length": 9727.0, "completions/mean_terminated_length": 9727.0, "completions/min_length": 5404.0, "completions/min_terminated_length": 5404.0, "entropy": 0.7589566335082054, "epoch": 0.022531325439705714, "frac_reward_zero_std": 0.0, "grad_norm": 0.01131840143352747, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 18700517.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997626543045044, "sampling/importance_sampling_ratio/min": 0.40493497252464294, "sampling/sampling_logp_difference/max": 0.9040288925170898, "sampling/sampling_logp_difference/mean": 0.025711549445986748, "step": 392 }, { "clip_ratio/high_max": 4.1948918806156144e-05, "clip_ratio/high_mean": 4.1948918806156144e-05, "clip_ratio/low_mean": 7.62939453125e-05, "clip_ratio/low_min": 7.62939453125e-05, "clip_ratio/region_mean": 0.00011824286411865614, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 7665.0, "completions/mean_length": 5646.625, "completions/mean_terminated_length": 4112.71435546875, "completions/min_length": 2722.0, "completions/min_terminated_length": 2722.0, "entropy": 0.3640553914010525, "epoch": 0.02258880331072537, "frac_reward_zero_std": 0.0, "grad_norm": 0.0058986893855035305, "learning_rate": 1e-05, "loss": 0.6721, "num_tokens": 18746890.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9096522331237793, "sampling/importance_sampling_ratio/mean": 0.9995434880256653, "sampling/importance_sampling_ratio/min": 0.13865266740322113, "sampling/sampling_logp_difference/max": 1.9757832288742065, "sampling/sampling_logp_difference/mean": 0.01803496852517128, "step": 393 }, { "clip_ratio/high_max": 8.52718403621111e-05, "clip_ratio/high_mean": 8.52718403621111e-05, "clip_ratio/low_mean": 1.787501787475776e-05, "clip_ratio/low_min": 1.787501787475776e-05, "clip_ratio/region_mean": 0.00010314685823686887, "completions/clipped_ratio": 0.0, "completions/max_length": 6993.0, "completions/max_terminated_length": 6993.0, "completions/mean_length": 4168.625, "completions/mean_terminated_length": 4168.625, "completions/min_length": 2657.0, "completions/min_terminated_length": 2657.0, "entropy": 0.5951997227966785, "epoch": 0.02264628118174503, "frac_reward_zero_std": 0.0, "grad_norm": 0.0068208263255655766, "learning_rate": 1e-05, "loss": 0.2391, "num_tokens": 18780935.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.404531478881836, "sampling/importance_sampling_ratio/mean": 1.0002244710922241, "sampling/importance_sampling_ratio/min": 0.5984583497047424, "sampling/sampling_logp_difference/max": 0.5133984088897705, "sampling/sampling_logp_difference/mean": 0.01755034737288952, "step": 394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 16136.0, "completions/mean_length": 13842.25, "completions/mean_terminated_length": 12317.2001953125, "completions/min_length": 6477.0, "completions/min_terminated_length": 6477.0, "entropy": 0.5238826386630535, "epoch": 0.022703759052764685, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18893713.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999638795852661, "sampling/importance_sampling_ratio/min": 0.25419166684150696, "sampling/sampling_logp_difference/max": 1.3696666955947876, "sampling/sampling_logp_difference/mean": 0.02123842015862465, "step": 395 }, { "clip_ratio/high_max": 0.00013980714356875978, "clip_ratio/high_mean": 0.00013980714356875978, "clip_ratio/low_mean": 0.0002534045561333187, "clip_ratio/low_min": 0.0002534045561333187, "clip_ratio/region_mean": 0.0003932116997020785, "completions/clipped_ratio": 0.0, "completions/max_length": 11665.0, "completions/max_terminated_length": 11665.0, "completions/mean_length": 6654.5, "completions/mean_terminated_length": 6654.5, "completions/min_length": 2381.0, "completions/min_terminated_length": 2381.0, "entropy": 0.7207529321312904, "epoch": 0.02276123692378434, "frac_reward_zero_std": 0.0, "grad_norm": 0.014938006177544594, "learning_rate": 1e-05, "loss": 0.4181, "num_tokens": 18948141.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6153500080108643, "sampling/importance_sampling_ratio/mean": 1.0001254081726074, "sampling/importance_sampling_ratio/min": 0.1812034398317337, "sampling/sampling_logp_difference/max": 1.7081348896026611, "sampling/sampling_logp_difference/mean": 0.028049036860466003, "step": 396 }, { "clip_ratio/high_max": 0.00010434056457597762, "clip_ratio/high_mean": 0.00010434056457597762, "clip_ratio/low_mean": 0.00019210199388908222, "clip_ratio/low_min": 0.00019210199388908222, "clip_ratio/region_mean": 0.00029644255846505985, "completions/clipped_ratio": 0.0, "completions/max_length": 4322.0, "completions/max_terminated_length": 4322.0, "completions/mean_length": 2145.0, "completions/mean_terminated_length": 2145.0, "completions/min_length": 957.0, "completions/min_terminated_length": 957.0, "entropy": 1.2026251778006554, "epoch": 0.022818714794804, "frac_reward_zero_std": 0.0, "grad_norm": 0.115672767162323, "learning_rate": 1e-05, "loss": 0.1559, "num_tokens": 18967597.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4467931985855103, "sampling/importance_sampling_ratio/mean": 1.0001999139785767, "sampling/importance_sampling_ratio/min": 0.32789283990859985, "sampling/sampling_logp_difference/max": 1.1150684356689453, "sampling/sampling_logp_difference/mean": 0.028453677892684937, "step": 397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1266.0, "completions/max_terminated_length": 1266.0, "completions/mean_length": 978.625, "completions/mean_terminated_length": 978.625, "completions/min_length": 561.0, "completions/min_terminated_length": 561.0, "entropy": 0.33559664711356163, "epoch": 0.022876192665823657, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18976562.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3459874391555786, "sampling/importance_sampling_ratio/mean": 0.9993994235992432, "sampling/importance_sampling_ratio/min": 0.5799973011016846, "sampling/sampling_logp_difference/max": 0.544731855392456, "sampling/sampling_logp_difference/mean": 0.01532378327101469, "step": 398 }, { "clip_ratio/high_max": 4.535308380582137e-05, "clip_ratio/high_mean": 4.535308380582137e-05, "clip_ratio/low_mean": 0.0008447676373180002, "clip_ratio/low_min": 0.0008447676373180002, "clip_ratio/region_mean": 0.0008901207211238216, "completions/clipped_ratio": 0.0, "completions/max_length": 15105.0, "completions/max_terminated_length": 15105.0, "completions/mean_length": 12117.5, "completions/mean_terminated_length": 12117.5, "completions/min_length": 9188.0, "completions/min_terminated_length": 9188.0, "entropy": 0.964058130979538, "epoch": 0.022933670536843316, "frac_reward_zero_std": 0.0, "grad_norm": 0.029942598193883896, "learning_rate": 1e-05, "loss": 0.0397, "num_tokens": 19075086.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9400761127471924, "sampling/importance_sampling_ratio/mean": 0.9999693632125854, "sampling/importance_sampling_ratio/min": 0.12511885166168213, "sampling/sampling_logp_difference/max": 2.0784912109375, "sampling/sampling_logp_difference/mean": 0.03425825387239456, "step": 399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5271.0, "completions/max_terminated_length": 5271.0, "completions/mean_length": 3118.0, "completions/mean_terminated_length": 3118.0, "completions/min_length": 1730.0, "completions/min_terminated_length": 1730.0, "entropy": 0.3079453557729721, "epoch": 0.022991148407862972, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19101014.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.570227861404419, "sampling/importance_sampling_ratio/mean": 1.0002251863479614, "sampling/importance_sampling_ratio/min": 0.6216328144073486, "sampling/sampling_logp_difference/max": 0.4754056930541992, "sampling/sampling_logp_difference/mean": 0.014316723681986332, "step": 400 }, { "clip_ratio/high_max": 5.466923084895825e-05, "clip_ratio/high_mean": 5.466923084895825e-05, "clip_ratio/low_mean": 0.0005235423232079484, "clip_ratio/low_min": 0.0005235423232079484, "clip_ratio/region_mean": 0.0005782115540569066, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13242.0, "completions/mean_length": 9912.0, "completions/mean_terminated_length": 8987.4287109375, "completions/min_length": 5808.0, "completions/min_terminated_length": 5808.0, "entropy": 0.7004808411002159, "epoch": 0.02304862627888263, "frac_reward_zero_std": 0.0, "grad_norm": 0.021963581442832947, "learning_rate": 1e-05, "loss": 0.0552, "num_tokens": 19181342.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002297163009644, "sampling/importance_sampling_ratio/min": 0.15124423801898956, "sampling/sampling_logp_difference/max": 1.8888592720031738, "sampling/sampling_logp_difference/mean": 0.023938024416565895, "step": 401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4148.0, "completions/max_terminated_length": 4148.0, "completions/mean_length": 3192.625, "completions/mean_terminated_length": 3192.625, "completions/min_length": 2401.0, "completions/min_terminated_length": 2401.0, "entropy": 0.5008226558566093, "epoch": 0.023106104149902287, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19208043.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3460251092910767, "sampling/importance_sampling_ratio/mean": 1.0003421306610107, "sampling/importance_sampling_ratio/min": 0.5640384554862976, "sampling/sampling_logp_difference/max": 0.5726327896118164, "sampling/sampling_logp_difference/mean": 0.019691096618771553, "step": 402 }, { "clip_ratio/high_max": 2.632687392178923e-05, "clip_ratio/high_mean": 2.632687392178923e-05, "clip_ratio/low_mean": 0.00033804898339440115, "clip_ratio/low_min": 0.00033804898339440115, "clip_ratio/region_mean": 0.0003643758573161904, "completions/clipped_ratio": 0.0, "completions/max_length": 6754.0, "completions/max_terminated_length": 6754.0, "completions/mean_length": 4741.875, "completions/mean_terminated_length": 4741.875, "completions/min_length": 3627.0, "completions/min_terminated_length": 3627.0, "entropy": 1.0446897074580193, "epoch": 0.023163582020921947, "frac_reward_zero_std": 0.0, "grad_norm": 0.027849644422531128, "learning_rate": 1e-05, "loss": 0.069, "num_tokens": 19247746.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.5717452764511108, "sampling/importance_sampling_ratio/mean": 1.0000735521316528, "sampling/importance_sampling_ratio/min": 0.5520835518836975, "sampling/sampling_logp_difference/max": 0.5940558910369873, "sampling/sampling_logp_difference/mean": 0.03194881230592728, "step": 403 }, { "clip_ratio/high_max": 0.0001590799347468419, "clip_ratio/high_mean": 0.0001590799347468419, "clip_ratio/low_mean": 7.156488572945818e-05, "clip_ratio/low_min": 7.156488572945818e-05, "clip_ratio/region_mean": 0.00023064482047630008, "completions/clipped_ratio": 0.0, "completions/max_length": 10480.0, "completions/max_terminated_length": 10480.0, "completions/mean_length": 5560.625, "completions/mean_terminated_length": 5560.625, "completions/min_length": 3504.0, "completions/min_terminated_length": 3504.0, "entropy": 0.3376346956938505, "epoch": 0.023221059891941603, "frac_reward_zero_std": 0.0, "grad_norm": 0.00477626221254468, "learning_rate": 1e-05, "loss": 0.3124, "num_tokens": 19293247.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4549741744995117, "sampling/importance_sampling_ratio/mean": 1.0000038146972656, "sampling/importance_sampling_ratio/min": 0.3097218871116638, "sampling/sampling_logp_difference/max": 1.1720805168151855, "sampling/sampling_logp_difference/mean": 0.013973518274724483, "step": 404 }, { "clip_ratio/high_max": 3.463563189143315e-05, "clip_ratio/high_mean": 3.463563189143315e-05, "clip_ratio/low_mean": 0.0003400525311008096, "clip_ratio/low_min": 0.0003400525311008096, "clip_ratio/region_mean": 0.0003746881629922427, "completions/clipped_ratio": 0.0, "completions/max_length": 5307.0, "completions/max_terminated_length": 5307.0, "completions/mean_length": 3020.75, "completions/mean_terminated_length": 3020.75, "completions/min_length": 1991.0, "completions/min_terminated_length": 1991.0, "entropy": 0.31760530546307564, "epoch": 0.02327853776296126, "frac_reward_zero_std": 0.0, "grad_norm": 0.01940259523689747, "learning_rate": 1e-05, "loss": -0.2548, "num_tokens": 19318221.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4378602504730225, "sampling/importance_sampling_ratio/mean": 0.9998482465744019, "sampling/importance_sampling_ratio/min": 0.6357212662696838, "sampling/sampling_logp_difference/max": 0.45299506187438965, "sampling/sampling_logp_difference/mean": 0.013742378912866116, "step": 405 }, { "clip_ratio/high_max": 2.612876232888084e-05, "clip_ratio/high_mean": 2.612876232888084e-05, "clip_ratio/low_mean": 0.00046183585618564393, "clip_ratio/low_min": 0.00046183585618564393, "clip_ratio/region_mean": 0.00048796461851452477, "completions/clipped_ratio": 0.0, "completions/max_length": 10908.0, "completions/max_terminated_length": 10908.0, "completions/mean_length": 8740.75, "completions/mean_terminated_length": 8740.75, "completions/min_length": 5959.0, "completions/min_terminated_length": 5959.0, "entropy": 1.2719523832201958, "epoch": 0.023336015633980918, "frac_reward_zero_std": 0.0, "grad_norm": 0.06205248832702637, "learning_rate": 1e-05, "loss": 0.0602, "num_tokens": 19389651.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000325441360474, "sampling/importance_sampling_ratio/min": 0.3509225845336914, "sampling/sampling_logp_difference/max": 1.0471895933151245, "sampling/sampling_logp_difference/mean": 0.033512599766254425, "step": 406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4998.0, "completions/max_terminated_length": 4998.0, "completions/mean_length": 3092.125, "completions/mean_terminated_length": 3092.125, "completions/min_length": 1531.0, "completions/min_terminated_length": 1531.0, "entropy": 0.2838469259440899, "epoch": 0.023393493505000574, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19415244.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4130116701126099, "sampling/importance_sampling_ratio/mean": 1.0002003908157349, "sampling/importance_sampling_ratio/min": 0.4827224612236023, "sampling/sampling_logp_difference/max": 0.7283134460449219, "sampling/sampling_logp_difference/mean": 0.011639527045190334, "step": 407 }, { "clip_ratio/high_max": 0.00012426235116436146, "clip_ratio/high_mean": 0.00012426235116436146, "clip_ratio/low_mean": 0.0002611955424072221, "clip_ratio/low_min": 0.0002611955424072221, "clip_ratio/region_mean": 0.00038545789357158355, "completions/clipped_ratio": 0.0, "completions/max_length": 8004.0, "completions/max_terminated_length": 8004.0, "completions/mean_length": 6059.375, "completions/mean_terminated_length": 6059.375, "completions/min_length": 3504.0, "completions/min_terminated_length": 3504.0, "entropy": 0.3146762400865555, "epoch": 0.023450971376020233, "frac_reward_zero_std": 0.0, "grad_norm": 0.011039726436138153, "learning_rate": 1e-05, "loss": -0.1041, "num_tokens": 19464743.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000091314315796, "sampling/importance_sampling_ratio/min": 0.5943339467048645, "sampling/sampling_logp_difference/max": 1.1753723621368408, "sampling/sampling_logp_difference/mean": 0.013002103194594383, "step": 408 }, { "clip_ratio/high_max": 8.714617433724925e-05, "clip_ratio/high_mean": 8.714617433724925e-05, "clip_ratio/low_mean": 4.341031308285892e-05, "clip_ratio/low_min": 4.341031308285892e-05, "clip_ratio/region_mean": 0.00013055648742010817, "completions/clipped_ratio": 0.0, "completions/max_length": 11518.0, "completions/max_terminated_length": 11518.0, "completions/mean_length": 5280.0, "completions/mean_terminated_length": 5280.0, "completions/min_length": 898.0, "completions/min_terminated_length": 898.0, "entropy": 0.6443157829344273, "epoch": 0.02350844924703989, "frac_reward_zero_std": 0.0, "grad_norm": 0.013067906722426414, "learning_rate": 1e-05, "loss": 0.0948, "num_tokens": 19508063.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5871800184249878, "sampling/importance_sampling_ratio/mean": 0.9999516010284424, "sampling/importance_sampling_ratio/min": 0.37245309352874756, "sampling/sampling_logp_difference/max": 0.9876441955566406, "sampling/sampling_logp_difference/mean": 0.02346200682222843, "step": 409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016735495501052355, "clip_ratio/low_min": 0.00016735495501052355, "clip_ratio/region_mean": 0.00016735495501052355, "completions/clipped_ratio": 0.0, "completions/max_length": 14712.0, "completions/max_terminated_length": 14712.0, "completions/mean_length": 8260.125, "completions/mean_terminated_length": 8260.125, "completions/min_length": 3139.0, "completions/min_terminated_length": 3139.0, "entropy": 0.802754320204258, "epoch": 0.02356592711805955, "frac_reward_zero_std": 0.0, "grad_norm": 0.011645487509667873, "learning_rate": 1e-05, "loss": 0.1717, "num_tokens": 19575168.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000720024108887, "sampling/importance_sampling_ratio/min": 0.2969121038913727, "sampling/sampling_logp_difference/max": 2.245997190475464, "sampling/sampling_logp_difference/mean": 0.021546609699726105, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6560.0, "completions/max_terminated_length": 6560.0, "completions/mean_length": 4040.875, "completions/mean_terminated_length": 4040.875, "completions/min_length": 1674.0, "completions/min_terminated_length": 1674.0, "entropy": 0.5578320845961571, "epoch": 0.023623404989079205, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19608279.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.575091004371643, "sampling/importance_sampling_ratio/mean": 0.9999189376831055, "sampling/importance_sampling_ratio/min": 0.6190906167030334, "sampling/sampling_logp_difference/max": 0.4795036315917969, "sampling/sampling_logp_difference/mean": 0.020353052765130997, "step": 411 }, { "clip_ratio/high_max": 0.00010627352912706556, "clip_ratio/high_mean": 0.00010627352912706556, "clip_ratio/low_mean": 5.2323146519484e-05, "clip_ratio/low_min": 5.2323146519484e-05, "clip_ratio/region_mean": 0.00015859667564654956, "completions/clipped_ratio": 0.0, "completions/max_length": 12693.0, "completions/max_terminated_length": 12693.0, "completions/mean_length": 6344.5, "completions/mean_terminated_length": 6344.5, "completions/min_length": 3364.0, "completions/min_terminated_length": 3364.0, "entropy": 0.5004016980528831, "epoch": 0.02368088286009886, "frac_reward_zero_std": 0.0, "grad_norm": 0.045842453837394714, "learning_rate": 1e-05, "loss": -0.0875, "num_tokens": 19659931.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998654127120972, "sampling/importance_sampling_ratio/min": 0.41087549924850464, "sampling/sampling_logp_difference/max": 1.0791079998016357, "sampling/sampling_logp_difference/mean": 0.019905027002096176, "step": 412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 16074.0, "completions/mean_length": 14651.625, "completions/mean_terminated_length": 14074.1669921875, "completions/min_length": 10867.0, "completions/min_terminated_length": 10867.0, "entropy": 0.6848966851830482, "epoch": 0.02373836073111852, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19778648.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000201940536499, "sampling/importance_sampling_ratio/min": 0.10975662618875504, "sampling/sampling_logp_difference/max": 2.2094898223876953, "sampling/sampling_logp_difference/mean": 0.024020234122872353, "step": 413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021074144387966953, "clip_ratio/low_min": 0.00021074144387966953, "clip_ratio/region_mean": 0.00021074144387966953, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 16260.0, "completions/mean_length": 13597.375, "completions/mean_terminated_length": 13199.2861328125, "completions/min_length": 9625.0, "completions/min_terminated_length": 9625.0, "entropy": 0.8754015639424324, "epoch": 0.023795838602138176, "frac_reward_zero_std": 0.0, "grad_norm": 0.008069061674177647, "learning_rate": 1e-05, "loss": 0.0094, "num_tokens": 19888467.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999531507492065, "sampling/importance_sampling_ratio/min": 0.013737646862864494, "sampling/sampling_logp_difference/max": 4.2876152992248535, "sampling/sampling_logp_difference/mean": 0.02309914119541645, "step": 414 }, { "clip_ratio/high_max": 1.565043203299865e-05, "clip_ratio/high_mean": 1.565043203299865e-05, "clip_ratio/low_mean": 0.0006899575782881584, "clip_ratio/low_min": 0.0006899575782881584, "clip_ratio/region_mean": 0.000705608010321157, "completions/clipped_ratio": 0.0, "completions/max_length": 15027.0, "completions/max_terminated_length": 15027.0, "completions/mean_length": 11184.125, "completions/mean_terminated_length": 11184.125, "completions/min_length": 7987.0, "completions/min_terminated_length": 7987.0, "entropy": 1.2949485927820206, "epoch": 0.023853316473157835, "frac_reward_zero_std": 0.0, "grad_norm": 0.008786063641309738, "learning_rate": 1e-05, "loss": 0.1009, "num_tokens": 19979628.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000450611114502, "sampling/importance_sampling_ratio/min": 0.2494448721408844, "sampling/sampling_logp_difference/max": 1.3885173797607422, "sampling/sampling_logp_difference/mean": 0.03763988986611366, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3552.0, "completions/max_terminated_length": 3552.0, "completions/mean_length": 2262.875, "completions/mean_terminated_length": 2262.875, "completions/min_length": 1275.0, "completions/min_terminated_length": 1275.0, "entropy": 0.4064488708972931, "epoch": 0.02391079434417749, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19998563.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4219017028808594, "sampling/importance_sampling_ratio/mean": 0.9998508095741272, "sampling/importance_sampling_ratio/min": 0.6928720474243164, "sampling/sampling_logp_difference/max": 0.3669099807739258, "sampling/sampling_logp_difference/mean": 0.012062828056514263, "step": 416 }, { "clip_ratio/high_max": 0.0001556326469653868, "clip_ratio/high_mean": 0.0001556326469653868, "clip_ratio/low_mean": 5.8449451898923144e-05, "clip_ratio/low_min": 5.8449451898923144e-05, "clip_ratio/region_mean": 0.00021408209886430996, "completions/clipped_ratio": 0.0, "completions/max_length": 11744.0, "completions/max_terminated_length": 11744.0, "completions/mean_length": 9722.5, "completions/mean_terminated_length": 9722.5, "completions/min_length": 7513.0, "completions/min_terminated_length": 7513.0, "entropy": 0.9047807976603508, "epoch": 0.02396827221519715, "frac_reward_zero_std": 0.0, "grad_norm": 0.0071815080009400845, "learning_rate": 1e-05, "loss": 0.0351, "num_tokens": 20077431.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6673693656921387, "sampling/importance_sampling_ratio/mean": 0.9999994039535522, "sampling/importance_sampling_ratio/min": 0.3000313341617584, "sampling/sampling_logp_difference/max": 1.2038683891296387, "sampling/sampling_logp_difference/mean": 0.029282165691256523, "step": 417 }, { "clip_ratio/high_max": 2.9005685064475983e-05, "clip_ratio/high_mean": 2.9005685064475983e-05, "clip_ratio/low_mean": 0.000624057647655718, "clip_ratio/low_min": 0.000624057647655718, "clip_ratio/region_mean": 0.000653063332720194, "completions/clipped_ratio": 0.0, "completions/max_length": 14529.0, "completions/max_terminated_length": 14529.0, "completions/mean_length": 9068.25, "completions/mean_terminated_length": 9068.25, "completions/min_length": 2787.0, "completions/min_terminated_length": 2787.0, "entropy": 0.6444983296096325, "epoch": 0.024025750086216806, "frac_reward_zero_std": 0.0, "grad_norm": 0.0046965391375124454, "learning_rate": 1e-05, "loss": 0.0176, "num_tokens": 20151305.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003751516342163, "sampling/importance_sampling_ratio/min": 0.45572802424430847, "sampling/sampling_logp_difference/max": 0.955262303352356, "sampling/sampling_logp_difference/mean": 0.023841775953769684, "step": 418 }, { "clip_ratio/high_max": 0.00013555289660871495, "clip_ratio/high_mean": 0.00013555289660871495, "clip_ratio/low_mean": 0.0002302800421603024, "clip_ratio/low_min": 0.0002302800421603024, "clip_ratio/region_mean": 0.00036583293876901735, "completions/clipped_ratio": 0.0, "completions/max_length": 7007.0, "completions/max_terminated_length": 7007.0, "completions/mean_length": 3841.5, "completions/mean_terminated_length": 3841.5, "completions/min_length": 886.0, "completions/min_terminated_length": 886.0, "entropy": 0.37390831112861633, "epoch": 0.024083227957236462, "frac_reward_zero_std": 0.0, "grad_norm": 0.011447721160948277, "learning_rate": 1e-05, "loss": 0.0144, "num_tokens": 20182813.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.58346688747406, "sampling/importance_sampling_ratio/mean": 1.0001040697097778, "sampling/importance_sampling_ratio/min": 0.4571685492992401, "sampling/sampling_logp_difference/max": 0.782703161239624, "sampling/sampling_logp_difference/mean": 0.015756886452436447, "step": 419 }, { "clip_ratio/high_max": 3.7204606996965595e-05, "clip_ratio/high_mean": 3.7204606996965595e-05, "clip_ratio/low_mean": 0.000520950517966412, "clip_ratio/low_min": 0.000520950517966412, "clip_ratio/region_mean": 0.0005581551249633776, "completions/clipped_ratio": 0.0, "completions/max_length": 11133.0, "completions/max_terminated_length": 11133.0, "completions/mean_length": 6676.25, "completions/mean_terminated_length": 6676.25, "completions/min_length": 1333.0, "completions/min_terminated_length": 1333.0, "entropy": 0.7315829992294312, "epoch": 0.024140705828256122, "frac_reward_zero_std": 0.0, "grad_norm": 0.013631900772452354, "learning_rate": 1e-05, "loss": -0.1044, "num_tokens": 20237207.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000150203704834, "sampling/importance_sampling_ratio/min": 0.2815264165401459, "sampling/sampling_logp_difference/max": 1.267529010772705, "sampling/sampling_logp_difference/mean": 0.028012793511152267, "step": 420 }, { "clip_ratio/high_max": 8.560589230910409e-05, "clip_ratio/high_mean": 8.560589230910409e-05, "clip_ratio/low_mean": 0.00015359168173745275, "clip_ratio/low_min": 0.00015359168173745275, "clip_ratio/region_mean": 0.00023919757404655684, "completions/clipped_ratio": 0.0, "completions/max_length": 10580.0, "completions/max_terminated_length": 10580.0, "completions/mean_length": 6822.625, "completions/mean_terminated_length": 6822.625, "completions/min_length": 4128.0, "completions/min_terminated_length": 4128.0, "entropy": 0.45269355177879333, "epoch": 0.024198183699275778, "frac_reward_zero_std": 0.0, "grad_norm": 0.005463574081659317, "learning_rate": 1e-05, "loss": 0.195, "num_tokens": 20294660.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6221908330917358, "sampling/importance_sampling_ratio/mean": 1.0004749298095703, "sampling/importance_sampling_ratio/min": 0.5730248093605042, "sampling/sampling_logp_difference/max": 0.5568262934684753, "sampling/sampling_logp_difference/mean": 0.019623752683401108, "step": 421 }, { "clip_ratio/high_max": 0.0001267301668121945, "clip_ratio/high_mean": 0.0001267301668121945, "clip_ratio/low_mean": 0.00020027384016430005, "clip_ratio/low_min": 0.00020027384016430005, "clip_ratio/region_mean": 0.00032700400697649457, "completions/clipped_ratio": 0.0, "completions/max_length": 7208.0, "completions/max_terminated_length": 7208.0, "completions/mean_length": 4787.75, "completions/mean_terminated_length": 4787.75, "completions/min_length": 2899.0, "completions/min_terminated_length": 2899.0, "entropy": 0.7233065217733383, "epoch": 0.024255661570295437, "frac_reward_zero_std": 0.0, "grad_norm": 0.05743607133626938, "learning_rate": 1e-05, "loss": 0.1347, "num_tokens": 20334202.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9416543245315552, "sampling/importance_sampling_ratio/mean": 1.0002912282943726, "sampling/importance_sampling_ratio/min": 0.4011287987232208, "sampling/sampling_logp_difference/max": 0.9134727716445923, "sampling/sampling_logp_difference/mean": 0.028952833265066147, "step": 422 }, { "clip_ratio/high_max": 2.2445681679528207e-05, "clip_ratio/high_mean": 2.2445681679528207e-05, "clip_ratio/low_mean": 0.00024523546744603664, "clip_ratio/low_min": 0.00024523546744603664, "clip_ratio/region_mean": 0.00026768114912556484, "completions/clipped_ratio": 0.0, "completions/max_length": 10331.0, "completions/max_terminated_length": 10331.0, "completions/mean_length": 4971.0, "completions/mean_terminated_length": 4971.0, "completions/min_length": 2063.0, "completions/min_terminated_length": 2063.0, "entropy": 0.6473056003451347, "epoch": 0.024313139441315093, "frac_reward_zero_std": 0.0, "grad_norm": 0.022175902500748634, "learning_rate": 1e-05, "loss": 0.211, "num_tokens": 20374786.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.5078468322753906, "sampling/importance_sampling_ratio/mean": 0.9998683929443359, "sampling/importance_sampling_ratio/min": 0.5398330092430115, "sampling/sampling_logp_difference/max": 0.6164953708648682, "sampling/sampling_logp_difference/mean": 0.023362314328551292, "step": 423 }, { "clip_ratio/high_max": 0.00014690128955408, "clip_ratio/high_mean": 0.00014690128955408, "clip_ratio/low_mean": 0.00022953185725782532, "clip_ratio/low_min": 0.00022953185725782532, "clip_ratio/region_mean": 0.0003764331468119053, "completions/clipped_ratio": 0.0, "completions/max_length": 13965.0, "completions/max_terminated_length": 13965.0, "completions/mean_length": 6154.25, "completions/mean_terminated_length": 6154.25, "completions/min_length": 2629.0, "completions/min_terminated_length": 2629.0, "entropy": 0.2938068602234125, "epoch": 0.024370617312334752, "frac_reward_zero_std": 0.0, "grad_norm": 0.02145836502313614, "learning_rate": 1e-05, "loss": 0.4481, "num_tokens": 20425172.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.697704553604126, "sampling/importance_sampling_ratio/mean": 0.9996499419212341, "sampling/importance_sampling_ratio/min": 0.4860590994358063, "sampling/sampling_logp_difference/max": 0.7214250564575195, "sampling/sampling_logp_difference/mean": 0.013533429242670536, "step": 424 }, { "clip_ratio/high_max": 2.4418832254013978e-05, "clip_ratio/high_mean": 2.4418832254013978e-05, "clip_ratio/low_mean": 0.0005891155597055331, "clip_ratio/low_min": 0.0005891155597055331, "clip_ratio/region_mean": 0.0006135343919595471, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 15780.0, "completions/mean_length": 13765.625, "completions/mean_terminated_length": 12892.833984375, "completions/min_length": 8168.0, "completions/min_terminated_length": 8168.0, "entropy": 0.6838430911302567, "epoch": 0.02442809518335441, "frac_reward_zero_std": 0.0, "grad_norm": 0.008852551691234112, "learning_rate": 1e-05, "loss": 0.1788, "num_tokens": 20536209.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999184608459473, "sampling/importance_sampling_ratio/min": 0.2145192176103592, "sampling/sampling_logp_difference/max": 1.539355993270874, "sampling/sampling_logp_difference/mean": 0.024426208809018135, "step": 425 }, { "clip_ratio/high_max": 0.00016085648167063482, "clip_ratio/high_mean": 0.00016085648167063482, "clip_ratio/low_mean": 8.392333984375e-05, "clip_ratio/low_min": 8.392333984375e-05, "clip_ratio/region_mean": 0.0002447798215143848, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13085.0, "completions/mean_length": 9232.875, "completions/mean_terminated_length": 8211.2861328125, "completions/min_length": 4249.0, "completions/min_terminated_length": 4249.0, "entropy": 0.3341218587011099, "epoch": 0.024485573054374064, "frac_reward_zero_std": 0.0, "grad_norm": 0.0050441003404557705, "learning_rate": 1e-05, "loss": 0.2738, "num_tokens": 20611248.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999803900718689, "sampling/importance_sampling_ratio/min": 0.3071734607219696, "sampling/sampling_logp_difference/max": 1.180342674255371, "sampling/sampling_logp_difference/mean": 0.01568899303674698, "step": 426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7967.0, "completions/max_terminated_length": 7967.0, "completions/mean_length": 5727.625, "completions/mean_terminated_length": 5727.625, "completions/min_length": 4229.0, "completions/min_terminated_length": 4229.0, "entropy": 0.3350247237831354, "epoch": 0.024543050925393724, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20658421.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6528373956680298, "sampling/importance_sampling_ratio/mean": 1.000126838684082, "sampling/importance_sampling_ratio/min": 0.517755389213562, "sampling/sampling_logp_difference/max": 0.6582523584365845, "sampling/sampling_logp_difference/mean": 0.014429958537220955, "step": 427 }, { "clip_ratio/high_max": 7.744733738945797e-05, "clip_ratio/high_mean": 7.744733738945797e-05, "clip_ratio/low_mean": 0.0001059763271769043, "clip_ratio/low_min": 0.0001059763271769043, "clip_ratio/region_mean": 0.00018342366456636228, "completions/clipped_ratio": 0.0, "completions/max_length": 4645.0, "completions/max_terminated_length": 4645.0, "completions/mean_length": 3572.125, "completions/mean_terminated_length": 3572.125, "completions/min_length": 2342.0, "completions/min_terminated_length": 2342.0, "entropy": 0.2327709961682558, "epoch": 0.02460052879641338, "frac_reward_zero_std": 0.0, "grad_norm": 0.010757486335933208, "learning_rate": 1e-05, "loss": -0.0153, "num_tokens": 20687950.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.2831183671951294, "sampling/importance_sampling_ratio/mean": 1.0001705884933472, "sampling/importance_sampling_ratio/min": 0.3317119777202606, "sampling/sampling_logp_difference/max": 1.1034882068634033, "sampling/sampling_logp_difference/mean": 0.009886321611702442, "step": 428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13145.0, "completions/max_terminated_length": 13145.0, "completions/mean_length": 9479.625, "completions/mean_terminated_length": 9479.625, "completions/min_length": 5774.0, "completions/min_terminated_length": 5774.0, "entropy": 0.8704848438501358, "epoch": 0.02465800666743304, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20764843.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000956058502197, "sampling/importance_sampling_ratio/min": 0.5414952635765076, "sampling/sampling_logp_difference/max": 1.0219764709472656, "sampling/sampling_logp_difference/mean": 0.023348411545157433, "step": 429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7330.0, "completions/max_terminated_length": 7330.0, "completions/mean_length": 3063.75, "completions/mean_terminated_length": 3063.75, "completions/min_length": 1098.0, "completions/min_terminated_length": 1098.0, "entropy": 0.44027850963175297, "epoch": 0.024715484538452695, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20790465.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8898735046386719, "sampling/importance_sampling_ratio/mean": 1.0003888607025146, "sampling/importance_sampling_ratio/min": 0.5528064966201782, "sampling/sampling_logp_difference/max": 0.636509895324707, "sampling/sampling_logp_difference/mean": 0.021011190488934517, "step": 430 }, { "clip_ratio/high_max": 9.095011228055228e-05, "clip_ratio/high_mean": 9.095011228055228e-05, "clip_ratio/low_mean": 0.00013116473564878106, "clip_ratio/low_min": 0.00013116473564878106, "clip_ratio/region_mean": 0.00022211484792933334, "completions/clipped_ratio": 0.0, "completions/max_length": 6139.0, "completions/max_terminated_length": 6139.0, "completions/mean_length": 4173.125, "completions/mean_terminated_length": 4173.125, "completions/min_length": 1248.0, "completions/min_terminated_length": 1248.0, "entropy": 0.4412926509976387, "epoch": 0.024772962409472354, "frac_reward_zero_std": 0.0, "grad_norm": 0.008337810635566711, "learning_rate": 1e-05, "loss": -0.1114, "num_tokens": 20824850.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4961583614349365, "sampling/importance_sampling_ratio/mean": 1.0003280639648438, "sampling/importance_sampling_ratio/min": 0.6040873527526855, "sampling/sampling_logp_difference/max": 0.5040364265441895, "sampling/sampling_logp_difference/mean": 0.015653381124138832, "step": 431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4904.0, "completions/max_terminated_length": 4904.0, "completions/mean_length": 2487.75, "completions/mean_terminated_length": 2487.75, "completions/min_length": 1118.0, "completions/min_terminated_length": 1118.0, "entropy": 0.5677931979298592, "epoch": 0.02483044028049201, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20845560.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5420756340026855, "sampling/importance_sampling_ratio/mean": 1.0001436471939087, "sampling/importance_sampling_ratio/min": 0.5958837270736694, "sampling/sampling_logp_difference/max": 0.5177097320556641, "sampling/sampling_logp_difference/mean": 0.02164706215262413, "step": 432 }, { "clip_ratio/high_max": 9.885607778414851e-05, "clip_ratio/high_mean": 9.885607778414851e-05, "clip_ratio/low_mean": 0.00026702880859375, "clip_ratio/low_min": 0.00026702880859375, "clip_ratio/region_mean": 0.0003658848863778985, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 13501.0, "completions/mean_length": 12829.5, "completions/mean_terminated_length": 10696.7998046875, "completions/min_length": 7107.0, "completions/min_terminated_length": 7107.0, "entropy": 0.6649357154965401, "epoch": 0.02488791815151167, "frac_reward_zero_std": 0.0, "grad_norm": 0.021279316395521164, "learning_rate": 1e-05, "loss": 0.2007, "num_tokens": 20949396.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.933716058731079, "sampling/importance_sampling_ratio/mean": 1.0002251863479614, "sampling/importance_sampling_ratio/min": 0.36596012115478516, "sampling/sampling_logp_difference/max": 1.0052309036254883, "sampling/sampling_logp_difference/mean": 0.02312537096440792, "step": 433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6995.0, "completions/max_terminated_length": 6995.0, "completions/mean_length": 4439.75, "completions/mean_terminated_length": 4439.75, "completions/min_length": 1721.0, "completions/min_terminated_length": 1721.0, "entropy": 0.3750717956572771, "epoch": 0.024945396022531326, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20985786.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4755791425704956, "sampling/importance_sampling_ratio/mean": 0.9998895525932312, "sampling/importance_sampling_ratio/min": 0.6152413487434387, "sampling/sampling_logp_difference/max": 0.48574066162109375, "sampling/sampling_logp_difference/mean": 0.015745513141155243, "step": 434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14915.0, "completions/max_terminated_length": 14915.0, "completions/mean_length": 9068.5, "completions/mean_terminated_length": 9068.5, "completions/min_length": 5947.0, "completions/min_terminated_length": 5947.0, "entropy": 0.4036018028855324, "epoch": 0.02500287389355098, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21059198.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001212358474731, "sampling/importance_sampling_ratio/min": 0.4523788094520569, "sampling/sampling_logp_difference/max": 0.8566110134124756, "sampling/sampling_logp_difference/mean": 0.016451718285679817, "step": 435 }, { "clip_ratio/high_max": 2.6483050532988273e-05, "clip_ratio/high_mean": 2.6483050532988273e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 2.6483050532988273e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 11219.0, "completions/max_terminated_length": 11219.0, "completions/mean_length": 3920.75, "completions/mean_terminated_length": 3920.75, "completions/min_length": 2068.0, "completions/min_terminated_length": 2068.0, "entropy": 0.30207252874970436, "epoch": 0.02506035176457064, "frac_reward_zero_std": 0.0, "grad_norm": 0.019070561975240707, "learning_rate": 1e-05, "loss": -0.3176, "num_tokens": 21091332.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4534658193588257, "sampling/importance_sampling_ratio/mean": 1.0002200603485107, "sampling/importance_sampling_ratio/min": 0.5835214257240295, "sampling/sampling_logp_difference/max": 0.5386741161346436, "sampling/sampling_logp_difference/mean": 0.013538790866732597, "step": 436 }, { "clip_ratio/high_max": 8.412201714236289e-05, "clip_ratio/high_mean": 8.412201714236289e-05, "clip_ratio/low_mean": 0.00011005135456798598, "clip_ratio/low_min": 0.00011005135456798598, "clip_ratio/region_mean": 0.00019417337171034887, "completions/clipped_ratio": 0.0, "completions/max_length": 6815.0, "completions/max_terminated_length": 6815.0, "completions/mean_length": 4433.0, "completions/mean_terminated_length": 4433.0, "completions/min_length": 2936.0, "completions/min_terminated_length": 2936.0, "entropy": 0.5785276852548122, "epoch": 0.025117829635590297, "frac_reward_zero_std": 0.0, "grad_norm": 0.00692575890570879, "learning_rate": 1e-05, "loss": 0.1902, "num_tokens": 21128260.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5965887308120728, "sampling/importance_sampling_ratio/mean": 0.9999819993972778, "sampling/importance_sampling_ratio/min": 0.5772815942764282, "sampling/sampling_logp_difference/max": 0.5494251251220703, "sampling/sampling_logp_difference/mean": 0.020583458244800568, "step": 437 }, { "clip_ratio/high_max": 0.00018167695816373453, "clip_ratio/high_mean": 0.00018167695816373453, "clip_ratio/low_mean": 0.00010382059554103762, "clip_ratio/low_min": 0.00010382059554103762, "clip_ratio/region_mean": 0.00028549755370477214, "completions/clipped_ratio": 0.0, "completions/max_length": 2774.0, "completions/max_terminated_length": 2774.0, "completions/mean_length": 1935.5, "completions/mean_terminated_length": 1935.5, "completions/min_length": 1090.0, "completions/min_terminated_length": 1090.0, "entropy": 0.2605552598834038, "epoch": 0.025175307506609956, "frac_reward_zero_std": 0.0, "grad_norm": 0.013630878180265427, "learning_rate": 1e-05, "loss": -0.1338, "num_tokens": 21145048.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6154388189315796, "sampling/importance_sampling_ratio/mean": 0.9999661445617676, "sampling/importance_sampling_ratio/min": 0.6618703007698059, "sampling/sampling_logp_difference/max": 0.47960662841796875, "sampling/sampling_logp_difference/mean": 0.011951710097491741, "step": 438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7486.0, "completions/max_terminated_length": 7486.0, "completions/mean_length": 5581.875, "completions/mean_terminated_length": 5581.875, "completions/min_length": 3620.0, "completions/min_terminated_length": 3620.0, "entropy": 0.18832128401845694, "epoch": 0.025232785377629612, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21191823.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3808515071868896, "sampling/importance_sampling_ratio/mean": 1.000009298324585, "sampling/importance_sampling_ratio/min": 0.5028315782546997, "sampling/sampling_logp_difference/max": 0.6875, "sampling/sampling_logp_difference/mean": 0.008609802462160587, "step": 439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9760.0, "completions/max_terminated_length": 9760.0, "completions/mean_length": 5481.375, "completions/mean_terminated_length": 5481.375, "completions/min_length": 2329.0, "completions/min_terminated_length": 2329.0, "entropy": 0.7463670670986176, "epoch": 0.02529026324864927, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21236754.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4439177513122559, "sampling/importance_sampling_ratio/mean": 0.9998909831047058, "sampling/importance_sampling_ratio/min": 0.46668437123298645, "sampling/sampling_logp_difference/max": 0.7621021270751953, "sampling/sampling_logp_difference/mean": 0.027332153171300888, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14763.0, "completions/mean_length": 12928.625, "completions/mean_terminated_length": 11776.833984375, "completions/min_length": 8289.0, "completions/min_terminated_length": 8289.0, "entropy": 0.8742486983537674, "epoch": 0.025347741119668928, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21341943.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998680353164673, "sampling/importance_sampling_ratio/min": 0.0002018306840909645, "sampling/sampling_logp_difference/max": 8.508081436157227, "sampling/sampling_logp_difference/mean": 0.031038979068398476, "step": 441 }, { "clip_ratio/high_max": 2.4747574570938013e-05, "clip_ratio/high_mean": 2.4747574570938013e-05, "clip_ratio/low_mean": 0.0001732190194161376, "clip_ratio/low_min": 0.0001732190194161376, "clip_ratio/region_mean": 0.0001979665939870756, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12717.0, "completions/mean_length": 9281.125, "completions/mean_terminated_length": 8266.4287109375, "completions/min_length": 5051.0, "completions/min_terminated_length": 5051.0, "entropy": 0.37633494660258293, "epoch": 0.025405218990688583, "frac_reward_zero_std": 0.0, "grad_norm": 0.009374288842082024, "learning_rate": 1e-05, "loss": 0.3569, "num_tokens": 21417792.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000054955482483, "sampling/importance_sampling_ratio/min": 9.635421065468108e-07, "sampling/sampling_logp_difference/max": 13.852649688720703, "sampling/sampling_logp_difference/mean": 0.01673848181962967, "step": 442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2495.0, "completions/max_terminated_length": 2495.0, "completions/mean_length": 1921.375, "completions/mean_terminated_length": 1921.375, "completions/min_length": 1267.0, "completions/min_terminated_length": 1267.0, "entropy": 0.3145306296646595, "epoch": 0.025462696861708243, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21433963.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3491744995117188, "sampling/importance_sampling_ratio/mean": 1.00028395652771, "sampling/importance_sampling_ratio/min": 0.5357253551483154, "sampling/sampling_logp_difference/max": 0.6241335868835449, "sampling/sampling_logp_difference/mean": 0.013191049918532372, "step": 443 }, { "clip_ratio/high_max": 0.00011003005056409165, "clip_ratio/high_mean": 0.00011003005056409165, "clip_ratio/low_mean": 3.907471182174049e-05, "clip_ratio/low_min": 3.907471182174049e-05, "clip_ratio/region_mean": 0.00014910476238583215, "completions/clipped_ratio": 0.0, "completions/max_length": 3692.0, "completions/max_terminated_length": 3692.0, "completions/mean_length": 2988.625, "completions/mean_terminated_length": 2988.625, "completions/min_length": 1483.0, "completions/min_terminated_length": 1483.0, "entropy": 0.2208611536771059, "epoch": 0.0255201747327279, "frac_reward_zero_std": 0.0, "grad_norm": 0.006359876599162817, "learning_rate": 1e-05, "loss": 0.0249, "num_tokens": 21459168.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5963165760040283, "sampling/importance_sampling_ratio/mean": 1.000028371810913, "sampling/importance_sampling_ratio/min": 0.6824864745140076, "sampling/sampling_logp_difference/max": 0.4676988124847412, "sampling/sampling_logp_difference/mean": 0.009486507624387741, "step": 444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 2.2449712560046464e-05, "clip_ratio/low_min": 2.2449712560046464e-05, "clip_ratio/region_mean": 2.2449712560046464e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6965.0, "completions/max_terminated_length": 6965.0, "completions/mean_length": 4913.875, "completions/mean_terminated_length": 4913.875, "completions/min_length": 3344.0, "completions/min_terminated_length": 3344.0, "entropy": 0.7445646338164806, "epoch": 0.025577652603747558, "frac_reward_zero_std": 0.0, "grad_norm": 0.0071185920387506485, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 21499639.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5986356735229492, "sampling/importance_sampling_ratio/mean": 0.9998476505279541, "sampling/importance_sampling_ratio/min": 0.6074582934379578, "sampling/sampling_logp_difference/max": 0.498471736907959, "sampling/sampling_logp_difference/mean": 0.018018942326307297, "step": 445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8990.0, "completions/max_terminated_length": 8990.0, "completions/mean_length": 7207.875, "completions/mean_terminated_length": 7207.875, "completions/min_length": 5745.0, "completions/min_terminated_length": 5745.0, "entropy": 0.6107673943042755, "epoch": 0.025635130474767214, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21558302.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001622438430786, "sampling/importance_sampling_ratio/min": 0.42563557624816895, "sampling/sampling_logp_difference/max": 0.953639030456543, "sampling/sampling_logp_difference/mean": 0.01979878358542919, "step": 446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9706.0, "completions/max_terminated_length": 9706.0, "completions/mean_length": 6625.5, "completions/mean_terminated_length": 6625.5, "completions/min_length": 2281.0, "completions/min_terminated_length": 2281.0, "entropy": 1.3155563473701477, "epoch": 0.025692608345786874, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21613538.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7799874544143677, "sampling/importance_sampling_ratio/mean": 0.9997109770774841, "sampling/importance_sampling_ratio/min": 0.5182348489761353, "sampling/sampling_logp_difference/max": 0.6573266983032227, "sampling/sampling_logp_difference/mean": 0.029717689380049706, "step": 447 }, { "clip_ratio/high_max": 6.18738977209432e-05, "clip_ratio/high_mean": 6.18738977209432e-05, "clip_ratio/low_mean": 0.0006023716705385596, "clip_ratio/low_min": 0.0006023716705385596, "clip_ratio/region_mean": 0.0006642455682595028, "completions/clipped_ratio": 0.0, "completions/max_length": 11615.0, "completions/max_terminated_length": 11615.0, "completions/mean_length": 8738.625, "completions/mean_terminated_length": 8738.625, "completions/min_length": 2999.0, "completions/min_terminated_length": 2999.0, "entropy": 0.659493874758482, "epoch": 0.02575008621680653, "frac_reward_zero_std": 0.0, "grad_norm": 0.02282099425792694, "learning_rate": 1e-05, "loss": -0.0112, "num_tokens": 21684527.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000520944595337, "sampling/importance_sampling_ratio/min": 0.007554459385573864, "sampling/sampling_logp_difference/max": 4.885617256164551, "sampling/sampling_logp_difference/mean": 0.024753792211413383, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2289.0, "completions/max_terminated_length": 2289.0, "completions/mean_length": 1831.375, "completions/mean_terminated_length": 1831.375, "completions/min_length": 1260.0, "completions/min_terminated_length": 1260.0, "entropy": 0.2874491699039936, "epoch": 0.025807564087826185, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21701402.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3110114336013794, "sampling/importance_sampling_ratio/mean": 0.9998665452003479, "sampling/importance_sampling_ratio/min": 0.6000904440879822, "sampling/sampling_logp_difference/max": 0.5106749534606934, "sampling/sampling_logp_difference/mean": 0.011926800943911076, "step": 449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3337.0, "completions/max_terminated_length": 3337.0, "completions/mean_length": 2186.375, "completions/mean_terminated_length": 2186.375, "completions/min_length": 1329.0, "completions/min_terminated_length": 1329.0, "entropy": 0.30876625422388315, "epoch": 0.025865041958845845, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21719853.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3667190074920654, "sampling/importance_sampling_ratio/mean": 0.999986469745636, "sampling/importance_sampling_ratio/min": 0.5996760129928589, "sampling/sampling_logp_difference/max": 0.5113657712936401, "sampling/sampling_logp_difference/mean": 0.013923629187047482, "step": 450 }, { "clip_ratio/high_max": 0.00010394073979114182, "clip_ratio/high_mean": 0.00010394073979114182, "clip_ratio/low_mean": 0.00010789814405143261, "clip_ratio/low_min": 0.00010789814405143261, "clip_ratio/region_mean": 0.00021183888384257443, "completions/clipped_ratio": 0.0, "completions/max_length": 11284.0, "completions/max_terminated_length": 11284.0, "completions/mean_length": 6918.0, "completions/mean_terminated_length": 6918.0, "completions/min_length": 3510.0, "completions/min_terminated_length": 3510.0, "entropy": 0.32617291435599327, "epoch": 0.0259225198298655, "frac_reward_zero_std": 0.0, "grad_norm": 0.007784546818584204, "learning_rate": 1e-05, "loss": -0.1168, "num_tokens": 21776301.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6965603828430176, "sampling/importance_sampling_ratio/mean": 0.9999083876609802, "sampling/importance_sampling_ratio/min": 0.45862019062042236, "sampling/sampling_logp_difference/max": 0.7795329093933105, "sampling/sampling_logp_difference/mean": 0.014778019860386848, "step": 451 }, { "clip_ratio/high_max": 0.00036039591941516846, "clip_ratio/high_mean": 0.00036039591941516846, "clip_ratio/low_mean": 0.00016512550064362586, "clip_ratio/low_min": 0.00016512550064362586, "clip_ratio/region_mean": 0.0005255214200587943, "completions/clipped_ratio": 0.0, "completions/max_length": 1830.0, "completions/max_terminated_length": 1830.0, "completions/mean_length": 1146.375, "completions/mean_terminated_length": 1146.375, "completions/min_length": 757.0, "completions/min_terminated_length": 757.0, "entropy": 0.36676783487200737, "epoch": 0.02597999770088516, "frac_reward_zero_std": 0.0, "grad_norm": 0.1551930457353592, "learning_rate": 1e-05, "loss": -0.1203, "num_tokens": 21786624.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4105974435806274, "sampling/importance_sampling_ratio/mean": 1.0005935430526733, "sampling/importance_sampling_ratio/min": 0.5231972932815552, "sampling/sampling_logp_difference/max": 0.647796630859375, "sampling/sampling_logp_difference/mean": 0.016509389504790306, "step": 452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12260.0, "completions/max_terminated_length": 12260.0, "completions/mean_length": 8123.75, "completions/mean_terminated_length": 8123.75, "completions/min_length": 5628.0, "completions/min_terminated_length": 5628.0, "entropy": 0.8492995873093605, "epoch": 0.026037475571904816, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21852710.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002620220184326, "sampling/importance_sampling_ratio/min": 0.39434200525283813, "sampling/sampling_logp_difference/max": 1.6590116024017334, "sampling/sampling_logp_difference/mean": 0.023842215538024902, "step": 453 }, { "clip_ratio/high_max": 0.00021226795979600865, "clip_ratio/high_mean": 0.00021226795979600865, "clip_ratio/low_mean": 7.415757136186585e-05, "clip_ratio/low_min": 7.415757136186585e-05, "clip_ratio/region_mean": 0.0002864255311578745, "completions/clipped_ratio": 0.0, "completions/max_length": 15258.0, "completions/max_terminated_length": 15258.0, "completions/mean_length": 7037.0, "completions/mean_terminated_length": 7037.0, "completions/min_length": 3892.0, "completions/min_terminated_length": 3892.0, "entropy": 0.5697024390101433, "epoch": 0.026094953442924475, "frac_reward_zero_std": 0.0, "grad_norm": 0.0062552159652113914, "learning_rate": 1e-05, "loss": 0.0699, "num_tokens": 21909982.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996772408485413, "sampling/importance_sampling_ratio/min": 0.18077947199344635, "sampling/sampling_logp_difference/max": 1.710477352142334, "sampling/sampling_logp_difference/mean": 0.02124391682446003, "step": 454 }, { "clip_ratio/high_max": 0.0001177287249447545, "clip_ratio/high_mean": 0.0001177287249447545, "clip_ratio/low_mean": 5.022433470003307e-05, "clip_ratio/low_min": 5.022433470003307e-05, "clip_ratio/region_mean": 0.00016795305964478757, "completions/clipped_ratio": 0.0, "completions/max_length": 14933.0, "completions/max_terminated_length": 14933.0, "completions/mean_length": 6386.125, "completions/mean_terminated_length": 6386.125, "completions/min_length": 3251.0, "completions/min_terminated_length": 3251.0, "entropy": 0.3480612672865391, "epoch": 0.02615243131394413, "frac_reward_zero_std": 0.0, "grad_norm": 0.006207325495779514, "learning_rate": 1e-05, "loss": 0.4729, "num_tokens": 21962583.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999215602874756, "sampling/importance_sampling_ratio/min": 0.3246854245662689, "sampling/sampling_logp_difference/max": 1.1248984336853027, "sampling/sampling_logp_difference/mean": 0.015614835545420647, "step": 455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6037.0, "completions/max_terminated_length": 6037.0, "completions/mean_length": 2807.0, "completions/mean_terminated_length": 2807.0, "completions/min_length": 1630.0, "completions/min_terminated_length": 1630.0, "entropy": 0.4020255170762539, "epoch": 0.026209909184963787, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 21986991.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4531421661376953, "sampling/importance_sampling_ratio/mean": 0.9996038675308228, "sampling/importance_sampling_ratio/min": 0.6206403374671936, "sampling/sampling_logp_difference/max": 0.4770035743713379, "sampling/sampling_logp_difference/mean": 0.013629144988954067, "step": 456 }, { "clip_ratio/high_max": 0.00010171175199502613, "clip_ratio/high_mean": 0.00010171175199502613, "clip_ratio/low_mean": 6.430040957638994e-05, "clip_ratio/low_min": 6.430040957638994e-05, "clip_ratio/region_mean": 0.00016601216157141607, "completions/clipped_ratio": 0.0, "completions/max_length": 14204.0, "completions/max_terminated_length": 14204.0, "completions/mean_length": 8440.375, "completions/mean_terminated_length": 8440.375, "completions/min_length": 4524.0, "completions/min_terminated_length": 4524.0, "entropy": 0.5252612680196762, "epoch": 0.026267387055983447, "frac_reward_zero_std": 0.0, "grad_norm": 0.007233121432363987, "learning_rate": 1e-05, "loss": 0.0536, "num_tokens": 22055802.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6282354593276978, "sampling/importance_sampling_ratio/mean": 1.0000444650650024, "sampling/importance_sampling_ratio/min": 0.2853838801383972, "sampling/sampling_logp_difference/max": 1.253920078277588, "sampling/sampling_logp_difference/mean": 0.02232937142252922, "step": 457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010700904749683104, "clip_ratio/low_min": 0.00010700904749683104, "clip_ratio/region_mean": 0.00010700904749683104, "completions/clipped_ratio": 0.0, "completions/max_length": 8769.0, "completions/max_terminated_length": 8769.0, "completions/mean_length": 4938.75, "completions/mean_terminated_length": 4938.75, "completions/min_length": 1072.0, "completions/min_terminated_length": 1072.0, "entropy": 0.5232226587831974, "epoch": 0.026324864927003103, "frac_reward_zero_std": 0.0, "grad_norm": 0.020089680328965187, "learning_rate": 1e-05, "loss": 0.0249, "num_tokens": 22096592.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6376785039901733, "sampling/importance_sampling_ratio/mean": 1.000291109085083, "sampling/importance_sampling_ratio/min": 0.2560388743877411, "sampling/sampling_logp_difference/max": 1.3624259233474731, "sampling/sampling_logp_difference/mean": 0.01678202673792839, "step": 458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012275764129299205, "clip_ratio/low_min": 0.00012275764129299205, "clip_ratio/region_mean": 0.00012275764129299205, "completions/clipped_ratio": 0.0, "completions/max_length": 14034.0, "completions/max_terminated_length": 14034.0, "completions/mean_length": 7140.875, "completions/mean_terminated_length": 7140.875, "completions/min_length": 3312.0, "completions/min_terminated_length": 3312.0, "entropy": 0.5556174069643021, "epoch": 0.026382342798022762, "frac_reward_zero_std": 0.0, "grad_norm": 0.020406965166330338, "learning_rate": 1e-05, "loss": 0.3139, "num_tokens": 22155039.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998794794082642, "sampling/importance_sampling_ratio/min": 0.4274432957172394, "sampling/sampling_logp_difference/max": 0.8499336242675781, "sampling/sampling_logp_difference/mean": 0.017594872042536736, "step": 459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2209.0, "completions/max_terminated_length": 2209.0, "completions/mean_length": 1376.5, "completions/mean_terminated_length": 1376.5, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.42941916920244694, "epoch": 0.026439820669042418, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22167115.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4358333349227905, "sampling/importance_sampling_ratio/mean": 0.9994628429412842, "sampling/importance_sampling_ratio/min": 0.6410631537437439, "sampling/sampling_logp_difference/max": 0.4446272850036621, "sampling/sampling_logp_difference/mean": 0.01788030005991459, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14561.0, "completions/max_terminated_length": 14561.0, "completions/mean_length": 12358.0, "completions/mean_terminated_length": 12358.0, "completions/min_length": 8595.0, "completions/min_terminated_length": 8595.0, "entropy": 0.8948144093155861, "epoch": 0.026497298540062077, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22266907.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000369548797607, "sampling/importance_sampling_ratio/min": 0.1848621815443039, "sampling/sampling_logp_difference/max": 1.6881446838378906, "sampling/sampling_logp_difference/mean": 0.03346993029117584, "step": 461 }, { "clip_ratio/high_max": 0.00016611522733001038, "clip_ratio/high_mean": 0.00016611522733001038, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00016611522733001038, "completions/clipped_ratio": 0.0, "completions/max_length": 2903.0, "completions/max_terminated_length": 2903.0, "completions/mean_length": 1705.375, "completions/mean_terminated_length": 1705.375, "completions/min_length": 804.0, "completions/min_terminated_length": 804.0, "entropy": 0.2837061323225498, "epoch": 0.026554776411081733, "frac_reward_zero_std": 0.0, "grad_norm": 0.009395787492394447, "learning_rate": 1e-05, "loss": -0.1491, "num_tokens": 22281606.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.340754747390747, "sampling/importance_sampling_ratio/mean": 1.0000104904174805, "sampling/importance_sampling_ratio/min": 0.6883840560913086, "sampling/sampling_logp_difference/max": 0.37340831756591797, "sampling/sampling_logp_difference/mean": 0.01224634051322937, "step": 462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12146.0, "completions/max_terminated_length": 12146.0, "completions/mean_length": 6735.0, "completions/mean_terminated_length": 6735.0, "completions/min_length": 3194.0, "completions/min_terminated_length": 3194.0, "entropy": 0.7924304902553558, "epoch": 0.02661225428210139, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22336646.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8254384994506836, "sampling/importance_sampling_ratio/mean": 0.9998587369918823, "sampling/importance_sampling_ratio/min": 0.48047584295272827, "sampling/sampling_logp_difference/max": 0.732978343963623, "sampling/sampling_logp_difference/mean": 0.02267562970519066, "step": 463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5731.0, "completions/max_terminated_length": 5731.0, "completions/mean_length": 3318.125, "completions/mean_terminated_length": 3318.125, "completions/min_length": 2141.0, "completions/min_terminated_length": 2141.0, "entropy": 0.4460943043231964, "epoch": 0.02666973215312105, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22364415.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5287853479385376, "sampling/importance_sampling_ratio/mean": 1.0001696348190308, "sampling/importance_sampling_ratio/min": 0.43343740701675415, "sampling/sampling_logp_difference/max": 0.8360079526901245, "sampling/sampling_logp_difference/mean": 0.014565175399184227, "step": 464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11477.0, "completions/max_terminated_length": 11477.0, "completions/mean_length": 8231.375, "completions/mean_terminated_length": 8231.375, "completions/min_length": 4362.0, "completions/min_terminated_length": 4362.0, "entropy": 0.850938007235527, "epoch": 0.026727210024140705, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22431722.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997738003730774, "sampling/importance_sampling_ratio/min": 0.39070361852645874, "sampling/sampling_logp_difference/max": 0.9398059844970703, "sampling/sampling_logp_difference/mean": 0.029489165171980858, "step": 465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14860.0, "completions/max_terminated_length": 14860.0, "completions/mean_length": 8764.75, "completions/mean_terminated_length": 8764.75, "completions/min_length": 5482.0, "completions/min_terminated_length": 5482.0, "entropy": 0.5348370894789696, "epoch": 0.026784687895160364, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22502592.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6334214210510254, "sampling/importance_sampling_ratio/mean": 0.9999181628227234, "sampling/importance_sampling_ratio/min": 0.2320336103439331, "sampling/sampling_logp_difference/max": 1.4608731269836426, "sampling/sampling_logp_difference/mean": 0.020580865442752838, "step": 466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2754.0, "completions/max_terminated_length": 2754.0, "completions/mean_length": 2042.625, "completions/mean_terminated_length": 2042.625, "completions/min_length": 1483.0, "completions/min_terminated_length": 1483.0, "entropy": 0.1900317259132862, "epoch": 0.02684216576618002, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22520365.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.639816164970398, "sampling/importance_sampling_ratio/mean": 0.9998511075973511, "sampling/importance_sampling_ratio/min": 0.6222306489944458, "sampling/sampling_logp_difference/max": 0.4945840835571289, "sampling/sampling_logp_difference/mean": 0.00861221645027399, "step": 467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1526.0, "completions/max_terminated_length": 1526.0, "completions/mean_length": 1028.125, "completions/mean_terminated_length": 1028.125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "entropy": 0.4898565150797367, "epoch": 0.02689964363719968, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22530166.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000059962272644, "sampling/importance_sampling_ratio/min": 0.5672351121902466, "sampling/sampling_logp_difference/max": 1.330442190170288, "sampling/sampling_logp_difference/mean": 0.02235967107117176, "step": 468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11362.0, "completions/max_terminated_length": 11362.0, "completions/mean_length": 7199.25, "completions/mean_terminated_length": 7199.25, "completions/min_length": 3571.0, "completions/min_terminated_length": 3571.0, "entropy": 0.7652010135352612, "epoch": 0.026957121508219335, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22588608.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5251785516738892, "sampling/importance_sampling_ratio/mean": 0.999806821346283, "sampling/importance_sampling_ratio/min": 0.5348965525627136, "sampling/sampling_logp_difference/max": 0.6256818771362305, "sampling/sampling_logp_difference/mean": 0.028109097853302956, "step": 469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11268.0, "completions/max_terminated_length": 11268.0, "completions/mean_length": 8559.75, "completions/mean_terminated_length": 8559.75, "completions/min_length": 6053.0, "completions/min_terminated_length": 6053.0, "entropy": 0.7304617203772068, "epoch": 0.027014599379238995, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22658062.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000112056732178, "sampling/importance_sampling_ratio/min": 0.4830860495567322, "sampling/sampling_logp_difference/max": 0.7403964996337891, "sampling/sampling_logp_difference/mean": 0.02655942179262638, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11310.0, "completions/max_terminated_length": 11310.0, "completions/mean_length": 6771.875, "completions/mean_terminated_length": 6771.875, "completions/min_length": 4234.0, "completions/min_terminated_length": 4234.0, "entropy": 0.4115558788180351, "epoch": 0.02707207725025865, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22713893.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002235174179077, "sampling/importance_sampling_ratio/min": 0.3621920943260193, "sampling/sampling_logp_difference/max": 1.015580654144287, "sampling/sampling_logp_difference/mean": 0.017113743349909782, "step": 471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6806.0, "completions/max_terminated_length": 6806.0, "completions/mean_length": 4121.625, "completions/mean_terminated_length": 4121.625, "completions/min_length": 2350.0, "completions/min_terminated_length": 2350.0, "entropy": 0.5099250301718712, "epoch": 0.027129555121278306, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22747762.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5958448648452759, "sampling/importance_sampling_ratio/mean": 0.999858558177948, "sampling/importance_sampling_ratio/min": 0.6475029587745667, "sampling/sampling_logp_difference/max": 0.4674032926559448, "sampling/sampling_logp_difference/mean": 0.015033961273729801, "step": 472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4152.0, "completions/max_terminated_length": 4152.0, "completions/mean_length": 2881.375, "completions/mean_terminated_length": 2881.375, "completions/min_length": 1063.0, "completions/min_terminated_length": 1063.0, "entropy": 0.4350298345088959, "epoch": 0.027187032992297966, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22772069.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3541855812072754, "sampling/importance_sampling_ratio/mean": 1.0003026723861694, "sampling/importance_sampling_ratio/min": 0.4431356191635132, "sampling/sampling_logp_difference/max": 0.8138794302940369, "sampling/sampling_logp_difference/mean": 0.013945409096777439, "step": 473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2685.0, "completions/max_terminated_length": 2685.0, "completions/mean_length": 1239.375, "completions/mean_terminated_length": 1239.375, "completions/min_length": 768.0, "completions/min_terminated_length": 768.0, "entropy": 0.38876313529908657, "epoch": 0.027244510863317622, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22783096.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.001139760017395, "sampling/importance_sampling_ratio/min": 0.5878161787986755, "sampling/sampling_logp_difference/max": 0.7716981172561646, "sampling/sampling_logp_difference/mean": 0.017898164689540863, "step": 474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1941.0, "completions/max_terminated_length": 1941.0, "completions/mean_length": 1576.125, "completions/mean_terminated_length": 1576.125, "completions/min_length": 1156.0, "completions/min_terminated_length": 1156.0, "entropy": 0.29721128195524216, "epoch": 0.02730198873433728, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22796777.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.33108389377594, "sampling/importance_sampling_ratio/mean": 1.0004914999008179, "sampling/importance_sampling_ratio/min": 0.6335042715072632, "sampling/sampling_logp_difference/max": 0.45648860931396484, "sampling/sampling_logp_difference/mean": 0.01284761168062687, "step": 475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3524.0, "completions/max_terminated_length": 3524.0, "completions/mean_length": 1846.5, "completions/mean_terminated_length": 1846.5, "completions/min_length": 1132.0, "completions/min_terminated_length": 1132.0, "entropy": 0.3397200219333172, "epoch": 0.027359466605356937, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22812477.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3227969408035278, "sampling/importance_sampling_ratio/mean": 0.9997376203536987, "sampling/importance_sampling_ratio/min": 0.5311287045478821, "sampling/sampling_logp_difference/max": 0.6327509880065918, "sampling/sampling_logp_difference/mean": 0.01491733267903328, "step": 476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2348.0, "completions/max_terminated_length": 2348.0, "completions/mean_length": 1724.75, "completions/mean_terminated_length": 1724.75, "completions/min_length": 1021.0, "completions/min_terminated_length": 1021.0, "entropy": 0.2508810628205538, "epoch": 0.027416944476376597, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22827179.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.278839349746704, "sampling/importance_sampling_ratio/mean": 0.9998406171798706, "sampling/importance_sampling_ratio/min": 0.6765270233154297, "sampling/sampling_logp_difference/max": 0.39078283309936523, "sampling/sampling_logp_difference/mean": 0.010592957958579063, "step": 477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14360.0, "completions/max_terminated_length": 14360.0, "completions/mean_length": 11501.125, "completions/mean_terminated_length": 11501.125, "completions/min_length": 7187.0, "completions/min_terminated_length": 7187.0, "entropy": 1.016406163573265, "epoch": 0.027474422347396252, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 22920164.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996692538261414, "sampling/importance_sampling_ratio/min": 0.023657720535993576, "sampling/sampling_logp_difference/max": 3.744065761566162, "sampling/sampling_logp_difference/mean": 0.026961635798215866, "step": 478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001397428713971749, "clip_ratio/low_min": 0.0001397428713971749, "clip_ratio/region_mean": 0.0001397428713971749, "completions/clipped_ratio": 0.0, "completions/max_length": 4426.0, "completions/max_terminated_length": 4426.0, "completions/mean_length": 3070.875, "completions/mean_terminated_length": 3070.875, "completions/min_length": 1494.0, "completions/min_terminated_length": 1494.0, "entropy": 0.5050739087164402, "epoch": 0.02753190021841591, "frac_reward_zero_std": 0.0, "grad_norm": 0.022422660142183304, "learning_rate": 1e-05, "loss": -0.251, "num_tokens": 22946243.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.352946400642395, "sampling/importance_sampling_ratio/mean": 0.9993551969528198, "sampling/importance_sampling_ratio/min": 0.64349365234375, "sampling/sampling_logp_difference/max": 0.4408431053161621, "sampling/sampling_logp_difference/mean": 0.01553282979875803, "step": 479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13683.0, "completions/max_terminated_length": 13683.0, "completions/mean_length": 9254.0, "completions/mean_terminated_length": 9254.0, "completions/min_length": 7146.0, "completions/min_terminated_length": 7146.0, "entropy": 0.870044969022274, "epoch": 0.027589378089435568, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23020995.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001555681228638, "sampling/importance_sampling_ratio/min": 0.480752557516098, "sampling/sampling_logp_difference/max": 0.7817888259887695, "sampling/sampling_logp_difference/mean": 0.026576273143291473, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3018.0, "completions/max_terminated_length": 3018.0, "completions/mean_length": 2079.75, "completions/mean_terminated_length": 2079.75, "completions/min_length": 1483.0, "completions/min_terminated_length": 1483.0, "entropy": 0.2843241859227419, "epoch": 0.027646855960455224, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23039457.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2747890949249268, "sampling/importance_sampling_ratio/mean": 1.000415563583374, "sampling/importance_sampling_ratio/min": 0.6076636910438538, "sampling/sampling_logp_difference/max": 0.49813365936279297, "sampling/sampling_logp_difference/mean": 0.012646266259253025, "step": 481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6854.0, "completions/max_terminated_length": 6854.0, "completions/mean_length": 4758.875, "completions/mean_terminated_length": 4758.875, "completions/min_length": 3788.0, "completions/min_terminated_length": 3788.0, "entropy": 0.4736669920384884, "epoch": 0.027704333831474883, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23078576.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.597193717956543, "sampling/importance_sampling_ratio/mean": 0.9998964071273804, "sampling/importance_sampling_ratio/min": 0.5398942232131958, "sampling/sampling_logp_difference/max": 0.6163821220397949, "sampling/sampling_logp_difference/mean": 0.012928749434649944, "step": 482 }, { "clip_ratio/high_max": 8.57110590004595e-05, "clip_ratio/high_mean": 8.57110590004595e-05, "clip_ratio/low_mean": 0.00044208051258465275, "clip_ratio/low_min": 0.00044208051258465275, "clip_ratio/region_mean": 0.0005277915715851123, "completions/clipped_ratio": 0.0, "completions/max_length": 10690.0, "completions/max_terminated_length": 10690.0, "completions/mean_length": 5721.5, "completions/mean_terminated_length": 5721.5, "completions/min_length": 3146.0, "completions/min_terminated_length": 3146.0, "entropy": 0.9015582054853439, "epoch": 0.02776181170249454, "frac_reward_zero_std": 0.0, "grad_norm": 0.020446304231882095, "learning_rate": 1e-05, "loss": -0.0063, "num_tokens": 23126004.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.7612286806106567, "sampling/importance_sampling_ratio/mean": 0.9996693730354309, "sampling/importance_sampling_ratio/min": 0.4542199671268463, "sampling/sampling_logp_difference/max": 0.7891737222671509, "sampling/sampling_logp_difference/mean": 0.03362022340297699, "step": 483 }, { "clip_ratio/high_max": 6.721661156916525e-05, "clip_ratio/high_mean": 6.721661156916525e-05, "clip_ratio/low_mean": 7.993861072463915e-05, "clip_ratio/low_min": 7.993861072463915e-05, "clip_ratio/region_mean": 0.0001471552222938044, "completions/clipped_ratio": 0.0, "completions/max_length": 15637.0, "completions/max_terminated_length": 15637.0, "completions/mean_length": 6268.75, "completions/mean_terminated_length": 6268.75, "completions/min_length": 3491.0, "completions/min_terminated_length": 3491.0, "entropy": 0.4071271996945143, "epoch": 0.0278192895735142, "frac_reward_zero_std": 0.0, "grad_norm": 0.004760902840644121, "learning_rate": 1e-05, "loss": 0.5282, "num_tokens": 23177418.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9609125852584839, "sampling/importance_sampling_ratio/mean": 0.9998213052749634, "sampling/importance_sampling_ratio/min": 0.26579052209854126, "sampling/sampling_logp_difference/max": 1.3250467777252197, "sampling/sampling_logp_difference/mean": 0.01721394993364811, "step": 484 }, { "clip_ratio/high_max": 8.207975042751059e-05, "clip_ratio/high_mean": 8.207975042751059e-05, "clip_ratio/low_mean": 0.0001684256421867758, "clip_ratio/low_min": 0.0001684256421867758, "clip_ratio/region_mean": 0.0002505053926142864, "completions/clipped_ratio": 0.0, "completions/max_length": 11609.0, "completions/max_terminated_length": 11609.0, "completions/mean_length": 6639.25, "completions/mean_terminated_length": 6639.25, "completions/min_length": 4246.0, "completions/min_terminated_length": 4246.0, "entropy": 0.4360942356288433, "epoch": 0.027876767444533854, "frac_reward_zero_std": 0.0, "grad_norm": 0.010879365727305412, "learning_rate": 1e-05, "loss": 0.177, "num_tokens": 23231260.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8237727880477905, "sampling/importance_sampling_ratio/mean": 1.0000526905059814, "sampling/importance_sampling_ratio/min": 0.4868440628051758, "sampling/sampling_logp_difference/max": 0.7198114395141602, "sampling/sampling_logp_difference/mean": 0.01597915217280388, "step": 485 }, { "clip_ratio/high_max": 0.0001207146342494525, "clip_ratio/high_mean": 0.0001207146342494525, "clip_ratio/low_mean": 0.00012236906331963837, "clip_ratio/low_min": 0.00012236906331963837, "clip_ratio/region_mean": 0.00024308369756909087, "completions/clipped_ratio": 0.0, "completions/max_length": 3952.0, "completions/max_terminated_length": 3952.0, "completions/mean_length": 2210.625, "completions/mean_terminated_length": 2210.625, "completions/min_length": 1225.0, "completions/min_terminated_length": 1225.0, "entropy": 0.557428639382124, "epoch": 0.02793424531555351, "frac_reward_zero_std": 0.0, "grad_norm": 0.017054788768291473, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 23250313.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3488322496414185, "sampling/importance_sampling_ratio/mean": 1.0004565715789795, "sampling/importance_sampling_ratio/min": 0.5910559892654419, "sampling/sampling_logp_difference/max": 0.5258445143699646, "sampling/sampling_logp_difference/mean": 0.017454911023378372, "step": 486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16030.0, "completions/max_terminated_length": 16030.0, "completions/mean_length": 13478.25, "completions/mean_terminated_length": 13478.25, "completions/min_length": 10660.0, "completions/min_terminated_length": 10660.0, "entropy": 0.9878299161791801, "epoch": 0.02799172318657317, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23359907.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001929998397827, "sampling/importance_sampling_ratio/min": 0.41012296080589294, "sampling/sampling_logp_difference/max": 0.8912982940673828, "sampling/sampling_logp_difference/mean": 0.030822424218058586, "step": 487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5339.0, "completions/max_terminated_length": 5339.0, "completions/mean_length": 3032.0, "completions/mean_terminated_length": 3032.0, "completions/min_length": 978.0, "completions/min_terminated_length": 978.0, "entropy": 0.7438951432704926, "epoch": 0.028049201057592826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23385315.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4101923704147339, "sampling/importance_sampling_ratio/mean": 0.9995203614234924, "sampling/importance_sampling_ratio/min": 0.42016395926475525, "sampling/sampling_logp_difference/max": 0.8671103119850159, "sampling/sampling_logp_difference/mean": 0.021243689581751823, "step": 488 }, { "clip_ratio/high_max": 0.00016840477474033833, "clip_ratio/high_mean": 0.00016840477474033833, "clip_ratio/low_mean": 2.255096478620544e-05, "clip_ratio/low_min": 2.255096478620544e-05, "clip_ratio/region_mean": 0.00019095573952654377, "completions/clipped_ratio": 0.0, "completions/max_length": 5974.0, "completions/max_terminated_length": 5974.0, "completions/mean_length": 4167.125, "completions/mean_terminated_length": 4167.125, "completions/min_length": 2826.0, "completions/min_terminated_length": 2826.0, "entropy": 0.42411772534251213, "epoch": 0.028106678928612485, "frac_reward_zero_std": 0.0, "grad_norm": 0.01035399828106165, "learning_rate": 1e-05, "loss": 0.1167, "num_tokens": 23419876.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4058266878128052, "sampling/importance_sampling_ratio/mean": 1.0002118349075317, "sampling/importance_sampling_ratio/min": 0.3586447238922119, "sampling/sampling_logp_difference/max": 1.0254230499267578, "sampling/sampling_logp_difference/mean": 0.017031384631991386, "step": 489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3024.0, "completions/max_terminated_length": 3024.0, "completions/mean_length": 1588.125, "completions/mean_terminated_length": 1588.125, "completions/min_length": 754.0, "completions/min_terminated_length": 754.0, "entropy": 0.3231074959039688, "epoch": 0.02816415679963214, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23433549.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4304660558700562, "sampling/importance_sampling_ratio/mean": 1.0003496408462524, "sampling/importance_sampling_ratio/min": 0.6068309545516968, "sampling/sampling_logp_difference/max": 0.49950504302978516, "sampling/sampling_logp_difference/mean": 0.014455264434218407, "step": 490 }, { "clip_ratio/high_max": 9.99200619844487e-06, "clip_ratio/high_mean": 9.99200619844487e-06, "clip_ratio/low_mean": 0.0006146665982669219, "clip_ratio/low_min": 0.0006146665982669219, "clip_ratio/region_mean": 0.0006246586044653668, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15235.0, "completions/mean_length": 13781.125, "completions/mean_terminated_length": 12219.400390625, "completions/min_length": 10145.0, "completions/min_terminated_length": 10145.0, "entropy": 0.3590470626950264, "epoch": 0.0282216346706518, "frac_reward_zero_std": 0.0, "grad_norm": 0.005382860079407692, "learning_rate": 1e-05, "loss": 0.0323, "num_tokens": 23545838.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001873970031738, "sampling/importance_sampling_ratio/min": 0.1353372186422348, "sampling/sampling_logp_difference/max": 1.999985694885254, "sampling/sampling_logp_difference/mean": 0.01698617823421955, "step": 491 }, { "clip_ratio/high_max": 5.8022722441819496e-05, "clip_ratio/high_mean": 5.8022722441819496e-05, "clip_ratio/low_mean": 0.001055316490237601, "clip_ratio/low_min": 0.001055316490237601, "clip_ratio/region_mean": 0.0011133392126794206, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12816.0, "completions/mean_length": 10489.625, "completions/mean_terminated_length": 9647.572265625, "completions/min_length": 8512.0, "completions/min_terminated_length": 8512.0, "entropy": 0.6383561491966248, "epoch": 0.028279112541671456, "frac_reward_zero_std": 0.0, "grad_norm": 0.009697741828858852, "learning_rate": 1e-05, "loss": 0.098, "num_tokens": 23630611.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003139972686768, "sampling/importance_sampling_ratio/min": 0.29774343967437744, "sampling/sampling_logp_difference/max": 1.2115230560302734, "sampling/sampling_logp_difference/mean": 0.02427665889263153, "step": 492 }, { "clip_ratio/high_max": 1.3520821994461585e-05, "clip_ratio/high_mean": 1.3520821994461585e-05, "clip_ratio/low_mean": 0.0007133609578886535, "clip_ratio/low_min": 0.0007133609578886535, "clip_ratio/region_mean": 0.0007268817798831151, "completions/clipped_ratio": 0.0, "completions/max_length": 11801.0, "completions/max_terminated_length": 11801.0, "completions/mean_length": 10556.0, "completions/mean_terminated_length": 10556.0, "completions/min_length": 9245.0, "completions/min_terminated_length": 9245.0, "entropy": 1.3196925669908524, "epoch": 0.028336590412691112, "frac_reward_zero_std": 0.0, "grad_norm": 0.00755338603630662, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 23716099.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000447034835815, "sampling/importance_sampling_ratio/min": 0.4495033025741577, "sampling/sampling_logp_difference/max": 0.7996120452880859, "sampling/sampling_logp_difference/mean": 0.03670940920710564, "step": 493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12623.0, "completions/max_terminated_length": 12623.0, "completions/mean_length": 8810.125, "completions/mean_terminated_length": 8810.125, "completions/min_length": 6119.0, "completions/min_terminated_length": 6119.0, "entropy": 0.748796284198761, "epoch": 0.02839406828371077, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23788116.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9936128854751587, "sampling/importance_sampling_ratio/mean": 1.0001407861709595, "sampling/importance_sampling_ratio/min": 0.5555142760276794, "sampling/sampling_logp_difference/max": 0.689948558807373, "sampling/sampling_logp_difference/mean": 0.025567520409822464, "step": 494 }, { "clip_ratio/high_max": 8.218006041715853e-05, "clip_ratio/high_mean": 8.218006041715853e-05, "clip_ratio/low_mean": 5.5847202020231634e-05, "clip_ratio/low_min": 5.5847202020231634e-05, "clip_ratio/region_mean": 0.00013802726243739016, "completions/clipped_ratio": 0.0, "completions/max_length": 10511.0, "completions/max_terminated_length": 10511.0, "completions/mean_length": 8325.375, "completions/mean_terminated_length": 8325.375, "completions/min_length": 6508.0, "completions/min_terminated_length": 6508.0, "entropy": 0.569528117775917, "epoch": 0.028451546154730428, "frac_reward_zero_std": 0.0, "grad_norm": 0.005489890463650227, "learning_rate": 1e-05, "loss": 0.0267, "num_tokens": 23855703.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003366470336914, "sampling/importance_sampling_ratio/min": 0.37893322110176086, "sampling/sampling_logp_difference/max": 0.9703953266143799, "sampling/sampling_logp_difference/mean": 0.020501164719462395, "step": 495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13860.0, "completions/max_terminated_length": 13860.0, "completions/mean_length": 5334.125, "completions/mean_terminated_length": 5334.125, "completions/min_length": 2312.0, "completions/min_terminated_length": 2312.0, "entropy": 0.5351596400141716, "epoch": 0.028509024025750087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23899200.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6597518920898438, "sampling/importance_sampling_ratio/mean": 1.0000720024108887, "sampling/importance_sampling_ratio/min": 0.24629662930965424, "sampling/sampling_logp_difference/max": 1.4012186527252197, "sampling/sampling_logp_difference/mean": 0.015156200155615807, "step": 496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2975.0, "completions/max_terminated_length": 2975.0, "completions/mean_length": 1838.125, "completions/mean_terminated_length": 1838.125, "completions/min_length": 1260.0, "completions/min_terminated_length": 1260.0, "entropy": 0.23078706115484238, "epoch": 0.028566501896769743, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23914809.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2703499794006348, "sampling/importance_sampling_ratio/mean": 0.9996809363365173, "sampling/importance_sampling_ratio/min": 0.5686377882957458, "sampling/sampling_logp_difference/max": 0.5645116567611694, "sampling/sampling_logp_difference/mean": 0.010511109605431557, "step": 497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15032.0, "completions/max_terminated_length": 15032.0, "completions/mean_length": 7930.0, "completions/mean_terminated_length": 7930.0, "completions/min_length": 2650.0, "completions/min_terminated_length": 2650.0, "entropy": 0.8748474940657616, "epoch": 0.028623979767789402, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 23979425.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9730079174041748, "sampling/importance_sampling_ratio/mean": 0.9999067783355713, "sampling/importance_sampling_ratio/min": 0.2752572000026703, "sampling/sampling_logp_difference/max": 1.2900493144989014, "sampling/sampling_logp_difference/mean": 0.024676917120814323, "step": 498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015094849004526623, "clip_ratio/low_min": 0.00015094849004526623, "clip_ratio/region_mean": 0.00015094849004526623, "completions/clipped_ratio": 0.0, "completions/max_length": 10074.0, "completions/max_terminated_length": 10074.0, "completions/mean_length": 5790.5, "completions/mean_terminated_length": 5790.5, "completions/min_length": 1818.0, "completions/min_terminated_length": 1818.0, "entropy": 0.6304151639342308, "epoch": 0.028681457638809058, "frac_reward_zero_std": 0.0, "grad_norm": 0.025980690494179726, "learning_rate": 1e-05, "loss": 0.0543, "num_tokens": 24026821.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999737560749054, "sampling/importance_sampling_ratio/min": 0.34373846650123596, "sampling/sampling_logp_difference/max": 1.493558645248413, "sampling/sampling_logp_difference/mean": 0.018344547599554062, "step": 499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10268.0, "completions/max_terminated_length": 10268.0, "completions/mean_length": 7145.625, "completions/mean_terminated_length": 7145.625, "completions/min_length": 4421.0, "completions/min_terminated_length": 4421.0, "entropy": 1.3390654921531677, "epoch": 0.028738935509828718, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24085714.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002769231796265, "sampling/importance_sampling_ratio/min": 0.4710195064544678, "sampling/sampling_logp_difference/max": 0.9900603294372559, "sampling/sampling_logp_difference/mean": 0.029477065429091454, "step": 500 }, { "clip_ratio/high_max": 9.443172166356817e-05, "clip_ratio/high_mean": 9.443172166356817e-05, "clip_ratio/low_mean": 0.0001649851656111423, "clip_ratio/low_min": 0.0001649851656111423, "clip_ratio/region_mean": 0.00025941688727471046, "completions/clipped_ratio": 0.0, "completions/max_length": 10452.0, "completions/max_terminated_length": 10452.0, "completions/mean_length": 4857.75, "completions/mean_terminated_length": 4857.75, "completions/min_length": 2025.0, "completions/min_terminated_length": 2025.0, "entropy": 0.4218490272760391, "epoch": 0.028796413380848374, "frac_reward_zero_std": 0.0, "grad_norm": 0.014390205033123493, "learning_rate": 1e-05, "loss": 0.2487, "num_tokens": 24125448.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6404558420181274, "sampling/importance_sampling_ratio/mean": 1.0005238056182861, "sampling/importance_sampling_ratio/min": 0.4971098303794861, "sampling/sampling_logp_difference/max": 0.6989443302154541, "sampling/sampling_logp_difference/mean": 0.01898825541138649, "step": 501 }, { "clip_ratio/high_max": 1.6306828911183402e-05, "clip_ratio/high_mean": 1.6306828911183402e-05, "clip_ratio/low_mean": 8.022757174330764e-05, "clip_ratio/low_min": 8.022757174330764e-05, "clip_ratio/region_mean": 9.653440065449104e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 15331.0, "completions/max_terminated_length": 15331.0, "completions/mean_length": 7210.375, "completions/mean_terminated_length": 7210.375, "completions/min_length": 2991.0, "completions/min_terminated_length": 2991.0, "entropy": 0.4630022421479225, "epoch": 0.02885389125186803, "frac_reward_zero_std": 0.0, "grad_norm": 0.007532189134508371, "learning_rate": 1e-05, "loss": 0.2951, "num_tokens": 24184539.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996907711029053, "sampling/importance_sampling_ratio/min": 0.3317732512950897, "sampling/sampling_logp_difference/max": 1.1033034324645996, "sampling/sampling_logp_difference/mean": 0.015898901969194412, "step": 502 }, { "clip_ratio/high_max": 7.776827169436729e-05, "clip_ratio/high_mean": 7.776827169436729e-05, "clip_ratio/low_mean": 0.0004161382021266036, "clip_ratio/low_min": 0.0004161382021266036, "clip_ratio/region_mean": 0.0004939064738209709, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 11740.0, "completions/mean_length": 10123.875, "completions/mean_terminated_length": 9229.572265625, "completions/min_length": 7444.0, "completions/min_terminated_length": 7444.0, "entropy": 0.5822692811489105, "epoch": 0.02891136912288769, "frac_reward_zero_std": 0.0, "grad_norm": 0.014252358116209507, "learning_rate": 1e-05, "loss": 0.0932, "num_tokens": 24266602.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.85968816280365, "sampling/importance_sampling_ratio/mean": 0.9999411702156067, "sampling/importance_sampling_ratio/min": 0.2939281463623047, "sampling/sampling_logp_difference/max": 1.2244199514389038, "sampling/sampling_logp_difference/mean": 0.022970611229538918, "step": 503 }, { "clip_ratio/high_max": 5.5250749028346036e-05, "clip_ratio/high_mean": 5.5250749028346036e-05, "clip_ratio/low_mean": 0.0005120095156598836, "clip_ratio/low_min": 0.0005120095156598836, "clip_ratio/region_mean": 0.0005672602646882297, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14483.0, "completions/mean_length": 12207.0, "completions/mean_terminated_length": 11610.2861328125, "completions/min_length": 5119.0, "completions/min_terminated_length": 5119.0, "entropy": 0.7222501114010811, "epoch": 0.028968846993907345, "frac_reward_zero_std": 0.0, "grad_norm": 0.014984922483563423, "learning_rate": 1e-05, "loss": 0.0742, "num_tokens": 24365946.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000762939453125, "sampling/importance_sampling_ratio/min": 0.2893992066383362, "sampling/sampling_logp_difference/max": 1.2399482727050781, "sampling/sampling_logp_difference/mean": 0.024225441738963127, "step": 504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2659.0, "completions/max_terminated_length": 2659.0, "completions/mean_length": 1596.0, "completions/mean_terminated_length": 1596.0, "completions/min_length": 926.0, "completions/min_terminated_length": 926.0, "entropy": 0.3962612822651863, "epoch": 0.029026324864927004, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24379986.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3249133825302124, "sampling/importance_sampling_ratio/mean": 1.0003875494003296, "sampling/importance_sampling_ratio/min": 0.6831625699996948, "sampling/sampling_logp_difference/max": 0.38102245330810547, "sampling/sampling_logp_difference/mean": 0.013281316496431828, "step": 505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3931.0, "completions/max_terminated_length": 3931.0, "completions/mean_length": 3004.25, "completions/mean_terminated_length": 3004.25, "completions/min_length": 1691.0, "completions/min_terminated_length": 1691.0, "entropy": 0.46810152754187584, "epoch": 0.02908380273594666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24404932.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.459431529045105, "sampling/importance_sampling_ratio/mean": 0.9995197653770447, "sampling/importance_sampling_ratio/min": 0.5531719923019409, "sampling/sampling_logp_difference/max": 0.5920863151550293, "sampling/sampling_logp_difference/mean": 0.020342981442809105, "step": 506 }, { "clip_ratio/high_max": 2.5797131456783973e-05, "clip_ratio/high_mean": 2.5797131456783973e-05, "clip_ratio/low_mean": 0.0001815050891309511, "clip_ratio/low_min": 0.0001815050891309511, "clip_ratio/region_mean": 0.00020730222058773506, "completions/clipped_ratio": 0.0, "completions/max_length": 9691.0, "completions/max_terminated_length": 9691.0, "completions/mean_length": 7357.125, "completions/mean_terminated_length": 7357.125, "completions/min_length": 5695.0, "completions/min_terminated_length": 5695.0, "entropy": 1.2582993060350418, "epoch": 0.02914128060696632, "frac_reward_zero_std": 0.0, "grad_norm": 0.013779567554593086, "learning_rate": 1e-05, "loss": -0.112, "num_tokens": 24464925.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.775281310081482, "sampling/importance_sampling_ratio/mean": 0.9998346567153931, "sampling/importance_sampling_ratio/min": 0.28037261962890625, "sampling/sampling_logp_difference/max": 1.2716357707977295, "sampling/sampling_logp_difference/mean": 0.030548227950930595, "step": 507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10245.0, "completions/max_terminated_length": 10245.0, "completions/mean_length": 6751.75, "completions/mean_terminated_length": 6751.75, "completions/min_length": 2975.0, "completions/min_terminated_length": 2975.0, "entropy": 0.8135130777955055, "epoch": 0.029198758477985975, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24520483.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998216032981873, "sampling/importance_sampling_ratio/min": 0.19519375264644623, "sampling/sampling_logp_difference/max": 1.6337625980377197, "sampling/sampling_logp_difference/mean": 0.021174805238842964, "step": 508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9227.0, "completions/max_terminated_length": 9227.0, "completions/mean_length": 7027.375, "completions/mean_terminated_length": 7027.375, "completions/min_length": 4381.0, "completions/min_terminated_length": 4381.0, "entropy": 0.5941646546125412, "epoch": 0.02925623634900563, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24577558.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7379595041275024, "sampling/importance_sampling_ratio/mean": 0.99977046251297, "sampling/importance_sampling_ratio/min": 0.3878824710845947, "sampling/sampling_logp_difference/max": 0.9470529556274414, "sampling/sampling_logp_difference/mean": 0.020396653562784195, "step": 509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1676.0, "completions/max_terminated_length": 1676.0, "completions/mean_length": 884.5, "completions/mean_terminated_length": 884.5, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "entropy": 0.27856239303946495, "epoch": 0.02931371422002529, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24585994.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2593625783920288, "sampling/importance_sampling_ratio/mean": 1.0001964569091797, "sampling/importance_sampling_ratio/min": 0.4726683497428894, "sampling/sampling_logp_difference/max": 0.7493612766265869, "sampling/sampling_logp_difference/mean": 0.013685714453458786, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15352.0, "completions/mean_length": 14449.375, "completions/mean_terminated_length": 13288.6005859375, "completions/min_length": 12290.0, "completions/min_terminated_length": 12290.0, "entropy": 0.7529455497860909, "epoch": 0.029371192091044947, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24704045.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999600052833557, "sampling/importance_sampling_ratio/min": 0.1564919352531433, "sampling/sampling_logp_difference/max": 1.8547508716583252, "sampling/sampling_logp_difference/mean": 0.024882635101675987, "step": 511 }, { "clip_ratio/high_max": 0.00015584083666908555, "clip_ratio/high_mean": 0.00015584083666908555, "clip_ratio/low_mean": 0.0003105180585407652, "clip_ratio/low_min": 0.0003105180585407652, "clip_ratio/region_mean": 0.00046635889520985074, "completions/clipped_ratio": 0.0, "completions/max_length": 15903.0, "completions/max_terminated_length": 15903.0, "completions/mean_length": 9703.5, "completions/mean_terminated_length": 9703.5, "completions/min_length": 4475.0, "completions/min_terminated_length": 4475.0, "entropy": 0.5617002211511135, "epoch": 0.029428669962064606, "frac_reward_zero_std": 0.0, "grad_norm": 0.014774056151509285, "learning_rate": 1e-05, "loss": 0.1992, "num_tokens": 24782641.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998946785926819, "sampling/importance_sampling_ratio/min": 0.16020439565181732, "sampling/sampling_logp_difference/max": 1.8313047885894775, "sampling/sampling_logp_difference/mean": 0.020321058109402657, "step": 512 }, { "clip_ratio/high_max": 1.3700131603400223e-05, "clip_ratio/high_mean": 1.3700131603400223e-05, "clip_ratio/low_mean": 0.00016415268146374729, "clip_ratio/low_min": 0.00016415268146374729, "clip_ratio/region_mean": 0.0001778528130671475, "completions/clipped_ratio": 0.0, "completions/max_length": 12495.0, "completions/max_terminated_length": 12495.0, "completions/mean_length": 7462.125, "completions/mean_terminated_length": 7462.125, "completions/min_length": 1041.0, "completions/min_terminated_length": 1041.0, "entropy": 0.8099309280514717, "epoch": 0.029486147833084262, "frac_reward_zero_std": 0.0, "grad_norm": 0.010028025135397911, "learning_rate": 1e-05, "loss": -0.0788, "num_tokens": 24844130.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998405575752258, "sampling/importance_sampling_ratio/min": 0.4794335663318634, "sampling/sampling_logp_difference/max": 0.9951958656311035, "sampling/sampling_logp_difference/mean": 0.023017920553684235, "step": 513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5331.0, "completions/max_terminated_length": 5331.0, "completions/mean_length": 3015.875, "completions/mean_terminated_length": 3015.875, "completions/min_length": 1888.0, "completions/min_terminated_length": 1888.0, "entropy": 0.41553497686982155, "epoch": 0.02954362570410392, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24870433.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4040570259094238, "sampling/importance_sampling_ratio/mean": 1.000180721282959, "sampling/importance_sampling_ratio/min": 0.6158313155174255, "sampling/sampling_logp_difference/max": 0.48478221893310547, "sampling/sampling_logp_difference/mean": 0.017517894506454468, "step": 514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9485.0, "completions/max_terminated_length": 9485.0, "completions/mean_length": 5718.25, "completions/mean_terminated_length": 5718.25, "completions/min_length": 1134.0, "completions/min_terminated_length": 1134.0, "entropy": 1.5000159665942192, "epoch": 0.029601103575123577, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24917627.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4485816955566406, "sampling/importance_sampling_ratio/mean": 0.999874472618103, "sampling/importance_sampling_ratio/min": 0.4224056303501129, "sampling/sampling_logp_difference/max": 0.8617892265319824, "sampling/sampling_logp_difference/mean": 0.03128407895565033, "step": 515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4253.0, "completions/max_terminated_length": 4253.0, "completions/mean_length": 3634.125, "completions/mean_terminated_length": 3634.125, "completions/min_length": 3104.0, "completions/min_terminated_length": 3104.0, "entropy": 0.4635204151272774, "epoch": 0.029658581446143233, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 24947500.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3518959283828735, "sampling/importance_sampling_ratio/mean": 0.9999291896820068, "sampling/importance_sampling_ratio/min": 0.5894184708595276, "sampling/sampling_logp_difference/max": 0.5286188125610352, "sampling/sampling_logp_difference/mean": 0.016389012336730957, "step": 516 }, { "clip_ratio/high_max": 0.000108700145574403, "clip_ratio/high_mean": 0.000108700145574403, "clip_ratio/low_mean": 0.00011426684795878828, "clip_ratio/low_min": 0.00011426684795878828, "clip_ratio/region_mean": 0.00022296699353319127, "completions/clipped_ratio": 0.0, "completions/max_length": 9711.0, "completions/max_terminated_length": 9711.0, "completions/mean_length": 7623.625, "completions/mean_terminated_length": 7623.625, "completions/min_length": 4969.0, "completions/min_terminated_length": 4969.0, "entropy": 0.8240789994597435, "epoch": 0.029716059317162893, "frac_reward_zero_std": 0.0, "grad_norm": 0.01151200383901596, "learning_rate": 1e-05, "loss": 0.0704, "num_tokens": 25009649.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9267656803131104, "sampling/importance_sampling_ratio/mean": 0.9998918771743774, "sampling/importance_sampling_ratio/min": 0.49004605412483215, "sampling/sampling_logp_difference/max": 0.7132558822631836, "sampling/sampling_logp_difference/mean": 0.024903230369091034, "step": 517 }, { "clip_ratio/high_max": 3.167764953104779e-05, "clip_ratio/high_mean": 3.167764953104779e-05, "clip_ratio/low_mean": 3.570408443920314e-05, "clip_ratio/low_min": 3.570408443920314e-05, "clip_ratio/region_mean": 6.738173397025093e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6400.0, "completions/max_terminated_length": 6400.0, "completions/mean_length": 3796.125, "completions/mean_terminated_length": 3796.125, "completions/min_length": 2425.0, "completions/min_terminated_length": 2425.0, "entropy": 0.3162513505667448, "epoch": 0.02977353718818255, "frac_reward_zero_std": 0.0, "grad_norm": 0.016649702563881874, "learning_rate": 1e-05, "loss": -0.0261, "num_tokens": 25041626.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4987657070159912, "sampling/importance_sampling_ratio/mean": 1.0002034902572632, "sampling/importance_sampling_ratio/min": 0.6169796586036682, "sampling/sampling_logp_difference/max": 0.48291921615600586, "sampling/sampling_logp_difference/mean": 0.010826973244547844, "step": 518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10812.0, "completions/max_terminated_length": 10812.0, "completions/mean_length": 4711.0, "completions/mean_terminated_length": 4711.0, "completions/min_length": 2224.0, "completions/min_terminated_length": 2224.0, "entropy": 0.5308945141732693, "epoch": 0.029831015059202208, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25081034.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.75653076171875, "sampling/importance_sampling_ratio/mean": 0.9997653365135193, "sampling/importance_sampling_ratio/min": 0.3093548119068146, "sampling/sampling_logp_difference/max": 1.1732664108276367, "sampling/sampling_logp_difference/mean": 0.01582672819495201, "step": 519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4469.0, "completions/max_terminated_length": 4469.0, "completions/mean_length": 2266.375, "completions/mean_terminated_length": 2266.375, "completions/min_length": 1308.0, "completions/min_terminated_length": 1308.0, "entropy": 0.39835625514388084, "epoch": 0.029888492930221864, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25099869.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4820570945739746, "sampling/importance_sampling_ratio/mean": 0.9994468688964844, "sampling/importance_sampling_ratio/min": 0.6810091137886047, "sampling/sampling_logp_difference/max": 0.39343106746673584, "sampling/sampling_logp_difference/mean": 0.012017875909805298, "step": 520 }, { "clip_ratio/high_max": 6.734220733051188e-05, "clip_ratio/high_mean": 6.734220733051188e-05, "clip_ratio/low_mean": 5.90318777540233e-05, "clip_ratio/low_min": 5.90318777540233e-05, "clip_ratio/region_mean": 0.00012637408508453518, "completions/clipped_ratio": 0.0, "completions/max_length": 7119.0, "completions/max_terminated_length": 7119.0, "completions/mean_length": 5200.0, "completions/mean_terminated_length": 5200.0, "completions/min_length": 3149.0, "completions/min_terminated_length": 3149.0, "entropy": 0.932475708425045, "epoch": 0.029945970801241523, "frac_reward_zero_std": 0.0, "grad_norm": 0.01645560748875141, "learning_rate": 1e-05, "loss": -0.1567, "num_tokens": 25142645.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.975502610206604, "sampling/importance_sampling_ratio/mean": 0.9996724724769592, "sampling/importance_sampling_ratio/min": 0.5454188585281372, "sampling/sampling_logp_difference/max": 0.6808228492736816, "sampling/sampling_logp_difference/mean": 0.024106526747345924, "step": 521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12674.0, "completions/max_terminated_length": 12674.0, "completions/mean_length": 6347.5, "completions/mean_terminated_length": 6347.5, "completions/min_length": 3475.0, "completions/min_terminated_length": 3475.0, "entropy": 0.7964662276208401, "epoch": 0.03000344867226118, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25194665.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000587701797485, "sampling/importance_sampling_ratio/min": 0.5250390768051147, "sampling/sampling_logp_difference/max": 1.6287641525268555, "sampling/sampling_logp_difference/mean": 0.021775493398308754, "step": 522 }, { "clip_ratio/high_max": 6.732512156304438e-05, "clip_ratio/high_mean": 6.732512156304438e-05, "clip_ratio/low_mean": 0.0003050125887966715, "clip_ratio/low_min": 0.0003050125887966715, "clip_ratio/region_mean": 0.00037233771035971586, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15956.0, "completions/mean_length": 9150.75, "completions/mean_terminated_length": 8117.4287109375, "completions/min_length": 4102.0, "completions/min_terminated_length": 4102.0, "entropy": 0.5763437412679195, "epoch": 0.030060926543280835, "frac_reward_zero_std": 0.0, "grad_norm": 0.009423363953828812, "learning_rate": 1e-05, "loss": 0.4037, "num_tokens": 25269599.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998732209205627, "sampling/importance_sampling_ratio/min": 0.056408416479825974, "sampling/sampling_logp_difference/max": 2.8751368522644043, "sampling/sampling_logp_difference/mean": 0.023485755547881126, "step": 523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4061.0, "completions/max_terminated_length": 4061.0, "completions/mean_length": 2743.5, "completions/mean_terminated_length": 2743.5, "completions/min_length": 2199.0, "completions/min_terminated_length": 2199.0, "entropy": 0.2120806910097599, "epoch": 0.030118404414300495, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25292747.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.317800760269165, "sampling/importance_sampling_ratio/mean": 1.000043511390686, "sampling/importance_sampling_ratio/min": 0.6889001131057739, "sampling/sampling_logp_difference/max": 0.37265896797180176, "sampling/sampling_logp_difference/mean": 0.00854382012039423, "step": 524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0004927924674120732, "clip_ratio/low_min": 0.0004927924674120732, "clip_ratio/region_mean": 0.0004927924674120732, "completions/clipped_ratio": 0.0, "completions/max_length": 14011.0, "completions/max_terminated_length": 14011.0, "completions/mean_length": 9724.25, "completions/mean_terminated_length": 9724.25, "completions/min_length": 5912.0, "completions/min_terminated_length": 5912.0, "entropy": 1.079609528183937, "epoch": 0.03017588228532015, "frac_reward_zero_std": 0.0, "grad_norm": 0.006379496771842241, "learning_rate": 1e-05, "loss": 0.0673, "num_tokens": 25371813.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999752044677734, "sampling/importance_sampling_ratio/min": 0.4720262289047241, "sampling/sampling_logp_difference/max": 0.9534740447998047, "sampling/sampling_logp_difference/mean": 0.032003190368413925, "step": 525 }, { "clip_ratio/high_max": 1.640635309740901e-05, "clip_ratio/high_mean": 1.640635309740901e-05, "clip_ratio/low_mean": 0.0003436263014009455, "clip_ratio/low_min": 0.0003436263014009455, "clip_ratio/region_mean": 0.00036003265449835453, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15250.0, "completions/mean_length": 13140.5, "completions/mean_terminated_length": 11194.400390625, "completions/min_length": 5292.0, "completions/min_terminated_length": 5292.0, "entropy": 0.6533002927899361, "epoch": 0.03023336015633981, "frac_reward_zero_std": 0.0, "grad_norm": 0.010692539624869823, "learning_rate": 1e-05, "loss": 0.2745, "num_tokens": 25477793.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998891949653625, "sampling/importance_sampling_ratio/min": 0.25593504309654236, "sampling/sampling_logp_difference/max": 1.3628315925598145, "sampling/sampling_logp_difference/mean": 0.02106543257832527, "step": 526 }, { "clip_ratio/high_max": 6.990562269493239e-05, "clip_ratio/high_mean": 6.990562269493239e-05, "clip_ratio/low_mean": 0.0003605697274906561, "clip_ratio/low_min": 0.0003605697274906561, "clip_ratio/region_mean": 0.00043047535018558847, "completions/clipped_ratio": 0.0, "completions/max_length": 10260.0, "completions/max_terminated_length": 10260.0, "completions/mean_length": 6054.375, "completions/mean_terminated_length": 6054.375, "completions/min_length": 1385.0, "completions/min_terminated_length": 1385.0, "entropy": 0.7893617749214172, "epoch": 0.030290838027359466, "frac_reward_zero_std": 0.0, "grad_norm": 0.017319226637482643, "learning_rate": 1e-05, "loss": 0.2011, "num_tokens": 25527860.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4795838594436646, "sampling/importance_sampling_ratio/mean": 0.9998732805252075, "sampling/importance_sampling_ratio/min": 0.4862327575683594, "sampling/sampling_logp_difference/max": 0.7210679054260254, "sampling/sampling_logp_difference/mean": 0.02935694344341755, "step": 527 }, { "clip_ratio/high_max": 0.00015416615678987, "clip_ratio/high_mean": 0.00015416615678987, "clip_ratio/low_mean": 0.00021387422748375684, "clip_ratio/low_min": 0.00021387422748375684, "clip_ratio/region_mean": 0.00036804038427362684, "completions/clipped_ratio": 0.0, "completions/max_length": 7759.0, "completions/max_terminated_length": 7759.0, "completions/mean_length": 6577.0, "completions/mean_terminated_length": 6577.0, "completions/min_length": 4464.0, "completions/min_terminated_length": 4464.0, "entropy": 0.3716000635176897, "epoch": 0.030348315898379125, "frac_reward_zero_std": 0.0, "grad_norm": 0.014287028461694717, "learning_rate": 1e-05, "loss": -0.1007, "num_tokens": 25581460.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.491353988647461, "sampling/importance_sampling_ratio/mean": 1.0000569820404053, "sampling/importance_sampling_ratio/min": 0.48764222860336304, "sampling/sampling_logp_difference/max": 0.7181732654571533, "sampling/sampling_logp_difference/mean": 0.0157511904835701, "step": 528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11032.0, "completions/max_terminated_length": 11032.0, "completions/mean_length": 9941.0, "completions/mean_terminated_length": 9941.0, "completions/min_length": 7598.0, "completions/min_terminated_length": 7598.0, "entropy": 1.1432203650474548, "epoch": 0.03040579376939878, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25661940.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9362651109695435, "sampling/importance_sampling_ratio/mean": 1.0002930164337158, "sampling/importance_sampling_ratio/min": 0.19722844660282135, "sampling/sampling_logp_difference/max": 1.6233925819396973, "sampling/sampling_logp_difference/mean": 0.032692648470401764, "step": 529 }, { "clip_ratio/high_max": 7.840077341825236e-05, "clip_ratio/high_mean": 7.840077341825236e-05, "clip_ratio/low_mean": 0.0001308221362705808, "clip_ratio/low_min": 0.0001308221362705808, "clip_ratio/region_mean": 0.00020922290968883317, "completions/clipped_ratio": 0.0, "completions/max_length": 13244.0, "completions/max_terminated_length": 13244.0, "completions/mean_length": 8941.75, "completions/mean_terminated_length": 8941.75, "completions/min_length": 4762.0, "completions/min_terminated_length": 4762.0, "entropy": 0.5170811600983143, "epoch": 0.03046327164041844, "frac_reward_zero_std": 0.0, "grad_norm": 0.026918554678559303, "learning_rate": 1e-05, "loss": 0.1153, "num_tokens": 25735154.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000345706939697, "sampling/importance_sampling_ratio/min": 0.19450077414512634, "sampling/sampling_logp_difference/max": 1.6373190879821777, "sampling/sampling_logp_difference/mean": 0.016961760818958282, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11578.0, "completions/max_terminated_length": 11578.0, "completions/mean_length": 9032.0, "completions/mean_terminated_length": 9032.0, "completions/min_length": 7280.0, "completions/min_terminated_length": 7280.0, "entropy": 0.8715998902916908, "epoch": 0.030520749511438097, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25809218.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002477169036865, "sampling/importance_sampling_ratio/min": 0.3181779682636261, "sampling/sampling_logp_difference/max": 1.1451444625854492, "sampling/sampling_logp_difference/mean": 0.02723650261759758, "step": 531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10805.0, "completions/max_terminated_length": 10805.0, "completions/mean_length": 6243.625, "completions/mean_terminated_length": 6243.625, "completions/min_length": 4468.0, "completions/min_terminated_length": 4468.0, "entropy": 0.44305991381406784, "epoch": 0.030578227382457752, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25860415.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.539061188697815, "sampling/importance_sampling_ratio/mean": 0.9996513724327087, "sampling/importance_sampling_ratio/min": 0.2045818716287613, "sampling/sampling_logp_difference/max": 1.5867869853973389, "sampling/sampling_logp_difference/mean": 0.018062395974993706, "step": 532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9178.0, "completions/max_terminated_length": 9178.0, "completions/mean_length": 4305.75, "completions/mean_terminated_length": 4305.75, "completions/min_length": 2004.0, "completions/min_terminated_length": 2004.0, "entropy": 0.726465031504631, "epoch": 0.030635705253477412, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25896157.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004148483276367, "sampling/importance_sampling_ratio/min": 0.49740368127822876, "sampling/sampling_logp_difference/max": 0.8762345314025879, "sampling/sampling_logp_difference/mean": 0.020223364233970642, "step": 533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3170.0, "completions/max_terminated_length": 3170.0, "completions/mean_length": 2441.125, "completions/mean_terminated_length": 2441.125, "completions/min_length": 1859.0, "completions/min_terminated_length": 1859.0, "entropy": 0.21888789907097816, "epoch": 0.030693183124497068, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25917342.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.359049916267395, "sampling/importance_sampling_ratio/mean": 1.0004068613052368, "sampling/importance_sampling_ratio/min": 0.6980496048927307, "sampling/sampling_logp_difference/max": 0.3594651222229004, "sampling/sampling_logp_difference/mean": 0.008858595043420792, "step": 534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3214.0, "completions/max_terminated_length": 3214.0, "completions/mean_length": 2559.75, "completions/mean_terminated_length": 2559.75, "completions/min_length": 1920.0, "completions/min_terminated_length": 1920.0, "entropy": 0.2667273599654436, "epoch": 0.030750660995516727, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25938916.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3679293394088745, "sampling/importance_sampling_ratio/mean": 1.00014328956604, "sampling/importance_sampling_ratio/min": 0.6829566955566406, "sampling/sampling_logp_difference/max": 0.38132381439208984, "sampling/sampling_logp_difference/mean": 0.010634428821504116, "step": 535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5091.0, "completions/max_terminated_length": 5091.0, "completions/mean_length": 2904.75, "completions/mean_terminated_length": 2904.75, "completions/min_length": 2074.0, "completions/min_terminated_length": 2074.0, "entropy": 0.33710989728569984, "epoch": 0.030808138866536383, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25963002.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3362371921539307, "sampling/importance_sampling_ratio/mean": 0.9998267292976379, "sampling/importance_sampling_ratio/min": 0.6263135671615601, "sampling/sampling_logp_difference/max": 0.46790409088134766, "sampling/sampling_logp_difference/mean": 0.01330847479403019, "step": 536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4675.0, "completions/max_terminated_length": 4675.0, "completions/mean_length": 3451.875, "completions/mean_terminated_length": 3451.875, "completions/min_length": 1792.0, "completions/min_terminated_length": 1792.0, "entropy": 0.6891240365803242, "epoch": 0.030865616737556043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 25991385.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4058934450149536, "sampling/importance_sampling_ratio/mean": 1.0001932382583618, "sampling/importance_sampling_ratio/min": 0.5406275987625122, "sampling/sampling_logp_difference/max": 0.6150245666503906, "sampling/sampling_logp_difference/mean": 0.023480074480175972, "step": 537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7788.0, "completions/max_terminated_length": 7788.0, "completions/mean_length": 5403.625, "completions/mean_terminated_length": 5403.625, "completions/min_length": 2901.0, "completions/min_terminated_length": 2901.0, "entropy": 0.5850558616220951, "epoch": 0.0309230946085757, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26035286.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001170635223389, "sampling/importance_sampling_ratio/min": 0.4724728763103485, "sampling/sampling_logp_difference/max": 1.1240196228027344, "sampling/sampling_logp_difference/mean": 0.015387811698019505, "step": 538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4958.0, "completions/max_terminated_length": 4958.0, "completions/mean_length": 3688.375, "completions/mean_terminated_length": 3688.375, "completions/min_length": 2695.0, "completions/min_terminated_length": 2695.0, "entropy": 0.3979345727711916, "epoch": 0.030980572479595354, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26065657.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3764249086380005, "sampling/importance_sampling_ratio/mean": 1.0001006126403809, "sampling/importance_sampling_ratio/min": 0.6127292513847351, "sampling/sampling_logp_difference/max": 0.48983216285705566, "sampling/sampling_logp_difference/mean": 0.01582927256822586, "step": 539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.778089977568015e-05, "clip_ratio/low_min": 8.778089977568015e-05, "clip_ratio/region_mean": 8.778089977568015e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 2246.0, "completions/max_terminated_length": 2246.0, "completions/mean_length": 1731.375, "completions/mean_terminated_length": 1731.375, "completions/min_length": 912.0, "completions/min_terminated_length": 912.0, "entropy": 0.43122661486268044, "epoch": 0.031038050350615014, "frac_reward_zero_std": 0.0, "grad_norm": 0.025397062301635742, "learning_rate": 1e-05, "loss": -0.1756, "num_tokens": 26080724.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.3162853717803955, "sampling/importance_sampling_ratio/mean": 1.0002543926239014, "sampling/importance_sampling_ratio/min": 0.6237183213233948, "sampling/sampling_logp_difference/max": 0.47205638885498047, "sampling/sampling_logp_difference/mean": 0.018269170075654984, "step": 540 }, { "clip_ratio/high_max": 4.667284338211175e-05, "clip_ratio/high_mean": 4.667284338211175e-05, "clip_ratio/low_mean": 0.00012969970703125, "clip_ratio/low_min": 0.00012969970703125, "clip_ratio/region_mean": 0.00017637255041336175, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12208.0, "completions/mean_length": 6765.125, "completions/mean_terminated_length": 5391.0, "completions/min_length": 2185.0, "completions/min_terminated_length": 2185.0, "entropy": 0.4485978316515684, "epoch": 0.03109552822163467, "frac_reward_zero_std": 0.0, "grad_norm": 0.09193140268325806, "learning_rate": 1e-05, "loss": 0.5025, "num_tokens": 26135517.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997003078460693, "sampling/importance_sampling_ratio/min": 0.23442094027996063, "sampling/sampling_logp_difference/max": 1.4506369829177856, "sampling/sampling_logp_difference/mean": 0.02020702324807644, "step": 541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2062.0, "completions/max_terminated_length": 2062.0, "completions/mean_length": 1290.25, "completions/mean_terminated_length": 1290.25, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.3065439872443676, "epoch": 0.03115300609265433, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26147215.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3034846782684326, "sampling/importance_sampling_ratio/mean": 1.0002012252807617, "sampling/importance_sampling_ratio/min": 0.5934768319129944, "sampling/sampling_logp_difference/max": 0.5217571258544922, "sampling/sampling_logp_difference/mean": 0.014665590599179268, "step": 542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8280.0, "completions/max_terminated_length": 8280.0, "completions/mean_length": 3343.0, "completions/mean_terminated_length": 3343.0, "completions/min_length": 1585.0, "completions/min_terminated_length": 1585.0, "entropy": 0.5018441341817379, "epoch": 0.031210483963673985, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26175031.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998016357421875, "sampling/importance_sampling_ratio/min": 0.555431067943573, "sampling/sampling_logp_difference/max": 0.7488462924957275, "sampling/sampling_logp_difference/mean": 0.016006747260689735, "step": 543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14868.0, "completions/max_terminated_length": 14868.0, "completions/mean_length": 11940.125, "completions/mean_terminated_length": 11940.125, "completions/min_length": 8299.0, "completions/min_terminated_length": 8299.0, "entropy": 0.8669895231723785, "epoch": 0.031267961834693644, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26271776.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000169277191162, "sampling/importance_sampling_ratio/min": 0.42007341980934143, "sampling/sampling_logp_difference/max": 1.1415109634399414, "sampling/sampling_logp_difference/mean": 0.028770722448825836, "step": 544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10331.0, "completions/max_terminated_length": 10331.0, "completions/mean_length": 5932.875, "completions/mean_terminated_length": 5932.875, "completions/min_length": 2475.0, "completions/min_terminated_length": 2475.0, "entropy": 0.43243067897856236, "epoch": 0.031325439705713304, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26320239.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7835361957550049, "sampling/importance_sampling_ratio/mean": 1.0001643896102905, "sampling/importance_sampling_ratio/min": 0.5707191228866577, "sampling/sampling_logp_difference/max": 0.5785980224609375, "sampling/sampling_logp_difference/mean": 0.016995299607515335, "step": 545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5199.0, "completions/max_terminated_length": 5199.0, "completions/mean_length": 2501.125, "completions/mean_terminated_length": 2501.125, "completions/min_length": 1163.0, "completions/min_terminated_length": 1163.0, "entropy": 0.37866940535604954, "epoch": 0.031382917576732956, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26340944.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4671776294708252, "sampling/importance_sampling_ratio/mean": 0.9997065663337708, "sampling/importance_sampling_ratio/min": 0.5567675232887268, "sampling/sampling_logp_difference/max": 0.5856075286865234, "sampling/sampling_logp_difference/mean": 0.016660301014780998, "step": 546 }, { "clip_ratio/high_max": 4.625774818123318e-05, "clip_ratio/high_mean": 4.625774818123318e-05, "clip_ratio/low_mean": 0.0005768008268205449, "clip_ratio/low_min": 0.0005768008268205449, "clip_ratio/region_mean": 0.0006230585750017781, "completions/clipped_ratio": 0.0, "completions/max_length": 14392.0, "completions/max_terminated_length": 14392.0, "completions/mean_length": 11711.5, "completions/mean_terminated_length": 11711.5, "completions/min_length": 10353.0, "completions/min_terminated_length": 10353.0, "entropy": 1.127445548772812, "epoch": 0.031440395447752616, "frac_reward_zero_std": 0.0, "grad_norm": 0.007289466448128223, "learning_rate": 1e-05, "loss": 0.0273, "num_tokens": 26435892.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000074863433838, "sampling/importance_sampling_ratio/min": 0.37040817737579346, "sampling/sampling_logp_difference/max": 0.9931497573852539, "sampling/sampling_logp_difference/mean": 0.03271519020199776, "step": 547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7694.0, "completions/max_terminated_length": 7694.0, "completions/mean_length": 4812.625, "completions/mean_terminated_length": 4812.625, "completions/min_length": 2769.0, "completions/min_terminated_length": 2769.0, "entropy": 0.406748129054904, "epoch": 0.031497873318772275, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26475817.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3370963335037231, "sampling/importance_sampling_ratio/mean": 0.999840497970581, "sampling/importance_sampling_ratio/min": 0.5931328535079956, "sampling/sampling_logp_difference/max": 0.5223369598388672, "sampling/sampling_logp_difference/mean": 0.012553131207823753, "step": 548 }, { "clip_ratio/high_max": 5.752835022576619e-05, "clip_ratio/high_mean": 5.752835022576619e-05, "clip_ratio/low_mean": 0.00046199192729545757, "clip_ratio/low_min": 0.00046199192729545757, "clip_ratio/region_mean": 0.0005195202775212238, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 11286.0, "completions/mean_length": 10621.875, "completions/mean_terminated_length": 9798.71484375, "completions/min_length": 8583.0, "completions/min_terminated_length": 8583.0, "entropy": 0.7983961999416351, "epoch": 0.03155535118979193, "frac_reward_zero_std": 0.0, "grad_norm": 0.016102788969874382, "learning_rate": 1e-05, "loss": -0.0115, "num_tokens": 26561888.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8847070932388306, "sampling/importance_sampling_ratio/mean": 0.9998630285263062, "sampling/importance_sampling_ratio/min": 0.28585535287857056, "sampling/sampling_logp_difference/max": 1.2522692680358887, "sampling/sampling_logp_difference/mean": 0.028470125049352646, "step": 549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12183.0, "completions/max_terminated_length": 12183.0, "completions/mean_length": 10100.0, "completions/mean_terminated_length": 10100.0, "completions/min_length": 8776.0, "completions/min_terminated_length": 8776.0, "entropy": 1.3207355961203575, "epoch": 0.03161282906081159, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 26643680.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8347219228744507, "sampling/importance_sampling_ratio/mean": 0.9998499155044556, "sampling/importance_sampling_ratio/min": 0.24634848535060883, "sampling/sampling_logp_difference/max": 1.401008129119873, "sampling/sampling_logp_difference/mean": 0.031560782343149185, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00040513855128665455, "clip_ratio/low_min": 0.00040513855128665455, "clip_ratio/region_mean": 0.00040513855128665455, "completions/clipped_ratio": 0.0, "completions/max_length": 6572.0, "completions/max_terminated_length": 6572.0, "completions/mean_length": 4079.625, "completions/mean_terminated_length": 4079.625, "completions/min_length": 2154.0, "completions/min_terminated_length": 2154.0, "entropy": 0.6699302792549133, "epoch": 0.031670306931831246, "frac_reward_zero_std": 0.0, "grad_norm": 0.011394458822906017, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 26677829.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.825103759765625, "sampling/importance_sampling_ratio/mean": 1.0001373291015625, "sampling/importance_sampling_ratio/min": 0.611719012260437, "sampling/sampling_logp_difference/max": 0.6016368865966797, "sampling/sampling_logp_difference/mean": 0.01900627464056015, "step": 551 }, { "clip_ratio/high_max": 0.000285563863144489, "clip_ratio/high_mean": 0.000285563863144489, "clip_ratio/low_mean": 0.00028458890665206127, "clip_ratio/low_min": 0.00028458890665206127, "clip_ratio/region_mean": 0.0005701527697965503, "completions/clipped_ratio": 0.0, "completions/max_length": 11037.0, "completions/max_terminated_length": 11037.0, "completions/mean_length": 7845.25, "completions/mean_terminated_length": 7845.25, "completions/min_length": 4319.0, "completions/min_terminated_length": 4319.0, "entropy": 0.9123466908931732, "epoch": 0.031727784802850906, "frac_reward_zero_std": 0.0, "grad_norm": 0.020108914002776146, "learning_rate": 1e-05, "loss": 0.0876, "num_tokens": 26742063.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002487897872925, "sampling/importance_sampling_ratio/min": 0.44681787490844727, "sampling/sampling_logp_difference/max": 0.8798071146011353, "sampling/sampling_logp_difference/mean": 0.03306012228131294, "step": 552 }, { "clip_ratio/high_max": 0.0001038736809277907, "clip_ratio/high_mean": 0.0001038736809277907, "clip_ratio/low_mean": 0.0003119798711850308, "clip_ratio/low_min": 0.0003119798711850308, "clip_ratio/region_mean": 0.0004158535521128215, "completions/clipped_ratio": 0.0, "completions/max_length": 7810.0, "completions/max_terminated_length": 7810.0, "completions/mean_length": 5391.375, "completions/mean_terminated_length": 5391.375, "completions/min_length": 2700.0, "completions/min_terminated_length": 2700.0, "entropy": 0.5226490460336208, "epoch": 0.03178526267387056, "frac_reward_zero_std": 0.0, "grad_norm": 0.020251518115401268, "learning_rate": 1e-05, "loss": 0.0072, "num_tokens": 26786074.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.5146822929382324, "sampling/importance_sampling_ratio/mean": 0.9999815821647644, "sampling/importance_sampling_ratio/min": 0.5456681251525879, "sampling/sampling_logp_difference/max": 0.6057443618774414, "sampling/sampling_logp_difference/mean": 0.02046748623251915, "step": 553 }, { "clip_ratio/high_max": 1.740341031108983e-05, "clip_ratio/high_mean": 1.740341031108983e-05, "clip_ratio/low_mean": 0.000733494212909136, "clip_ratio/low_min": 0.000733494212909136, "clip_ratio/region_mean": 0.0007508976232202258, "completions/clipped_ratio": 0.75, "completions/max_length": 16384.0, "completions/max_terminated_length": 14365.0, "completions/mean_length": 15804.5, "completions/mean_terminated_length": 14066.0, "completions/min_length": 13767.0, "completions/min_terminated_length": 13767.0, "entropy": 0.6251523271203041, "epoch": 0.03184274054489022, "frac_reward_zero_std": 0.0, "grad_norm": 0.00569904176518321, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 26913950.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000016689300537, "sampling/importance_sampling_ratio/min": 0.09686436504125595, "sampling/sampling_logp_difference/max": 2.3344435691833496, "sampling/sampling_logp_difference/mean": 0.022980913519859314, "step": 554 }, { "clip_ratio/high_max": 1.7365935491397977e-05, "clip_ratio/high_mean": 1.7365935491397977e-05, "clip_ratio/low_mean": 0.0006256720516830683, "clip_ratio/low_min": 0.0006256720516830683, "clip_ratio/region_mean": 0.0006430379871744663, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 14712.0, "completions/mean_length": 14299.125, "completions/mean_terminated_length": 13604.1669921875, "completions/min_length": 11781.0, "completions/min_terminated_length": 11781.0, "entropy": 0.7783675789833069, "epoch": 0.03190021841590988, "frac_reward_zero_std": 0.0, "grad_norm": 0.007816760800778866, "learning_rate": 1e-05, "loss": -0.0024, "num_tokens": 27029951.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000168800354004, "sampling/importance_sampling_ratio/min": 0.27642184495925903, "sampling/sampling_logp_difference/max": 1.2858271598815918, "sampling/sampling_logp_difference/mean": 0.024905677884817123, "step": 555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5884.0, "completions/max_terminated_length": 5884.0, "completions/mean_length": 3079.5, "completions/mean_terminated_length": 3079.5, "completions/min_length": 1069.0, "completions/min_terminated_length": 1069.0, "entropy": 0.4540298841893673, "epoch": 0.03195769628692953, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27055723.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.876967191696167, "sampling/importance_sampling_ratio/mean": 1.000065803527832, "sampling/importance_sampling_ratio/min": 0.6275302767753601, "sampling/sampling_logp_difference/max": 0.6296572685241699, "sampling/sampling_logp_difference/mean": 0.01692120172083378, "step": 556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16001.0, "completions/max_terminated_length": 16001.0, "completions/mean_length": 11362.125, "completions/mean_terminated_length": 11362.125, "completions/min_length": 8171.0, "completions/min_terminated_length": 8171.0, "entropy": 0.7231776863336563, "epoch": 0.03201517415794919, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27148028.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000840425491333, "sampling/importance_sampling_ratio/min": 0.2771872580051422, "sampling/sampling_logp_difference/max": 1.2830619812011719, "sampling/sampling_logp_difference/mean": 0.02536618895828724, "step": 557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6062.0, "completions/max_terminated_length": 6062.0, "completions/mean_length": 4407.0, "completions/mean_terminated_length": 4407.0, "completions/min_length": 2839.0, "completions/min_terminated_length": 2839.0, "entropy": 0.27537715062499046, "epoch": 0.03207265202896885, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27184364.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5452710390090942, "sampling/importance_sampling_ratio/mean": 1.0002614259719849, "sampling/importance_sampling_ratio/min": 0.37730151414871216, "sampling/sampling_logp_difference/max": 0.9747107028961182, "sampling/sampling_logp_difference/mean": 0.01256801001727581, "step": 558 }, { "clip_ratio/high_max": 4.54817582067335e-05, "clip_ratio/high_mean": 4.54817582067335e-05, "clip_ratio/low_mean": 0.0006661408442596439, "clip_ratio/low_min": 0.0006661408442596439, "clip_ratio/region_mean": 0.0007116226024663774, "completions/clipped_ratio": 0.0, "completions/max_length": 16202.0, "completions/max_terminated_length": 16202.0, "completions/mean_length": 10014.25, "completions/mean_terminated_length": 10014.25, "completions/min_length": 5414.0, "completions/min_terminated_length": 5414.0, "entropy": 1.008294090628624, "epoch": 0.03213012989998851, "frac_reward_zero_std": 0.0, "grad_norm": 0.011675787158310413, "learning_rate": 1e-05, "loss": -0.2011, "num_tokens": 27265294.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.7396332025527954, "sampling/importance_sampling_ratio/mean": 1.0002118349075317, "sampling/importance_sampling_ratio/min": 0.0755876898765564, "sampling/sampling_logp_difference/max": 2.5824618339538574, "sampling/sampling_logp_difference/mean": 0.029534026980400085, "step": 559 }, { "clip_ratio/high_max": 2.252658123325091e-05, "clip_ratio/high_mean": 2.252658123325091e-05, "clip_ratio/low_mean": 0.00013239652616903186, "clip_ratio/low_min": 0.00013239652616903186, "clip_ratio/region_mean": 0.00015492310740228277, "completions/clipped_ratio": 0.0, "completions/max_length": 8241.0, "completions/max_terminated_length": 8241.0, "completions/mean_length": 5694.375, "completions/mean_terminated_length": 5694.375, "completions/min_length": 4215.0, "completions/min_terminated_length": 4215.0, "entropy": 0.2622173074632883, "epoch": 0.03218760777100816, "frac_reward_zero_std": 0.0, "grad_norm": 0.029902080073952675, "learning_rate": 1e-05, "loss": 0.0032, "num_tokens": 27311745.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6120625734329224, "sampling/importance_sampling_ratio/mean": 1.0001996755599976, "sampling/importance_sampling_ratio/min": 0.6292364001274109, "sampling/sampling_logp_difference/max": 0.47751450538635254, "sampling/sampling_logp_difference/mean": 0.010917848907411098, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3649.0, "completions/max_terminated_length": 3649.0, "completions/mean_length": 2169.625, "completions/mean_terminated_length": 2169.625, "completions/min_length": 884.0, "completions/min_terminated_length": 884.0, "entropy": 0.217912208288908, "epoch": 0.03224508564202782, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27330158.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.362808108329773, "sampling/importance_sampling_ratio/mean": 0.9996784329414368, "sampling/importance_sampling_ratio/min": 0.6077929735183716, "sampling/sampling_logp_difference/max": 0.4979209899902344, "sampling/sampling_logp_difference/mean": 0.01014635618776083, "step": 561 }, { "clip_ratio/high_max": 0.00023204212811833713, "clip_ratio/high_mean": 0.00023204212811833713, "clip_ratio/low_mean": 0.0002970689092762768, "clip_ratio/low_min": 0.0002970689092762768, "clip_ratio/region_mean": 0.000529111037394614, "completions/clipped_ratio": 0.0, "completions/max_length": 8291.0, "completions/max_terminated_length": 8291.0, "completions/mean_length": 5924.875, "completions/mean_terminated_length": 5924.875, "completions/min_length": 3787.0, "completions/min_terminated_length": 3787.0, "entropy": 0.6691635362803936, "epoch": 0.03230256351304748, "frac_reward_zero_std": 0.0, "grad_norm": 0.008634035475552082, "learning_rate": 1e-05, "loss": -0.1276, "num_tokens": 27378389.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9441900253295898, "sampling/importance_sampling_ratio/mean": 1.0003318786621094, "sampling/importance_sampling_ratio/min": 0.3514730632305145, "sampling/sampling_logp_difference/max": 1.0456222295761108, "sampling/sampling_logp_difference/mean": 0.023020021617412567, "step": 562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5532.0, "completions/max_terminated_length": 5532.0, "completions/mean_length": 3391.25, "completions/mean_terminated_length": 3391.25, "completions/min_length": 1759.0, "completions/min_terminated_length": 1759.0, "entropy": 1.4503809213638306, "epoch": 0.03236004138406713, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27407231.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5266939401626587, "sampling/importance_sampling_ratio/mean": 1.0000207424163818, "sampling/importance_sampling_ratio/min": 0.6551684141159058, "sampling/sampling_logp_difference/max": 0.42310452461242676, "sampling/sampling_logp_difference/mean": 0.03122049756348133, "step": 563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 1.851303386501968e-05, "clip_ratio/low_min": 1.851303386501968e-05, "clip_ratio/region_mean": 1.851303386501968e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6752.0, "completions/max_terminated_length": 6752.0, "completions/mean_length": 4286.375, "completions/mean_terminated_length": 4286.375, "completions/min_length": 3230.0, "completions/min_terminated_length": 3230.0, "entropy": 0.5639854185283184, "epoch": 0.03241751925508679, "frac_reward_zero_std": 0.0, "grad_norm": 0.08701658993959427, "learning_rate": 1e-05, "loss": 0.2032, "num_tokens": 27442314.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5835564136505127, "sampling/importance_sampling_ratio/mean": 0.9998155832290649, "sampling/importance_sampling_ratio/min": 0.43028634786605835, "sampling/sampling_logp_difference/max": 0.8433043956756592, "sampling/sampling_logp_difference/mean": 0.014913749881088734, "step": 564 }, { "clip_ratio/high_max": 1.7660355297266506e-05, "clip_ratio/high_mean": 1.7660355297266506e-05, "clip_ratio/low_mean": 2.10774815059267e-05, "clip_ratio/low_min": 2.10774815059267e-05, "clip_ratio/region_mean": 3.8737836803193204e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 11861.0, "completions/max_terminated_length": 11861.0, "completions/mean_length": 5827.375, "completions/mean_terminated_length": 5827.375, "completions/min_length": 2766.0, "completions/min_terminated_length": 2766.0, "entropy": 0.7557978555560112, "epoch": 0.03247499712610645, "frac_reward_zero_std": 0.0, "grad_norm": 0.009867901913821697, "learning_rate": 1e-05, "loss": 0.1927, "num_tokens": 27490309.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5746175050735474, "sampling/importance_sampling_ratio/mean": 1.0000483989715576, "sampling/importance_sampling_ratio/min": 0.08365873247385025, "sampling/sampling_logp_difference/max": 2.4810094833374023, "sampling/sampling_logp_difference/mean": 0.021323755383491516, "step": 565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9862.0, "completions/max_terminated_length": 9862.0, "completions/mean_length": 6101.75, "completions/mean_terminated_length": 6101.75, "completions/min_length": 4143.0, "completions/min_terminated_length": 4143.0, "entropy": 0.45524169504642487, "epoch": 0.03253247499712611, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27539923.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4786040782928467, "sampling/importance_sampling_ratio/mean": 1.000051736831665, "sampling/importance_sampling_ratio/min": 0.5340578556060791, "sampling/sampling_logp_difference/max": 0.627251148223877, "sampling/sampling_logp_difference/mean": 0.016592375934123993, "step": 566 }, { "clip_ratio/high_max": 0.0001332753572569345, "clip_ratio/high_mean": 0.0001332753572569345, "clip_ratio/low_mean": 8.392333984375e-05, "clip_ratio/low_min": 8.392333984375e-05, "clip_ratio/region_mean": 0.0002171986971006845, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 9590.0, "completions/mean_length": 8802.25, "completions/mean_terminated_length": 7719.14306640625, "completions/min_length": 4527.0, "completions/min_terminated_length": 4527.0, "entropy": 0.7129413932561874, "epoch": 0.03258995286814576, "frac_reward_zero_std": 0.0, "grad_norm": 0.07194610685110092, "learning_rate": 1e-05, "loss": 0.3045, "num_tokens": 27612101.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997614026069641, "sampling/importance_sampling_ratio/min": 0.4759815037250519, "sampling/sampling_logp_difference/max": 0.9844303131103516, "sampling/sampling_logp_difference/mean": 0.02401205711066723, "step": 567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13311.0, "completions/max_terminated_length": 13311.0, "completions/mean_length": 9598.875, "completions/mean_terminated_length": 9598.875, "completions/min_length": 3185.0, "completions/min_terminated_length": 3185.0, "entropy": 1.1379680186510086, "epoch": 0.03264743073916542, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27690012.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999382495880127, "sampling/importance_sampling_ratio/min": 0.32948538661003113, "sampling/sampling_logp_difference/max": 1.1102232933044434, "sampling/sampling_logp_difference/mean": 0.02715124748647213, "step": 568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14169.0, "completions/max_terminated_length": 14169.0, "completions/mean_length": 10503.625, "completions/mean_terminated_length": 10503.625, "completions/min_length": 7785.0, "completions/min_terminated_length": 7785.0, "entropy": 1.1986949443817139, "epoch": 0.03270490861018508, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27775057.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8444825410842896, "sampling/importance_sampling_ratio/mean": 0.9999459385871887, "sampling/importance_sampling_ratio/min": 0.31394320726394653, "sampling/sampling_logp_difference/max": 1.1585431098937988, "sampling/sampling_logp_difference/mean": 0.034226298332214355, "step": 569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 14594.0, "completions/mean_length": 14919.375, "completions/mean_terminated_length": 13454.75, "completions/min_length": 12108.0, "completions/min_terminated_length": 12108.0, "entropy": 0.4883462227880955, "epoch": 0.03276238648120473, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27896108.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000072717666626, "sampling/importance_sampling_ratio/min": 0.07481062412261963, "sampling/sampling_logp_difference/max": 2.5927953720092773, "sampling/sampling_logp_difference/mean": 0.0203559510409832, "step": 570 }, { "clip_ratio/high_max": 3.197748810634948e-05, "clip_ratio/high_mean": 3.197748810634948e-05, "clip_ratio/low_mean": 9.897070412989706e-05, "clip_ratio/low_min": 9.897070412989706e-05, "clip_ratio/region_mean": 0.00013094819223624654, "completions/clipped_ratio": 0.0, "completions/max_length": 4189.0, "completions/max_terminated_length": 4189.0, "completions/mean_length": 2474.75, "completions/mean_terminated_length": 2474.75, "completions/min_length": 1237.0, "completions/min_terminated_length": 1237.0, "entropy": 0.27096972055733204, "epoch": 0.03281986435222439, "frac_reward_zero_std": 0.0, "grad_norm": 0.006724013015627861, "learning_rate": 1e-05, "loss": -0.1731, "num_tokens": 27916762.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3275405168533325, "sampling/importance_sampling_ratio/mean": 1.0002199411392212, "sampling/importance_sampling_ratio/min": 0.6540438532829285, "sampling/sampling_logp_difference/max": 0.42458081245422363, "sampling/sampling_logp_difference/mean": 0.011759305372834206, "step": 571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3571.0, "completions/max_terminated_length": 3571.0, "completions/mean_length": 2710.375, "completions/mean_terminated_length": 2710.375, "completions/min_length": 1772.0, "completions/min_terminated_length": 1772.0, "entropy": 0.3044627793133259, "epoch": 0.03287734222324405, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27939205.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4333112239837646, "sampling/importance_sampling_ratio/mean": 0.9999540448188782, "sampling/importance_sampling_ratio/min": 0.5170538425445557, "sampling/sampling_logp_difference/max": 0.6596082448959351, "sampling/sampling_logp_difference/mean": 0.013198107481002808, "step": 572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4330.0, "completions/max_terminated_length": 4330.0, "completions/mean_length": 2604.0, "completions/mean_terminated_length": 2604.0, "completions/min_length": 1475.0, "completions/min_terminated_length": 1475.0, "entropy": 0.37607937678694725, "epoch": 0.03293482009426371, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27960693.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2526540756225586, "sampling/importance_sampling_ratio/mean": 0.9998202919960022, "sampling/importance_sampling_ratio/min": 0.6126984357833862, "sampling/sampling_logp_difference/max": 0.4898824691772461, "sampling/sampling_logp_difference/mean": 0.014478504657745361, "step": 573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5820.0, "completions/max_terminated_length": 5820.0, "completions/mean_length": 2501.5, "completions/mean_terminated_length": 2501.5, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "entropy": 0.6340821757912636, "epoch": 0.032992297965283364, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 27981657.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0006377696990967, "sampling/importance_sampling_ratio/min": 0.5598527789115906, "sampling/sampling_logp_difference/max": 2.0551321506500244, "sampling/sampling_logp_difference/mean": 0.02611786127090454, "step": 574 }, { "clip_ratio/high_max": 5.595344555331394e-05, "clip_ratio/high_mean": 5.595344555331394e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.595344555331394e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 4290.0, "completions/max_terminated_length": 4290.0, "completions/mean_length": 1981.0, "completions/mean_terminated_length": 1981.0, "completions/min_length": 978.0, "completions/min_terminated_length": 978.0, "entropy": 0.4312672056257725, "epoch": 0.03304977583630302, "frac_reward_zero_std": 0.0, "grad_norm": 0.008908388204872608, "learning_rate": 1e-05, "loss": -0.1301, "num_tokens": 27998633.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4045640230178833, "sampling/importance_sampling_ratio/mean": 0.9997453093528748, "sampling/importance_sampling_ratio/min": 0.6376234292984009, "sampling/sampling_logp_difference/max": 0.45000743865966797, "sampling/sampling_logp_difference/mean": 0.014045635238289833, "step": 575 }, { "clip_ratio/high_max": 4.603465094987769e-05, "clip_ratio/high_mean": 4.603465094987769e-05, "clip_ratio/low_mean": 0.0005794132048322354, "clip_ratio/low_min": 0.0005794132048322354, "clip_ratio/region_mean": 0.000625447855782113, "completions/clipped_ratio": 0.625, "completions/max_length": 16384.0, "completions/max_terminated_length": 16160.0, "completions/mean_length": 14580.875, "completions/mean_terminated_length": 11575.6669921875, "completions/min_length": 5512.0, "completions/min_terminated_length": 5512.0, "entropy": 0.6104663833975792, "epoch": 0.03310725370732268, "frac_reward_zero_std": 0.0, "grad_norm": 0.027196569368243217, "learning_rate": 1e-05, "loss": -0.001, "num_tokens": 28116416.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999819993972778, "sampling/importance_sampling_ratio/min": 0.1977550983428955, "sampling/sampling_logp_difference/max": 1.6207258701324463, "sampling/sampling_logp_difference/mean": 0.022195711731910706, "step": 576 }, { "clip_ratio/high_max": 0.0001135228740167804, "clip_ratio/high_mean": 0.0001135228740167804, "clip_ratio/low_mean": 0.0002313375152880326, "clip_ratio/low_min": 0.0002313375152880326, "clip_ratio/region_mean": 0.000344860389304813, "completions/clipped_ratio": 0.0, "completions/max_length": 14296.0, "completions/max_terminated_length": 14296.0, "completions/mean_length": 7867.5, "completions/mean_terminated_length": 7867.5, "completions/min_length": 2346.0, "completions/min_terminated_length": 2346.0, "entropy": 0.5577292367815971, "epoch": 0.033164731578342335, "frac_reward_zero_std": 0.0, "grad_norm": 0.01272398978471756, "learning_rate": 1e-05, "loss": 0.2425, "num_tokens": 28180620.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998690485954285, "sampling/importance_sampling_ratio/min": 0.3565557301044464, "sampling/sampling_logp_difference/max": 1.137618064880371, "sampling/sampling_logp_difference/mean": 0.021759258583188057, "step": 577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5939.0, "completions/max_terminated_length": 5939.0, "completions/mean_length": 4477.75, "completions/mean_terminated_length": 4477.75, "completions/min_length": 2956.0, "completions/min_terminated_length": 2956.0, "entropy": 0.3239993769675493, "epoch": 0.033222209449361995, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28217298.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3068655729293823, "sampling/importance_sampling_ratio/mean": 0.9998660087585449, "sampling/importance_sampling_ratio/min": 0.5894641280174255, "sampling/sampling_logp_difference/max": 0.5285414457321167, "sampling/sampling_logp_difference/mean": 0.013875147327780724, "step": 578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4556.0, "completions/max_terminated_length": 4556.0, "completions/mean_length": 3082.25, "completions/mean_terminated_length": 3082.25, "completions/min_length": 1829.0, "completions/min_terminated_length": 1829.0, "entropy": 1.0218954682350159, "epoch": 0.033279687320381654, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28246924.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5577107667922974, "sampling/importance_sampling_ratio/mean": 0.9996776580810547, "sampling/importance_sampling_ratio/min": 0.5805104374885559, "sampling/sampling_logp_difference/max": 0.5438475608825684, "sampling/sampling_logp_difference/mean": 0.03262925148010254, "step": 579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13441.0, "completions/max_terminated_length": 13441.0, "completions/mean_length": 5557.75, "completions/mean_terminated_length": 5557.75, "completions/min_length": 1983.0, "completions/min_terminated_length": 1983.0, "entropy": 0.3323168959468603, "epoch": 0.03333716519140131, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28292650.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5502843856811523, "sampling/importance_sampling_ratio/mean": 1.0001157522201538, "sampling/importance_sampling_ratio/min": 0.5263938903808594, "sampling/sampling_logp_difference/max": 0.6417055130004883, "sampling/sampling_logp_difference/mean": 0.014897220768034458, "step": 580 }, { "clip_ratio/high_max": 0.0001226265521836467, "clip_ratio/high_mean": 0.0001226265521836467, "clip_ratio/low_mean": 4.661135608330369e-05, "clip_ratio/low_min": 4.661135608330369e-05, "clip_ratio/region_mean": 0.0001692379082669504, "completions/clipped_ratio": 0.0, "completions/max_length": 10727.0, "completions/max_terminated_length": 10727.0, "completions/mean_length": 5124.75, "completions/mean_terminated_length": 5124.75, "completions/min_length": 2884.0, "completions/min_terminated_length": 2884.0, "entropy": 0.28536986373364925, "epoch": 0.033394643062420966, "frac_reward_zero_std": 0.0, "grad_norm": 0.009833471849560738, "learning_rate": 1e-05, "loss": 0.3864, "num_tokens": 28334848.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8411531448364258, "sampling/importance_sampling_ratio/mean": 0.999947726726532, "sampling/importance_sampling_ratio/min": 0.544079065322876, "sampling/sampling_logp_difference/max": 0.6103920936584473, "sampling/sampling_logp_difference/mean": 0.01447758823633194, "step": 581 }, { "clip_ratio/high_max": 1.4386005204869434e-05, "clip_ratio/high_mean": 1.4386005204869434e-05, "clip_ratio/low_mean": 1.2572922969411593e-05, "clip_ratio/low_min": 1.2572922969411593e-05, "clip_ratio/region_mean": 2.6958928174281027e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 9942.0, "completions/max_terminated_length": 9942.0, "completions/mean_length": 4292.25, "completions/mean_terminated_length": 4292.25, "completions/min_length": 707.0, "completions/min_terminated_length": 707.0, "entropy": 0.9875373095273972, "epoch": 0.033452120933440625, "frac_reward_zero_std": 0.0, "grad_norm": 0.02404913865029812, "learning_rate": 1e-05, "loss": 0.1552, "num_tokens": 28370554.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997151494026184, "sampling/importance_sampling_ratio/min": 0.5137315392494202, "sampling/sampling_logp_difference/max": 1.003482460975647, "sampling/sampling_logp_difference/mean": 0.026508359238505363, "step": 582 }, { "clip_ratio/high_max": 0.00013962076900497777, "clip_ratio/high_mean": 0.00013962076900497777, "clip_ratio/low_mean": 0.0001890673847810831, "clip_ratio/low_min": 0.0001890673847810831, "clip_ratio/region_mean": 0.00032868815378606087, "completions/clipped_ratio": 0.0, "completions/max_length": 8471.0, "completions/max_terminated_length": 8471.0, "completions/mean_length": 6555.875, "completions/mean_terminated_length": 6555.875, "completions/min_length": 4629.0, "completions/min_terminated_length": 4629.0, "entropy": 0.5242008939385414, "epoch": 0.033509598804460285, "frac_reward_zero_std": 0.0, "grad_norm": 0.011201899498701096, "learning_rate": 1e-05, "loss": 0.0831, "num_tokens": 28424425.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.522914171218872, "sampling/importance_sampling_ratio/mean": 0.9998530149459839, "sampling/importance_sampling_ratio/min": 0.1309383511543274, "sampling/sampling_logp_difference/max": 2.0330286026000977, "sampling/sampling_logp_difference/mean": 0.01950019784271717, "step": 583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8798.0, "completions/max_terminated_length": 8798.0, "completions/mean_length": 6118.125, "completions/mean_terminated_length": 6118.125, "completions/min_length": 2994.0, "completions/min_terminated_length": 2994.0, "entropy": 0.33216336742043495, "epoch": 0.03356707667547994, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28476538.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.611647605895996, "sampling/importance_sampling_ratio/mean": 0.9997901916503906, "sampling/importance_sampling_ratio/min": 0.5181024074554443, "sampling/sampling_logp_difference/max": 0.6575822830200195, "sampling/sampling_logp_difference/mean": 0.014396095648407936, "step": 584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5823.0, "completions/max_terminated_length": 5823.0, "completions/mean_length": 3631.0, "completions/mean_terminated_length": 3631.0, "completions/min_length": 2419.0, "completions/min_terminated_length": 2419.0, "entropy": 0.4794558174908161, "epoch": 0.033624554546499597, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28506218.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7523808479309082, "sampling/importance_sampling_ratio/mean": 1.0003423690795898, "sampling/importance_sampling_ratio/min": 0.6068167686462402, "sampling/sampling_logp_difference/max": 0.5609753131866455, "sampling/sampling_logp_difference/mean": 0.017627373337745667, "step": 585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.751045344164595e-05, "clip_ratio/low_min": 4.751045344164595e-05, "clip_ratio/region_mean": 4.751045344164595e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 11402.0, "completions/max_terminated_length": 11402.0, "completions/mean_length": 7818.875, "completions/mean_terminated_length": 7818.875, "completions/min_length": 4552.0, "completions/min_terminated_length": 4552.0, "entropy": 0.8490485176444054, "epoch": 0.033682032417519256, "frac_reward_zero_std": 0.0, "grad_norm": 0.008899558335542679, "learning_rate": 1e-05, "loss": 0.1221, "num_tokens": 28569657.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002703666687012, "sampling/importance_sampling_ratio/min": 0.4663603603839874, "sampling/sampling_logp_difference/max": 0.7821841239929199, "sampling/sampling_logp_difference/mean": 0.023546023294329643, "step": 586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10799.0, "completions/max_terminated_length": 10799.0, "completions/mean_length": 4907.625, "completions/mean_terminated_length": 4907.625, "completions/min_length": 2972.0, "completions/min_terminated_length": 2972.0, "entropy": 0.5072892606258392, "epoch": 0.033739510288538915, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28609878.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001423358917236, "sampling/importance_sampling_ratio/min": 0.45848244428634644, "sampling/sampling_logp_difference/max": 0.7798333168029785, "sampling/sampling_logp_difference/mean": 0.020117294043302536, "step": 587 }, { "clip_ratio/high_max": 3.053248656215146e-05, "clip_ratio/high_mean": 3.053248656215146e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 3.053248656215146e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 5954.0, "completions/max_terminated_length": 5954.0, "completions/mean_length": 3908.75, "completions/mean_terminated_length": 3908.75, "completions/min_length": 2401.0, "completions/min_terminated_length": 2401.0, "entropy": 0.5021892823278904, "epoch": 0.03379698815955857, "frac_reward_zero_std": 0.0, "grad_norm": 0.05622486770153046, "learning_rate": 1e-05, "loss": -0.0993, "num_tokens": 28642052.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3737815618515015, "sampling/importance_sampling_ratio/mean": 1.000217080116272, "sampling/importance_sampling_ratio/min": 0.5733225345611572, "sampling/sampling_logp_difference/max": 0.5563068389892578, "sampling/sampling_logp_difference/mean": 0.015061739832162857, "step": 588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00048635396160534583, "clip_ratio/low_min": 0.00048635396160534583, "clip_ratio/region_mean": 0.00048635396160534583, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 15475.0, "completions/mean_length": 13414.625, "completions/mean_terminated_length": 12424.833984375, "completions/min_length": 10277.0, "completions/min_terminated_length": 10277.0, "entropy": 0.5952613316476345, "epoch": 0.03385446603057823, "frac_reward_zero_std": 0.0, "grad_norm": 0.0053983223624527454, "learning_rate": 1e-05, "loss": 0.0667, "num_tokens": 28750753.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999475479125977, "sampling/importance_sampling_ratio/min": 0.31103649735450745, "sampling/sampling_logp_difference/max": 1.430211067199707, "sampling/sampling_logp_difference/mean": 0.020132360979914665, "step": 589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16013.0, "completions/max_terminated_length": 16013.0, "completions/mean_length": 11769.0, "completions/mean_terminated_length": 11769.0, "completions/min_length": 7974.0, "completions/min_terminated_length": 7974.0, "entropy": 0.68447620049119, "epoch": 0.03391194390159789, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28846745.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000179409980774, "sampling/importance_sampling_ratio/min": 0.36562904715538025, "sampling/sampling_logp_difference/max": 1.0061359405517578, "sampling/sampling_logp_difference/mean": 0.02607608214020729, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15866.0, "completions/mean_length": 13849.75, "completions/mean_terminated_length": 12329.2001953125, "completions/min_length": 6174.0, "completions/min_terminated_length": 6174.0, "entropy": 0.7128940373659134, "epoch": 0.03396942177261754, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28958479.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997577667236328, "sampling/importance_sampling_ratio/min": 0.2632654905319214, "sampling/sampling_logp_difference/max": 1.334592342376709, "sampling/sampling_logp_difference/mean": 0.025293508544564247, "step": 591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5245.0, "completions/max_terminated_length": 5245.0, "completions/mean_length": 3443.625, "completions/mean_terminated_length": 3443.625, "completions/min_length": 2227.0, "completions/min_terminated_length": 2227.0, "entropy": 0.3500296510756016, "epoch": 0.0340268996436372, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 28987252.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3872319459915161, "sampling/importance_sampling_ratio/mean": 0.9999179244041443, "sampling/importance_sampling_ratio/min": 0.6304831504821777, "sampling/sampling_logp_difference/max": 0.46126890182495117, "sampling/sampling_logp_difference/mean": 0.011495915241539478, "step": 592 }, { "clip_ratio/high_max": 2.918767768278485e-05, "clip_ratio/high_mean": 2.918767768278485e-05, "clip_ratio/low_mean": 0.00019780035381700145, "clip_ratio/low_min": 0.00019780035381700145, "clip_ratio/region_mean": 0.0002269880314997863, "completions/clipped_ratio": 0.0, "completions/max_length": 14519.0, "completions/max_terminated_length": 14519.0, "completions/mean_length": 9642.375, "completions/mean_terminated_length": 9642.375, "completions/min_length": 4627.0, "completions/min_terminated_length": 4627.0, "entropy": 0.7309829741716385, "epoch": 0.03408437751465686, "frac_reward_zero_std": 0.0, "grad_norm": 0.022312970831990242, "learning_rate": 1e-05, "loss": 0.2667, "num_tokens": 29065919.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998202919960022, "sampling/importance_sampling_ratio/min": 0.10352642834186554, "sampling/sampling_logp_difference/max": 2.2679283618927, "sampling/sampling_logp_difference/mean": 0.021985406056046486, "step": 593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14776.0, "completions/max_terminated_length": 14776.0, "completions/mean_length": 10161.625, "completions/mean_terminated_length": 10161.625, "completions/min_length": 6813.0, "completions/min_terminated_length": 6813.0, "entropy": 0.8121396787464619, "epoch": 0.03414185538567652, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29148220.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999736547470093, "sampling/importance_sampling_ratio/min": 0.4193873405456543, "sampling/sampling_logp_difference/max": 0.8689603805541992, "sampling/sampling_logp_difference/mean": 0.028623400256037712, "step": 594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7715395087143406e-05, "clip_ratio/low_min": 5.7715395087143406e-05, "clip_ratio/region_mean": 5.7715395087143406e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10829.0, "completions/max_terminated_length": 10829.0, "completions/mean_length": 6251.0, "completions/mean_terminated_length": 6251.0, "completions/min_length": 2634.0, "completions/min_terminated_length": 2634.0, "entropy": 0.3372673988342285, "epoch": 0.03419933325669617, "frac_reward_zero_std": 0.0, "grad_norm": 0.006354357115924358, "learning_rate": 1e-05, "loss": 0.2585, "num_tokens": 29199092.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997925162315369, "sampling/importance_sampling_ratio/min": 0.45151516795158386, "sampling/sampling_logp_difference/max": 0.9773533344268799, "sampling/sampling_logp_difference/mean": 0.014827420003712177, "step": 595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13528.0, "completions/max_terminated_length": 13528.0, "completions/mean_length": 6157.0, "completions/mean_terminated_length": 6157.0, "completions/min_length": 1573.0, "completions/min_terminated_length": 1573.0, "entropy": 0.6303038783371449, "epoch": 0.03425681112771583, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29249820.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997907876968384, "sampling/importance_sampling_ratio/min": 0.43775463104248047, "sampling/sampling_logp_difference/max": 0.9805684089660645, "sampling/sampling_logp_difference/mean": 0.026151971891522408, "step": 596 }, { "clip_ratio/high_max": 2.1258503693388775e-05, "clip_ratio/high_mean": 2.1258503693388775e-05, "clip_ratio/low_mean": 2.0589688574546017e-05, "clip_ratio/low_min": 2.0589688574546017e-05, "clip_ratio/region_mean": 4.184819226793479e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 8761.0, "completions/max_terminated_length": 8761.0, "completions/mean_length": 5538.125, "completions/mean_terminated_length": 5538.125, "completions/min_length": 2891.0, "completions/min_terminated_length": 2891.0, "entropy": 0.7887991666793823, "epoch": 0.03431428899873549, "frac_reward_zero_std": 0.0, "grad_norm": 0.006347157992422581, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 29295509.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.611868977546692, "sampling/importance_sampling_ratio/mean": 0.999934732913971, "sampling/importance_sampling_ratio/min": 0.5496208071708679, "sampling/sampling_logp_difference/max": 0.5985267162322998, "sampling/sampling_logp_difference/mean": 0.020369742065668106, "step": 597 }, { "clip_ratio/high_max": 5.1809893193421885e-05, "clip_ratio/high_mean": 5.1809893193421885e-05, "clip_ratio/low_mean": 0.00034243783011334017, "clip_ratio/low_min": 0.00034243783011334017, "clip_ratio/region_mean": 0.00039424772330676205, "completions/clipped_ratio": 0.0, "completions/max_length": 9087.0, "completions/max_terminated_length": 9087.0, "completions/mean_length": 6438.875, "completions/mean_terminated_length": 6438.875, "completions/min_length": 2442.0, "completions/min_terminated_length": 2442.0, "entropy": 0.5232025049626827, "epoch": 0.03437176686975514, "frac_reward_zero_std": 0.0, "grad_norm": 0.012352300807833672, "learning_rate": 1e-05, "loss": 0.1593, "num_tokens": 29348628.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6394954919815063, "sampling/importance_sampling_ratio/mean": 1.0000801086425781, "sampling/importance_sampling_ratio/min": 0.40205124020576477, "sampling/sampling_logp_difference/max": 0.9111757278442383, "sampling/sampling_logp_difference/mean": 0.020501915365457535, "step": 598 }, { "clip_ratio/high_max": 4.492400330491364e-05, "clip_ratio/high_mean": 4.492400330491364e-05, "clip_ratio/low_mean": 0.0005091654420539271, "clip_ratio/low_min": 0.0005091654420539271, "clip_ratio/region_mean": 0.0005540894453588407, "completions/clipped_ratio": 0.0, "completions/max_length": 12791.0, "completions/max_terminated_length": 12791.0, "completions/mean_length": 9780.875, "completions/mean_terminated_length": 9780.875, "completions/min_length": 7468.0, "completions/min_terminated_length": 7468.0, "entropy": 1.260837011039257, "epoch": 0.0344292447407748, "frac_reward_zero_std": 0.0, "grad_norm": 0.033661991357803345, "learning_rate": 1e-05, "loss": 0.0827, "num_tokens": 29428043.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000119686126709, "sampling/importance_sampling_ratio/min": 0.41858381032943726, "sampling/sampling_logp_difference/max": 0.8949170112609863, "sampling/sampling_logp_difference/mean": 0.03273266181349754, "step": 599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3599.0, "completions/max_terminated_length": 3599.0, "completions/mean_length": 1525.25, "completions/mean_terminated_length": 1525.25, "completions/min_length": 944.0, "completions/min_terminated_length": 944.0, "entropy": 0.20385275594890118, "epoch": 0.03448672261179446, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29441069.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.334586262702942, "sampling/importance_sampling_ratio/mean": 0.9994500279426575, "sampling/importance_sampling_ratio/min": 0.6543148159980774, "sampling/sampling_logp_difference/max": 0.4241666793823242, "sampling/sampling_logp_difference/mean": 0.009200350381433964, "step": 600 }, { "clip_ratio/high_max": 6.272678001550958e-05, "clip_ratio/high_mean": 6.272678001550958e-05, "clip_ratio/low_mean": 0.00018193727009929717, "clip_ratio/low_min": 0.00018193727009929717, "clip_ratio/region_mean": 0.00024466405011480674, "completions/clipped_ratio": 0.0, "completions/max_length": 13741.0, "completions/max_terminated_length": 13741.0, "completions/mean_length": 5186.625, "completions/mean_terminated_length": 5186.625, "completions/min_length": 2930.0, "completions/min_terminated_length": 2930.0, "entropy": 0.4094797223806381, "epoch": 0.03454420048281412, "frac_reward_zero_std": 0.0, "grad_norm": 0.1166185811161995, "learning_rate": 1e-05, "loss": 0.5825, "num_tokens": 29483602.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000827312469482, "sampling/importance_sampling_ratio/min": 0.19107843935489655, "sampling/sampling_logp_difference/max": 1.6550712585449219, "sampling/sampling_logp_difference/mean": 0.018584206700325012, "step": 601 }, { "clip_ratio/high_max": 4.015419108327478e-05, "clip_ratio/high_mean": 4.015419108327478e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.015419108327478e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 3686.0, "completions/max_terminated_length": 3686.0, "completions/mean_length": 2682.5, "completions/mean_terminated_length": 2682.5, "completions/min_length": 1643.0, "completions/min_terminated_length": 1643.0, "entropy": 0.32614005729556084, "epoch": 0.03460167835383377, "frac_reward_zero_std": 0.0, "grad_norm": 0.008030612953007221, "learning_rate": 1e-05, "loss": -0.1371, "num_tokens": 29505878.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3646286725997925, "sampling/importance_sampling_ratio/mean": 1.0001521110534668, "sampling/importance_sampling_ratio/min": 0.7037291526794434, "sampling/sampling_logp_difference/max": 0.3513617515563965, "sampling/sampling_logp_difference/mean": 0.010575047694146633, "step": 602 }, { "clip_ratio/high_max": 1.3867317647964228e-05, "clip_ratio/high_mean": 1.3867317647964228e-05, "clip_ratio/low_mean": 4.564910068438621e-05, "clip_ratio/low_min": 4.564910068438621e-05, "clip_ratio/region_mean": 5.951641833235044e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 11945.0, "completions/max_terminated_length": 11945.0, "completions/mean_length": 9154.875, "completions/mean_terminated_length": 9154.875, "completions/min_length": 7527.0, "completions/min_terminated_length": 7527.0, "entropy": 0.6813321709632874, "epoch": 0.03465915622485343, "frac_reward_zero_std": 0.0, "grad_norm": 0.02618837170302868, "learning_rate": 1e-05, "loss": -0.0216, "num_tokens": 29579917.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999310970306396, "sampling/importance_sampling_ratio/min": 0.4370376765727997, "sampling/sampling_logp_difference/max": 0.8931584358215332, "sampling/sampling_logp_difference/mean": 0.017381031066179276, "step": 603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14957.0, "completions/max_terminated_length": 14957.0, "completions/mean_length": 9390.625, "completions/mean_terminated_length": 9390.625, "completions/min_length": 1818.0, "completions/min_terminated_length": 1818.0, "entropy": 0.9319348335266113, "epoch": 0.03471663409587309, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29655842.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000106692314148, "sampling/importance_sampling_ratio/min": 0.4158443510532379, "sampling/sampling_logp_difference/max": 0.996164083480835, "sampling/sampling_logp_difference/mean": 0.028765833005309105, "step": 604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.875, "completions/max_length": 16384.0, "completions/max_terminated_length": 9962.0, "completions/mean_length": 15581.25, "completions/mean_terminated_length": 9962.0, "completions/min_length": 9962.0, "completions/min_terminated_length": 9962.0, "entropy": 0.5301873199641705, "epoch": 0.03477411196689274, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29782284.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997626543045044, "sampling/importance_sampling_ratio/min": 0.1687299758195877, "sampling/sampling_logp_difference/max": 1.7794556617736816, "sampling/sampling_logp_difference/mean": 0.021705882623791695, "step": 605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3263.0, "completions/max_terminated_length": 3263.0, "completions/mean_length": 2013.0, "completions/mean_terminated_length": 2013.0, "completions/min_length": 1070.0, "completions/min_terminated_length": 1070.0, "entropy": 0.6546818614006042, "epoch": 0.0348315898379124, "frac_reward_zero_std": 0.0, "grad_norm": 0.01744985021650791, "learning_rate": 1e-05, "loss": -0.0815, "num_tokens": 29799580.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4481335878372192, "sampling/importance_sampling_ratio/mean": 1.000276803970337, "sampling/importance_sampling_ratio/min": 0.6946722865104675, "sampling/sampling_logp_difference/max": 0.37027549743652344, "sampling/sampling_logp_difference/mean": 0.019837072119116783, "step": 606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 16190.0, "completions/mean_length": 14334.0, "completions/mean_terminated_length": 12284.0, "completions/min_length": 6729.0, "completions/min_terminated_length": 6729.0, "entropy": 0.6369915828108788, "epoch": 0.03488906770893206, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29915444.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998630881309509, "sampling/importance_sampling_ratio/min": 0.04873182997107506, "sampling/sampling_logp_difference/max": 3.021422863006592, "sampling/sampling_logp_difference/mean": 0.023937057703733444, "step": 607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4821.0, "completions/max_terminated_length": 4821.0, "completions/mean_length": 3964.625, "completions/mean_terminated_length": 3964.625, "completions/min_length": 3072.0, "completions/min_terminated_length": 3072.0, "entropy": 0.29254538379609585, "epoch": 0.03494654557995172, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29948337.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4031208753585815, "sampling/importance_sampling_ratio/mean": 0.9998210668563843, "sampling/importance_sampling_ratio/min": 0.6215870380401611, "sampling/sampling_logp_difference/max": 0.4754793643951416, "sampling/sampling_logp_difference/mean": 0.012456352822482586, "step": 608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3947.0, "completions/max_terminated_length": 3947.0, "completions/mean_length": 2879.375, "completions/mean_terminated_length": 2879.375, "completions/min_length": 1759.0, "completions/min_terminated_length": 1759.0, "entropy": 0.3981276024132967, "epoch": 0.035004023450971374, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29972588.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5246267318725586, "sampling/importance_sampling_ratio/mean": 0.9998361468315125, "sampling/importance_sampling_ratio/min": 0.550699770450592, "sampling/sampling_logp_difference/max": 0.5965654850006104, "sampling/sampling_logp_difference/mean": 0.016137398779392242, "step": 609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4285.0, "completions/max_terminated_length": 4285.0, "completions/mean_length": 1911.0, "completions/mean_terminated_length": 1911.0, "completions/min_length": 884.0, "completions/min_terminated_length": 884.0, "entropy": 0.47064877301454544, "epoch": 0.03506150132199103, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 29989380.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0005251169204712, "sampling/importance_sampling_ratio/min": 0.5568405389785767, "sampling/sampling_logp_difference/max": 0.7033340930938721, "sampling/sampling_logp_difference/mean": 0.021054988726973534, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022418411936087068, "clip_ratio/low_min": 0.00022418411936087068, "clip_ratio/region_mean": 0.00022418411936087068, "completions/clipped_ratio": 0.0, "completions/max_length": 10074.0, "completions/max_terminated_length": 10074.0, "completions/mean_length": 4916.0, "completions/mean_terminated_length": 4916.0, "completions/min_length": 1834.0, "completions/min_terminated_length": 1834.0, "entropy": 0.8414740897715092, "epoch": 0.03511897919301069, "frac_reward_zero_std": 0.0, "grad_norm": 0.005442914552986622, "learning_rate": 1e-05, "loss": -0.3711, "num_tokens": 30029524.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.453122854232788, "sampling/importance_sampling_ratio/mean": 0.9997959733009338, "sampling/importance_sampling_ratio/min": 0.5262638926506042, "sampling/sampling_logp_difference/max": 0.6419525146484375, "sampling/sampling_logp_difference/mean": 0.022994283586740494, "step": 611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8505.0, "completions/max_terminated_length": 8505.0, "completions/mean_length": 6475.5, "completions/mean_terminated_length": 6475.5, "completions/min_length": 3912.0, "completions/min_terminated_length": 3912.0, "entropy": 0.7211964651942253, "epoch": 0.03517645706403035, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30083464.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001176595687866, "sampling/importance_sampling_ratio/min": 0.387180358171463, "sampling/sampling_logp_difference/max": 0.9488646984100342, "sampling/sampling_logp_difference/mean": 0.020306039601564407, "step": 612 }, { "clip_ratio/high_max": 7.106310658855364e-05, "clip_ratio/high_mean": 7.106310658855364e-05, "clip_ratio/low_mean": 6.234413740457967e-05, "clip_ratio/low_min": 6.234413740457967e-05, "clip_ratio/region_mean": 0.0001334072439931333, "completions/clipped_ratio": 0.0, "completions/max_length": 2530.0, "completions/max_terminated_length": 2530.0, "completions/mean_length": 1808.125, "completions/mean_terminated_length": 1808.125, "completions/min_length": 1051.0, "completions/min_terminated_length": 1051.0, "entropy": 0.43920112028717995, "epoch": 0.035233934935050004, "frac_reward_zero_std": 0.0, "grad_norm": 0.013828789815306664, "learning_rate": 1e-05, "loss": 0.0387, "num_tokens": 30099753.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3013701438903809, "sampling/importance_sampling_ratio/mean": 1.000001311302185, "sampling/importance_sampling_ratio/min": 0.6757234930992126, "sampling/sampling_logp_difference/max": 0.3919713497161865, "sampling/sampling_logp_difference/mean": 0.014990867115557194, "step": 613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8216.0, "completions/max_terminated_length": 8216.0, "completions/mean_length": 6418.375, "completions/mean_terminated_length": 6418.375, "completions/min_length": 3757.0, "completions/min_terminated_length": 3757.0, "entropy": 0.8101372644305229, "epoch": 0.035291412806069664, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30151908.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4621585607528687, "sampling/importance_sampling_ratio/mean": 1.0002144575119019, "sampling/importance_sampling_ratio/min": 0.08840185403823853, "sampling/sampling_logp_difference/max": 2.4258623123168945, "sampling/sampling_logp_difference/mean": 0.024160563945770264, "step": 614 }, { "clip_ratio/high_max": 0.0001267733096028678, "clip_ratio/high_mean": 0.0001267733096028678, "clip_ratio/low_mean": 0.0003276848074165173, "clip_ratio/low_min": 0.0003276848074165173, "clip_ratio/region_mean": 0.0004544581170193851, "completions/clipped_ratio": 0.0, "completions/max_length": 12137.0, "completions/max_terminated_length": 12137.0, "completions/mean_length": 9474.625, "completions/mean_terminated_length": 9474.625, "completions/min_length": 4545.0, "completions/min_terminated_length": 4545.0, "entropy": 0.8706517145037651, "epoch": 0.03534889067708932, "frac_reward_zero_std": 0.0, "grad_norm": 0.00955223198980093, "learning_rate": 1e-05, "loss": 0.0154, "num_tokens": 30228561.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999082684516907, "sampling/importance_sampling_ratio/min": 0.06683102250099182, "sampling/sampling_logp_difference/max": 2.705587863922119, "sampling/sampling_logp_difference/mean": 0.028916819021105766, "step": 615 }, { "clip_ratio/high_max": 1.4719736100232694e-05, "clip_ratio/high_mean": 1.4719736100232694e-05, "clip_ratio/low_mean": 2.1208008547546342e-05, "clip_ratio/low_min": 2.1208008547546342e-05, "clip_ratio/region_mean": 3.5927744647779036e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 8492.0, "completions/max_terminated_length": 8492.0, "completions/mean_length": 4805.25, "completions/mean_terminated_length": 4805.25, "completions/min_length": 1864.0, "completions/min_terminated_length": 1864.0, "entropy": 1.3442905098199844, "epoch": 0.035406368548108975, "frac_reward_zero_std": 0.0, "grad_norm": 0.04488203302025795, "learning_rate": 1e-05, "loss": 0.1404, "num_tokens": 30268011.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4817619323730469, "sampling/importance_sampling_ratio/mean": 1.000339150428772, "sampling/importance_sampling_ratio/min": 0.642075777053833, "sampling/sampling_logp_difference/max": 0.44304895401000977, "sampling/sampling_logp_difference/mean": 0.02852136828005314, "step": 616 }, { "clip_ratio/high_max": 2.7837577363243327e-05, "clip_ratio/high_mean": 2.7837577363243327e-05, "clip_ratio/low_mean": 0.0001142160253948532, "clip_ratio/low_min": 0.0001142160253948532, "clip_ratio/region_mean": 0.00014205360275809653, "completions/clipped_ratio": 0.0, "completions/max_length": 14084.0, "completions/max_terminated_length": 14084.0, "completions/mean_length": 9474.875, "completions/mean_terminated_length": 9474.875, "completions/min_length": 1076.0, "completions/min_terminated_length": 1076.0, "entropy": 0.8050813674926758, "epoch": 0.035463846419128635, "frac_reward_zero_std": 0.0, "grad_norm": 0.013000767678022385, "learning_rate": 1e-05, "loss": -0.0651, "num_tokens": 30344666.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998914003372192, "sampling/importance_sampling_ratio/min": 0.2526843249797821, "sampling/sampling_logp_difference/max": 1.3756142854690552, "sampling/sampling_logp_difference/mean": 0.021909477189183235, "step": 617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10101.0, "completions/max_terminated_length": 10101.0, "completions/mean_length": 6487.75, "completions/mean_terminated_length": 6487.75, "completions/min_length": 4447.0, "completions/min_terminated_length": 4447.0, "entropy": 0.8418927937746048, "epoch": 0.035521324290148294, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30397336.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6516129970550537, "sampling/importance_sampling_ratio/mean": 1.0003292560577393, "sampling/importance_sampling_ratio/min": 0.028141897171735764, "sampling/sampling_logp_difference/max": 3.570495843887329, "sampling/sampling_logp_difference/mean": 0.02818719483911991, "step": 618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9095.0, "completions/max_terminated_length": 9095.0, "completions/mean_length": 6555.875, "completions/mean_terminated_length": 6555.875, "completions/min_length": 3374.0, "completions/min_terminated_length": 3374.0, "entropy": 0.4752928353846073, "epoch": 0.035578802161167954, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30450959.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7362655401229858, "sampling/importance_sampling_ratio/mean": 0.9999092817306519, "sampling/importance_sampling_ratio/min": 0.46842682361602783, "sampling/sampling_logp_difference/max": 0.7583754062652588, "sampling/sampling_logp_difference/mean": 0.017896175384521484, "step": 619 }, { "clip_ratio/high_max": 0.00010132539500773419, "clip_ratio/high_mean": 0.00010132539500773419, "clip_ratio/low_mean": 0.00021149046733626164, "clip_ratio/low_min": 0.00021149046733626164, "clip_ratio/region_mean": 0.00031281586234399583, "completions/clipped_ratio": 0.0, "completions/max_length": 9682.0, "completions/max_terminated_length": 9682.0, "completions/mean_length": 6095.875, "completions/mean_terminated_length": 6095.875, "completions/min_length": 2350.0, "completions/min_terminated_length": 2350.0, "entropy": 0.4976836070418358, "epoch": 0.035636280032187606, "frac_reward_zero_std": 0.0, "grad_norm": 0.04633909836411476, "learning_rate": 1e-05, "loss": 0.2209, "num_tokens": 30500790.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6743172407150269, "sampling/importance_sampling_ratio/mean": 1.0000988245010376, "sampling/importance_sampling_ratio/min": 0.03994845971465111, "sampling/sampling_logp_difference/max": 3.220165252685547, "sampling/sampling_logp_difference/mean": 0.020818788558244705, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13246.0, "completions/max_terminated_length": 13246.0, "completions/mean_length": 9948.375, "completions/mean_terminated_length": 9948.375, "completions/min_length": 2319.0, "completions/min_terminated_length": 2319.0, "entropy": 0.6574407070875168, "epoch": 0.035693757903207265, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30581337.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000053882598877, "sampling/importance_sampling_ratio/min": 0.2318928986787796, "sampling/sampling_logp_difference/max": 1.461479663848877, "sampling/sampling_logp_difference/mean": 0.028139887377619743, "step": 621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3408.0, "completions/max_terminated_length": 3408.0, "completions/mean_length": 2078.75, "completions/mean_terminated_length": 2078.75, "completions/min_length": 1400.0, "completions/min_terminated_length": 1400.0, "entropy": 0.2989246752113104, "epoch": 0.035751235774226925, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30599367.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3633450269699097, "sampling/importance_sampling_ratio/mean": 0.9999565482139587, "sampling/importance_sampling_ratio/min": 0.6855412721633911, "sampling/sampling_logp_difference/max": 0.3775465488433838, "sampling/sampling_logp_difference/mean": 0.013121597468852997, "step": 622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7455.0, "completions/max_terminated_length": 7455.0, "completions/mean_length": 5101.875, "completions/mean_terminated_length": 5101.875, "completions/min_length": 3354.0, "completions/min_terminated_length": 3354.0, "entropy": 0.2908296585083008, "epoch": 0.03580871364524658, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30641078.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5281180143356323, "sampling/importance_sampling_ratio/mean": 0.9997532367706299, "sampling/importance_sampling_ratio/min": 0.40564975142478943, "sampling/sampling_logp_difference/max": 0.902265191078186, "sampling/sampling_logp_difference/mean": 0.013133072294294834, "step": 623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6878.0, "completions/max_terminated_length": 6878.0, "completions/mean_length": 3665.25, "completions/mean_terminated_length": 3665.25, "completions/min_length": 1998.0, "completions/min_terminated_length": 1998.0, "entropy": 0.4997982829809189, "epoch": 0.03586619151626624, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30671784.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.482682466506958, "sampling/importance_sampling_ratio/mean": 0.9998175501823425, "sampling/importance_sampling_ratio/min": 0.6116805076599121, "sampling/sampling_logp_difference/max": 0.4915452003479004, "sampling/sampling_logp_difference/mean": 0.015914345160126686, "step": 624 }, { "clip_ratio/high_max": 9.951664287655149e-05, "clip_ratio/high_mean": 9.951664287655149e-05, "clip_ratio/low_mean": 6.253126775845885e-05, "clip_ratio/low_min": 6.253126775845885e-05, "clip_ratio/region_mean": 0.00016204791063501034, "completions/clipped_ratio": 0.0, "completions/max_length": 10027.0, "completions/max_terminated_length": 10027.0, "completions/mean_length": 6736.375, "completions/mean_terminated_length": 6736.375, "completions/min_length": 3956.0, "completions/min_terminated_length": 3956.0, "entropy": 0.4235830921679735, "epoch": 0.035923669387285896, "frac_reward_zero_std": 0.0, "grad_norm": 0.008596781641244888, "learning_rate": 1e-05, "loss": -0.1433, "num_tokens": 30726587.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.987758994102478, "sampling/importance_sampling_ratio/mean": 0.9999027848243713, "sampling/importance_sampling_ratio/min": 0.28737959265708923, "sampling/sampling_logp_difference/max": 1.2469513416290283, "sampling/sampling_logp_difference/mean": 0.017401665449142456, "step": 625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3125.0, "completions/max_terminated_length": 3125.0, "completions/mean_length": 2240.625, "completions/mean_terminated_length": 2240.625, "completions/min_length": 1545.0, "completions/min_terminated_length": 1545.0, "entropy": 0.27014053612947464, "epoch": 0.035981147258305556, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30745872.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.422957181930542, "sampling/importance_sampling_ratio/mean": 0.9996587038040161, "sampling/importance_sampling_ratio/min": 0.5549529194831848, "sampling/sampling_logp_difference/max": 0.588871955871582, "sampling/sampling_logp_difference/mean": 0.012328863143920898, "step": 626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7049.0, "completions/max_terminated_length": 7049.0, "completions/mean_length": 4789.75, "completions/mean_terminated_length": 4789.75, "completions/min_length": 3448.0, "completions/min_terminated_length": 3448.0, "entropy": 0.9613191336393356, "epoch": 0.03603862512932521, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30785742.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5561383962631226, "sampling/importance_sampling_ratio/mean": 0.9999451637268066, "sampling/importance_sampling_ratio/min": 0.5691731572151184, "sampling/sampling_logp_difference/max": 0.5635705590248108, "sampling/sampling_logp_difference/mean": 0.02334233559668064, "step": 627 }, { "clip_ratio/high_max": 1.0570824088063091e-05, "clip_ratio/high_mean": 1.0570824088063091e-05, "clip_ratio/low_mean": 0.0005046323458373081, "clip_ratio/low_min": 0.0005046323458373081, "clip_ratio/region_mean": 0.0005152031699253712, "completions/clipped_ratio": 0.0, "completions/max_length": 11825.0, "completions/max_terminated_length": 11825.0, "completions/mean_length": 6922.5, "completions/mean_terminated_length": 6922.5, "completions/min_length": 3187.0, "completions/min_terminated_length": 3187.0, "entropy": 0.7306752167642117, "epoch": 0.03609610300034487, "frac_reward_zero_std": 0.0, "grad_norm": 0.04242414981126785, "learning_rate": 1e-05, "loss": 0.0029, "num_tokens": 30842170.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997965097427368, "sampling/importance_sampling_ratio/min": 0.4445940852165222, "sampling/sampling_logp_difference/max": 1.2198967933654785, "sampling/sampling_logp_difference/mean": 0.02574812062084675, "step": 628 }, { "clip_ratio/high_max": 0.00015604176587658003, "clip_ratio/high_mean": 0.00015604176587658003, "clip_ratio/low_mean": 0.00024582105106674135, "clip_ratio/low_min": 0.00024582105106674135, "clip_ratio/region_mean": 0.0004018628169433214, "completions/clipped_ratio": 0.0, "completions/max_length": 1679.0, "completions/max_terminated_length": 1679.0, "completions/mean_length": 1238.875, "completions/mean_terminated_length": 1238.875, "completions/min_length": 730.0, "completions/min_terminated_length": 730.0, "entropy": 0.5561301298439503, "epoch": 0.03615358087136453, "frac_reward_zero_std": 0.0, "grad_norm": 0.025145193561911583, "learning_rate": 1e-05, "loss": -0.0636, "num_tokens": 30852905.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3101199865341187, "sampling/importance_sampling_ratio/mean": 0.999557375907898, "sampling/importance_sampling_ratio/min": 0.6831345558166504, "sampling/sampling_logp_difference/max": 0.38106346130371094, "sampling/sampling_logp_difference/mean": 0.022644085809588432, "step": 629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7080.0, "completions/max_terminated_length": 7080.0, "completions/mean_length": 4361.375, "completions/mean_terminated_length": 4361.375, "completions/min_length": 2176.0, "completions/min_terminated_length": 2176.0, "entropy": 0.5513044223189354, "epoch": 0.03621105874238418, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30888884.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5646969079971313, "sampling/importance_sampling_ratio/mean": 1.0001299381256104, "sampling/importance_sampling_ratio/min": 0.6017554998397827, "sampling/sampling_logp_difference/max": 0.507904052734375, "sampling/sampling_logp_difference/mean": 0.02093680016696453, "step": 630 }, { "clip_ratio/high_max": 0.0001117119663831545, "clip_ratio/high_mean": 0.0001117119663831545, "clip_ratio/low_mean": 0.00028732088685501367, "clip_ratio/low_min": 0.00028732088685501367, "clip_ratio/region_mean": 0.00039903285323816817, "completions/clipped_ratio": 0.0, "completions/max_length": 14229.0, "completions/max_terminated_length": 14229.0, "completions/mean_length": 7521.5, "completions/mean_terminated_length": 7521.5, "completions/min_length": 3601.0, "completions/min_terminated_length": 3601.0, "entropy": 0.4962787926197052, "epoch": 0.03626853661340384, "frac_reward_zero_std": 0.0, "grad_norm": 0.03398900479078293, "learning_rate": 1e-05, "loss": 0.2421, "num_tokens": 30950312.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001864433288574, "sampling/importance_sampling_ratio/min": 0.5368049740791321, "sampling/sampling_logp_difference/max": 0.7254056930541992, "sampling/sampling_logp_difference/mean": 0.02080281637609005, "step": 631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4979.0, "completions/max_terminated_length": 4979.0, "completions/mean_length": 2755.375, "completions/mean_terminated_length": 2755.375, "completions/min_length": 1737.0, "completions/min_terminated_length": 1737.0, "entropy": 0.36767720989882946, "epoch": 0.0363260144844235, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 30975347.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6318131685256958, "sampling/importance_sampling_ratio/mean": 0.9997876286506653, "sampling/importance_sampling_ratio/min": 0.5845015048980713, "sampling/sampling_logp_difference/max": 0.5369958877563477, "sampling/sampling_logp_difference/mean": 0.016707640141248703, "step": 632 }, { "clip_ratio/high_max": 3.844970706268214e-05, "clip_ratio/high_mean": 3.844970706268214e-05, "clip_ratio/low_mean": 0.00021832776837982237, "clip_ratio/low_min": 0.00021832776837982237, "clip_ratio/region_mean": 0.0002567774754425045, "completions/clipped_ratio": 0.0, "completions/max_length": 4832.0, "completions/max_terminated_length": 4832.0, "completions/mean_length": 2538.5, "completions/mean_terminated_length": 2538.5, "completions/min_length": 1024.0, "completions/min_terminated_length": 1024.0, "entropy": 0.3899029716849327, "epoch": 0.03638349235544316, "frac_reward_zero_std": 0.0, "grad_norm": 0.028107285499572754, "learning_rate": 1e-05, "loss": -0.08, "num_tokens": 30996623.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.3855682611465454, "sampling/importance_sampling_ratio/mean": 1.0003924369812012, "sampling/importance_sampling_ratio/min": 0.6096794009208679, "sampling/sampling_logp_difference/max": 0.49482202529907227, "sampling/sampling_logp_difference/mean": 0.016877148300409317, "step": 633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14282.0, "completions/mean_length": 11356.0, "completions/mean_terminated_length": 10637.71484375, "completions/min_length": 6873.0, "completions/min_terminated_length": 6873.0, "entropy": 0.9494549110531807, "epoch": 0.03644097022646281, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31088711.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998199343681335, "sampling/importance_sampling_ratio/min": 0.13677631318569183, "sampling/sampling_logp_difference/max": 1.9894084930419922, "sampling/sampling_logp_difference/mean": 0.030628858134150505, "step": 634 }, { "clip_ratio/high_max": 0.00017078808559745084, "clip_ratio/high_mean": 0.00017078808559745084, "clip_ratio/low_mean": 7.034987356746569e-05, "clip_ratio/low_min": 7.034987356746569e-05, "clip_ratio/region_mean": 0.00024113795916491654, "completions/clipped_ratio": 0.0, "completions/max_length": 10661.0, "completions/max_terminated_length": 10661.0, "completions/mean_length": 5707.5, "completions/mean_terminated_length": 5707.5, "completions/min_length": 3604.0, "completions/min_terminated_length": 3604.0, "entropy": 0.48223840817809105, "epoch": 0.03649844809748247, "frac_reward_zero_std": 0.0, "grad_norm": 0.0070729670114815235, "learning_rate": 1e-05, "loss": 0.3067, "num_tokens": 31135627.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9997117519378662, "sampling/importance_sampling_ratio/mean": 0.9997674822807312, "sampling/importance_sampling_ratio/min": 0.4563618004322052, "sampling/sampling_logp_difference/max": 0.7844693660736084, "sampling/sampling_logp_difference/mean": 0.019021878018975258, "step": 635 }, { "clip_ratio/high_max": 0.00014165624452289194, "clip_ratio/high_mean": 0.00014165624452289194, "clip_ratio/low_mean": 3.7015102861914784e-05, "clip_ratio/low_min": 3.7015102861914784e-05, "clip_ratio/region_mean": 0.00017867134738480672, "completions/clipped_ratio": 0.0, "completions/max_length": 10054.0, "completions/max_terminated_length": 10054.0, "completions/mean_length": 6902.0, "completions/mean_terminated_length": 6902.0, "completions/min_length": 3377.0, "completions/min_terminated_length": 3377.0, "entropy": 0.5428228601813316, "epoch": 0.03655592596850213, "frac_reward_zero_std": 0.0, "grad_norm": 0.007322242949157953, "learning_rate": 1e-05, "loss": -0.1804, "num_tokens": 31191939.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.89657461643219, "sampling/importance_sampling_ratio/mean": 1.000014305114746, "sampling/importance_sampling_ratio/min": 0.1402890980243683, "sampling/sampling_logp_difference/max": 1.964050054550171, "sampling/sampling_logp_difference/mean": 0.02175675891339779, "step": 636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7303.0, "completions/max_terminated_length": 7303.0, "completions/mean_length": 4214.25, "completions/mean_terminated_length": 4214.25, "completions/min_length": 1506.0, "completions/min_terminated_length": 1506.0, "entropy": 0.4480618126690388, "epoch": 0.03661340383952178, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31226445.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4272229671478271, "sampling/importance_sampling_ratio/mean": 1.0000189542770386, "sampling/importance_sampling_ratio/min": 0.5564479231834412, "sampling/sampling_logp_difference/max": 0.586181640625, "sampling/sampling_logp_difference/mean": 0.019137555733323097, "step": 637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15738.0, "completions/max_terminated_length": 15738.0, "completions/mean_length": 8606.125, "completions/mean_terminated_length": 8606.125, "completions/min_length": 5057.0, "completions/min_terminated_length": 5057.0, "entropy": 1.0016021132469177, "epoch": 0.03667088171054144, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31296486.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998247027397156, "sampling/importance_sampling_ratio/min": 0.3685136139392853, "sampling/sampling_logp_difference/max": 0.9982776641845703, "sampling/sampling_logp_difference/mean": 0.031207874417304993, "step": 638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1933.0, "completions/max_terminated_length": 1933.0, "completions/mean_length": 1130.375, "completions/mean_terminated_length": 1130.375, "completions/min_length": 803.0, "completions/min_terminated_length": 803.0, "entropy": 0.34428837709128857, "epoch": 0.0367283595815611, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31306681.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4450687170028687, "sampling/importance_sampling_ratio/mean": 0.999875545501709, "sampling/importance_sampling_ratio/min": 0.608730673789978, "sampling/sampling_logp_difference/max": 0.49637937545776367, "sampling/sampling_logp_difference/mean": 0.015093890018761158, "step": 639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4220.0, "completions/max_terminated_length": 4220.0, "completions/mean_length": 3131.625, "completions/mean_terminated_length": 3131.625, "completions/min_length": 2298.0, "completions/min_terminated_length": 2298.0, "entropy": 0.3271697387099266, "epoch": 0.03678583745258076, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31332470.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999399781227112, "sampling/importance_sampling_ratio/min": 0.6118444800376892, "sampling/sampling_logp_difference/max": 0.7423067092895508, "sampling/sampling_logp_difference/mean": 0.013562855310738087, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8782.0, "completions/max_terminated_length": 8782.0, "completions/mean_length": 4073.875, "completions/mean_terminated_length": 4073.875, "completions/min_length": 2656.0, "completions/min_terminated_length": 2656.0, "entropy": 0.28052452206611633, "epoch": 0.03684331532360041, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31366053.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5160768032073975, "sampling/importance_sampling_ratio/mean": 0.9997190237045288, "sampling/importance_sampling_ratio/min": 0.6058594584465027, "sampling/sampling_logp_difference/max": 0.5011072158813477, "sampling/sampling_logp_difference/mean": 0.011799659579992294, "step": 641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12843.0, "completions/max_terminated_length": 12843.0, "completions/mean_length": 7506.0, "completions/mean_terminated_length": 7506.0, "completions/min_length": 1168.0, "completions/min_terminated_length": 1168.0, "entropy": 0.7255656868219376, "epoch": 0.03690079319462007, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31428005.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8286902904510498, "sampling/importance_sampling_ratio/mean": 1.0001459121704102, "sampling/importance_sampling_ratio/min": 0.2612954080104828, "sampling/sampling_logp_difference/max": 1.3421037197113037, "sampling/sampling_logp_difference/mean": 0.026665309444069862, "step": 642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1565.0, "completions/max_terminated_length": 1565.0, "completions/mean_length": 1258.625, "completions/mean_terminated_length": 1258.625, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.25606208853423595, "epoch": 0.03695827106563973, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31439074.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4041321277618408, "sampling/importance_sampling_ratio/mean": 1.0002100467681885, "sampling/importance_sampling_ratio/min": 0.6901095509529114, "sampling/sampling_logp_difference/max": 0.37090492248535156, "sampling/sampling_logp_difference/mean": 0.010658006183803082, "step": 643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5756.0, "completions/max_terminated_length": 5756.0, "completions/mean_length": 3021.625, "completions/mean_terminated_length": 3021.625, "completions/min_length": 1667.0, "completions/min_terminated_length": 1667.0, "entropy": 0.3396462220698595, "epoch": 0.03701574893665938, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31464671.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5129506587982178, "sampling/importance_sampling_ratio/mean": 1.000061273574829, "sampling/importance_sampling_ratio/min": 0.6020646691322327, "sampling/sampling_logp_difference/max": 0.5073903799057007, "sampling/sampling_logp_difference/mean": 0.015328392386436462, "step": 644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025276739506807644, "clip_ratio/low_min": 0.00025276739506807644, "clip_ratio/region_mean": 0.00025276739506807644, "completions/clipped_ratio": 0.0, "completions/max_length": 12352.0, "completions/max_terminated_length": 12352.0, "completions/mean_length": 9202.125, "completions/mean_terminated_length": 9202.125, "completions/min_length": 2436.0, "completions/min_terminated_length": 2436.0, "entropy": 0.8207555264234543, "epoch": 0.03707322680767904, "frac_reward_zero_std": 0.0, "grad_norm": 0.00827634148299694, "learning_rate": 1e-05, "loss": -0.0154, "num_tokens": 31539320.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999133944511414, "sampling/importance_sampling_ratio/min": 0.23281870782375336, "sampling/sampling_logp_difference/max": 1.4574952125549316, "sampling/sampling_logp_difference/mean": 0.02198132872581482, "step": 645 }, { "clip_ratio/high_max": 1.0605803254293278e-05, "clip_ratio/high_mean": 1.0605803254293278e-05, "clip_ratio/low_mean": 0.0007031539062154479, "clip_ratio/low_min": 0.0007031539062154479, "clip_ratio/region_mean": 0.0007137597094697412, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14455.0, "completions/mean_length": 12227.875, "completions/mean_terminated_length": 11634.1435546875, "completions/min_length": 7661.0, "completions/min_terminated_length": 7661.0, "entropy": 0.8926212266087532, "epoch": 0.0371307046786987, "frac_reward_zero_std": 0.0, "grad_norm": 0.012510129250586033, "learning_rate": 1e-05, "loss": 0.1104, "num_tokens": 31638375.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.8638911247253418, "sampling/importance_sampling_ratio/mean": 0.999682605266571, "sampling/importance_sampling_ratio/min": 0.0004884360823780298, "sampling/sampling_logp_difference/max": 7.624301910400391, "sampling/sampling_logp_difference/mean": 0.030755989253520966, "step": 646 }, { "clip_ratio/high_max": 6.94213140377542e-05, "clip_ratio/high_mean": 6.94213140377542e-05, "clip_ratio/low_mean": 0.0006168327490740921, "clip_ratio/low_min": 0.0006168327490740921, "clip_ratio/region_mean": 0.0006862540631118463, "completions/clipped_ratio": 0.0, "completions/max_length": 12073.0, "completions/max_terminated_length": 12073.0, "completions/mean_length": 6840.0, "completions/mean_terminated_length": 6840.0, "completions/min_length": 2184.0, "completions/min_terminated_length": 2184.0, "entropy": 0.5328716840595007, "epoch": 0.03718818254971836, "frac_reward_zero_std": 0.0, "grad_norm": 0.006825166754424572, "learning_rate": 1e-05, "loss": -0.2403, "num_tokens": 31694063.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.615504264831543, "sampling/importance_sampling_ratio/mean": 0.9995995759963989, "sampling/importance_sampling_ratio/min": 0.4787727892398834, "sampling/sampling_logp_difference/max": 0.7365291714668274, "sampling/sampling_logp_difference/mean": 0.021908694878220558, "step": 647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6260.0, "completions/max_terminated_length": 6260.0, "completions/mean_length": 4119.0, "completions/mean_terminated_length": 4119.0, "completions/min_length": 3026.0, "completions/min_terminated_length": 3026.0, "entropy": 0.5343853086233139, "epoch": 0.037245660420738014, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31728111.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6207329034805298, "sampling/importance_sampling_ratio/mean": 0.9997470378875732, "sampling/importance_sampling_ratio/min": 0.5611321926116943, "sampling/sampling_logp_difference/max": 0.5777988433837891, "sampling/sampling_logp_difference/mean": 0.019489815458655357, "step": 648 }, { "clip_ratio/high_max": 1.0199086318607442e-05, "clip_ratio/high_mean": 1.0199086318607442e-05, "clip_ratio/low_mean": 0.0007925065146991983, "clip_ratio/low_min": 0.0007925065146991983, "clip_ratio/region_mean": 0.0008027056010178057, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 16068.0, "completions/mean_length": 14343.375, "completions/mean_terminated_length": 13663.1669921875, "completions/min_length": 10579.0, "completions/min_terminated_length": 10579.0, "entropy": 0.8473053500056267, "epoch": 0.03730313829175767, "frac_reward_zero_std": 0.0, "grad_norm": 0.007272316142916679, "learning_rate": 1e-05, "loss": 0.0514, "num_tokens": 31844370.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998790621757507, "sampling/importance_sampling_ratio/min": 0.36918404698371887, "sampling/sampling_logp_difference/max": 1.1674456596374512, "sampling/sampling_logp_difference/mean": 0.028136812150478363, "step": 649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12058.0, "completions/max_terminated_length": 12058.0, "completions/mean_length": 10764.75, "completions/mean_terminated_length": 10764.75, "completions/min_length": 8857.0, "completions/min_terminated_length": 8857.0, "entropy": 1.032348372042179, "epoch": 0.03736061616277733, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 31931680.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000051498413086, "sampling/importance_sampling_ratio/min": 0.05681953579187393, "sampling/sampling_logp_difference/max": 2.867875099182129, "sampling/sampling_logp_difference/mean": 0.027168579399585724, "step": 650 }, { "clip_ratio/high_max": 6.181249773362651e-05, "clip_ratio/high_mean": 6.181249773362651e-05, "clip_ratio/low_mean": 9.385793964611366e-05, "clip_ratio/low_min": 9.385793964611366e-05, "clip_ratio/region_mean": 0.00015567043737974018, "completions/clipped_ratio": 0.0, "completions/max_length": 10608.0, "completions/max_terminated_length": 10608.0, "completions/mean_length": 5501.625, "completions/mean_terminated_length": 5501.625, "completions/min_length": 2475.0, "completions/min_terminated_length": 2475.0, "entropy": 0.27897660434246063, "epoch": 0.037418094033796985, "frac_reward_zero_std": 0.0, "grad_norm": 0.006079891696572304, "learning_rate": 1e-05, "loss": 0.0739, "num_tokens": 31976805.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.631583571434021, "sampling/importance_sampling_ratio/mean": 0.9998789429664612, "sampling/importance_sampling_ratio/min": 0.5440642833709717, "sampling/sampling_logp_difference/max": 0.6086878776550293, "sampling/sampling_logp_difference/mean": 0.013075186870992184, "step": 651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9332.0, "completions/max_terminated_length": 9332.0, "completions/mean_length": 4771.75, "completions/mean_terminated_length": 4771.75, "completions/min_length": 1879.0, "completions/min_terminated_length": 1879.0, "entropy": 0.33066012151539326, "epoch": 0.037475571904816644, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32016211.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5278139114379883, "sampling/importance_sampling_ratio/mean": 0.9999580979347229, "sampling/importance_sampling_ratio/min": 0.5082956552505493, "sampling/sampling_logp_difference/max": 0.676692008972168, "sampling/sampling_logp_difference/mean": 0.015965519472956657, "step": 652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5189.0, "completions/max_terminated_length": 5189.0, "completions/mean_length": 3232.375, "completions/mean_terminated_length": 3232.375, "completions/min_length": 1872.0, "completions/min_terminated_length": 1872.0, "entropy": 0.459980808198452, "epoch": 0.037533049775836304, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32042830.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.66094172000885, "sampling/importance_sampling_ratio/mean": 1.0002100467681885, "sampling/importance_sampling_ratio/min": 0.6095602512359619, "sampling/sampling_logp_difference/max": 0.5073847770690918, "sampling/sampling_logp_difference/mean": 0.019693497568368912, "step": 653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6369.0, "completions/max_terminated_length": 6369.0, "completions/mean_length": 3249.0, "completions/mean_terminated_length": 3249.0, "completions/min_length": 953.0, "completions/min_terminated_length": 953.0, "entropy": 0.4465507138520479, "epoch": 0.03759052764685596, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32069542.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4097540378570557, "sampling/importance_sampling_ratio/mean": 0.9997287392616272, "sampling/importance_sampling_ratio/min": 0.6412166357040405, "sampling/sampling_logp_difference/max": 0.44438791275024414, "sampling/sampling_logp_difference/mean": 0.017448458820581436, "step": 654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4656.0, "completions/max_terminated_length": 4656.0, "completions/mean_length": 3473.0, "completions/mean_terminated_length": 3473.0, "completions/min_length": 2085.0, "completions/min_terminated_length": 2085.0, "entropy": 0.3590561877936125, "epoch": 0.037648005517875616, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32098478.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4240132570266724, "sampling/importance_sampling_ratio/mean": 0.9996775388717651, "sampling/importance_sampling_ratio/min": 0.6096674799919128, "sampling/sampling_logp_difference/max": 0.4948415756225586, "sampling/sampling_logp_difference/mean": 0.013298592530190945, "step": 655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3872.0, "completions/max_terminated_length": 3872.0, "completions/mean_length": 2196.5, "completions/mean_terminated_length": 2196.5, "completions/min_length": 1109.0, "completions/min_terminated_length": 1109.0, "entropy": 0.3919911291450262, "epoch": 0.037705483388895275, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32117242.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2846709489822388, "sampling/importance_sampling_ratio/mean": 0.9998692870140076, "sampling/importance_sampling_ratio/min": 0.5643377900123596, "sampling/sampling_logp_difference/max": 0.5721023082733154, "sampling/sampling_logp_difference/mean": 0.016193049028515816, "step": 656 }, { "clip_ratio/high_max": 1.6898742615012452e-05, "clip_ratio/high_mean": 1.6898742615012452e-05, "clip_ratio/low_mean": 0.0006029009782650974, "clip_ratio/low_min": 0.0006029009782650974, "clip_ratio/region_mean": 0.0006197997208801098, "completions/clipped_ratio": 0.0, "completions/max_length": 9414.0, "completions/max_terminated_length": 9414.0, "completions/mean_length": 6923.125, "completions/mean_terminated_length": 6923.125, "completions/min_length": 3289.0, "completions/min_terminated_length": 3289.0, "entropy": 0.6096305660903454, "epoch": 0.037762961259914934, "frac_reward_zero_std": 0.0, "grad_norm": 0.03939204290509224, "learning_rate": 1e-05, "loss": 0.0233, "num_tokens": 32175099.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999470114707947, "sampling/importance_sampling_ratio/min": 0.2801990211009979, "sampling/sampling_logp_difference/max": 1.2722551822662354, "sampling/sampling_logp_difference/mean": 0.023543942719697952, "step": 657 }, { "clip_ratio/high_max": 6.583962385775521e-05, "clip_ratio/high_mean": 6.583962385775521e-05, "clip_ratio/low_mean": 2.7673235308611766e-05, "clip_ratio/low_min": 2.7673235308611766e-05, "clip_ratio/region_mean": 9.351285916636698e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 4550.0, "completions/max_terminated_length": 4550.0, "completions/mean_length": 3096.75, "completions/mean_terminated_length": 3096.75, "completions/min_length": 1688.0, "completions/min_terminated_length": 1688.0, "entropy": 0.3205137122422457, "epoch": 0.03782043913093459, "frac_reward_zero_std": 0.0, "grad_norm": 0.010451900772750378, "learning_rate": 1e-05, "loss": 0.1624, "num_tokens": 32201289.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.3689239025115967, "sampling/importance_sampling_ratio/mean": 0.9997068643569946, "sampling/importance_sampling_ratio/min": 0.6163736581802368, "sampling/sampling_logp_difference/max": 0.4839019775390625, "sampling/sampling_logp_difference/mean": 0.014021601527929306, "step": 658 }, { "clip_ratio/high_max": 7.058823393890634e-05, "clip_ratio/high_mean": 7.058823393890634e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.058823393890634e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10625.0, "completions/max_terminated_length": 10625.0, "completions/mean_length": 3746.625, "completions/mean_terminated_length": 3746.625, "completions/min_length": 615.0, "completions/min_terminated_length": 615.0, "entropy": 0.4980541616678238, "epoch": 0.037877917001954246, "frac_reward_zero_std": 0.0, "grad_norm": 0.05415203794836998, "learning_rate": 1e-05, "loss": -0.7338, "num_tokens": 32232134.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.6332781314849854, "sampling/importance_sampling_ratio/mean": 1.000425934791565, "sampling/importance_sampling_ratio/min": 0.20834365487098694, "sampling/sampling_logp_difference/max": 1.5685663223266602, "sampling/sampling_logp_difference/mean": 0.02229555882513523, "step": 659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2485.0, "completions/max_terminated_length": 2485.0, "completions/mean_length": 1889.5, "completions/mean_terminated_length": 1889.5, "completions/min_length": 1555.0, "completions/min_terminated_length": 1555.0, "entropy": 0.27757626585662365, "epoch": 0.037935394872973906, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32248786.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.404600739479065, "sampling/importance_sampling_ratio/mean": 0.9997795224189758, "sampling/importance_sampling_ratio/min": 0.6911824941635132, "sampling/sampling_logp_difference/max": 0.3693513870239258, "sampling/sampling_logp_difference/mean": 0.01197406928986311, "step": 660 }, { "clip_ratio/high_max": 7.698351691942662e-05, "clip_ratio/high_mean": 7.698351691942662e-05, "clip_ratio/low_mean": 0.00032798682514112443, "clip_ratio/low_min": 0.00032798682514112443, "clip_ratio/region_mean": 0.00040497034206055105, "completions/clipped_ratio": 0.0, "completions/max_length": 12712.0, "completions/max_terminated_length": 12712.0, "completions/mean_length": 8898.875, "completions/mean_terminated_length": 8898.875, "completions/min_length": 889.0, "completions/min_terminated_length": 889.0, "entropy": 0.9743796847760677, "epoch": 0.037992872743993565, "frac_reward_zero_std": 0.0, "grad_norm": 0.023000622168183327, "learning_rate": 1e-05, "loss": -0.0914, "num_tokens": 32320841.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000112771987915, "sampling/importance_sampling_ratio/min": 0.2698608636856079, "sampling/sampling_logp_difference/max": 1.3098487854003906, "sampling/sampling_logp_difference/mean": 0.03133612871170044, "step": 661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5503.0, "completions/max_terminated_length": 5503.0, "completions/mean_length": 4371.0, "completions/mean_terminated_length": 4371.0, "completions/min_length": 2243.0, "completions/min_terminated_length": 2243.0, "entropy": 0.4780786894261837, "epoch": 0.03805035061501322, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32357441.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4757083654403687, "sampling/importance_sampling_ratio/mean": 1.0000330209732056, "sampling/importance_sampling_ratio/min": 0.6170393824577332, "sampling/sampling_logp_difference/max": 0.4828224182128906, "sampling/sampling_logp_difference/mean": 0.018893638625741005, "step": 662 }, { "clip_ratio/high_max": 0.00014642454880231526, "clip_ratio/high_mean": 0.00014642454880231526, "clip_ratio/low_mean": 0.00032221698347711936, "clip_ratio/low_min": 0.00032221698347711936, "clip_ratio/region_mean": 0.0004686415322794346, "completions/clipped_ratio": 0.0, "completions/max_length": 15354.0, "completions/max_terminated_length": 15354.0, "completions/mean_length": 10650.75, "completions/mean_terminated_length": 10650.75, "completions/min_length": 6780.0, "completions/min_terminated_length": 6780.0, "entropy": 1.0497223064303398, "epoch": 0.03810782848603288, "frac_reward_zero_std": 0.0, "grad_norm": 0.01746147871017456, "learning_rate": 1e-05, "loss": 0.1221, "num_tokens": 32443919.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000566244125366, "sampling/importance_sampling_ratio/min": 0.08749648183584213, "sampling/sampling_logp_difference/max": 2.436156749725342, "sampling/sampling_logp_difference/mean": 0.03377619385719299, "step": 663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2039.0, "completions/max_terminated_length": 2039.0, "completions/mean_length": 1290.25, "completions/mean_terminated_length": 1290.25, "completions/min_length": 484.0, "completions/min_terminated_length": 484.0, "entropy": 0.48637060821056366, "epoch": 0.038165306357052536, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32455537.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3380074501037598, "sampling/importance_sampling_ratio/mean": 0.9999166131019592, "sampling/importance_sampling_ratio/min": 0.6255008578300476, "sampling/sampling_logp_difference/max": 0.46920251846313477, "sampling/sampling_logp_difference/mean": 0.014542607590556145, "step": 664 }, { "clip_ratio/high_max": 9.940319978340995e-05, "clip_ratio/high_mean": 9.940319978340995e-05, "clip_ratio/low_mean": 0.00027485240570968017, "clip_ratio/low_min": 0.00027485240570968017, "clip_ratio/region_mean": 0.0003742556054930901, "completions/clipped_ratio": 0.0, "completions/max_length": 7288.0, "completions/max_terminated_length": 7288.0, "completions/mean_length": 5577.75, "completions/mean_terminated_length": 5577.75, "completions/min_length": 4316.0, "completions/min_terminated_length": 4316.0, "entropy": 0.3677636533975601, "epoch": 0.03822278422807219, "frac_reward_zero_std": 0.0, "grad_norm": 0.028435494750738144, "learning_rate": 1e-05, "loss": 0.1259, "num_tokens": 32501975.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.8607910871505737, "sampling/importance_sampling_ratio/mean": 0.9998171925544739, "sampling/importance_sampling_ratio/min": 0.5201219916343689, "sampling/sampling_logp_difference/max": 0.6536918878555298, "sampling/sampling_logp_difference/mean": 0.014673839323222637, "step": 665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6103.0, "completions/max_terminated_length": 6103.0, "completions/mean_length": 3594.875, "completions/mean_terminated_length": 3594.875, "completions/min_length": 1624.0, "completions/min_terminated_length": 1624.0, "entropy": 0.5236541256308556, "epoch": 0.03828026209909185, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32531398.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4908353090286255, "sampling/importance_sampling_ratio/mean": 0.9998536109924316, "sampling/importance_sampling_ratio/min": 0.6307196021080017, "sampling/sampling_logp_difference/max": 0.4608938694000244, "sampling/sampling_logp_difference/mean": 0.020223848521709442, "step": 666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10825.0, "completions/max_terminated_length": 10825.0, "completions/mean_length": 4053.375, "completions/mean_terminated_length": 4053.375, "completions/min_length": 1600.0, "completions/min_terminated_length": 1600.0, "entropy": 0.5823912918567657, "epoch": 0.03833773997011151, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32565401.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.831857442855835, "sampling/importance_sampling_ratio/mean": 0.9999573826789856, "sampling/importance_sampling_ratio/min": 0.5057035088539124, "sampling/sampling_logp_difference/max": 0.6818046569824219, "sampling/sampling_logp_difference/mean": 0.022258469834923744, "step": 667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7894.0, "completions/max_terminated_length": 7894.0, "completions/mean_length": 6322.25, "completions/mean_terminated_length": 6322.25, "completions/min_length": 5268.0, "completions/min_terminated_length": 5268.0, "entropy": 0.3588370196521282, "epoch": 0.03839521784113117, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32617499.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5773845911026, "sampling/importance_sampling_ratio/mean": 1.0001798868179321, "sampling/importance_sampling_ratio/min": 0.5621546506881714, "sampling/sampling_logp_difference/max": 0.5759782791137695, "sampling/sampling_logp_difference/mean": 0.014516005292534828, "step": 668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3645.0, "completions/max_terminated_length": 3645.0, "completions/mean_length": 2183.375, "completions/mean_terminated_length": 2183.375, "completions/min_length": 1064.0, "completions/min_terminated_length": 1064.0, "entropy": 0.43765346333384514, "epoch": 0.03845269571215082, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32635942.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4591889381408691, "sampling/importance_sampling_ratio/mean": 0.9996445178985596, "sampling/importance_sampling_ratio/min": 0.623860239982605, "sampling/sampling_logp_difference/max": 0.4718289375305176, "sampling/sampling_logp_difference/mean": 0.01857232116162777, "step": 669 }, { "clip_ratio/high_max": 0.00015413603614433669, "clip_ratio/high_mean": 0.00015413603614433669, "clip_ratio/low_mean": 0.00021569597447523847, "clip_ratio/low_min": 0.00021569597447523847, "clip_ratio/region_mean": 0.00036983201061957516, "completions/clipped_ratio": 0.0, "completions/max_length": 10828.0, "completions/max_terminated_length": 10828.0, "completions/mean_length": 6755.25, "completions/mean_terminated_length": 6755.25, "completions/min_length": 4850.0, "completions/min_terminated_length": 4850.0, "entropy": 0.6177436001598835, "epoch": 0.03851017358317048, "frac_reward_zero_std": 0.0, "grad_norm": 0.0151678416877985, "learning_rate": 1e-05, "loss": -0.0212, "num_tokens": 32692680.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002573728561401, "sampling/importance_sampling_ratio/min": 0.5178200006484985, "sampling/sampling_logp_difference/max": 0.7427375316619873, "sampling/sampling_logp_difference/mean": 0.024688497185707092, "step": 670 }, { "clip_ratio/high_max": 3.6542105590342544e-05, "clip_ratio/high_mean": 3.6542105590342544e-05, "clip_ratio/low_mean": 0.0005743766814703122, "clip_ratio/low_min": 0.0005743766814703122, "clip_ratio/region_mean": 0.0006109187870606547, "completions/clipped_ratio": 0.0, "completions/max_length": 9235.0, "completions/max_terminated_length": 9235.0, "completions/mean_length": 6995.125, "completions/mean_terminated_length": 6995.125, "completions/min_length": 1965.0, "completions/min_terminated_length": 1965.0, "entropy": 0.6776020042598248, "epoch": 0.03856765145419014, "frac_reward_zero_std": 0.0, "grad_norm": 0.017563335597515106, "learning_rate": 1e-05, "loss": -0.0917, "num_tokens": 32749769.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.8356059789657593, "sampling/importance_sampling_ratio/mean": 1.0002498626708984, "sampling/importance_sampling_ratio/min": 0.4345121681690216, "sampling/sampling_logp_difference/max": 0.833531379699707, "sampling/sampling_logp_difference/mean": 0.02470722235739231, "step": 671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10612.0, "completions/max_terminated_length": 10612.0, "completions/mean_length": 7585.125, "completions/mean_terminated_length": 7585.125, "completions/min_length": 5516.0, "completions/min_terminated_length": 5516.0, "entropy": 0.7838869988918304, "epoch": 0.0386251293252098, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32811538.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.521096110343933, "sampling/importance_sampling_ratio/mean": 0.9998078346252441, "sampling/importance_sampling_ratio/min": 0.5315068960189819, "sampling/sampling_logp_difference/max": 0.6320390701293945, "sampling/sampling_logp_difference/mean": 0.0278419628739357, "step": 672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2868.0, "completions/max_terminated_length": 2868.0, "completions/mean_length": 2171.375, "completions/mean_terminated_length": 2171.375, "completions/min_length": 1519.0, "completions/min_terminated_length": 1519.0, "entropy": 0.23898638412356377, "epoch": 0.03868260719622945, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32830229.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4613124132156372, "sampling/importance_sampling_ratio/mean": 1.0008420944213867, "sampling/importance_sampling_ratio/min": 0.6819443702697754, "sampling/sampling_logp_difference/max": 0.38280725479125977, "sampling/sampling_logp_difference/mean": 0.010130731388926506, "step": 673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5391.0, "completions/max_terminated_length": 5391.0, "completions/mean_length": 2565.125, "completions/mean_terminated_length": 2565.125, "completions/min_length": 1361.0, "completions/min_terminated_length": 1361.0, "entropy": 0.34914788603782654, "epoch": 0.03874008506724911, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32855590.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3615074157714844, "sampling/importance_sampling_ratio/mean": 0.9999479651451111, "sampling/importance_sampling_ratio/min": 0.6529754996299744, "sampling/sampling_logp_difference/max": 0.42621564865112305, "sampling/sampling_logp_difference/mean": 0.014581763185560703, "step": 674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7936.0, "completions/max_terminated_length": 7936.0, "completions/mean_length": 4431.25, "completions/mean_terminated_length": 4431.25, "completions/min_length": 2476.0, "completions/min_terminated_length": 2476.0, "entropy": 0.7659710869193077, "epoch": 0.03879756293826877, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 32891904.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003544092178345, "sampling/importance_sampling_ratio/min": 0.6181734800338745, "sampling/sampling_logp_difference/max": 0.7092313766479492, "sampling/sampling_logp_difference/mean": 0.026135122403502464, "step": 675 }, { "clip_ratio/high_max": 9.624111044104211e-05, "clip_ratio/high_mean": 9.624111044104211e-05, "clip_ratio/low_mean": 0.0001634950895095244, "clip_ratio/low_min": 0.0001634950895095244, "clip_ratio/region_mean": 0.0002597361999505665, "completions/clipped_ratio": 0.0, "completions/max_length": 7907.0, "completions/max_terminated_length": 7907.0, "completions/mean_length": 6195.0, "completions/mean_terminated_length": 6195.0, "completions/min_length": 3833.0, "completions/min_terminated_length": 3833.0, "entropy": 0.42524387314915657, "epoch": 0.03885504080928842, "frac_reward_zero_std": 0.0, "grad_norm": 0.03270287811756134, "learning_rate": 1e-05, "loss": -0.0713, "num_tokens": 32942816.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998810887336731, "sampling/importance_sampling_ratio/min": 0.3291226625442505, "sampling/sampling_logp_difference/max": 1.1113247871398926, "sampling/sampling_logp_difference/mean": 0.017316991463303566, "step": 676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13464.0, "completions/max_terminated_length": 13464.0, "completions/mean_length": 10994.0, "completions/mean_terminated_length": 10994.0, "completions/min_length": 8666.0, "completions/min_terminated_length": 8666.0, "entropy": 1.1970580369234085, "epoch": 0.03891251868030808, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33031624.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998262524604797, "sampling/importance_sampling_ratio/min": 0.18210645020008087, "sampling/sampling_logp_difference/max": 1.7031638622283936, "sampling/sampling_logp_difference/mean": 0.033924952149391174, "step": 677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6647.0, "completions/max_terminated_length": 6647.0, "completions/mean_length": 5000.75, "completions/mean_terminated_length": 5000.75, "completions/min_length": 3936.0, "completions/min_terminated_length": 3936.0, "entropy": 0.30226829275488853, "epoch": 0.03896999655132774, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33072886.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999596357345581, "sampling/importance_sampling_ratio/min": 0.44488099217414856, "sampling/sampling_logp_difference/max": 0.8099484443664551, "sampling/sampling_logp_difference/mean": 0.01302533783018589, "step": 678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13463.0, "completions/max_terminated_length": 13463.0, "completions/mean_length": 3916.0, "completions/mean_terminated_length": 3916.0, "completions/min_length": 1495.0, "completions/min_terminated_length": 1495.0, "entropy": 0.4489707089960575, "epoch": 0.0390274744223474, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33104894.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002237558364868, "sampling/importance_sampling_ratio/min": 0.6260717511177063, "sampling/sampling_logp_difference/max": 1.0247268676757812, "sampling/sampling_logp_difference/mean": 0.019337687641382217, "step": 679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10198.0, "completions/max_terminated_length": 10198.0, "completions/mean_length": 5768.75, "completions/mean_terminated_length": 5768.75, "completions/min_length": 3202.0, "completions/min_terminated_length": 3202.0, "entropy": 0.6122239828109741, "epoch": 0.03908495229336705, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33152308.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7636010646820068, "sampling/importance_sampling_ratio/mean": 1.000188946723938, "sampling/importance_sampling_ratio/min": 0.3669360280036926, "sampling/sampling_logp_difference/max": 1.0025677680969238, "sampling/sampling_logp_difference/mean": 0.023170841857790947, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7347.0, "completions/max_terminated_length": 7347.0, "completions/mean_length": 5382.625, "completions/mean_terminated_length": 5382.625, "completions/min_length": 3880.0, "completions/min_terminated_length": 3880.0, "entropy": 0.3156553637236357, "epoch": 0.03914243016438671, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33196441.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4277715682983398, "sampling/importance_sampling_ratio/mean": 1.0001729726791382, "sampling/importance_sampling_ratio/min": 0.41980209946632385, "sampling/sampling_logp_difference/max": 0.8679718971252441, "sampling/sampling_logp_difference/mean": 0.01181100308895111, "step": 681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4533.0, "completions/max_terminated_length": 4533.0, "completions/mean_length": 3031.5, "completions/mean_terminated_length": 3031.5, "completions/min_length": 1969.0, "completions/min_terminated_length": 1969.0, "entropy": 0.5651755183935165, "epoch": 0.03919990803540637, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33221773.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5667259693145752, "sampling/importance_sampling_ratio/mean": 1.0002094507217407, "sampling/importance_sampling_ratio/min": 0.6560506224632263, "sampling/sampling_logp_difference/max": 0.44898808002471924, "sampling/sampling_logp_difference/mean": 0.01682489737868309, "step": 682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6297.0, "completions/max_terminated_length": 6297.0, "completions/mean_length": 3567.875, "completions/mean_terminated_length": 3567.875, "completions/min_length": 2270.0, "completions/min_terminated_length": 2270.0, "entropy": 0.36461544781923294, "epoch": 0.03925738590642602, "frac_reward_zero_std": 0.0, "grad_norm": 0.008703446947038174, "learning_rate": 1e-05, "loss": -0.1287, "num_tokens": 33251684.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001412630081177, "sampling/importance_sampling_ratio/min": 0.6068717837333679, "sampling/sampling_logp_difference/max": 1.1031246185302734, "sampling/sampling_logp_difference/mean": 0.01144314557313919, "step": 683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4761.0, "completions/max_terminated_length": 4761.0, "completions/mean_length": 2861.625, "completions/mean_terminated_length": 2861.625, "completions/min_length": 2091.0, "completions/min_terminated_length": 2091.0, "entropy": 0.40712008252739906, "epoch": 0.03931486377744568, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33275481.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4589375257492065, "sampling/importance_sampling_ratio/mean": 1.000318169593811, "sampling/importance_sampling_ratio/min": 0.6887938380241394, "sampling/sampling_logp_difference/max": 0.37770843505859375, "sampling/sampling_logp_difference/mean": 0.012673290446400642, "step": 684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9499.0, "completions/max_terminated_length": 9499.0, "completions/mean_length": 7287.0, "completions/mean_terminated_length": 7287.0, "completions/min_length": 4820.0, "completions/min_terminated_length": 4820.0, "entropy": 0.8849638998508453, "epoch": 0.03937234164846534, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33335385.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001122951507568, "sampling/importance_sampling_ratio/min": 0.07736177742481232, "sampling/sampling_logp_difference/max": 2.55926251411438, "sampling/sampling_logp_difference/mean": 0.026662811636924744, "step": 685 }, { "clip_ratio/high_max": 0.00020184914319543168, "clip_ratio/high_mean": 0.00020184914319543168, "clip_ratio/low_mean": 0.00015800524488440715, "clip_ratio/low_min": 0.00015800524488440715, "clip_ratio/region_mean": 0.00035985438807983883, "completions/clipped_ratio": 0.0, "completions/max_length": 8523.0, "completions/max_terminated_length": 8523.0, "completions/mean_length": 5732.0, "completions/mean_terminated_length": 5732.0, "completions/min_length": 3946.0, "completions/min_terminated_length": 3946.0, "entropy": 0.6059318371117115, "epoch": 0.039429819519485, "frac_reward_zero_std": 0.0, "grad_norm": 0.011816953308880329, "learning_rate": 1e-05, "loss": -0.0966, "num_tokens": 33383721.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6127854585647583, "sampling/importance_sampling_ratio/mean": 1.000256061553955, "sampling/importance_sampling_ratio/min": 0.40270552039146423, "sampling/sampling_logp_difference/max": 0.9095497131347656, "sampling/sampling_logp_difference/mean": 0.02260206826031208, "step": 686 }, { "clip_ratio/high_max": 0.0001106429444917012, "clip_ratio/high_mean": 0.0001106429444917012, "clip_ratio/low_mean": 0.00011195700790267438, "clip_ratio/low_min": 0.00011195700790267438, "clip_ratio/region_mean": 0.00022259995239437558, "completions/clipped_ratio": 0.0, "completions/max_length": 8932.0, "completions/max_terminated_length": 8932.0, "completions/mean_length": 3880.0, "completions/mean_terminated_length": 3880.0, "completions/min_length": 1381.0, "completions/min_terminated_length": 1381.0, "entropy": 0.6672930978238583, "epoch": 0.039487297390504654, "frac_reward_zero_std": 0.0, "grad_norm": 0.008529053069651127, "learning_rate": 1e-05, "loss": 0.4601, "num_tokens": 33415809.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002813339233398, "sampling/importance_sampling_ratio/min": 0.5746598243713379, "sampling/sampling_logp_difference/max": 1.2730731964111328, "sampling/sampling_logp_difference/mean": 0.026221957057714462, "step": 687 }, { "clip_ratio/high_max": 0.00010883017239393666, "clip_ratio/high_mean": 0.00010883017239393666, "clip_ratio/low_mean": 5.691649130312726e-05, "clip_ratio/low_min": 5.691649130312726e-05, "clip_ratio/region_mean": 0.00016574666369706392, "completions/clipped_ratio": 0.0, "completions/max_length": 11562.0, "completions/max_terminated_length": 11562.0, "completions/mean_length": 8732.375, "completions/mean_terminated_length": 8732.375, "completions/min_length": 6215.0, "completions/min_terminated_length": 6215.0, "entropy": 0.4940112605690956, "epoch": 0.03954477526152431, "frac_reward_zero_std": 0.0, "grad_norm": 0.055429477244615555, "learning_rate": 1e-05, "loss": 0.0909, "num_tokens": 33486532.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.9368542432785034, "sampling/importance_sampling_ratio/mean": 0.9996843934059143, "sampling/importance_sampling_ratio/min": 0.4573914110660553, "sampling/sampling_logp_difference/max": 0.7822158336639404, "sampling/sampling_logp_difference/mean": 0.017242681235074997, "step": 688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3760.0, "completions/max_terminated_length": 3760.0, "completions/mean_length": 2686.0, "completions/mean_terminated_length": 2686.0, "completions/min_length": 1888.0, "completions/min_terminated_length": 1888.0, "entropy": 0.5823257826268673, "epoch": 0.03960225313254397, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33509748.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.547451376914978, "sampling/importance_sampling_ratio/mean": 0.9992647171020508, "sampling/importance_sampling_ratio/min": 0.5946010947227478, "sampling/sampling_logp_difference/max": 0.519864559173584, "sampling/sampling_logp_difference/mean": 0.021205391734838486, "step": 689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9689.0, "completions/max_terminated_length": 9689.0, "completions/mean_length": 5061.75, "completions/mean_terminated_length": 5061.75, "completions/min_length": 3379.0, "completions/min_terminated_length": 3379.0, "entropy": 0.3352288715541363, "epoch": 0.039659731003563625, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33551298.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5500582456588745, "sampling/importance_sampling_ratio/mean": 1.000038504600525, "sampling/importance_sampling_ratio/min": 0.5508604049682617, "sampling/sampling_logp_difference/max": 0.5962738990783691, "sampling/sampling_logp_difference/mean": 0.013936171308159828, "step": 690 }, { "clip_ratio/high_max": 3.461752748989966e-05, "clip_ratio/high_mean": 3.461752748989966e-05, "clip_ratio/low_mean": 6.168013169371989e-05, "clip_ratio/low_min": 6.168013169371989e-05, "clip_ratio/region_mean": 9.629765918361954e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 13501.0, "completions/max_terminated_length": 13501.0, "completions/mean_length": 8161.375, "completions/mean_terminated_length": 8161.375, "completions/min_length": 5466.0, "completions/min_terminated_length": 5466.0, "entropy": 1.392954371869564, "epoch": 0.039717208874583285, "frac_reward_zero_std": 0.0, "grad_norm": 0.016017109155654907, "learning_rate": 1e-05, "loss": 0.0264, "num_tokens": 33618485.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001697540283203, "sampling/importance_sampling_ratio/min": 0.09820054471492767, "sampling/sampling_logp_difference/max": 2.3207435607910156, "sampling/sampling_logp_difference/mean": 0.02959430404007435, "step": 691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2393.0, "completions/max_terminated_length": 2393.0, "completions/mean_length": 1943.625, "completions/mean_terminated_length": 1943.625, "completions/min_length": 935.0, "completions/min_terminated_length": 935.0, "entropy": 0.2567156068980694, "epoch": 0.039774686745602944, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33634738.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3141580820083618, "sampling/importance_sampling_ratio/mean": 0.9999099969863892, "sampling/importance_sampling_ratio/min": 0.6668732762336731, "sampling/sampling_logp_difference/max": 0.4051551818847656, "sampling/sampling_logp_difference/mean": 0.010601194575428963, "step": 692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000468082151201088, "clip_ratio/low_min": 0.000468082151201088, "clip_ratio/region_mean": 0.000468082151201088, "completions/clipped_ratio": 0.0, "completions/max_length": 7899.0, "completions/max_terminated_length": 7899.0, "completions/mean_length": 6070.875, "completions/mean_terminated_length": 6070.875, "completions/min_length": 3963.0, "completions/min_terminated_length": 3963.0, "entropy": 0.35571036115288734, "epoch": 0.0398321646166226, "frac_reward_zero_std": 0.0, "grad_norm": 0.03677962347865105, "learning_rate": 1e-05, "loss": 0.0764, "num_tokens": 33685121.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7124580144882202, "sampling/importance_sampling_ratio/mean": 1.000043272972107, "sampling/importance_sampling_ratio/min": 0.6065323948860168, "sampling/sampling_logp_difference/max": 0.5379297733306885, "sampling/sampling_logp_difference/mean": 0.014671890065073967, "step": 693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8086.0, "completions/max_terminated_length": 8086.0, "completions/mean_length": 5424.875, "completions/mean_terminated_length": 5424.875, "completions/min_length": 2582.0, "completions/min_terminated_length": 2582.0, "entropy": 0.3909875378012657, "epoch": 0.039889642487642256, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33729568.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.51531982421875, "sampling/importance_sampling_ratio/mean": 0.9995766282081604, "sampling/importance_sampling_ratio/min": 0.566514253616333, "sampling/sampling_logp_difference/max": 0.5682530403137207, "sampling/sampling_logp_difference/mean": 0.01627148874104023, "step": 694 }, { "clip_ratio/high_max": 4.5257060264702886e-05, "clip_ratio/high_mean": 4.5257060264702886e-05, "clip_ratio/low_mean": 3.753190321731381e-05, "clip_ratio/low_min": 3.753190321731381e-05, "clip_ratio/region_mean": 8.27889634820167e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 6661.0, "completions/max_terminated_length": 6661.0, "completions/mean_length": 2688.0, "completions/mean_terminated_length": 2688.0, "completions/min_length": 1150.0, "completions/min_terminated_length": 1150.0, "entropy": 0.7454186230897903, "epoch": 0.039947120358661915, "frac_reward_zero_std": 0.0, "grad_norm": 0.046566031873226166, "learning_rate": 1e-05, "loss": 0.16, "num_tokens": 33752112.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4530314207077026, "sampling/importance_sampling_ratio/mean": 0.9998077154159546, "sampling/importance_sampling_ratio/min": 0.4387819468975067, "sampling/sampling_logp_difference/max": 0.823752760887146, "sampling/sampling_logp_difference/mean": 0.020349618047475815, "step": 695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3779.0, "completions/max_terminated_length": 3779.0, "completions/mean_length": 2568.75, "completions/mean_terminated_length": 2568.75, "completions/min_length": 1461.0, "completions/min_terminated_length": 1461.0, "entropy": 0.19969465024769306, "epoch": 0.040004598229681575, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33773774.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3181301355361938, "sampling/importance_sampling_ratio/mean": 0.9999646544456482, "sampling/importance_sampling_ratio/min": 0.6209668517112732, "sampling/sampling_logp_difference/max": 0.47647762298583984, "sampling/sampling_logp_difference/mean": 0.008397262543439865, "step": 696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11052.0, "completions/max_terminated_length": 11052.0, "completions/mean_length": 4977.625, "completions/mean_terminated_length": 4977.625, "completions/min_length": 2632.0, "completions/min_terminated_length": 2632.0, "entropy": 0.7128969207406044, "epoch": 0.04006207610070123, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33814811.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6682052612304688, "sampling/importance_sampling_ratio/mean": 1.0000675916671753, "sampling/importance_sampling_ratio/min": 0.3981750011444092, "sampling/sampling_logp_difference/max": 0.9208636283874512, "sampling/sampling_logp_difference/mean": 0.021932139992713928, "step": 697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4771.0, "completions/max_terminated_length": 4771.0, "completions/mean_length": 3366.0, "completions/mean_terminated_length": 3366.0, "completions/min_length": 1955.0, "completions/min_terminated_length": 1955.0, "entropy": 0.49507489800453186, "epoch": 0.04011955397172089, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33842467.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5655663013458252, "sampling/importance_sampling_ratio/mean": 0.9996000528335571, "sampling/importance_sampling_ratio/min": 0.6176598072052002, "sampling/sampling_logp_difference/max": 0.48181748390197754, "sampling/sampling_logp_difference/mean": 0.015915554016828537, "step": 698 }, { "clip_ratio/high_max": 0.0002093834464176325, "clip_ratio/high_mean": 0.0002093834464176325, "clip_ratio/low_mean": 0.00022398296277970076, "clip_ratio/low_min": 0.00022398296277970076, "clip_ratio/region_mean": 0.00043336640919733327, "completions/clipped_ratio": 0.0, "completions/max_length": 7555.0, "completions/max_terminated_length": 7555.0, "completions/mean_length": 5141.75, "completions/mean_terminated_length": 5141.75, "completions/min_length": 3768.0, "completions/min_terminated_length": 3768.0, "entropy": 0.3876085914671421, "epoch": 0.040177031842740546, "frac_reward_zero_std": 0.0, "grad_norm": 0.011120148934423923, "learning_rate": 1e-05, "loss": -0.0258, "num_tokens": 33884761.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7247319221496582, "sampling/importance_sampling_ratio/mean": 0.9998995661735535, "sampling/importance_sampling_ratio/min": 0.5331995487213135, "sampling/sampling_logp_difference/max": 0.6288595199584961, "sampling/sampling_logp_difference/mean": 0.01583828590810299, "step": 699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8194.0, "completions/max_terminated_length": 8194.0, "completions/mean_length": 2488.875, "completions/mean_terminated_length": 2488.875, "completions/min_length": 1125.0, "completions/min_terminated_length": 1125.0, "entropy": 0.4824654534459114, "epoch": 0.040234509713760205, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 33905704.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.750427007675171, "sampling/importance_sampling_ratio/mean": 0.9995976090431213, "sampling/importance_sampling_ratio/min": 0.615777850151062, "sampling/sampling_logp_difference/max": 0.5598597526550293, "sampling/sampling_logp_difference/mean": 0.021767619997262955, "step": 700 }, { "clip_ratio/high_max": 2.5206694772350602e-05, "clip_ratio/high_mean": 2.5206694772350602e-05, "clip_ratio/low_mean": 0.0007288992856047116, "clip_ratio/low_min": 0.0007288992856047116, "clip_ratio/region_mean": 0.0007541059803770622, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14555.0, "completions/mean_length": 11815.75, "completions/mean_terminated_length": 11163.1435546875, "completions/min_length": 8267.0, "completions/min_terminated_length": 8267.0, "entropy": 0.9517037346959114, "epoch": 0.04029198758477986, "frac_reward_zero_std": 0.0, "grad_norm": 0.007710161153227091, "learning_rate": 1e-05, "loss": 0.0569, "num_tokens": 34001678.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000290870666504, "sampling/importance_sampling_ratio/min": 0.21266819536685944, "sampling/sampling_logp_difference/max": 1.5480220317840576, "sampling/sampling_logp_difference/mean": 0.029893921688199043, "step": 701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2894.0, "completions/max_terminated_length": 2894.0, "completions/mean_length": 1616.0, "completions/mean_terminated_length": 1616.0, "completions/min_length": 1117.0, "completions/min_terminated_length": 1117.0, "entropy": 0.31655081547796726, "epoch": 0.04034946545579952, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34015366.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2877933979034424, "sampling/importance_sampling_ratio/mean": 0.9999195337295532, "sampling/importance_sampling_ratio/min": 0.6954226493835449, "sampling/sampling_logp_difference/max": 0.3632354736328125, "sampling/sampling_logp_difference/mean": 0.013414298184216022, "step": 702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 769.625, "completions/mean_terminated_length": 769.625, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "entropy": 0.27478561364114285, "epoch": 0.04040694332681918, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34022531.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4855388402938843, "sampling/importance_sampling_ratio/mean": 1.0002121925354004, "sampling/importance_sampling_ratio/min": 0.7252554893493652, "sampling/sampling_logp_difference/max": 0.3957775831222534, "sampling/sampling_logp_difference/mean": 0.01212250255048275, "step": 703 }, { "clip_ratio/high_max": 1.60689032782102e-05, "clip_ratio/high_mean": 1.60689032782102e-05, "clip_ratio/low_mean": 0.00010499392374185845, "clip_ratio/low_min": 0.00010499392374185845, "clip_ratio/region_mean": 0.00012106282702006865, "completions/clipped_ratio": 0.0, "completions/max_length": 9504.0, "completions/max_terminated_length": 9504.0, "completions/mean_length": 6224.375, "completions/mean_terminated_length": 6224.375, "completions/min_length": 3157.0, "completions/min_terminated_length": 3157.0, "entropy": 1.265712484717369, "epoch": 0.04046442119783883, "frac_reward_zero_std": 0.0, "grad_norm": 0.01775595173239708, "learning_rate": 1e-05, "loss": -0.0961, "num_tokens": 34073670.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995413422584534, "sampling/importance_sampling_ratio/min": 0.484165757894516, "sampling/sampling_logp_difference/max": 1.0355019569396973, "sampling/sampling_logp_difference/mean": 0.029952924698591232, "step": 704 }, { "clip_ratio/high_max": 0.0001581618926138617, "clip_ratio/high_mean": 0.0001581618926138617, "clip_ratio/low_mean": 0.0002704128928598948, "clip_ratio/low_min": 0.0002704128928598948, "clip_ratio/region_mean": 0.0004285747854737565, "completions/clipped_ratio": 0.0, "completions/max_length": 14399.0, "completions/max_terminated_length": 14399.0, "completions/mean_length": 9567.75, "completions/mean_terminated_length": 9567.75, "completions/min_length": 6186.0, "completions/min_terminated_length": 6186.0, "entropy": 0.6653300747275352, "epoch": 0.04052189906885849, "frac_reward_zero_std": 0.0, "grad_norm": 0.035848502069711685, "learning_rate": 1e-05, "loss": 0.2547, "num_tokens": 34151036.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001119375228882, "sampling/importance_sampling_ratio/min": 0.43148738145828247, "sampling/sampling_logp_difference/max": 0.8515582084655762, "sampling/sampling_logp_difference/mean": 0.024204259738326073, "step": 705 }, { "clip_ratio/high_max": 0.00015623676335962955, "clip_ratio/high_mean": 0.00015623676335962955, "clip_ratio/low_mean": 0.00015163769421633333, "clip_ratio/low_min": 0.00015163769421633333, "clip_ratio/region_mean": 0.0003078744575759629, "completions/clipped_ratio": 0.0, "completions/max_length": 10781.0, "completions/max_terminated_length": 10781.0, "completions/mean_length": 6790.25, "completions/mean_terminated_length": 6790.25, "completions/min_length": 4821.0, "completions/min_terminated_length": 4821.0, "entropy": 0.7921525165438652, "epoch": 0.04057937693987815, "frac_reward_zero_std": 0.0, "grad_norm": 0.008926024660468102, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 34206622.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002115964889526, "sampling/importance_sampling_ratio/min": 0.24665939807891846, "sampling/sampling_logp_difference/max": 1.3997468948364258, "sampling/sampling_logp_difference/mean": 0.024822304025292397, "step": 706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8387.0, "completions/max_terminated_length": 8387.0, "completions/mean_length": 4797.75, "completions/mean_terminated_length": 4797.75, "completions/min_length": 3115.0, "completions/min_terminated_length": 3115.0, "entropy": 0.46194231510162354, "epoch": 0.04063685481089781, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34246108.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4877686500549316, "sampling/importance_sampling_ratio/mean": 1.0000929832458496, "sampling/importance_sampling_ratio/min": 0.27588385343551636, "sampling/sampling_logp_difference/max": 1.2877752780914307, "sampling/sampling_logp_difference/mean": 0.018465694040060043, "step": 707 }, { "clip_ratio/high_max": 0.00010743194980022963, "clip_ratio/high_mean": 0.00010743194980022963, "clip_ratio/low_mean": 0.00044837331370217726, "clip_ratio/low_min": 0.00044837331370217726, "clip_ratio/region_mean": 0.0005558052635024069, "completions/clipped_ratio": 0.0, "completions/max_length": 16046.0, "completions/max_terminated_length": 16046.0, "completions/mean_length": 13463.875, "completions/mean_terminated_length": 13463.875, "completions/min_length": 7465.0, "completions/min_terminated_length": 7465.0, "entropy": 0.5835207551717758, "epoch": 0.04069433268191746, "frac_reward_zero_std": 0.0, "grad_norm": 0.01515286322683096, "learning_rate": 1e-05, "loss": 0.1109, "num_tokens": 34355619.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.99961918592453, "sampling/importance_sampling_ratio/min": 0.2652982771396637, "sampling/sampling_logp_difference/max": 1.3269004821777344, "sampling/sampling_logp_difference/mean": 0.022717438638210297, "step": 708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5255.0, "completions/max_terminated_length": 5255.0, "completions/mean_length": 3748.125, "completions/mean_terminated_length": 3748.125, "completions/min_length": 2404.0, "completions/min_terminated_length": 2404.0, "entropy": 0.2801939845085144, "epoch": 0.04075181055293712, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34387228.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4169551134109497, "sampling/importance_sampling_ratio/mean": 1.00017511844635, "sampling/importance_sampling_ratio/min": 0.6282829642295837, "sampling/sampling_logp_difference/max": 0.4647645950317383, "sampling/sampling_logp_difference/mean": 0.011412669904530048, "step": 709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13015.0, "completions/max_terminated_length": 13015.0, "completions/mean_length": 6196.75, "completions/mean_terminated_length": 6196.75, "completions/min_length": 2745.0, "completions/min_terminated_length": 2745.0, "entropy": 0.30945868603885174, "epoch": 0.04080928842395678, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34437778.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.786260724067688, "sampling/importance_sampling_ratio/mean": 1.0003477334976196, "sampling/importance_sampling_ratio/min": 0.48065608739852905, "sampling/sampling_logp_difference/max": 0.7326033115386963, "sampling/sampling_logp_difference/mean": 0.013471252284944057, "step": 710 }, { "clip_ratio/high_max": 1.1674605048028752e-05, "clip_ratio/high_mean": 1.1674605048028752e-05, "clip_ratio/low_mean": 0.000622489293164108, "clip_ratio/low_min": 0.000622489293164108, "clip_ratio/region_mean": 0.0006341638982121367, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15498.0, "completions/mean_length": 12853.875, "completions/mean_terminated_length": 10735.7998046875, "completions/min_length": 7401.0, "completions/min_terminated_length": 7401.0, "entropy": 0.7121129371225834, "epoch": 0.04086676629497643, "frac_reward_zero_std": 0.0, "grad_norm": 0.010693341493606567, "learning_rate": 1e-05, "loss": 0.1595, "num_tokens": 34541865.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997833371162415, "sampling/importance_sampling_ratio/min": 0.055794231593608856, "sampling/sampling_logp_difference/max": 2.886084794998169, "sampling/sampling_logp_difference/mean": 0.02393028326332569, "step": 711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11685.0, "completions/max_terminated_length": 11685.0, "completions/mean_length": 7179.0, "completions/mean_terminated_length": 7179.0, "completions/min_length": 3413.0, "completions/min_terminated_length": 3413.0, "entropy": 1.0252695009112358, "epoch": 0.04092424416599609, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34601137.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9936015605926514, "sampling/importance_sampling_ratio/mean": 0.9998710751533508, "sampling/importance_sampling_ratio/min": 0.3348657190799713, "sampling/sampling_logp_difference/max": 1.0940256118774414, "sampling/sampling_logp_difference/mean": 0.028300868347287178, "step": 712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4483.0, "completions/max_terminated_length": 4483.0, "completions/mean_length": 2664.0, "completions/mean_terminated_length": 2664.0, "completions/min_length": 904.0, "completions/min_terminated_length": 904.0, "entropy": 0.40963806211948395, "epoch": 0.04098172203701575, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34623793.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4329776763916016, "sampling/importance_sampling_ratio/mean": 1.0002580881118774, "sampling/importance_sampling_ratio/min": 0.6091949939727783, "sampling/sampling_logp_difference/max": 0.4956169128417969, "sampling/sampling_logp_difference/mean": 0.017322929576039314, "step": 713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2038.0, "completions/max_terminated_length": 2038.0, "completions/mean_length": 1373.375, "completions/mean_terminated_length": 1373.375, "completions/min_length": 893.0, "completions/min_terminated_length": 893.0, "entropy": 0.4024125635623932, "epoch": 0.04103919990803541, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34636564.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.285967469215393, "sampling/importance_sampling_ratio/mean": 1.0001393556594849, "sampling/importance_sampling_ratio/min": 0.6914010643959045, "sampling/sampling_logp_difference/max": 0.3690352439880371, "sampling/sampling_logp_difference/mean": 0.014160930179059505, "step": 714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 16384.0, "completions/max_terminated_length": 14365.0, "completions/mean_length": 15336.5, "completions/mean_terminated_length": 12194.0, "completions/min_length": 10023.0, "completions/min_terminated_length": 10023.0, "entropy": 0.5720738880336285, "epoch": 0.04109667777905506, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34760608.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999816417694092, "sampling/importance_sampling_ratio/min": 0.10772384703159332, "sampling/sampling_logp_difference/max": 2.228184223175049, "sampling/sampling_logp_difference/mean": 0.020170021802186966, "step": 715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 14470.0, "completions/mean_length": 13615.0, "completions/mean_terminated_length": 11953.6005859375, "completions/min_length": 9337.0, "completions/min_terminated_length": 9337.0, "entropy": 0.7706930637359619, "epoch": 0.04115415565007472, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 34871160.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000240445137024, "sampling/importance_sampling_ratio/min": 0.027686169371008873, "sampling/sampling_logp_difference/max": 3.586822271347046, "sampling/sampling_logp_difference/mean": 0.026281150057911873, "step": 716 }, { "clip_ratio/high_max": 0.00011296495176793542, "clip_ratio/high_mean": 0.00011296495176793542, "clip_ratio/low_mean": 0.0002407500724075362, "clip_ratio/low_min": 0.0002407500724075362, "clip_ratio/region_mean": 0.0003537150241754716, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14633.0, "completions/mean_length": 8809.75, "completions/mean_terminated_length": 7727.71484375, "completions/min_length": 4345.0, "completions/min_terminated_length": 4345.0, "entropy": 0.44957026094198227, "epoch": 0.04121163352109438, "frac_reward_zero_std": 0.0, "grad_norm": 0.007761371321976185, "learning_rate": 1e-05, "loss": 0.4103, "num_tokens": 34943390.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998711943626404, "sampling/importance_sampling_ratio/min": 0.20378170907497406, "sampling/sampling_logp_difference/max": 1.5907058715820312, "sampling/sampling_logp_difference/mean": 0.02028319612145424, "step": 717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14348.0, "completions/max_terminated_length": 14348.0, "completions/mean_length": 8975.125, "completions/mean_terminated_length": 8975.125, "completions/min_length": 4669.0, "completions/min_terminated_length": 4669.0, "entropy": 1.0683716759085655, "epoch": 0.04126911139211403, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35017103.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.00001060962677, "sampling/importance_sampling_ratio/min": 0.12027474492788315, "sampling/sampling_logp_difference/max": 2.117976665496826, "sampling/sampling_logp_difference/mean": 0.027666814625263214, "step": 718 }, { "clip_ratio/high_max": 0.00016163456348294858, "clip_ratio/high_mean": 0.00016163456348294858, "clip_ratio/low_mean": 0.00011371677101124078, "clip_ratio/low_min": 0.00011371677101124078, "clip_ratio/region_mean": 0.00027535133449418936, "completions/clipped_ratio": 0.0, "completions/max_length": 9893.0, "completions/max_terminated_length": 9893.0, "completions/mean_length": 5716.5, "completions/mean_terminated_length": 5716.5, "completions/min_length": 3320.0, "completions/min_terminated_length": 3320.0, "entropy": 0.6190186478197575, "epoch": 0.04132658926313369, "frac_reward_zero_std": 0.0, "grad_norm": 0.00785940419882536, "learning_rate": 1e-05, "loss": 0.2583, "num_tokens": 35063611.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5323699712753296, "sampling/importance_sampling_ratio/mean": 0.999622642993927, "sampling/importance_sampling_ratio/min": 0.40493717789649963, "sampling/sampling_logp_difference/max": 0.9040234088897705, "sampling/sampling_logp_difference/mean": 0.02180623635649681, "step": 719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5821.0, "completions/max_terminated_length": 5821.0, "completions/mean_length": 3332.625, "completions/mean_terminated_length": 3332.625, "completions/min_length": 2111.0, "completions/min_terminated_length": 2111.0, "entropy": 0.46974773705005646, "epoch": 0.04138406713415335, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35091560.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5911545753479004, "sampling/importance_sampling_ratio/mean": 1.000192642211914, "sampling/importance_sampling_ratio/min": 0.6977284550666809, "sampling/sampling_logp_difference/max": 0.4644598960876465, "sampling/sampling_logp_difference/mean": 0.016217876225709915, "step": 720 }, { "clip_ratio/high_max": 4.110490044695325e-05, "clip_ratio/high_mean": 4.110490044695325e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.110490044695325e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 7240.0, "completions/max_terminated_length": 7240.0, "completions/mean_length": 4125.0, "completions/mean_terminated_length": 4125.0, "completions/min_length": 1687.0, "completions/min_terminated_length": 1687.0, "entropy": 0.6900503560900688, "epoch": 0.04144154500517301, "frac_reward_zero_std": 0.0, "grad_norm": 0.006700992118567228, "learning_rate": 1e-05, "loss": 0.0519, "num_tokens": 35125472.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4440008401870728, "sampling/importance_sampling_ratio/mean": 1.0000923871994019, "sampling/importance_sampling_ratio/min": 0.6921617388725281, "sampling/sampling_logp_difference/max": 0.3679356575012207, "sampling/sampling_logp_difference/mean": 0.01725039631128311, "step": 721 }, { "clip_ratio/high_max": 3.130478216917254e-05, "clip_ratio/high_mean": 3.130478216917254e-05, "clip_ratio/low_mean": 0.0006059787192498334, "clip_ratio/low_min": 0.0006059787192498334, "clip_ratio/region_mean": 0.0006372835014190059, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13982.0, "completions/mean_length": 12861.125, "completions/mean_terminated_length": 11686.833984375, "completions/min_length": 8026.0, "completions/min_terminated_length": 8026.0, "entropy": 0.613908689469099, "epoch": 0.041499022876192664, "frac_reward_zero_std": 0.0, "grad_norm": 0.04210219904780388, "learning_rate": 1e-05, "loss": 0.0241, "num_tokens": 35229185.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000654458999634, "sampling/importance_sampling_ratio/min": 0.09198317676782608, "sampling/sampling_logp_difference/max": 2.3861496448516846, "sampling/sampling_logp_difference/mean": 0.022381480783224106, "step": 722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5580.0, "completions/max_terminated_length": 5580.0, "completions/mean_length": 4443.125, "completions/mean_terminated_length": 4443.125, "completions/min_length": 3073.0, "completions/min_terminated_length": 3073.0, "entropy": 0.41248780488967896, "epoch": 0.04155650074721232, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35265842.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5729005336761475, "sampling/importance_sampling_ratio/mean": 1.0002816915512085, "sampling/importance_sampling_ratio/min": 0.5367231369018555, "sampling/sampling_logp_difference/max": 0.6222729682922363, "sampling/sampling_logp_difference/mean": 0.016163570806384087, "step": 723 }, { "clip_ratio/high_max": 4.961562081007287e-05, "clip_ratio/high_mean": 4.961562081007287e-05, "clip_ratio/low_mean": 0.00019073980365647003, "clip_ratio/low_min": 0.00019073980365647003, "clip_ratio/region_mean": 0.0002403554244665429, "completions/clipped_ratio": 0.0, "completions/max_length": 5504.0, "completions/max_terminated_length": 5504.0, "completions/mean_length": 4128.5, "completions/mean_terminated_length": 4128.5, "completions/min_length": 1338.0, "completions/min_terminated_length": 1338.0, "entropy": 0.2921385783702135, "epoch": 0.04161397861823198, "frac_reward_zero_std": 0.0, "grad_norm": 0.026594476774334908, "learning_rate": 1e-05, "loss": -0.1561, "num_tokens": 35300614.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.426784634590149, "sampling/importance_sampling_ratio/mean": 1.0003502368927002, "sampling/importance_sampling_ratio/min": 0.6033138632774353, "sampling/sampling_logp_difference/max": 0.5053176879882812, "sampling/sampling_logp_difference/mean": 0.012470947578549385, "step": 724 }, { "clip_ratio/high_max": 3.451131851761602e-05, "clip_ratio/high_mean": 3.451131851761602e-05, "clip_ratio/low_mean": 4.0167094994103536e-05, "clip_ratio/low_min": 4.0167094994103536e-05, "clip_ratio/region_mean": 7.467841351171955e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 10597.0, "completions/max_terminated_length": 10597.0, "completions/mean_length": 5033.5, "completions/mean_terminated_length": 5033.5, "completions/min_length": 3112.0, "completions/min_terminated_length": 3112.0, "entropy": 0.32643257454037666, "epoch": 0.041671456489251635, "frac_reward_zero_std": 0.0, "grad_norm": 0.012710866518318653, "learning_rate": 1e-05, "loss": -0.1349, "num_tokens": 35341554.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8153191804885864, "sampling/importance_sampling_ratio/mean": 1.0000742673873901, "sampling/importance_sampling_ratio/min": 0.5182788372039795, "sampling/sampling_logp_difference/max": 0.6572418212890625, "sampling/sampling_logp_difference/mean": 0.013508888892829418, "step": 725 }, { "clip_ratio/high_max": 2.7364272682461888e-05, "clip_ratio/high_mean": 2.7364272682461888e-05, "clip_ratio/low_mean": 0.00047822888518567197, "clip_ratio/low_min": 0.00047822888518567197, "clip_ratio/region_mean": 0.0005055931578681339, "completions/clipped_ratio": 0.0, "completions/max_length": 14225.0, "completions/max_terminated_length": 14225.0, "completions/mean_length": 11134.375, "completions/mean_terminated_length": 11134.375, "completions/min_length": 7511.0, "completions/min_terminated_length": 7511.0, "entropy": 0.7894294708967209, "epoch": 0.041728934360271294, "frac_reward_zero_std": 0.0, "grad_norm": 0.006197603419423103, "learning_rate": 1e-05, "loss": -0.0818, "num_tokens": 35431997.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8917529582977295, "sampling/importance_sampling_ratio/mean": 0.9999784231185913, "sampling/importance_sampling_ratio/min": 0.26045432686805725, "sampling/sampling_logp_difference/max": 1.3453278541564941, "sampling/sampling_logp_difference/mean": 0.025853218510746956, "step": 726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9139.0, "completions/max_terminated_length": 9139.0, "completions/mean_length": 8027.5, "completions/mean_terminated_length": 8027.5, "completions/min_length": 4464.0, "completions/min_terminated_length": 4464.0, "entropy": 1.3220058307051659, "epoch": 0.041786412231290954, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35496993.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5896265506744385, "sampling/importance_sampling_ratio/mean": 0.9997549653053284, "sampling/importance_sampling_ratio/min": 0.3792286515235901, "sampling/sampling_logp_difference/max": 0.9696159362792969, "sampling/sampling_logp_difference/mean": 0.03147401288151741, "step": 727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5001.0, "completions/max_terminated_length": 5001.0, "completions/mean_length": 2748.625, "completions/mean_terminated_length": 2748.625, "completions/min_length": 1392.0, "completions/min_terminated_length": 1392.0, "entropy": 0.41126847825944424, "epoch": 0.04184389010231061, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35519918.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4342248439788818, "sampling/importance_sampling_ratio/mean": 0.9995415210723877, "sampling/importance_sampling_ratio/min": 0.36883601546287537, "sampling/sampling_logp_difference/max": 0.9974031448364258, "sampling/sampling_logp_difference/mean": 0.01715588942170143, "step": 728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7621.0, "completions/max_terminated_length": 7621.0, "completions/mean_length": 5826.875, "completions/mean_terminated_length": 5826.875, "completions/min_length": 3099.0, "completions/min_terminated_length": 3099.0, "entropy": 0.45034367963671684, "epoch": 0.041901367973330265, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35567565.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8826528787612915, "sampling/importance_sampling_ratio/mean": 0.9997577667236328, "sampling/importance_sampling_ratio/min": 0.49221664667129517, "sampling/sampling_logp_difference/max": 0.7088363170623779, "sampling/sampling_logp_difference/mean": 0.01746448129415512, "step": 729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1540.0, "completions/max_terminated_length": 1540.0, "completions/mean_length": 815.5, "completions/mean_terminated_length": 815.5, "completions/min_length": 496.0, "completions/min_terminated_length": 496.0, "entropy": 0.2682587541639805, "epoch": 0.041958845844349925, "frac_reward_zero_std": 0.0, "grad_norm": 0.04569428786635399, "learning_rate": 1e-05, "loss": -0.0042, "num_tokens": 35575289.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6146409511566162, "sampling/importance_sampling_ratio/mean": 1.0007268190383911, "sampling/importance_sampling_ratio/min": 0.6108195185661316, "sampling/sampling_logp_difference/max": 0.4929537773132324, "sampling/sampling_logp_difference/mean": 0.013069335371255875, "step": 730 }, { "clip_ratio/high_max": 0.00015017349323898088, "clip_ratio/high_mean": 0.00015017349323898088, "clip_ratio/low_mean": 0.00033866037119878456, "clip_ratio/low_min": 0.00033866037119878456, "clip_ratio/region_mean": 0.0004888338644377654, "completions/clipped_ratio": 0.0, "completions/max_length": 13608.0, "completions/max_terminated_length": 13608.0, "completions/mean_length": 9953.75, "completions/mean_terminated_length": 9953.75, "completions/min_length": 5572.0, "completions/min_terminated_length": 5572.0, "entropy": 0.44230707362294197, "epoch": 0.042016323715369584, "frac_reward_zero_std": 0.0, "grad_norm": 0.01017897017300129, "learning_rate": 1e-05, "loss": 0.1495, "num_tokens": 35656087.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000959634780884, "sampling/importance_sampling_ratio/min": 0.34991180896759033, "sampling/sampling_logp_difference/max": 1.0500741004943848, "sampling/sampling_logp_difference/mean": 0.01938582770526409, "step": 731 }, { "clip_ratio/high_max": 0.00028460937392082997, "clip_ratio/high_mean": 0.00028460937392082997, "clip_ratio/low_mean": 0.00011256754078203812, "clip_ratio/low_min": 0.00011256754078203812, "clip_ratio/region_mean": 0.0003971769147028681, "completions/clipped_ratio": 0.0, "completions/max_length": 9994.0, "completions/max_terminated_length": 9994.0, "completions/mean_length": 4649.625, "completions/mean_terminated_length": 4649.625, "completions/min_length": 1875.0, "completions/min_terminated_length": 1875.0, "entropy": 0.3113875426352024, "epoch": 0.04207380158638924, "frac_reward_zero_std": 0.0, "grad_norm": 0.005080060102045536, "learning_rate": 1e-05, "loss": 0.407, "num_tokens": 35694684.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999955892562866, "sampling/importance_sampling_ratio/min": 0.371010422706604, "sampling/sampling_logp_difference/max": 1.3381919860839844, "sampling/sampling_logp_difference/mean": 0.01487748697400093, "step": 732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8570.0, "completions/max_terminated_length": 8570.0, "completions/mean_length": 5824.5, "completions/mean_terminated_length": 5824.5, "completions/min_length": 3032.0, "completions/min_terminated_length": 3032.0, "entropy": 0.8167114555835724, "epoch": 0.042131279457408896, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35742112.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.638521671295166, "sampling/importance_sampling_ratio/mean": 1.0000720024108887, "sampling/importance_sampling_ratio/min": 0.5551620125770569, "sampling/sampling_logp_difference/max": 0.5884952545166016, "sampling/sampling_logp_difference/mean": 0.028144417330622673, "step": 733 }, { "clip_ratio/high_max": 5.923052776779514e-05, "clip_ratio/high_mean": 5.923052776779514e-05, "clip_ratio/low_mean": 0.00012696800695266575, "clip_ratio/low_min": 0.00012696800695266575, "clip_ratio/region_mean": 0.00018619853472046088, "completions/clipped_ratio": 0.0, "completions/max_length": 13783.0, "completions/max_terminated_length": 13783.0, "completions/mean_length": 6044.125, "completions/mean_terminated_length": 6044.125, "completions/min_length": 3520.0, "completions/min_terminated_length": 3520.0, "entropy": 0.5173937827348709, "epoch": 0.042188757328428556, "frac_reward_zero_std": 0.0, "grad_norm": 0.007069787010550499, "learning_rate": 1e-05, "loss": 0.4529, "num_tokens": 35791729.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002763271331787, "sampling/importance_sampling_ratio/min": 0.4600960612297058, "sampling/sampling_logp_difference/max": 2.29365873336792, "sampling/sampling_logp_difference/mean": 0.021605221554636955, "step": 734 }, { "clip_ratio/high_max": 0.0001391499117744388, "clip_ratio/high_mean": 0.0001391499117744388, "clip_ratio/low_mean": 9.918212890625e-05, "clip_ratio/low_min": 9.918212890625e-05, "clip_ratio/region_mean": 0.0002383320406806888, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 9402.0, "completions/mean_length": 9174.5, "completions/mean_terminated_length": 8144.57177734375, "completions/min_length": 6748.0, "completions/min_terminated_length": 6748.0, "entropy": 0.3978094048798084, "epoch": 0.042246235199448215, "frac_reward_zero_std": 0.0, "grad_norm": 0.004910781979560852, "learning_rate": 1e-05, "loss": 0.278, "num_tokens": 35866853.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001506805419922, "sampling/importance_sampling_ratio/min": 0.032911330461502075, "sampling/sampling_logp_difference/max": 3.413938283920288, "sampling/sampling_logp_difference/mean": 0.016997680068016052, "step": 735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8155.0, "completions/max_terminated_length": 8155.0, "completions/mean_length": 6074.0, "completions/mean_terminated_length": 6074.0, "completions/min_length": 3486.0, "completions/min_terminated_length": 3486.0, "entropy": 0.5682268403470516, "epoch": 0.04230371307046787, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35916253.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.809597373008728, "sampling/importance_sampling_ratio/mean": 0.9998506307601929, "sampling/importance_sampling_ratio/min": 0.5570792555809021, "sampling/sampling_logp_difference/max": 0.593104362487793, "sampling/sampling_logp_difference/mean": 0.019280117005109787, "step": 736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7579.0, "completions/max_terminated_length": 7579.0, "completions/mean_length": 5406.125, "completions/mean_terminated_length": 5406.125, "completions/min_length": 2819.0, "completions/min_terminated_length": 2819.0, "entropy": 0.4998742453753948, "epoch": 0.04236119094148753, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 35961342.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8628578186035156, "sampling/importance_sampling_ratio/mean": 0.9997942447662354, "sampling/importance_sampling_ratio/min": 0.3487333357334137, "sampling/sampling_logp_difference/max": 1.0534477233886719, "sampling/sampling_logp_difference/mean": 0.019377753138542175, "step": 737 }, { "clip_ratio/high_max": 0.00010993094838340767, "clip_ratio/high_mean": 0.00010993094838340767, "clip_ratio/low_mean": 0.0006901673477841541, "clip_ratio/low_min": 0.0006901673477841541, "clip_ratio/region_mean": 0.0008000982961675618, "completions/clipped_ratio": 0.0, "completions/max_length": 5172.0, "completions/max_terminated_length": 5172.0, "completions/mean_length": 3657.25, "completions/mean_terminated_length": 3657.25, "completions/min_length": 1528.0, "completions/min_terminated_length": 1528.0, "entropy": 0.5930343978106976, "epoch": 0.042418668812507186, "frac_reward_zero_std": 0.0, "grad_norm": 0.019185570999979973, "learning_rate": 1e-05, "loss": 0.0612, "num_tokens": 36000776.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6273977756500244, "sampling/importance_sampling_ratio/mean": 1.0000773668289185, "sampling/importance_sampling_ratio/min": 0.6057080626487732, "sampling/sampling_logp_difference/max": 0.5013570785522461, "sampling/sampling_logp_difference/mean": 0.023027827963232994, "step": 738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4550.0, "completions/max_terminated_length": 4550.0, "completions/mean_length": 2885.0, "completions/mean_terminated_length": 2885.0, "completions/min_length": 942.0, "completions/min_terminated_length": 942.0, "entropy": 0.5166193172335625, "epoch": 0.042476146683526846, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36024728.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5046650171279907, "sampling/importance_sampling_ratio/mean": 0.99989914894104, "sampling/importance_sampling_ratio/min": 0.5714343786239624, "sampling/sampling_logp_difference/max": 0.559605598449707, "sampling/sampling_logp_difference/mean": 0.021570967510342598, "step": 739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6262.0, "completions/max_terminated_length": 6262.0, "completions/mean_length": 3477.25, "completions/mean_terminated_length": 3477.25, "completions/min_length": 2028.0, "completions/min_terminated_length": 2028.0, "entropy": 1.358054056763649, "epoch": 0.0425336245545465, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36054314.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.351204752922058, "sampling/importance_sampling_ratio/mean": 1.0000070333480835, "sampling/importance_sampling_ratio/min": 0.559296727180481, "sampling/sampling_logp_difference/max": 0.5810751914978027, "sampling/sampling_logp_difference/mean": 0.02934137172996998, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002633853864608682, "clip_ratio/low_min": 0.0002633853864608682, "clip_ratio/region_mean": 0.0002633853864608682, "completions/clipped_ratio": 0.0, "completions/max_length": 11377.0, "completions/max_terminated_length": 11377.0, "completions/mean_length": 6082.75, "completions/mean_terminated_length": 6082.75, "completions/min_length": 1249.0, "completions/min_terminated_length": 1249.0, "entropy": 0.6389830745756626, "epoch": 0.04259110242556616, "frac_reward_zero_std": 0.0, "grad_norm": 0.009630247950553894, "learning_rate": 1e-05, "loss": -0.2918, "num_tokens": 36104496.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001235008239746, "sampling/importance_sampling_ratio/min": 0.4028584957122803, "sampling/sampling_logp_difference/max": 1.661686897277832, "sampling/sampling_logp_difference/mean": 0.01897379383444786, "step": 741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8875.0, "completions/max_terminated_length": 8875.0, "completions/mean_length": 5781.5, "completions/mean_terminated_length": 5781.5, "completions/min_length": 3317.0, "completions/min_terminated_length": 3317.0, "entropy": 0.2835051864385605, "epoch": 0.04264858029658582, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36151884.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4807900190353394, "sampling/importance_sampling_ratio/mean": 0.9999252557754517, "sampling/importance_sampling_ratio/min": 0.47688576579093933, "sampling/sampling_logp_difference/max": 0.7404782772064209, "sampling/sampling_logp_difference/mean": 0.01191310491412878, "step": 742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14238.0, "completions/max_terminated_length": 14238.0, "completions/mean_length": 10056.25, "completions/mean_terminated_length": 10056.25, "completions/min_length": 5931.0, "completions/min_terminated_length": 5931.0, "entropy": 0.5038664862513542, "epoch": 0.04270605816760547, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36233654.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9971343278884888, "sampling/importance_sampling_ratio/mean": 1.0000650882720947, "sampling/importance_sampling_ratio/min": 0.43807902932167053, "sampling/sampling_logp_difference/max": 0.8253560066223145, "sampling/sampling_logp_difference/mean": 0.019881820306181908, "step": 743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7930.0, "completions/max_terminated_length": 7930.0, "completions/mean_length": 4383.125, "completions/mean_terminated_length": 4383.125, "completions/min_length": 2851.0, "completions/min_terminated_length": 2851.0, "entropy": 0.5845376513898373, "epoch": 0.04276353603862513, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36269847.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4925652742385864, "sampling/importance_sampling_ratio/mean": 1.0003907680511475, "sampling/importance_sampling_ratio/min": 0.5988203883171082, "sampling/sampling_logp_difference/max": 0.5127935409545898, "sampling/sampling_logp_difference/mean": 0.022330380976200104, "step": 744 }, { "clip_ratio/high_max": 0.00019135282946081134, "clip_ratio/high_mean": 0.00019135282946081134, "clip_ratio/low_mean": 0.00021997636940795928, "clip_ratio/low_min": 0.00021997636940795928, "clip_ratio/region_mean": 0.0004113291988687706, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13948.0, "completions/mean_length": 10948.875, "completions/mean_terminated_length": 10172.4287109375, "completions/min_length": 6118.0, "completions/min_terminated_length": 6118.0, "entropy": 0.5107805952429771, "epoch": 0.04282101390964479, "frac_reward_zero_std": 0.0, "grad_norm": 0.006762061268091202, "learning_rate": 1e-05, "loss": 0.191, "num_tokens": 36358526.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997686147689819, "sampling/importance_sampling_ratio/min": 0.22359634935855865, "sampling/sampling_logp_difference/max": 1.497912883758545, "sampling/sampling_logp_difference/mean": 0.02171766199171543, "step": 745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4627.0, "completions/max_terminated_length": 4627.0, "completions/mean_length": 2852.625, "completions/mean_terminated_length": 2852.625, "completions/min_length": 1922.0, "completions/min_terminated_length": 1922.0, "entropy": 0.26189582608640194, "epoch": 0.04287849178066445, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36382091.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3446849584579468, "sampling/importance_sampling_ratio/mean": 1.000154733657837, "sampling/importance_sampling_ratio/min": 0.6982633471488953, "sampling/sampling_logp_difference/max": 0.359158992767334, "sampling/sampling_logp_difference/mean": 0.011170906014740467, "step": 746 }, { "clip_ratio/high_max": 8.409540168941021e-05, "clip_ratio/high_mean": 8.409540168941021e-05, "clip_ratio/low_mean": 0.0003280408418504521, "clip_ratio/low_min": 0.0003280408418504521, "clip_ratio/region_mean": 0.00041213624353986233, "completions/clipped_ratio": 0.0, "completions/max_length": 16324.0, "completions/max_terminated_length": 16324.0, "completions/mean_length": 11410.25, "completions/mean_terminated_length": 11410.25, "completions/min_length": 9648.0, "completions/min_terminated_length": 9648.0, "entropy": 0.909414604306221, "epoch": 0.0429359696516841, "frac_reward_zero_std": 0.0, "grad_norm": 0.03144266456365585, "learning_rate": 1e-05, "loss": 0.0714, "num_tokens": 36474973.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001788139343262, "sampling/importance_sampling_ratio/min": 0.1302867829799652, "sampling/sampling_logp_difference/max": 2.0380172729492188, "sampling/sampling_logp_difference/mean": 0.030877703800797462, "step": 747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1975.0, "completions/max_terminated_length": 1975.0, "completions/mean_length": 1422.75, "completions/mean_terminated_length": 1422.75, "completions/min_length": 937.0, "completions/min_terminated_length": 937.0, "entropy": 0.18569425866007805, "epoch": 0.04299344752270376, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36487451.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2547622919082642, "sampling/importance_sampling_ratio/mean": 0.9994440674781799, "sampling/importance_sampling_ratio/min": 0.6839190721511841, "sampling/sampling_logp_difference/max": 0.379915714263916, "sampling/sampling_logp_difference/mean": 0.007429002318531275, "step": 748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9787.0, "completions/max_terminated_length": 9787.0, "completions/mean_length": 6890.375, "completions/mean_terminated_length": 6890.375, "completions/min_length": 2732.0, "completions/min_terminated_length": 2732.0, "entropy": 0.9186786785721779, "epoch": 0.04305092539372342, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36543478.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.750204086303711, "sampling/importance_sampling_ratio/mean": 0.9995497465133667, "sampling/importance_sampling_ratio/min": 0.4784961938858032, "sampling/sampling_logp_difference/max": 0.7371070384979248, "sampling/sampling_logp_difference/mean": 0.030549895018339157, "step": 749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6420.0, "completions/max_terminated_length": 6420.0, "completions/mean_length": 3031.75, "completions/mean_terminated_length": 3031.75, "completions/min_length": 1262.0, "completions/min_terminated_length": 1262.0, "entropy": 0.5197884812951088, "epoch": 0.04310840326474307, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36568788.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6078063249588013, "sampling/importance_sampling_ratio/mean": 1.0003070831298828, "sampling/importance_sampling_ratio/min": 0.6132988333702087, "sampling/sampling_logp_difference/max": 0.4889030456542969, "sampling/sampling_logp_difference/mean": 0.01749591901898384, "step": 750 }, { "clip_ratio/high_max": 5.22683603776386e-05, "clip_ratio/high_mean": 5.22683603776386e-05, "clip_ratio/low_mean": 0.0002829300574376248, "clip_ratio/low_min": 0.0002829300574376248, "clip_ratio/region_mean": 0.0003351984178152634, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14724.0, "completions/mean_length": 10519.25, "completions/mean_terminated_length": 9681.4287109375, "completions/min_length": 4374.0, "completions/min_terminated_length": 4374.0, "entropy": 0.43503354489803314, "epoch": 0.04316588113576273, "frac_reward_zero_std": 0.0, "grad_norm": 0.014100971631705761, "learning_rate": 1e-05, "loss": 0.3265, "num_tokens": 36654550.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999142289161682, "sampling/importance_sampling_ratio/min": 0.12263402342796326, "sampling/sampling_logp_difference/max": 2.098550796508789, "sampling/sampling_logp_difference/mean": 0.019631195813417435, "step": 751 }, { "clip_ratio/high_max": 0.00012976017387700267, "clip_ratio/high_mean": 0.00012976017387700267, "clip_ratio/low_mean": 0.00011444091796875, "clip_ratio/low_min": 0.00011444091796875, "clip_ratio/region_mean": 0.00024420109184575267, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 10783.0, "completions/mean_length": 8605.0, "completions/mean_terminated_length": 7493.71484375, "completions/min_length": 3763.0, "completions/min_terminated_length": 3763.0, "entropy": 0.34873415902256966, "epoch": 0.04322335900678239, "frac_reward_zero_std": 0.0, "grad_norm": 0.08116161823272705, "learning_rate": 1e-05, "loss": 0.3201, "num_tokens": 36724430.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002851486206055, "sampling/importance_sampling_ratio/min": 0.43439263105392456, "sampling/sampling_logp_difference/max": 0.8338065147399902, "sampling/sampling_logp_difference/mean": 0.015720469877123833, "step": 752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8387.0, "completions/max_terminated_length": 8387.0, "completions/mean_length": 2870.25, "completions/mean_terminated_length": 2870.25, "completions/min_length": 1653.0, "completions/min_terminated_length": 1653.0, "entropy": 0.2975548021495342, "epoch": 0.04328083687780205, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36748352.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4519537687301636, "sampling/importance_sampling_ratio/mean": 1.000065565109253, "sampling/importance_sampling_ratio/min": 0.6157928109169006, "sampling/sampling_logp_difference/max": 0.4848446846008301, "sampling/sampling_logp_difference/mean": 0.013579648919403553, "step": 753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7919.0, "completions/max_terminated_length": 7919.0, "completions/mean_length": 4499.0, "completions/mean_terminated_length": 4499.0, "completions/min_length": 2153.0, "completions/min_terminated_length": 2153.0, "entropy": 0.33532368391752243, "epoch": 0.0433383147488217, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36785704.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9725210666656494, "sampling/importance_sampling_ratio/mean": 1.000092625617981, "sampling/importance_sampling_ratio/min": 0.486767441034317, "sampling/sampling_logp_difference/max": 0.7199687957763672, "sampling/sampling_logp_difference/mean": 0.01376320794224739, "step": 754 }, { "clip_ratio/high_max": 0.00021123291207914008, "clip_ratio/high_mean": 0.00021123291207914008, "clip_ratio/low_mean": 0.0001068115234375, "clip_ratio/low_min": 0.0001068115234375, "clip_ratio/region_mean": 0.0003180444355166401, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 16342.0, "completions/mean_length": 13363.375, "completions/mean_terminated_length": 12356.5, "completions/min_length": 9058.0, "completions/min_terminated_length": 9058.0, "entropy": 0.7907493934035301, "epoch": 0.04339579261984136, "frac_reward_zero_std": 0.0, "grad_norm": 0.008395982906222343, "learning_rate": 1e-05, "loss": 0.1219, "num_tokens": 36893595.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999707818031311, "sampling/importance_sampling_ratio/min": 0.3421905040740967, "sampling/sampling_logp_difference/max": 1.0723876953125, "sampling/sampling_logp_difference/mean": 0.02502988465130329, "step": 755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2124.0, "completions/max_terminated_length": 2124.0, "completions/mean_length": 1946.625, "completions/mean_terminated_length": 1946.625, "completions/min_length": 1610.0, "completions/min_terminated_length": 1610.0, "entropy": 0.21949215792119503, "epoch": 0.04345327049086102, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36910032.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3764185905456543, "sampling/importance_sampling_ratio/mean": 1.0003129243850708, "sampling/importance_sampling_ratio/min": 0.6126875877380371, "sampling/sampling_logp_difference/max": 0.4899001121520996, "sampling/sampling_logp_difference/mean": 0.008816458284854889, "step": 756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14144.0, "completions/max_terminated_length": 14144.0, "completions/mean_length": 6182.625, "completions/mean_terminated_length": 6182.625, "completions/min_length": 3759.0, "completions/min_terminated_length": 3759.0, "entropy": 0.4224625453352928, "epoch": 0.04351074836188067, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 36960373.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000442385673523, "sampling/importance_sampling_ratio/min": 0.24169601500034332, "sampling/sampling_logp_difference/max": 1.420074462890625, "sampling/sampling_logp_difference/mean": 0.01654467172920704, "step": 757 }, { "clip_ratio/high_max": 2.2036955670046154e-05, "clip_ratio/high_mean": 2.2036955670046154e-05, "clip_ratio/low_mean": 0.0007877628231653944, "clip_ratio/low_min": 0.0007877628231653944, "clip_ratio/region_mean": 0.0008097997788354405, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15043.0, "completions/mean_length": 13502.625, "completions/mean_terminated_length": 11773.7998046875, "completions/min_length": 8676.0, "completions/min_terminated_length": 8676.0, "entropy": 0.5118184201419353, "epoch": 0.04356822623290033, "frac_reward_zero_std": 0.0, "grad_norm": 0.008944939821958542, "learning_rate": 1e-05, "loss": 0.039, "num_tokens": 37069434.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000118017196655, "sampling/importance_sampling_ratio/min": 0.25388526916503906, "sampling/sampling_logp_difference/max": 1.3708728551864624, "sampling/sampling_logp_difference/mean": 0.0219736211001873, "step": 758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13312.0, "completions/max_terminated_length": 13312.0, "completions/mean_length": 4839.125, "completions/mean_terminated_length": 4839.125, "completions/min_length": 2586.0, "completions/min_terminated_length": 2586.0, "entropy": 0.4705335944890976, "epoch": 0.04362570410391999, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37109091.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996165633201599, "sampling/importance_sampling_ratio/min": 0.4147474765777588, "sampling/sampling_logp_difference/max": 0.8800854682922363, "sampling/sampling_logp_difference/mean": 0.019713759422302246, "step": 759 }, { "clip_ratio/high_max": 0.0001946960765053518, "clip_ratio/high_mean": 0.0001946960765053518, "clip_ratio/low_mean": 0.0004161716133239679, "clip_ratio/low_min": 0.0004161716133239679, "clip_ratio/region_mean": 0.0006108676898293197, "completions/clipped_ratio": 0.0, "completions/max_length": 8366.0, "completions/max_terminated_length": 8366.0, "completions/mean_length": 5856.0, "completions/mean_terminated_length": 5856.0, "completions/min_length": 4361.0, "completions/min_terminated_length": 4361.0, "entropy": 0.5813636928796768, "epoch": 0.04368318197493965, "frac_reward_zero_std": 0.0, "grad_norm": 0.031191250309348106, "learning_rate": 1e-05, "loss": 0.1031, "num_tokens": 37157627.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001590251922607, "sampling/importance_sampling_ratio/min": 0.36866581439971924, "sampling/sampling_logp_difference/max": 0.9978647232055664, "sampling/sampling_logp_difference/mean": 0.021152706816792488, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4228.0, "completions/max_terminated_length": 4228.0, "completions/mean_length": 2374.125, "completions/mean_terminated_length": 2374.125, "completions/min_length": 1501.0, "completions/min_terminated_length": 1501.0, "entropy": 0.3551023006439209, "epoch": 0.043740659845959304, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37177532.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4402356147766113, "sampling/importance_sampling_ratio/mean": 0.999523937702179, "sampling/importance_sampling_ratio/min": 0.6553753614425659, "sampling/sampling_logp_difference/max": 0.4225471019744873, "sampling/sampling_logp_difference/mean": 0.015214038081467152, "step": 761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4362.0, "completions/max_terminated_length": 4362.0, "completions/mean_length": 3316.5, "completions/mean_terminated_length": 3316.5, "completions/min_length": 2263.0, "completions/min_terminated_length": 2263.0, "entropy": 0.38055382668972015, "epoch": 0.04379813771697896, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37205144.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.283323884010315, "sampling/importance_sampling_ratio/mean": 1.0004974603652954, "sampling/importance_sampling_ratio/min": 0.602974534034729, "sampling/sampling_logp_difference/max": 0.5058803558349609, "sampling/sampling_logp_difference/mean": 0.012063817121088505, "step": 762 }, { "clip_ratio/high_max": 8.852691098582e-05, "clip_ratio/high_mean": 8.852691098582e-05, "clip_ratio/low_mean": 0.00023787668033037335, "clip_ratio/low_min": 0.00023787668033037335, "clip_ratio/region_mean": 0.00032640359131619334, "completions/clipped_ratio": 0.0, "completions/max_length": 8771.0, "completions/max_terminated_length": 8771.0, "completions/mean_length": 5399.375, "completions/mean_terminated_length": 5399.375, "completions/min_length": 1412.0, "completions/min_terminated_length": 1412.0, "entropy": 0.696052324026823, "epoch": 0.04385561558799862, "frac_reward_zero_std": 0.0, "grad_norm": 0.019166020676493645, "learning_rate": 1e-05, "loss": 0.33, "num_tokens": 37249891.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.6513043642044067, "sampling/importance_sampling_ratio/mean": 1.0002875328063965, "sampling/importance_sampling_ratio/min": 0.5157813429832458, "sampling/sampling_logp_difference/max": 0.6620724201202393, "sampling/sampling_logp_difference/mean": 0.021381070837378502, "step": 763 }, { "clip_ratio/high_max": 0.00014261173419072293, "clip_ratio/high_mean": 0.00014261173419072293, "clip_ratio/low_mean": 4.57763671875e-05, "clip_ratio/low_min": 4.57763671875e-05, "clip_ratio/region_mean": 0.00018838810137822293, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 8977.0, "completions/mean_length": 7273.25, "completions/mean_terminated_length": 5971.71435546875, "completions/min_length": 3626.0, "completions/min_terminated_length": 3626.0, "entropy": 0.4026900418102741, "epoch": 0.043913093459018275, "frac_reward_zero_std": 0.0, "grad_norm": 0.006018485873937607, "learning_rate": 1e-05, "loss": 0.443, "num_tokens": 37309349.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.8775500059127808, "sampling/importance_sampling_ratio/mean": 1.0000756978988647, "sampling/importance_sampling_ratio/min": 0.413297563791275, "sampling/sampling_logp_difference/max": 0.8835874795913696, "sampling/sampling_logp_difference/mean": 0.016446389257907867, "step": 764 }, { "clip_ratio/high_max": 4.789751983480528e-05, "clip_ratio/high_mean": 4.789751983480528e-05, "clip_ratio/low_mean": 3.534568168106489e-05, "clip_ratio/low_min": 3.534568168106489e-05, "clip_ratio/region_mean": 8.324320151587017e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 7073.0, "completions/max_terminated_length": 7073.0, "completions/mean_length": 4512.625, "completions/mean_terminated_length": 4512.625, "completions/min_length": 2290.0, "completions/min_terminated_length": 2290.0, "entropy": 0.967737190425396, "epoch": 0.043970571330037934, "frac_reward_zero_std": 0.0, "grad_norm": 0.01556406356394291, "learning_rate": 1e-05, "loss": 0.2005, "num_tokens": 37346106.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4978071451187134, "sampling/importance_sampling_ratio/mean": 0.9999997615814209, "sampling/importance_sampling_ratio/min": 0.10077287256717682, "sampling/sampling_logp_difference/max": 2.2948861122131348, "sampling/sampling_logp_difference/mean": 0.024122994393110275, "step": 765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5290.0, "completions/max_terminated_length": 5290.0, "completions/mean_length": 4047.0, "completions/mean_terminated_length": 4047.0, "completions/min_length": 3545.0, "completions/min_terminated_length": 3545.0, "entropy": 0.18963348120450974, "epoch": 0.044028049201057594, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37379506.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2990062236785889, "sampling/importance_sampling_ratio/mean": 1.0001507997512817, "sampling/importance_sampling_ratio/min": 0.6098744869232178, "sampling/sampling_logp_difference/max": 0.49450206756591797, "sampling/sampling_logp_difference/mean": 0.008034742437303066, "step": 766 }, { "clip_ratio/high_max": 0.00028455945175664965, "clip_ratio/high_mean": 0.00028455945175664965, "clip_ratio/low_mean": 0.00012617159518413246, "clip_ratio/low_min": 0.00012617159518413246, "clip_ratio/region_mean": 0.0004107310469407821, "completions/clipped_ratio": 0.0, "completions/max_length": 13870.0, "completions/max_terminated_length": 13870.0, "completions/mean_length": 8122.875, "completions/mean_terminated_length": 8122.875, "completions/min_length": 3130.0, "completions/min_terminated_length": 3130.0, "entropy": 0.5542602054774761, "epoch": 0.04408552707207725, "frac_reward_zero_std": 0.0, "grad_norm": 0.00859770830720663, "learning_rate": 1e-05, "loss": 0.2501, "num_tokens": 37445593.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000147819519043, "sampling/importance_sampling_ratio/min": 0.3401208519935608, "sampling/sampling_logp_difference/max": 1.0784542560577393, "sampling/sampling_logp_difference/mean": 0.022987596690654755, "step": 767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4691.0, "completions/max_terminated_length": 4691.0, "completions/mean_length": 2352.375, "completions/mean_terminated_length": 2352.375, "completions/min_length": 1018.0, "completions/min_terminated_length": 1018.0, "entropy": 0.5703789293766022, "epoch": 0.044143004943096906, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37465276.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3524365425109863, "sampling/importance_sampling_ratio/mean": 1.0001156330108643, "sampling/importance_sampling_ratio/min": 0.6338674426078796, "sampling/sampling_logp_difference/max": 0.4559154510498047, "sampling/sampling_logp_difference/mean": 0.019540216773748398, "step": 768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13324.0, "completions/mean_length": 12252.375, "completions/mean_terminated_length": 10875.1669921875, "completions/min_length": 8931.0, "completions/min_terminated_length": 8931.0, "entropy": 0.49922408908605576, "epoch": 0.044200482814116565, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37564479.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998695850372314, "sampling/importance_sampling_ratio/min": 0.3071923553943634, "sampling/sampling_logp_difference/max": 1.180281162261963, "sampling/sampling_logp_difference/mean": 0.021538907662034035, "step": 769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5862.0, "completions/max_terminated_length": 5862.0, "completions/mean_length": 4644.125, "completions/mean_terminated_length": 4644.125, "completions/min_length": 3247.0, "completions/min_terminated_length": 3247.0, "entropy": 0.22926154360175133, "epoch": 0.044257960685136224, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37603328.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.361389398574829, "sampling/importance_sampling_ratio/mean": 0.9998611807823181, "sampling/importance_sampling_ratio/min": 0.5420757532119751, "sampling/sampling_logp_difference/max": 0.6123495101928711, "sampling/sampling_logp_difference/mean": 0.010305430740118027, "step": 770 }, { "clip_ratio/high_max": 6.998877870501019e-05, "clip_ratio/high_mean": 6.998877870501019e-05, "clip_ratio/low_mean": 0.0005368740385165438, "clip_ratio/low_min": 0.0005368740385165438, "clip_ratio/region_mean": 0.000606862817221554, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 13207.0, "completions/mean_length": 10579.125, "completions/mean_terminated_length": 8644.1669921875, "completions/min_length": 7021.0, "completions/min_terminated_length": 7021.0, "entropy": 0.36422669887542725, "epoch": 0.04431543855615588, "frac_reward_zero_std": 0.0, "grad_norm": 0.012930000200867653, "learning_rate": 1e-05, "loss": 0.2912, "num_tokens": 37689193.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000097393989563, "sampling/importance_sampling_ratio/min": 0.4010445475578308, "sampling/sampling_logp_difference/max": 0.9136828184127808, "sampling/sampling_logp_difference/mean": 0.01695477031171322, "step": 771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4675.0, "completions/max_terminated_length": 4675.0, "completions/mean_length": 2548.25, "completions/mean_terminated_length": 2548.25, "completions/min_length": 745.0, "completions/min_terminated_length": 745.0, "entropy": 0.5197523236274719, "epoch": 0.044372916427175536, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37710955.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3970613479614258, "sampling/importance_sampling_ratio/mean": 1.0001896619796753, "sampling/importance_sampling_ratio/min": 0.6220954060554504, "sampling/sampling_logp_difference/max": 0.47466182708740234, "sampling/sampling_logp_difference/mean": 0.020341981202363968, "step": 772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4318.0, "completions/max_terminated_length": 4318.0, "completions/mean_length": 2709.875, "completions/mean_terminated_length": 2709.875, "completions/min_length": 1635.0, "completions/min_terminated_length": 1635.0, "entropy": 0.2924350444227457, "epoch": 0.044430394298195196, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37734018.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4420242309570312, "sampling/importance_sampling_ratio/mean": 0.9999087452888489, "sampling/importance_sampling_ratio/min": 0.6248693466186523, "sampling/sampling_logp_difference/max": 0.4702126979827881, "sampling/sampling_logp_difference/mean": 0.012126212008297443, "step": 773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3886.0, "completions/max_terminated_length": 3886.0, "completions/mean_length": 2811.5, "completions/mean_terminated_length": 2811.5, "completions/min_length": 2028.0, "completions/min_terminated_length": 2028.0, "entropy": 0.2661086991429329, "epoch": 0.044487872169214855, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37757742.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4472851753234863, "sampling/importance_sampling_ratio/mean": 1.0003266334533691, "sampling/importance_sampling_ratio/min": 0.6800163984298706, "sampling/sampling_logp_difference/max": 0.38563835620880127, "sampling/sampling_logp_difference/mean": 0.011031088419258595, "step": 774 }, { "clip_ratio/high_max": 7.778934559610207e-05, "clip_ratio/high_mean": 7.778934559610207e-05, "clip_ratio/low_mean": 6.820662383688614e-05, "clip_ratio/low_min": 6.820662383688614e-05, "clip_ratio/region_mean": 0.0001459959694329882, "completions/clipped_ratio": 0.0, "completions/max_length": 12702.0, "completions/max_terminated_length": 12702.0, "completions/mean_length": 5068.375, "completions/mean_terminated_length": 5068.375, "completions/min_length": 3129.0, "completions/min_terminated_length": 3129.0, "entropy": 0.2981343027204275, "epoch": 0.04454535004023451, "frac_reward_zero_std": 0.0, "grad_norm": 0.006297721527516842, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 37799689.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5255681276321411, "sampling/importance_sampling_ratio/mean": 1.0001076459884644, "sampling/importance_sampling_ratio/min": 0.4588330090045929, "sampling/sampling_logp_difference/max": 0.7790689468383789, "sampling/sampling_logp_difference/mean": 0.014135951176285744, "step": 775 }, { "clip_ratio/high_max": 0.00012856211105827242, "clip_ratio/high_mean": 0.00012856211105827242, "clip_ratio/low_mean": 0.00012969970703125, "clip_ratio/low_min": 0.00012969970703125, "clip_ratio/region_mean": 0.0002582618180895224, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 12232.0, "completions/mean_length": 10404.5, "completions/mean_terminated_length": 9550.2861328125, "completions/min_length": 5956.0, "completions/min_terminated_length": 5956.0, "entropy": 0.464727483689785, "epoch": 0.04460282791125417, "frac_reward_zero_std": 0.0, "grad_norm": 0.004750373307615519, "learning_rate": 1e-05, "loss": 0.2031, "num_tokens": 37884269.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999650716781616, "sampling/importance_sampling_ratio/min": 0.3263804316520691, "sampling/sampling_logp_difference/max": 1.1196916103363037, "sampling/sampling_logp_difference/mean": 0.01922188326716423, "step": 776 }, { "clip_ratio/high_max": 6.034998477844056e-05, "clip_ratio/high_mean": 6.034998477844056e-05, "clip_ratio/low_mean": 5.325947859091684e-05, "clip_ratio/low_min": 5.325947859091684e-05, "clip_ratio/region_mean": 0.0001136094633693574, "completions/clipped_ratio": 0.0, "completions/max_length": 6452.0, "completions/max_terminated_length": 6452.0, "completions/mean_length": 3338.0, "completions/mean_terminated_length": 3338.0, "completions/min_length": 1638.0, "completions/min_terminated_length": 1638.0, "entropy": 0.7064624726772308, "epoch": 0.044660305782273826, "frac_reward_zero_std": 0.0, "grad_norm": 0.018658028915524483, "learning_rate": 1e-05, "loss": -0.1049, "num_tokens": 37912125.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.4113073348999023, "sampling/importance_sampling_ratio/mean": 1.0000672340393066, "sampling/importance_sampling_ratio/min": 0.631686806678772, "sampling/sampling_logp_difference/max": 0.45936155319213867, "sampling/sampling_logp_difference/mean": 0.020603559911251068, "step": 777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4752.0, "completions/max_terminated_length": 4752.0, "completions/mean_length": 2977.25, "completions/mean_terminated_length": 2977.25, "completions/min_length": 2180.0, "completions/min_terminated_length": 2180.0, "entropy": 0.29995296895504, "epoch": 0.04471778365329348, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37936999.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3514126539230347, "sampling/importance_sampling_ratio/mean": 1.0000920295715332, "sampling/importance_sampling_ratio/min": 0.6493895053863525, "sampling/sampling_logp_difference/max": 0.43172264099121094, "sampling/sampling_logp_difference/mean": 0.011893728747963905, "step": 778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7375.0, "completions/max_terminated_length": 7375.0, "completions/mean_length": 4347.75, "completions/mean_terminated_length": 4347.75, "completions/min_length": 3186.0, "completions/min_terminated_length": 3186.0, "entropy": 0.6138377264142036, "epoch": 0.04477526152431314, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37972917.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9793411493301392, "sampling/importance_sampling_ratio/mean": 0.9999080896377563, "sampling/importance_sampling_ratio/min": 0.5359624624252319, "sampling/sampling_logp_difference/max": 0.6827640533447266, "sampling/sampling_logp_difference/mean": 0.021382445469498634, "step": 779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3508.0, "completions/max_terminated_length": 3508.0, "completions/mean_length": 2299.5, "completions/mean_terminated_length": 2299.5, "completions/min_length": 1172.0, "completions/min_terminated_length": 1172.0, "entropy": 0.34076982736587524, "epoch": 0.0448327393953328, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 37992769.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4448204040527344, "sampling/importance_sampling_ratio/mean": 1.0002573728561401, "sampling/importance_sampling_ratio/min": 0.5460746884346008, "sampling/sampling_logp_difference/max": 0.6049995422363281, "sampling/sampling_logp_difference/mean": 0.013583878986537457, "step": 780 }, { "clip_ratio/high_max": 1.4298787391453516e-05, "clip_ratio/high_mean": 1.4298787391453516e-05, "clip_ratio/low_mean": 3.613761145970784e-05, "clip_ratio/low_min": 3.613761145970784e-05, "clip_ratio/region_mean": 5.043639885116136e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 8742.0, "completions/max_terminated_length": 8742.0, "completions/mean_length": 3825.125, "completions/mean_terminated_length": 3825.125, "completions/min_length": 1157.0, "completions/min_terminated_length": 1157.0, "entropy": 0.3691420219838619, "epoch": 0.04489021726635246, "frac_reward_zero_std": 0.0, "grad_norm": 0.014242730103433132, "learning_rate": 1e-05, "loss": -0.0339, "num_tokens": 38024546.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6956981420516968, "sampling/importance_sampling_ratio/mean": 1.0004985332489014, "sampling/importance_sampling_ratio/min": 0.6899121403694153, "sampling/sampling_logp_difference/max": 0.5280945301055908, "sampling/sampling_logp_difference/mean": 0.012799215503036976, "step": 781 }, { "clip_ratio/high_max": 0.00010273643601976801, "clip_ratio/high_mean": 0.00010273643601976801, "clip_ratio/low_mean": 5.6116721680155024e-05, "clip_ratio/low_min": 5.6116721680155024e-05, "clip_ratio/region_mean": 0.00015885315769992303, "completions/clipped_ratio": 0.0, "completions/max_length": 8627.0, "completions/max_terminated_length": 8627.0, "completions/mean_length": 5232.0, "completions/mean_terminated_length": 5232.0, "completions/min_length": 1077.0, "completions/min_terminated_length": 1077.0, "entropy": 0.45510344952344894, "epoch": 0.04494769513737211, "frac_reward_zero_std": 0.0, "grad_norm": 0.0040388149209320545, "learning_rate": 1e-05, "loss": -0.0526, "num_tokens": 38067330.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.962630033493042, "sampling/importance_sampling_ratio/mean": 1.000290036201477, "sampling/importance_sampling_ratio/min": 0.4475387930870056, "sampling/sampling_logp_difference/max": 0.8039920926094055, "sampling/sampling_logp_difference/mean": 0.01985006593167782, "step": 782 }, { "clip_ratio/high_max": 8.888563024811447e-05, "clip_ratio/high_mean": 8.888563024811447e-05, "clip_ratio/low_mean": 0.00016076021711342037, "clip_ratio/low_min": 0.00016076021711342037, "clip_ratio/region_mean": 0.00024964584736153483, "completions/clipped_ratio": 0.0, "completions/max_length": 6998.0, "completions/max_terminated_length": 6998.0, "completions/mean_length": 3390.875, "completions/mean_terminated_length": 3390.875, "completions/min_length": 2329.0, "completions/min_terminated_length": 2329.0, "entropy": 0.407876942306757, "epoch": 0.04500517300839177, "frac_reward_zero_std": 0.0, "grad_norm": 0.012164358049631119, "learning_rate": 1e-05, "loss": 0.376, "num_tokens": 38095769.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000251293182373, "sampling/importance_sampling_ratio/min": 0.3836857080459595, "sampling/sampling_logp_difference/max": 0.957931637763977, "sampling/sampling_logp_difference/mean": 0.016471004113554955, "step": 783 }, { "clip_ratio/high_max": 4.222859297442483e-05, "clip_ratio/high_mean": 4.222859297442483e-05, "clip_ratio/low_mean": 0.00016196399610635126, "clip_ratio/low_min": 0.00016196399610635126, "clip_ratio/region_mean": 0.0002041925890807761, "completions/clipped_ratio": 0.0, "completions/max_length": 10519.0, "completions/max_terminated_length": 10519.0, "completions/mean_length": 7874.75, "completions/mean_terminated_length": 7874.75, "completions/min_length": 4199.0, "completions/min_terminated_length": 4199.0, "entropy": 0.5717684365808964, "epoch": 0.04506265087941143, "frac_reward_zero_std": 0.0, "grad_norm": 0.010475960560142994, "learning_rate": 1e-05, "loss": 0.2328, "num_tokens": 38159871.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999060034751892, "sampling/importance_sampling_ratio/min": 0.485975444316864, "sampling/sampling_logp_difference/max": 0.8914182186126709, "sampling/sampling_logp_difference/mean": 0.018375419080257416, "step": 784 }, { "clip_ratio/high_max": 0.0001322526013609604, "clip_ratio/high_mean": 0.0001322526013609604, "clip_ratio/low_mean": 0.00024901179131120443, "clip_ratio/low_min": 0.00024901179131120443, "clip_ratio/region_mean": 0.00038126439267216483, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 11454.0, "completions/mean_length": 8733.625, "completions/mean_terminated_length": 7640.71484375, "completions/min_length": 3766.0, "completions/min_terminated_length": 3766.0, "entropy": 0.4154110997915268, "epoch": 0.04512012875043108, "frac_reward_zero_std": 0.0, "grad_norm": 0.006835148669779301, "learning_rate": 1e-05, "loss": 0.31, "num_tokens": 38230860.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000584125518799, "sampling/importance_sampling_ratio/min": 0.10598830878734589, "sampling/sampling_logp_difference/max": 2.2444264888763428, "sampling/sampling_logp_difference/mean": 0.01849970407783985, "step": 785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2748.0, "completions/max_terminated_length": 2748.0, "completions/mean_length": 2169.375, "completions/mean_terminated_length": 2169.375, "completions/min_length": 1077.0, "completions/min_terminated_length": 1077.0, "entropy": 0.2906048223376274, "epoch": 0.04517760662145074, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38248983.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3024460077285767, "sampling/importance_sampling_ratio/mean": 0.9997011423110962, "sampling/importance_sampling_ratio/min": 0.6844488382339478, "sampling/sampling_logp_difference/max": 0.37914133071899414, "sampling/sampling_logp_difference/mean": 0.011078257113695145, "step": 786 }, { "clip_ratio/high_max": 0.00012830259220208973, "clip_ratio/high_mean": 0.00012830259220208973, "clip_ratio/low_mean": 0.00013058239710517228, "clip_ratio/low_min": 0.00013058239710517228, "clip_ratio/region_mean": 0.000258884989307262, "completions/clipped_ratio": 0.0, "completions/max_length": 6525.0, "completions/max_terminated_length": 6525.0, "completions/mean_length": 3507.5, "completions/mean_terminated_length": 3507.5, "completions/min_length": 1090.0, "completions/min_terminated_length": 1090.0, "entropy": 0.44777627289295197, "epoch": 0.0452350844924704, "frac_reward_zero_std": 0.0, "grad_norm": 0.0108426408842206, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 38278179.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6227647066116333, "sampling/importance_sampling_ratio/mean": 0.9997349381446838, "sampling/importance_sampling_ratio/min": 0.22513534128665924, "sampling/sampling_logp_difference/max": 1.491053581237793, "sampling/sampling_logp_difference/mean": 0.01890227571129799, "step": 787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15687.0, "completions/mean_length": 10812.375, "completions/mean_terminated_length": 10016.4287109375, "completions/min_length": 5587.0, "completions/min_terminated_length": 5587.0, "entropy": 0.854988344013691, "epoch": 0.04529256236349006, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38366014.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000146627426147, "sampling/importance_sampling_ratio/min": 5.867866093467455e-06, "sampling/sampling_logp_difference/max": 12.046019554138184, "sampling/sampling_logp_difference/mean": 0.029087401926517487, "step": 788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12542.0, "completions/max_terminated_length": 12542.0, "completions/mean_length": 10510.25, "completions/mean_terminated_length": 10510.25, "completions/min_length": 8905.0, "completions/min_terminated_length": 8905.0, "entropy": 0.5115222707390785, "epoch": 0.04535004023450971, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38451800.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.810068130493164, "sampling/importance_sampling_ratio/mean": 0.9998155832290649, "sampling/importance_sampling_ratio/min": 0.25198617577552795, "sampling/sampling_logp_difference/max": 1.3783811330795288, "sampling/sampling_logp_difference/mean": 0.020504631102085114, "step": 789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5676.0, "completions/max_terminated_length": 5676.0, "completions/mean_length": 3371.625, "completions/mean_terminated_length": 3371.625, "completions/min_length": 1447.0, "completions/min_terminated_length": 1447.0, "entropy": 0.2942180745303631, "epoch": 0.04540751810552937, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38479917.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.633518099784851, "sampling/importance_sampling_ratio/mean": 0.999942421913147, "sampling/importance_sampling_ratio/min": 0.6596074104309082, "sampling/sampling_logp_difference/max": 0.4907360076904297, "sampling/sampling_logp_difference/mean": 0.012629660777747631, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7157.0, "completions/max_terminated_length": 7157.0, "completions/mean_length": 4268.625, "completions/mean_terminated_length": 4268.625, "completions/min_length": 2575.0, "completions/min_terminated_length": 2575.0, "entropy": 0.40843693166971207, "epoch": 0.04546499597654903, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38515442.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8158138990402222, "sampling/importance_sampling_ratio/mean": 1.0000542402267456, "sampling/importance_sampling_ratio/min": 0.35182052850723267, "sampling/sampling_logp_difference/max": 1.0446341037750244, "sampling/sampling_logp_difference/mean": 0.016960738226771355, "step": 791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14009.0, "completions/max_terminated_length": 14009.0, "completions/mean_length": 11369.125, "completions/mean_terminated_length": 11369.125, "completions/min_length": 9728.0, "completions/min_terminated_length": 9728.0, "entropy": 0.35169897973537445, "epoch": 0.04552247384756868, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38607155.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999160766601562, "sampling/importance_sampling_ratio/min": 0.4047966003417969, "sampling/sampling_logp_difference/max": 0.9043706059455872, "sampling/sampling_logp_difference/mean": 0.014958295039832592, "step": 792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3624.0, "completions/max_terminated_length": 3624.0, "completions/mean_length": 2392.0, "completions/mean_terminated_length": 2392.0, "completions/min_length": 1949.0, "completions/min_terminated_length": 1949.0, "entropy": 0.2754772696644068, "epoch": 0.04557995171858834, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38627347.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2956032752990723, "sampling/importance_sampling_ratio/mean": 0.99988853931427, "sampling/importance_sampling_ratio/min": 0.6721356511116028, "sampling/sampling_logp_difference/max": 0.39729511737823486, "sampling/sampling_logp_difference/mean": 0.010974922217428684, "step": 793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5172.0, "completions/max_terminated_length": 5172.0, "completions/mean_length": 2922.875, "completions/mean_terminated_length": 2922.875, "completions/min_length": 1694.0, "completions/min_terminated_length": 1694.0, "entropy": 0.48841486871242523, "epoch": 0.045637429589608, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38651586.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3947864770889282, "sampling/importance_sampling_ratio/mean": 1.0002597570419312, "sampling/importance_sampling_ratio/min": 0.6183609962463379, "sampling/sampling_logp_difference/max": 0.4806828498840332, "sampling/sampling_logp_difference/mean": 0.019173720851540565, "step": 794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15147.0, "completions/max_terminated_length": 15147.0, "completions/mean_length": 8771.375, "completions/mean_terminated_length": 8771.375, "completions/min_length": 5587.0, "completions/min_terminated_length": 5587.0, "entropy": 0.5390896871685982, "epoch": 0.04569490746062766, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38722773.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.739428162574768, "sampling/importance_sampling_ratio/mean": 1.0003201961517334, "sampling/importance_sampling_ratio/min": 0.22796274721622467, "sampling/sampling_logp_difference/max": 1.4785730838775635, "sampling/sampling_logp_difference/mean": 0.02151065692305565, "step": 795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2438.0, "completions/max_terminated_length": 2438.0, "completions/mean_length": 1215.625, "completions/mean_terminated_length": 1215.625, "completions/min_length": 497.0, "completions/min_terminated_length": 497.0, "entropy": 0.45283616706728935, "epoch": 0.04575238533164731, "frac_reward_zero_std": 0.0, "grad_norm": 0.016843244433403015, "learning_rate": 1e-05, "loss": -0.2089, "num_tokens": 38733122.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999513566493988, "sampling/importance_sampling_ratio/min": 0.5027235746383667, "sampling/sampling_logp_difference/max": 0.7487432956695557, "sampling/sampling_logp_difference/mean": 0.019416222348809242, "step": 796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001385381747240899, "clip_ratio/low_min": 0.0001385381747240899, "clip_ratio/region_mean": 0.0001385381747240899, "completions/clipped_ratio": 0.0, "completions/max_length": 13027.0, "completions/max_terminated_length": 13027.0, "completions/mean_length": 8732.125, "completions/mean_terminated_length": 8732.125, "completions/min_length": 2335.0, "completions/min_terminated_length": 2335.0, "entropy": 0.6959386542439461, "epoch": 0.04580986320266697, "frac_reward_zero_std": 0.0, "grad_norm": 0.017630675807595253, "learning_rate": 1e-05, "loss": -0.1302, "num_tokens": 38804027.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.9107762575149536, "sampling/importance_sampling_ratio/mean": 1.0001033544540405, "sampling/importance_sampling_ratio/min": 0.28761106729507446, "sampling/sampling_logp_difference/max": 1.2461462020874023, "sampling/sampling_logp_difference/mean": 0.02121792547404766, "step": 797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10016.0, "completions/max_terminated_length": 10016.0, "completions/mean_length": 6614.0, "completions/mean_terminated_length": 6614.0, "completions/min_length": 4811.0, "completions/min_terminated_length": 4811.0, "entropy": 0.5177438035607338, "epoch": 0.04586734107368663, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38858331.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002449750900269, "sampling/importance_sampling_ratio/min": 0.534162163734436, "sampling/sampling_logp_difference/max": 0.7237164974212646, "sampling/sampling_logp_difference/mean": 0.0207215603441, "step": 798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12173.0, "completions/max_terminated_length": 12173.0, "completions/mean_length": 8296.5, "completions/mean_terminated_length": 8296.5, "completions/min_length": 2731.0, "completions/min_terminated_length": 2731.0, "entropy": 0.5470918118953705, "epoch": 0.045924818944706285, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38925455.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8654152154922485, "sampling/importance_sampling_ratio/mean": 1.0001827478408813, "sampling/importance_sampling_ratio/min": 0.3128296434879303, "sampling/sampling_logp_difference/max": 1.1620965003967285, "sampling/sampling_logp_difference/mean": 0.02176014333963394, "step": 799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8145.0, "completions/max_terminated_length": 8145.0, "completions/mean_length": 4590.125, "completions/mean_terminated_length": 4590.125, "completions/min_length": 3269.0, "completions/min_terminated_length": 3269.0, "entropy": 0.39563508704304695, "epoch": 0.045982296815725944, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 38963088.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4661719799041748, "sampling/importance_sampling_ratio/mean": 0.9999491572380066, "sampling/importance_sampling_ratio/min": 0.5654613971710205, "sampling/sampling_logp_difference/max": 0.5701131820678711, "sampling/sampling_logp_difference/mean": 0.01634219102561474, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 13936.0, "completions/mean_length": 11334.875, "completions/mean_terminated_length": 10613.572265625, "completions/min_length": 7248.0, "completions/min_terminated_length": 7248.0, "entropy": 0.4925721846520901, "epoch": 0.0460397746867456, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39054607.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999300241470337, "sampling/importance_sampling_ratio/min": 0.14733903110027313, "sampling/sampling_logp_difference/max": 1.9150190353393555, "sampling/sampling_logp_difference/mean": 0.020871059969067574, "step": 801 }, { "clip_ratio/high_max": 0.00012803788467863342, "clip_ratio/high_mean": 0.00012803788467863342, "clip_ratio/low_mean": 0.00017646442938712426, "clip_ratio/low_min": 0.00017646442938712426, "clip_ratio/region_mean": 0.0003045023140657577, "completions/clipped_ratio": 0.0, "completions/max_length": 15311.0, "completions/max_terminated_length": 15311.0, "completions/mean_length": 10999.5, "completions/mean_terminated_length": 10999.5, "completions/min_length": 4909.0, "completions/min_terminated_length": 4909.0, "entropy": 0.6176016889512539, "epoch": 0.04609725255776526, "frac_reward_zero_std": 0.0, "grad_norm": 0.04150174930691719, "learning_rate": 1e-05, "loss": 0.0067, "num_tokens": 39143507.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000609159469604, "sampling/importance_sampling_ratio/min": 0.30463430285453796, "sampling/sampling_logp_difference/max": 1.188643217086792, "sampling/sampling_logp_difference/mean": 0.022827615961432457, "step": 802 }, { "clip_ratio/high_max": 0.00021882837427256163, "clip_ratio/high_mean": 0.00021882837427256163, "clip_ratio/low_mean": 8.580878784414381e-05, "clip_ratio/low_min": 8.580878784414381e-05, "clip_ratio/region_mean": 0.00030463716211670544, "completions/clipped_ratio": 0.0, "completions/max_length": 16024.0, "completions/max_terminated_length": 16024.0, "completions/mean_length": 11012.75, "completions/mean_terminated_length": 11012.75, "completions/min_length": 4352.0, "completions/min_terminated_length": 4352.0, "entropy": 0.46151210367679596, "epoch": 0.046154730428784915, "frac_reward_zero_std": 0.0, "grad_norm": 0.005399103742092848, "learning_rate": 1e-05, "loss": 0.1607, "num_tokens": 39234697.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997912645339966, "sampling/importance_sampling_ratio/min": 0.0003445757902227342, "sampling/sampling_logp_difference/max": 7.973196506500244, "sampling/sampling_logp_difference/mean": 0.02030712179839611, "step": 803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3733.0, "completions/max_terminated_length": 3733.0, "completions/mean_length": 2693.5, "completions/mean_terminated_length": 2693.5, "completions/min_length": 1593.0, "completions/min_terminated_length": 1593.0, "entropy": 0.48430654034018517, "epoch": 0.046212208299804575, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39257365.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6534119844436646, "sampling/importance_sampling_ratio/mean": 0.9998710751533508, "sampling/importance_sampling_ratio/min": 0.666053056716919, "sampling/sampling_logp_difference/max": 0.5028409957885742, "sampling/sampling_logp_difference/mean": 0.013904084451496601, "step": 804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12951.0, "completions/max_terminated_length": 12951.0, "completions/mean_length": 10497.875, "completions/mean_terminated_length": 10497.875, "completions/min_length": 8734.0, "completions/min_terminated_length": 8734.0, "entropy": 0.6118557974696159, "epoch": 0.046269686170824234, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39342796.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000072717666626, "sampling/importance_sampling_ratio/min": 0.02966795302927494, "sampling/sampling_logp_difference/max": 3.5176877975463867, "sampling/sampling_logp_difference/mean": 0.01937045156955719, "step": 805 }, { "clip_ratio/high_max": 4.458720104594249e-05, "clip_ratio/high_mean": 4.458720104594249e-05, "clip_ratio/low_mean": 0.0002442765107844025, "clip_ratio/low_min": 0.0002442765107844025, "clip_ratio/region_mean": 0.000288863711830345, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15103.0, "completions/mean_length": 8428.75, "completions/mean_terminated_length": 7292.2861328125, "completions/min_length": 2741.0, "completions/min_terminated_length": 2741.0, "entropy": 0.47017400339245796, "epoch": 0.046327164041843893, "frac_reward_zero_std": 0.0, "grad_norm": 0.01168083120137453, "learning_rate": 1e-05, "loss": 0.4685, "num_tokens": 39411746.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997585415840149, "sampling/importance_sampling_ratio/min": 0.1679404079914093, "sampling/sampling_logp_difference/max": 1.7841460704803467, "sampling/sampling_logp_difference/mean": 0.02056879550218582, "step": 806 }, { "clip_ratio/high_max": 0.00017228539218194783, "clip_ratio/high_mean": 0.00017228539218194783, "clip_ratio/low_mean": 0.00014124294102657586, "clip_ratio/low_min": 0.00014124294102657586, "clip_ratio/region_mean": 0.0003135283332085237, "completions/clipped_ratio": 0.0, "completions/max_length": 4538.0, "completions/max_terminated_length": 4538.0, "completions/mean_length": 3048.625, "completions/mean_terminated_length": 3048.625, "completions/min_length": 2244.0, "completions/min_terminated_length": 2244.0, "entropy": 0.32937372848391533, "epoch": 0.046384641912863546, "frac_reward_zero_std": 0.0, "grad_norm": 0.008672213181853294, "learning_rate": 1e-05, "loss": -0.0459, "num_tokens": 39437015.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5268707275390625, "sampling/importance_sampling_ratio/mean": 0.9995371103286743, "sampling/importance_sampling_ratio/min": 0.6304581761360168, "sampling/sampling_logp_difference/max": 0.46130847930908203, "sampling/sampling_logp_difference/mean": 0.012702378444373608, "step": 807 }, { "clip_ratio/high_max": 0.0001735958139761351, "clip_ratio/high_mean": 0.0001735958139761351, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001735958139761351, "completions/clipped_ratio": 0.0, "completions/max_length": 6779.0, "completions/max_terminated_length": 6779.0, "completions/mean_length": 4766.75, "completions/mean_terminated_length": 4766.75, "completions/min_length": 3797.0, "completions/min_terminated_length": 3797.0, "entropy": 0.316675066947937, "epoch": 0.046442119783883205, "frac_reward_zero_std": 0.0, "grad_norm": 0.008523392491042614, "learning_rate": 1e-05, "loss": -0.0585, "num_tokens": 39476453.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6360336542129517, "sampling/importance_sampling_ratio/mean": 0.9999793767929077, "sampling/importance_sampling_ratio/min": 0.606621503829956, "sampling/sampling_logp_difference/max": 0.4998502731323242, "sampling/sampling_logp_difference/mean": 0.012706439942121506, "step": 808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7883.0, "completions/max_terminated_length": 7883.0, "completions/mean_length": 4571.0, "completions/mean_terminated_length": 4571.0, "completions/min_length": 2471.0, "completions/min_terminated_length": 2471.0, "entropy": 0.5718880109488964, "epoch": 0.046499597654902865, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39514021.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8031493425369263, "sampling/importance_sampling_ratio/mean": 1.0001895427703857, "sampling/importance_sampling_ratio/min": 0.3812735676765442, "sampling/sampling_logp_difference/max": 0.964238166809082, "sampling/sampling_logp_difference/mean": 0.01977667585015297, "step": 809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10511.0, "completions/max_terminated_length": 10511.0, "completions/mean_length": 7692.875, "completions/mean_terminated_length": 7692.875, "completions/min_length": 4590.0, "completions/min_terminated_length": 4590.0, "entropy": 0.7922063767910004, "epoch": 0.04655707552592252, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39576884.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997202157974243, "sampling/importance_sampling_ratio/min": 3.507722794893198e-05, "sampling/sampling_logp_difference/max": 10.25795841217041, "sampling/sampling_logp_difference/mean": 0.02425456792116165, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8177.0, "completions/max_terminated_length": 8177.0, "completions/mean_length": 5479.375, "completions/mean_terminated_length": 5479.375, "completions/min_length": 3871.0, "completions/min_terminated_length": 3871.0, "entropy": 0.3888476938009262, "epoch": 0.04661455339694218, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39621967.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9853969812393188, "sampling/importance_sampling_ratio/mean": 0.999758243560791, "sampling/importance_sampling_ratio/min": 0.6413594484329224, "sampling/sampling_logp_difference/max": 0.6858189105987549, "sampling/sampling_logp_difference/mean": 0.015474299900233746, "step": 811 }, { "clip_ratio/high_max": 2.919196595030371e-05, "clip_ratio/high_mean": 2.919196595030371e-05, "clip_ratio/low_mean": 0.0002884998011722928, "clip_ratio/low_min": 0.0002884998011722928, "clip_ratio/region_mean": 0.0003176917671225965, "completions/clipped_ratio": 0.0, "completions/max_length": 6662.0, "completions/max_terminated_length": 6662.0, "completions/mean_length": 4015.75, "completions/mean_terminated_length": 4015.75, "completions/min_length": 2439.0, "completions/min_terminated_length": 2439.0, "entropy": 0.5624052807688713, "epoch": 0.046672031267961836, "frac_reward_zero_std": 0.0, "grad_norm": 0.02600041590631008, "learning_rate": 1e-05, "loss": 0.1687, "num_tokens": 39655565.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.455366849899292, "sampling/importance_sampling_ratio/mean": 1.0001300573349, "sampling/importance_sampling_ratio/min": 0.5268458724021912, "sampling/sampling_logp_difference/max": 0.6408472061157227, "sampling/sampling_logp_difference/mean": 0.015504877083003521, "step": 812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0008639032275823411, "clip_ratio/low_min": 0.0008639032275823411, "clip_ratio/region_mean": 0.0008639032275823411, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 16234.0, "completions/mean_length": 14524.125, "completions/mean_terminated_length": 12664.25, "completions/min_length": 9105.0, "completions/min_terminated_length": 9105.0, "entropy": 0.5268431082367897, "epoch": 0.046729509138981495, "frac_reward_zero_std": 0.0, "grad_norm": 0.00783497653901577, "learning_rate": 1e-05, "loss": 0.1318, "num_tokens": 39774110.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999324679374695, "sampling/importance_sampling_ratio/min": 0.06411267071962357, "sampling/sampling_logp_difference/max": 2.7471132278442383, "sampling/sampling_logp_difference/mean": 0.02291969023644924, "step": 813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6868.0, "completions/max_terminated_length": 6868.0, "completions/mean_length": 5054.5, "completions/mean_terminated_length": 5054.5, "completions/min_length": 2968.0, "completions/min_terminated_length": 2968.0, "entropy": 0.4201580137014389, "epoch": 0.04678698701000115, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39816482.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6069447994232178, "sampling/importance_sampling_ratio/mean": 1.0003595352172852, "sampling/importance_sampling_ratio/min": 0.5436651706695557, "sampling/sampling_logp_difference/max": 0.6094217300415039, "sampling/sampling_logp_difference/mean": 0.016735922545194626, "step": 814 }, { "clip_ratio/high_max": 5.1415316193015315e-05, "clip_ratio/high_mean": 5.1415316193015315e-05, "clip_ratio/low_mean": 0.00033372340840287507, "clip_ratio/low_min": 0.00033372340840287507, "clip_ratio/region_mean": 0.0003851387245958904, "completions/clipped_ratio": 0.0, "completions/max_length": 5954.0, "completions/max_terminated_length": 5954.0, "completions/mean_length": 4577.0, "completions/mean_terminated_length": 4577.0, "completions/min_length": 3494.0, "completions/min_terminated_length": 3494.0, "entropy": 0.38223249465227127, "epoch": 0.04684446488102081, "frac_reward_zero_std": 0.0, "grad_norm": 0.012586724013090134, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 39854730.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.4735217094421387, "sampling/importance_sampling_ratio/mean": 1.0001871585845947, "sampling/importance_sampling_ratio/min": 0.6286534070968628, "sampling/sampling_logp_difference/max": 0.4641752243041992, "sampling/sampling_logp_difference/mean": 0.014774711802601814, "step": 815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3160.0, "completions/max_terminated_length": 3160.0, "completions/mean_length": 1987.125, "completions/mean_terminated_length": 1987.125, "completions/min_length": 789.0, "completions/min_terminated_length": 789.0, "entropy": 0.5257394835352898, "epoch": 0.04690194275204047, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39872299.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9991129636764526, "sampling/importance_sampling_ratio/min": 0.035076286643743515, "sampling/sampling_logp_difference/max": 3.3502299785614014, "sampling/sampling_logp_difference/mean": 0.02310028485953808, "step": 816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3111.0, "completions/max_terminated_length": 3111.0, "completions/mean_length": 2095.75, "completions/mean_terminated_length": 2095.75, "completions/min_length": 1284.0, "completions/min_terminated_length": 1284.0, "entropy": 0.2581703457981348, "epoch": 0.04695942062306012, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39890265.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4020533561706543, "sampling/importance_sampling_ratio/mean": 1.000020980834961, "sampling/importance_sampling_ratio/min": 0.6329248547554016, "sampling/sampling_logp_difference/max": 0.45740365982055664, "sampling/sampling_logp_difference/mean": 0.011570588685572147, "step": 817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7487.0, "completions/max_terminated_length": 7487.0, "completions/mean_length": 5824.875, "completions/mean_terminated_length": 5824.875, "completions/min_length": 3188.0, "completions/min_terminated_length": 3188.0, "entropy": 0.666115328669548, "epoch": 0.04701689849407978, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 39937504.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6528207063674927, "sampling/importance_sampling_ratio/mean": 1.0001704692840576, "sampling/importance_sampling_ratio/min": 0.6116548180580139, "sampling/sampling_logp_difference/max": 0.5024833679199219, "sampling/sampling_logp_difference/mean": 0.023771263659000397, "step": 818 }, { "clip_ratio/high_max": 0.0001977244046429405, "clip_ratio/high_mean": 0.0001977244046429405, "clip_ratio/low_mean": 0.00030129457445582375, "clip_ratio/low_min": 0.00030129457445582375, "clip_ratio/region_mean": 0.0004990189790987642, "completions/clipped_ratio": 0.0, "completions/max_length": 5463.0, "completions/max_terminated_length": 5463.0, "completions/mean_length": 4053.625, "completions/mean_terminated_length": 4053.625, "completions/min_length": 2778.0, "completions/min_terminated_length": 2778.0, "entropy": 0.4197586849331856, "epoch": 0.04707437636509944, "frac_reward_zero_std": 0.0, "grad_norm": 0.01711401157081127, "learning_rate": 1e-05, "loss": -0.0274, "num_tokens": 39973413.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.509365439414978, "sampling/importance_sampling_ratio/mean": 1.0000146627426147, "sampling/importance_sampling_ratio/min": 0.5575687885284424, "sampling/sampling_logp_difference/max": 0.5841693878173828, "sampling/sampling_logp_difference/mean": 0.016802938655018806, "step": 819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6550.0, "completions/max_terminated_length": 6550.0, "completions/mean_length": 3607.375, "completions/mean_terminated_length": 3607.375, "completions/min_length": 2357.0, "completions/min_terminated_length": 2357.0, "entropy": 0.3655742257833481, "epoch": 0.0471318542361191, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40003288.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4944053888320923, "sampling/importance_sampling_ratio/mean": 0.9998652935028076, "sampling/importance_sampling_ratio/min": 0.6139565110206604, "sampling/sampling_logp_difference/max": 0.48783111572265625, "sampling/sampling_logp_difference/mean": 0.015501335263252258, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13170.0, "completions/max_terminated_length": 13170.0, "completions/mean_length": 11619.25, "completions/mean_terminated_length": 11619.25, "completions/min_length": 9660.0, "completions/min_terminated_length": 9660.0, "entropy": 1.0866622403264046, "epoch": 0.04718933210713875, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40098362.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999122023582458, "sampling/importance_sampling_ratio/min": 0.08082595467567444, "sampling/sampling_logp_difference/max": 2.5154571533203125, "sampling/sampling_logp_difference/mean": 0.029451778158545494, "step": 821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10625.0, "completions/max_terminated_length": 10625.0, "completions/mean_length": 7619.75, "completions/mean_terminated_length": 7619.75, "completions/min_length": 4890.0, "completions/min_terminated_length": 4890.0, "entropy": 0.7763458155095577, "epoch": 0.04724680997815841, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40160192.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.8428149223327637, "sampling/importance_sampling_ratio/mean": 0.9997913837432861, "sampling/importance_sampling_ratio/min": 0.2217591106891632, "sampling/sampling_logp_difference/max": 1.5061635971069336, "sampling/sampling_logp_difference/mean": 0.02494068443775177, "step": 822 }, { "clip_ratio/high_max": 7.39419356250437e-05, "clip_ratio/high_mean": 7.39419356250437e-05, "clip_ratio/low_mean": 0.0001259022974409163, "clip_ratio/low_min": 0.0001259022974409163, "clip_ratio/region_mean": 0.00019984423306596, "completions/clipped_ratio": 0.0, "completions/max_length": 10638.0, "completions/max_terminated_length": 10638.0, "completions/mean_length": 6919.875, "completions/mean_terminated_length": 6919.875, "completions/min_length": 4934.0, "completions/min_terminated_length": 4934.0, "entropy": 0.5519363228231668, "epoch": 0.04730428784917807, "frac_reward_zero_std": 0.0, "grad_norm": 0.006461009848862886, "learning_rate": 1e-05, "loss": -0.0491, "num_tokens": 40216391.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.678276538848877, "sampling/importance_sampling_ratio/mean": 1.0000466108322144, "sampling/importance_sampling_ratio/min": 0.5927403569221497, "sampling/sampling_logp_difference/max": 0.5229988098144531, "sampling/sampling_logp_difference/mean": 0.019587228074669838, "step": 823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4283.0, "completions/max_terminated_length": 4283.0, "completions/mean_length": 2406.375, "completions/mean_terminated_length": 2406.375, "completions/min_length": 1445.0, "completions/min_terminated_length": 1445.0, "entropy": 0.7290726378560066, "epoch": 0.04736176572019772, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40237434.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3492921590805054, "sampling/importance_sampling_ratio/mean": 1.0002464056015015, "sampling/importance_sampling_ratio/min": 0.6790226101875305, "sampling/sampling_logp_difference/max": 0.38710081577301025, "sampling/sampling_logp_difference/mean": 0.019938737154006958, "step": 824 }, { "clip_ratio/high_max": 6.880117143737152e-05, "clip_ratio/high_mean": 6.880117143737152e-05, "clip_ratio/low_mean": 0.0002770135397440754, "clip_ratio/low_min": 0.0002770135397440754, "clip_ratio/region_mean": 0.0003458147111814469, "completions/clipped_ratio": 0.0, "completions/max_length": 13795.0, "completions/max_terminated_length": 13795.0, "completions/mean_length": 9337.625, "completions/mean_terminated_length": 9337.625, "completions/min_length": 4849.0, "completions/min_terminated_length": 4849.0, "entropy": 0.6878882050514221, "epoch": 0.04741924359121738, "frac_reward_zero_std": 0.0, "grad_norm": 0.015541315078735352, "learning_rate": 1e-05, "loss": 0.1089, "num_tokens": 40313159.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998970627784729, "sampling/importance_sampling_ratio/min": 0.3748171329498291, "sampling/sampling_logp_difference/max": 1.3108296394348145, "sampling/sampling_logp_difference/mean": 0.02515934221446514, "step": 825 }, { "clip_ratio/high_max": 0.00012176586642453913, "clip_ratio/high_mean": 0.00012176586642453913, "clip_ratio/low_mean": 0.0001878770490293391, "clip_ratio/low_min": 0.0001878770490293391, "clip_ratio/region_mean": 0.00030964291545387823, "completions/clipped_ratio": 0.0, "completions/max_length": 10120.0, "completions/max_terminated_length": 10120.0, "completions/mean_length": 4946.625, "completions/mean_terminated_length": 4946.625, "completions/min_length": 3033.0, "completions/min_terminated_length": 3033.0, "entropy": 0.5604685321450233, "epoch": 0.04747672146223704, "frac_reward_zero_std": 0.0, "grad_norm": 0.014558911323547363, "learning_rate": 1e-05, "loss": 0.2748, "num_tokens": 40354540.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9606750011444092, "sampling/importance_sampling_ratio/mean": 1.000071406364441, "sampling/importance_sampling_ratio/min": 0.511546790599823, "sampling/sampling_logp_difference/max": 0.6732888221740723, "sampling/sampling_logp_difference/mean": 0.021396879106760025, "step": 826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8860.0, "completions/max_terminated_length": 8860.0, "completions/mean_length": 4189.75, "completions/mean_terminated_length": 4189.75, "completions/min_length": 1084.0, "completions/min_terminated_length": 1084.0, "entropy": 0.5112106576561928, "epoch": 0.0475341993332567, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40389258.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999927282333374, "sampling/importance_sampling_ratio/min": 0.5225496888160706, "sampling/sampling_logp_difference/max": 0.7384638786315918, "sampling/sampling_logp_difference/mean": 0.020961347967386246, "step": 827 }, { "clip_ratio/high_max": 1.4709343304275535e-05, "clip_ratio/high_mean": 1.4709343304275535e-05, "clip_ratio/low_mean": 0.000620686354523059, "clip_ratio/low_min": 0.000620686354523059, "clip_ratio/region_mean": 0.0006353956978273345, "completions/clipped_ratio": 0.0, "completions/max_length": 10271.0, "completions/max_terminated_length": 10271.0, "completions/mean_length": 6616.375, "completions/mean_terminated_length": 6616.375, "completions/min_length": 4093.0, "completions/min_terminated_length": 4093.0, "entropy": 0.8901090696454048, "epoch": 0.04759167720427635, "frac_reward_zero_std": 0.0, "grad_norm": 0.012111762538552284, "learning_rate": 1e-05, "loss": -0.2257, "num_tokens": 40443541.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6525077819824219, "sampling/importance_sampling_ratio/mean": 0.9998406767845154, "sampling/importance_sampling_ratio/min": 0.47819602489471436, "sampling/sampling_logp_difference/max": 0.7377345561981201, "sampling/sampling_logp_difference/mean": 0.02911006659269333, "step": 828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4708.0, "completions/max_terminated_length": 4708.0, "completions/mean_length": 3663.125, "completions/mean_terminated_length": 3663.125, "completions/min_length": 2586.0, "completions/min_terminated_length": 2586.0, "entropy": 0.1860565161332488, "epoch": 0.04764915507529601, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40474334.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3648717403411865, "sampling/importance_sampling_ratio/mean": 0.9995778203010559, "sampling/importance_sampling_ratio/min": 0.4727814197540283, "sampling/sampling_logp_difference/max": 0.749122142791748, "sampling/sampling_logp_difference/mean": 0.0076871900819242, "step": 829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00026209683528577443, "clip_ratio/low_min": 0.00026209683528577443, "clip_ratio/region_mean": 0.00026209683528577443, "completions/clipped_ratio": 0.0, "completions/max_length": 7806.0, "completions/max_terminated_length": 7806.0, "completions/mean_length": 4064.875, "completions/mean_terminated_length": 4064.875, "completions/min_length": 1457.0, "completions/min_terminated_length": 1457.0, "entropy": 0.7798083312809467, "epoch": 0.04770663294631567, "frac_reward_zero_std": 0.0, "grad_norm": 0.0238056443631649, "learning_rate": 1e-05, "loss": -0.2503, "num_tokens": 40508485.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.467602014541626, "sampling/importance_sampling_ratio/mean": 0.9999377727508545, "sampling/importance_sampling_ratio/min": 0.6361068487167358, "sampling/sampling_logp_difference/max": 0.4523887634277344, "sampling/sampling_logp_difference/mean": 0.020134108141064644, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10481.0, "completions/max_terminated_length": 10481.0, "completions/mean_length": 6071.375, "completions/mean_terminated_length": 6071.375, "completions/min_length": 2701.0, "completions/min_terminated_length": 2701.0, "entropy": 1.1997693330049515, "epoch": 0.04776411081733532, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40558232.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6214420795440674, "sampling/importance_sampling_ratio/mean": 1.0001976490020752, "sampling/importance_sampling_ratio/min": 0.29392698407173157, "sampling/sampling_logp_difference/max": 1.224423885345459, "sampling/sampling_logp_difference/mean": 0.028441641479730606, "step": 831 }, { "clip_ratio/high_max": 0.00013510547069017775, "clip_ratio/high_mean": 0.00013510547069017775, "clip_ratio/low_mean": 4.892368087894283e-05, "clip_ratio/low_min": 4.892368087894283e-05, "clip_ratio/region_mean": 0.00018402915156912059, "completions/clipped_ratio": 0.0, "completions/max_length": 11216.0, "completions/max_terminated_length": 11216.0, "completions/mean_length": 3188.125, "completions/mean_terminated_length": 3188.125, "completions/min_length": 792.0, "completions/min_terminated_length": 792.0, "entropy": 0.5740125738084316, "epoch": 0.04782158868835498, "frac_reward_zero_std": 0.0, "grad_norm": 0.005405042320489883, "learning_rate": 1e-05, "loss": 0.2136, "num_tokens": 40584585.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5483968257904053, "sampling/importance_sampling_ratio/mean": 0.9997877478599548, "sampling/importance_sampling_ratio/min": 0.5343135595321655, "sampling/sampling_logp_difference/max": 0.626772403717041, "sampling/sampling_logp_difference/mean": 0.025576895102858543, "step": 832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3707.0, "completions/max_terminated_length": 3707.0, "completions/mean_length": 2523.5, "completions/mean_terminated_length": 2523.5, "completions/min_length": 1193.0, "completions/min_terminated_length": 1193.0, "entropy": 0.4515904374420643, "epoch": 0.04787906655937464, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40605621.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4757473468780518, "sampling/importance_sampling_ratio/mean": 1.0001509189605713, "sampling/importance_sampling_ratio/min": 0.6167693138122559, "sampling/sampling_logp_difference/max": 0.4832601547241211, "sampling/sampling_logp_difference/mean": 0.017882777377963066, "step": 833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7732.0, "completions/max_terminated_length": 7732.0, "completions/mean_length": 5135.375, "completions/mean_terminated_length": 5135.375, "completions/min_length": 1713.0, "completions/min_terminated_length": 1713.0, "entropy": 1.5831322520971298, "epoch": 0.0479365444303943, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40648112.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.7494943141937256, "sampling/importance_sampling_ratio/mean": 1.000225305557251, "sampling/importance_sampling_ratio/min": 0.48198986053466797, "sampling/sampling_logp_difference/max": 0.7298322319984436, "sampling/sampling_logp_difference/mean": 0.0324496291577816, "step": 834 }, { "clip_ratio/high_max": 0.0001332303618255537, "clip_ratio/high_mean": 0.0001332303618255537, "clip_ratio/low_mean": 0.0005785578468930908, "clip_ratio/low_min": 0.0005785578468930908, "clip_ratio/region_mean": 0.0007117882087186445, "completions/clipped_ratio": 0.0, "completions/max_length": 15771.0, "completions/max_terminated_length": 15771.0, "completions/mean_length": 11042.0, "completions/mean_terminated_length": 11042.0, "completions/min_length": 4754.0, "completions/min_terminated_length": 4754.0, "entropy": 0.8602729216217995, "epoch": 0.047994022301413954, "frac_reward_zero_std": 0.0, "grad_norm": 0.02577975206077099, "learning_rate": 1e-05, "loss": -0.0774, "num_tokens": 40737280.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.8252519369125366, "sampling/importance_sampling_ratio/mean": 0.9999959468841553, "sampling/importance_sampling_ratio/min": 0.4800814092159271, "sampling/sampling_logp_difference/max": 0.7337995767593384, "sampling/sampling_logp_difference/mean": 0.029321759939193726, "step": 835 }, { "clip_ratio/high_max": 0.0001423236262780847, "clip_ratio/high_mean": 0.0001423236262780847, "clip_ratio/low_mean": 0.00043645731784636155, "clip_ratio/low_min": 0.00043645731784636155, "clip_ratio/region_mean": 0.0005787809441244463, "completions/clipped_ratio": 0.0, "completions/max_length": 13784.0, "completions/max_terminated_length": 13784.0, "completions/mean_length": 7225.125, "completions/mean_terminated_length": 7225.125, "completions/min_length": 2818.0, "completions/min_terminated_length": 2818.0, "entropy": 0.5370804183185101, "epoch": 0.04805150017243361, "frac_reward_zero_std": 0.0, "grad_norm": 0.05066005885601044, "learning_rate": 1e-05, "loss": 0.2722, "num_tokens": 40796425.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000069499015808, "sampling/importance_sampling_ratio/min": 0.12644974887371063, "sampling/sampling_logp_difference/max": 2.0679101943969727, "sampling/sampling_logp_difference/mean": 0.023509005084633827, "step": 836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1232.0, "completions/max_terminated_length": 1232.0, "completions/mean_length": 692.25, "completions/mean_terminated_length": 692.25, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "entropy": 0.26157683692872524, "epoch": 0.04810897804345327, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40803171.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3890184164047241, "sampling/importance_sampling_ratio/mean": 1.0002567768096924, "sampling/importance_sampling_ratio/min": 0.7186996936798096, "sampling/sampling_logp_difference/max": 0.33031177520751953, "sampling/sampling_logp_difference/mean": 0.011813824065029621, "step": 837 }, { "clip_ratio/high_max": 8.923252607928589e-05, "clip_ratio/high_mean": 8.923252607928589e-05, "clip_ratio/low_mean": 8.392333984375e-05, "clip_ratio/low_min": 8.392333984375e-05, "clip_ratio/region_mean": 0.0001731558659230359, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 6700.0, "completions/mean_length": 5102.125, "completions/mean_terminated_length": 3490.4287109375, "completions/min_length": 2028.0, "completions/min_terminated_length": 2028.0, "entropy": 0.31383365765213966, "epoch": 0.048166455914472925, "frac_reward_zero_std": 0.0, "grad_norm": 0.005847550462931395, "learning_rate": 1e-05, "loss": 0.7812, "num_tokens": 40844996.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002087354660034, "sampling/importance_sampling_ratio/min": 0.5403235554695129, "sampling/sampling_logp_difference/max": 0.8489856719970703, "sampling/sampling_logp_difference/mean": 0.01445184275507927, "step": 838 }, { "clip_ratio/high_max": 0.00011266245746810455, "clip_ratio/high_mean": 0.00011266245746810455, "clip_ratio/low_mean": 0.0002288818359375, "clip_ratio/low_min": 0.0002288818359375, "clip_ratio/region_mean": 0.00034154429340560455, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 11315.0, "completions/mean_length": 12269.125, "completions/mean_terminated_length": 9800.2001953125, "completions/min_length": 7870.0, "completions/min_terminated_length": 7870.0, "entropy": 0.43832332640886307, "epoch": 0.048223933785492584, "frac_reward_zero_std": 0.0, "grad_norm": 0.025696445256471634, "learning_rate": 1e-05, "loss": 0.243, "num_tokens": 40944837.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998462796211243, "sampling/importance_sampling_ratio/min": 0.13480502367019653, "sampling/sampling_logp_difference/max": 2.0039258003234863, "sampling/sampling_logp_difference/mean": 0.017205210402607918, "step": 839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2316.0, "completions/max_terminated_length": 2316.0, "completions/mean_length": 1404.125, "completions/mean_terminated_length": 1404.125, "completions/min_length": 1044.0, "completions/min_terminated_length": 1044.0, "entropy": 0.29725291579961777, "epoch": 0.048281411656512244, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 40957158.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5018500089645386, "sampling/importance_sampling_ratio/mean": 1.0000519752502441, "sampling/importance_sampling_ratio/min": 0.5234063863754272, "sampling/sampling_logp_difference/max": 0.6473970413208008, "sampling/sampling_logp_difference/mean": 0.012851976789534092, "step": 840 }, { "clip_ratio/high_max": 0.00014753548020962626, "clip_ratio/high_mean": 0.00014753548020962626, "clip_ratio/low_mean": 0.0001099706714740023, "clip_ratio/low_min": 0.0001099706714740023, "clip_ratio/region_mean": 0.00025750615168362856, "completions/clipped_ratio": 0.0, "completions/max_length": 11561.0, "completions/max_terminated_length": 11561.0, "completions/mean_length": 6368.5, "completions/mean_terminated_length": 6368.5, "completions/min_length": 983.0, "completions/min_terminated_length": 983.0, "entropy": 0.5748859979212284, "epoch": 0.0483388895275319, "frac_reward_zero_std": 0.0, "grad_norm": 0.011653066612780094, "learning_rate": 1e-05, "loss": 0.0251, "num_tokens": 41008730.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.835068702697754, "sampling/importance_sampling_ratio/mean": 0.9999101758003235, "sampling/importance_sampling_ratio/min": 0.40269994735717773, "sampling/sampling_logp_difference/max": 0.9095635414123535, "sampling/sampling_logp_difference/mean": 0.02100183069705963, "step": 841 }, { "clip_ratio/high_max": 2.4808970920275897e-05, "clip_ratio/high_mean": 2.4808970920275897e-05, "clip_ratio/low_mean": 0.0005263937382551376, "clip_ratio/low_min": 0.0005263937382551376, "clip_ratio/region_mean": 0.0005512027091754135, "completions/clipped_ratio": 0.5, "completions/max_length": 16384.0, "completions/max_terminated_length": 15336.0, "completions/mean_length": 14456.0, "completions/mean_terminated_length": 12528.0, "completions/min_length": 10077.0, "completions/min_terminated_length": 10077.0, "entropy": 0.6874087527394295, "epoch": 0.048396367398551556, "frac_reward_zero_std": 0.0, "grad_norm": 0.007533515803515911, "learning_rate": 1e-05, "loss": 0.1071, "num_tokens": 41125658.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001729726791382, "sampling/importance_sampling_ratio/min": 0.27473217248916626, "sampling/sampling_logp_difference/max": 1.2919585704803467, "sampling/sampling_logp_difference/mean": 0.02346852608025074, "step": 842 }, { "clip_ratio/high_max": 0.00017548779032949824, "clip_ratio/high_mean": 0.00017548779032949824, "clip_ratio/low_mean": 0.00013662135461345315, "clip_ratio/low_min": 0.00013662135461345315, "clip_ratio/region_mean": 0.0003121091449429514, "completions/clipped_ratio": 0.0, "completions/max_length": 15289.0, "completions/max_terminated_length": 15289.0, "completions/mean_length": 9136.0, "completions/mean_terminated_length": 9136.0, "completions/min_length": 4885.0, "completions/min_terminated_length": 4885.0, "entropy": 0.7600704468786716, "epoch": 0.048453845269571215, "frac_reward_zero_std": 0.0, "grad_norm": 0.07290228456258774, "learning_rate": 1e-05, "loss": 0.2129, "num_tokens": 41199882.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000003576278687, "sampling/importance_sampling_ratio/min": 0.032247960567474365, "sampling/sampling_logp_difference/max": 3.434300422668457, "sampling/sampling_logp_difference/mean": 0.023679038509726524, "step": 843 }, { "clip_ratio/high_max": 1.346257431578124e-05, "clip_ratio/high_mean": 1.346257431578124e-05, "clip_ratio/low_mean": 0.0009665892139310017, "clip_ratio/low_min": 0.0009665892139310017, "clip_ratio/region_mean": 0.000980051788246783, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15901.0, "completions/mean_length": 11057.75, "completions/mean_terminated_length": 10296.857421875, "completions/min_length": 7544.0, "completions/min_terminated_length": 7544.0, "entropy": 0.7231486290693283, "epoch": 0.048511323140590874, "frac_reward_zero_std": 0.0, "grad_norm": 0.00933499913662672, "learning_rate": 1e-05, "loss": 0.0566, "num_tokens": 41289712.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003873109817505, "sampling/importance_sampling_ratio/min": 0.23354466259479523, "sampling/sampling_logp_difference/max": 1.4543819427490234, "sampling/sampling_logp_difference/mean": 0.027408158406615257, "step": 844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4645.0, "completions/max_terminated_length": 4645.0, "completions/mean_length": 3094.625, "completions/mean_terminated_length": 3094.625, "completions/min_length": 1248.0, "completions/min_terminated_length": 1248.0, "entropy": 0.6550607308745384, "epoch": 0.04856880101161053, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41315421.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.9551886320114136, "sampling/importance_sampling_ratio/mean": 0.9996981024742126, "sampling/importance_sampling_ratio/min": 0.5850485563278198, "sampling/sampling_logp_difference/max": 0.6704866886138916, "sampling/sampling_logp_difference/mean": 0.021493110805749893, "step": 845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12824.0, "completions/max_terminated_length": 12824.0, "completions/mean_length": 9190.625, "completions/mean_terminated_length": 9190.625, "completions/min_length": 5911.0, "completions/min_terminated_length": 5911.0, "entropy": 1.0740453079342842, "epoch": 0.048626278882630186, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41390218.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996057748794556, "sampling/importance_sampling_ratio/min": 0.18351244926452637, "sampling/sampling_logp_difference/max": 1.6954727172851562, "sampling/sampling_logp_difference/mean": 0.030437298119068146, "step": 846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3772.0, "completions/max_terminated_length": 3772.0, "completions/mean_length": 2648.125, "completions/mean_terminated_length": 2648.125, "completions/min_length": 1958.0, "completions/min_terminated_length": 1958.0, "entropy": 0.5212790332734585, "epoch": 0.048683756753649846, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41412907.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2867157459259033, "sampling/importance_sampling_ratio/mean": 0.9998852610588074, "sampling/importance_sampling_ratio/min": 0.6265954375267029, "sampling/sampling_logp_difference/max": 0.467454195022583, "sampling/sampling_logp_difference/mean": 0.014536884613335133, "step": 847 }, { "clip_ratio/high_max": 9.250123548554257e-05, "clip_ratio/high_mean": 9.250123548554257e-05, "clip_ratio/low_mean": 0.0004976047166564967, "clip_ratio/low_min": 0.0004976047166564967, "clip_ratio/region_mean": 0.0005901059521420393, "completions/clipped_ratio": 0.0, "completions/max_length": 13467.0, "completions/max_terminated_length": 13467.0, "completions/mean_length": 10680.5, "completions/mean_terminated_length": 10680.5, "completions/min_length": 8108.0, "completions/min_terminated_length": 8108.0, "entropy": 1.2461113184690475, "epoch": 0.048741234624669505, "frac_reward_zero_std": 0.0, "grad_norm": 0.010761252604424953, "learning_rate": 1e-05, "loss": 0.085, "num_tokens": 41499559.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997156262397766, "sampling/importance_sampling_ratio/min": 0.3138498365879059, "sampling/sampling_logp_difference/max": 1.1588406562805176, "sampling/sampling_logp_difference/mean": 0.03244992345571518, "step": 848 }, { "clip_ratio/high_max": 8.398293994105188e-05, "clip_ratio/high_mean": 8.398293994105188e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.398293994105188e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15537.0, "completions/mean_length": 8380.5, "completions/mean_terminated_length": 7237.14306640625, "completions/min_length": 3873.0, "completions/min_terminated_length": 3873.0, "entropy": 0.2859575841575861, "epoch": 0.04879871249568916, "frac_reward_zero_std": 0.0, "grad_norm": 0.008594042621552944, "learning_rate": 1e-05, "loss": -0.2744, "num_tokens": 41567843.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.9080530405044556, "sampling/importance_sampling_ratio/mean": 1.0000454187393188, "sampling/importance_sampling_ratio/min": 0.2176048308610916, "sampling/sampling_logp_difference/max": 1.5250746011734009, "sampling/sampling_logp_difference/mean": 0.013515960425138474, "step": 849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3119.0, "completions/max_terminated_length": 3119.0, "completions/mean_length": 2131.25, "completions/mean_terminated_length": 2131.25, "completions/min_length": 1290.0, "completions/min_terminated_length": 1290.0, "entropy": 0.2609759531915188, "epoch": 0.04885619036670882, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41585909.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2746500968933105, "sampling/importance_sampling_ratio/mean": 0.9998453259468079, "sampling/importance_sampling_ratio/min": 0.6176222562789917, "sampling/sampling_logp_difference/max": 0.48187828063964844, "sampling/sampling_logp_difference/mean": 0.011076872237026691, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 14980.0, "completions/mean_length": 11307.75, "completions/mean_terminated_length": 10582.572265625, "completions/min_length": 6041.0, "completions/min_terminated_length": 6041.0, "entropy": 0.9010175690054893, "epoch": 0.048913668237728476, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41677259.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001720190048218, "sampling/importance_sampling_ratio/min": 0.27008941769599915, "sampling/sampling_logp_difference/max": 1.309002161026001, "sampling/sampling_logp_difference/mean": 0.029708733782172203, "step": 851 }, { "clip_ratio/high_max": 0.00028325768289505504, "clip_ratio/high_mean": 0.00028325768289505504, "clip_ratio/low_mean": 0.0001486860874138074, "clip_ratio/low_min": 0.0001486860874138074, "clip_ratio/region_mean": 0.00043194377030886244, "completions/clipped_ratio": 0.0, "completions/max_length": 8348.0, "completions/max_terminated_length": 8348.0, "completions/mean_length": 4591.0, "completions/mean_terminated_length": 4591.0, "completions/min_length": 1292.0, "completions/min_terminated_length": 1292.0, "entropy": 0.5933905467391014, "epoch": 0.04897114610874813, "frac_reward_zero_std": 0.0, "grad_norm": 0.017782993614673615, "learning_rate": 1e-05, "loss": -0.0247, "num_tokens": 41715163.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.5106254816055298, "sampling/importance_sampling_ratio/mean": 1.0001640319824219, "sampling/importance_sampling_ratio/min": 0.6211827397346497, "sampling/sampling_logp_difference/max": 0.47613000869750977, "sampling/sampling_logp_difference/mean": 0.02104460448026657, "step": 852 }, { "clip_ratio/high_max": 7.08456300344551e-05, "clip_ratio/high_mean": 7.08456300344551e-05, "clip_ratio/low_mean": 0.0006373442593030632, "clip_ratio/low_min": 0.0006373442593030632, "clip_ratio/region_mean": 0.0007081898893375183, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 14195.0, "completions/mean_length": 12609.75, "completions/mean_terminated_length": 10345.2001953125, "completions/min_length": 7477.0, "completions/min_terminated_length": 7477.0, "entropy": 0.47567982599139214, "epoch": 0.04902862397976779, "frac_reward_zero_std": 0.0, "grad_norm": 0.014448332600295544, "learning_rate": 1e-05, "loss": 0.2058, "num_tokens": 41817425.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000337362289429, "sampling/importance_sampling_ratio/min": 0.19981612265110016, "sampling/sampling_logp_difference/max": 1.6103577613830566, "sampling/sampling_logp_difference/mean": 0.019261781126260757, "step": 853 }, { "clip_ratio/high_max": 1.3429307728074491e-05, "clip_ratio/high_mean": 1.3429307728074491e-05, "clip_ratio/low_mean": 0.000737236681743525, "clip_ratio/low_min": 0.000737236681743525, "clip_ratio/region_mean": 0.0007506659894715995, "completions/clipped_ratio": 0.0, "completions/max_length": 11029.0, "completions/max_terminated_length": 11029.0, "completions/mean_length": 7009.625, "completions/mean_terminated_length": 7009.625, "completions/min_length": 3964.0, "completions/min_terminated_length": 3964.0, "entropy": 0.5109223760664463, "epoch": 0.04908610185078745, "frac_reward_zero_std": 0.0, "grad_norm": 0.014890863560140133, "learning_rate": 1e-05, "loss": -0.0205, "num_tokens": 41874470.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.6614843606948853, "sampling/importance_sampling_ratio/mean": 0.9998973608016968, "sampling/importance_sampling_ratio/min": 0.28651162981987, "sampling/sampling_logp_difference/max": 1.2499761581420898, "sampling/sampling_logp_difference/mean": 0.02066435106098652, "step": 854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9242.0, "completions/max_terminated_length": 9242.0, "completions/mean_length": 5079.375, "completions/mean_terminated_length": 5079.375, "completions/min_length": 2845.0, "completions/min_terminated_length": 2845.0, "entropy": 0.8141607120633125, "epoch": 0.04914357972180711, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 41916889.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.4799855947494507, "sampling/sampling_logp_difference/max": 0.7513785362243652, "sampling/sampling_logp_difference/mean": 0.03267727419734001, "step": 855 }, { "clip_ratio/high_max": 0.00030041700665606186, "clip_ratio/high_mean": 0.00030041700665606186, "clip_ratio/low_mean": 0.00013357079296838492, "clip_ratio/low_min": 0.00013357079296838492, "clip_ratio/region_mean": 0.0004339877996244468, "completions/clipped_ratio": 0.0, "completions/max_length": 10089.0, "completions/max_terminated_length": 10089.0, "completions/mean_length": 5568.375, "completions/mean_terminated_length": 5568.375, "completions/min_length": 3980.0, "completions/min_terminated_length": 3980.0, "entropy": 0.728107213973999, "epoch": 0.04920105759282676, "frac_reward_zero_std": 0.0, "grad_norm": 0.009944768622517586, "learning_rate": 1e-05, "loss": 0.0033, "num_tokens": 41962572.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6252409219741821, "sampling/importance_sampling_ratio/mean": 1.000011920928955, "sampling/importance_sampling_ratio/min": 0.5747446417808533, "sampling/sampling_logp_difference/max": 0.5538294315338135, "sampling/sampling_logp_difference/mean": 0.026101958006620407, "step": 856 }, { "clip_ratio/high_max": 2.8901733458042145e-05, "clip_ratio/high_mean": 2.8901733458042145e-05, "clip_ratio/low_mean": 0.00021137000294402242, "clip_ratio/low_min": 0.00021137000294402242, "clip_ratio/region_mean": 0.00024027173640206456, "completions/clipped_ratio": 0.0, "completions/max_length": 4325.0, "completions/max_terminated_length": 4325.0, "completions/mean_length": 2578.625, "completions/mean_terminated_length": 2578.625, "completions/min_length": 1539.0, "completions/min_terminated_length": 1539.0, "entropy": 0.6042205579578876, "epoch": 0.04925853546384642, "frac_reward_zero_std": 0.0, "grad_norm": 0.03290915861725807, "learning_rate": 1e-05, "loss": -0.049, "num_tokens": 41984681.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.3172014951705933, "sampling/importance_sampling_ratio/mean": 0.9999310374259949, "sampling/importance_sampling_ratio/min": 0.5406526327133179, "sampling/sampling_logp_difference/max": 0.6149783134460449, "sampling/sampling_logp_difference/mean": 0.017368286848068237, "step": 857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8109.0, "completions/max_terminated_length": 8109.0, "completions/mean_length": 4775.125, "completions/mean_terminated_length": 4775.125, "completions/min_length": 2672.0, "completions/min_terminated_length": 2672.0, "entropy": 0.4232181943953037, "epoch": 0.04931601333486608, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42023650.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000287413597107, "sampling/importance_sampling_ratio/min": 0.6154567003250122, "sampling/sampling_logp_difference/max": 0.7010254859924316, "sampling/sampling_logp_difference/mean": 0.016251688823103905, "step": 858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11642.0, "completions/max_terminated_length": 11642.0, "completions/mean_length": 5772.5, "completions/mean_terminated_length": 5772.5, "completions/min_length": 1658.0, "completions/min_terminated_length": 1658.0, "entropy": 0.8299946524202824, "epoch": 0.04937349120588573, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42070782.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6497923135757446, "sampling/importance_sampling_ratio/mean": 0.999562680721283, "sampling/importance_sampling_ratio/min": 0.5525617003440857, "sampling/sampling_logp_difference/max": 0.5931901931762695, "sampling/sampling_logp_difference/mean": 0.027919327840209007, "step": 859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3451.0, "completions/max_terminated_length": 3451.0, "completions/mean_length": 2307.375, "completions/mean_terminated_length": 2307.375, "completions/min_length": 1564.0, "completions/min_terminated_length": 1564.0, "entropy": 0.22496127896010876, "epoch": 0.04943096907690539, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42090145.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3808553218841553, "sampling/importance_sampling_ratio/mean": 1.0001384019851685, "sampling/importance_sampling_ratio/min": 0.6539703011512756, "sampling/sampling_logp_difference/max": 0.42469334602355957, "sampling/sampling_logp_difference/mean": 0.009079709649085999, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 6959.0, "completions/max_terminated_length": 6959.0, "completions/mean_length": 4391.125, "completions/mean_terminated_length": 4391.125, "completions/min_length": 2587.0, "completions/min_terminated_length": 2587.0, "entropy": 0.24849341437220573, "epoch": 0.04948844694792505, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42126378.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999930739402771, "sampling/importance_sampling_ratio/min": 0.6111469864845276, "sampling/sampling_logp_difference/max": 0.9750795364379883, "sampling/sampling_logp_difference/mean": 0.010465294122695923, "step": 861 }, { "clip_ratio/high_max": 8.182639430742711e-05, "clip_ratio/high_mean": 8.182639430742711e-05, "clip_ratio/low_mean": 0.0005744469017372467, "clip_ratio/low_min": 0.0005744469017372467, "clip_ratio/region_mean": 0.0006562732960446738, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 14819.0, "completions/mean_length": 13806.75, "completions/mean_terminated_length": 12260.400390625, "completions/min_length": 8295.0, "completions/min_terminated_length": 8295.0, "entropy": 0.5159324482083321, "epoch": 0.04954592481894471, "frac_reward_zero_std": 0.0, "grad_norm": 0.01395700965076685, "learning_rate": 1e-05, "loss": 0.1395, "num_tokens": 42238248.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997313022613525, "sampling/importance_sampling_ratio/min": 0.0719979926943779, "sampling/sampling_logp_difference/max": 2.631117105484009, "sampling/sampling_logp_difference/mean": 0.020907243713736534, "step": 862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5283.0, "completions/max_terminated_length": 5283.0, "completions/mean_length": 3310.5, "completions/mean_terminated_length": 3310.5, "completions/min_length": 2184.0, "completions/min_terminated_length": 2184.0, "entropy": 0.4327528513967991, "epoch": 0.04960340268996436, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42265644.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4058609008789062, "sampling/importance_sampling_ratio/mean": 0.9999856948852539, "sampling/importance_sampling_ratio/min": 0.6020801067352295, "sampling/sampling_logp_difference/max": 0.5073647499084473, "sampling/sampling_logp_difference/mean": 0.01674344204366207, "step": 863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8570.0, "completions/max_terminated_length": 8570.0, "completions/mean_length": 3131.25, "completions/mean_terminated_length": 3131.25, "completions/min_length": 1276.0, "completions/min_terminated_length": 1276.0, "entropy": 0.3645509388297796, "epoch": 0.04966088056098402, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42291438.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000683069229126, "sampling/importance_sampling_ratio/min": 0.5521107912063599, "sampling/sampling_logp_difference/max": 0.8029875755310059, "sampling/sampling_logp_difference/mean": 0.014276611618697643, "step": 864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11691.0, "completions/max_terminated_length": 11691.0, "completions/mean_length": 8724.75, "completions/mean_terminated_length": 8724.75, "completions/min_length": 4671.0, "completions/min_terminated_length": 4671.0, "entropy": 0.5440570339560509, "epoch": 0.04971835843200368, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42362684.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003865957260132, "sampling/importance_sampling_ratio/min": 0.4315151870250702, "sampling/sampling_logp_difference/max": 1.0430023670196533, "sampling/sampling_logp_difference/mean": 0.021186521276831627, "step": 865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1969.0, "completions/max_terminated_length": 1969.0, "completions/mean_length": 1745.625, "completions/mean_terminated_length": 1745.625, "completions/min_length": 1347.0, "completions/min_terminated_length": 1347.0, "entropy": 0.23081133514642715, "epoch": 0.04977583630302334, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42377441.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.616547703742981, "sampling/importance_sampling_ratio/mean": 0.9998692274093628, "sampling/importance_sampling_ratio/min": 0.6369381546974182, "sampling/sampling_logp_difference/max": 0.48029279708862305, "sampling/sampling_logp_difference/mean": 0.009253324940800667, "step": 866 }, { "clip_ratio/high_max": 8.06503612693632e-06, "clip_ratio/high_mean": 8.06503612693632e-06, "clip_ratio/low_mean": 0.0010411409748485312, "clip_ratio/low_min": 0.0010411409748485312, "clip_ratio/region_mean": 0.0010492060109754675, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15499.0, "completions/mean_length": 12622.75, "completions/mean_terminated_length": 12085.4287109375, "completions/min_length": 6478.0, "completions/min_terminated_length": 6478.0, "entropy": 0.8774932026863098, "epoch": 0.04983331417404299, "frac_reward_zero_std": 0.0, "grad_norm": 0.009095371700823307, "learning_rate": 1e-05, "loss": -0.0803, "num_tokens": 42479703.0, "reward": 0.125, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999873042106628, "sampling/importance_sampling_ratio/min": 0.06402823328971863, "sampling/sampling_logp_difference/max": 2.7484312057495117, "sampling/sampling_logp_difference/mean": 0.03105144202709198, "step": 867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 11960.0, "completions/max_terminated_length": 11960.0, "completions/mean_length": 8840.625, "completions/mean_terminated_length": 8840.625, "completions/min_length": 5570.0, "completions/min_terminated_length": 5570.0, "entropy": 0.5907813012599945, "epoch": 0.04989079204506265, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42551724.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000455379486084, "sampling/importance_sampling_ratio/min": 0.43158450722694397, "sampling/sampling_logp_difference/max": 1.270559310913086, "sampling/sampling_logp_difference/mean": 0.02233121544122696, "step": 868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3482.0, "completions/max_terminated_length": 3482.0, "completions/mean_length": 1646.125, "completions/mean_terminated_length": 1646.125, "completions/min_length": 855.0, "completions/min_terminated_length": 855.0, "entropy": 0.48796676471829414, "epoch": 0.04994826991608231, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42566253.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3630291223526, "sampling/importance_sampling_ratio/mean": 0.9999997019767761, "sampling/importance_sampling_ratio/min": 0.6139880418777466, "sampling/sampling_logp_difference/max": 0.4877798557281494, "sampling/sampling_logp_difference/mean": 0.020210284739732742, "step": 869 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3315.0, "completions/max_terminated_length": 3315.0, "completions/mean_length": 2269.375, "completions/mean_terminated_length": 2269.375, "completions/min_length": 1329.0, "completions/min_terminated_length": 1329.0, "entropy": 0.280950590968132, "epoch": 0.05000574778710196, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42585416.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4039102792739868, "sampling/importance_sampling_ratio/mean": 0.999797523021698, "sampling/importance_sampling_ratio/min": 0.6312107443809509, "sampling/sampling_logp_difference/max": 0.4601154327392578, "sampling/sampling_logp_difference/mean": 0.011380360461771488, "step": 870 }, { "clip_ratio/high_max": 3.956213186029345e-05, "clip_ratio/high_mean": 3.956213186029345e-05, "clip_ratio/low_mean": 0.00019604176486609504, "clip_ratio/low_min": 0.00019604176486609504, "clip_ratio/region_mean": 0.00023560389672638848, "completions/clipped_ratio": 0.0, "completions/max_length": 12357.0, "completions/max_terminated_length": 12357.0, "completions/mean_length": 7380.625, "completions/mean_terminated_length": 7380.625, "completions/min_length": 3263.0, "completions/min_terminated_length": 3263.0, "entropy": 0.4952394440770149, "epoch": 0.05006322565812162, "frac_reward_zero_std": 0.0, "grad_norm": 0.025206970050930977, "learning_rate": 1e-05, "loss": 0.1449, "num_tokens": 42645621.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000628232955933, "sampling/importance_sampling_ratio/min": 0.39509356021881104, "sampling/sampling_logp_difference/max": 0.9286327362060547, "sampling/sampling_logp_difference/mean": 0.016573263332247734, "step": 871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13874.0, "completions/max_terminated_length": 13874.0, "completions/mean_length": 4435.625, "completions/mean_terminated_length": 4435.625, "completions/min_length": 1083.0, "completions/min_terminated_length": 1083.0, "entropy": 0.3616592064499855, "epoch": 0.05012070352914128, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42682282.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5590119361877441, "sampling/importance_sampling_ratio/mean": 0.9998528957366943, "sampling/importance_sampling_ratio/min": 0.509806215763092, "sampling/sampling_logp_difference/max": 0.6737246513366699, "sampling/sampling_logp_difference/mean": 0.018887242302298546, "step": 872 }, { "clip_ratio/high_max": 0.00018492788512958214, "clip_ratio/high_mean": 0.00018492788512958214, "clip_ratio/low_mean": 0.00019527735275914893, "clip_ratio/low_min": 0.00019527735275914893, "clip_ratio/region_mean": 0.00038020523788873106, "completions/clipped_ratio": 0.0, "completions/max_length": 11048.0, "completions/max_terminated_length": 11048.0, "completions/mean_length": 3477.375, "completions/mean_terminated_length": 3477.375, "completions/min_length": 1145.0, "completions/min_terminated_length": 1145.0, "entropy": 0.46324988082051277, "epoch": 0.05017818140016094, "frac_reward_zero_std": 0.0, "grad_norm": 0.07996657490730286, "learning_rate": 1e-05, "loss": 0.8725, "num_tokens": 42711837.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996238350868225, "sampling/importance_sampling_ratio/min": 0.2854886054992676, "sampling/sampling_logp_difference/max": 1.2535531520843506, "sampling/sampling_logp_difference/mean": 0.021634716540575027, "step": 873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6957175072748214e-05, "clip_ratio/low_min": 4.6957175072748214e-05, "clip_ratio/region_mean": 4.6957175072748214e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 5324.0, "completions/max_terminated_length": 5324.0, "completions/mean_length": 3691.75, "completions/mean_terminated_length": 3691.75, "completions/min_length": 2520.0, "completions/min_terminated_length": 2520.0, "entropy": 0.5134863629937172, "epoch": 0.050235659271180594, "frac_reward_zero_std": 0.0, "grad_norm": 0.012264709919691086, "learning_rate": 1e-05, "loss": 0.1561, "num_tokens": 42742483.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.527854323387146, "sampling/importance_sampling_ratio/mean": 1.0004197359085083, "sampling/importance_sampling_ratio/min": 0.6933304071426392, "sampling/sampling_logp_difference/max": 0.42386436462402344, "sampling/sampling_logp_difference/mean": 0.014306210912764072, "step": 874 }, { "clip_ratio/high_max": 0.00011534092845977284, "clip_ratio/high_mean": 0.00011534092845977284, "clip_ratio/low_mean": 8.392333984375e-05, "clip_ratio/low_min": 8.392333984375e-05, "clip_ratio/region_mean": 0.00019926426830352284, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 15044.0, "completions/mean_length": 8354.375, "completions/mean_terminated_length": 7207.2861328125, "completions/min_length": 3156.0, "completions/min_terminated_length": 3156.0, "entropy": 0.4865967780351639, "epoch": 0.05029313714220025, "frac_reward_zero_std": 0.0, "grad_norm": 0.08380790054798126, "learning_rate": 1e-05, "loss": 0.3399, "num_tokens": 42810662.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996609687805176, "sampling/importance_sampling_ratio/min": 0.42334502935409546, "sampling/sampling_logp_difference/max": 0.8777360916137695, "sampling/sampling_logp_difference/mean": 0.020147521048784256, "step": 875 }, { "clip_ratio/high_max": 0.00023532157501904294, "clip_ratio/high_mean": 0.00023532157501904294, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00023532157501904294, "completions/clipped_ratio": 0.0, "completions/max_length": 1745.0, "completions/max_terminated_length": 1745.0, "completions/mean_length": 1534.625, "completions/mean_terminated_length": 1534.625, "completions/min_length": 1203.0, "completions/min_terminated_length": 1203.0, "entropy": 0.34582608565688133, "epoch": 0.05035061501321991, "frac_reward_zero_std": 0.0, "grad_norm": 0.09692811965942383, "learning_rate": 1e-05, "loss": -0.0763, "num_tokens": 42823675.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.6467636823654175, "sampling/importance_sampling_ratio/mean": 0.9998302459716797, "sampling/importance_sampling_ratio/min": 0.6886875033378601, "sampling/sampling_logp_difference/max": 0.4988119602203369, "sampling/sampling_logp_difference/mean": 0.014031866565346718, "step": 876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022338571943691932, "clip_ratio/low_min": 0.00022338571943691932, "clip_ratio/region_mean": 0.00022338571943691932, "completions/clipped_ratio": 0.0, "completions/max_length": 8060.0, "completions/max_terminated_length": 8060.0, "completions/mean_length": 5448.125, "completions/mean_terminated_length": 5448.125, "completions/min_length": 3699.0, "completions/min_terminated_length": 3699.0, "entropy": 0.4809279851615429, "epoch": 0.050408092884239565, "frac_reward_zero_std": 0.0, "grad_norm": 0.01856175810098648, "learning_rate": 1e-05, "loss": 0.1233, "num_tokens": 42868708.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 1.5426536798477173, "sampling/importance_sampling_ratio/mean": 0.9998001456260681, "sampling/importance_sampling_ratio/min": 0.4641388952732086, "sampling/sampling_logp_difference/max": 0.7675714492797852, "sampling/sampling_logp_difference/mean": 0.015464321710169315, "step": 877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3965.0, "completions/max_terminated_length": 3965.0, "completions/mean_length": 2811.75, "completions/mean_terminated_length": 2811.75, "completions/min_length": 1122.0, "completions/min_terminated_length": 1122.0, "entropy": 0.5146245695650578, "epoch": 0.050465570755259224, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42891866.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4433928728103638, "sampling/importance_sampling_ratio/mean": 1.0000520944595337, "sampling/importance_sampling_ratio/min": 0.7131180763244629, "sampling/sampling_logp_difference/max": 0.36699652671813965, "sampling/sampling_logp_difference/mean": 0.014553203247487545, "step": 878 }, { "clip_ratio/high_max": 5.32821832166519e-05, "clip_ratio/high_mean": 5.32821832166519e-05, "clip_ratio/low_mean": 8.074935612967238e-05, "clip_ratio/low_min": 8.074935612967238e-05, "clip_ratio/region_mean": 0.00013403153934632428, "completions/clipped_ratio": 0.0, "completions/max_length": 2346.0, "completions/max_terminated_length": 2346.0, "completions/mean_length": 1769.875, "completions/mean_terminated_length": 1769.875, "completions/min_length": 1457.0, "completions/min_terminated_length": 1457.0, "entropy": 0.18315835110843182, "epoch": 0.050523048626278884, "frac_reward_zero_std": 0.0, "grad_norm": 0.009523826651275158, "learning_rate": 1e-05, "loss": -0.0444, "num_tokens": 42906889.0, "reward": 0.875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.5541505813598633, "sampling/importance_sampling_ratio/mean": 1.0000386238098145, "sampling/importance_sampling_ratio/min": 0.6114224791526794, "sampling/sampling_logp_difference/max": 0.49196720123291016, "sampling/sampling_logp_difference/mean": 0.008084428496658802, "step": 879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024305110491695814, "clip_ratio/low_min": 0.00024305110491695814, "clip_ratio/region_mean": 0.00024305110491695814, "completions/clipped_ratio": 0.0, "completions/max_length": 3995.0, "completions/max_terminated_length": 3995.0, "completions/mean_length": 2859.0, "completions/mean_terminated_length": 2859.0, "completions/min_length": 1726.0, "completions/min_terminated_length": 1726.0, "entropy": 0.882795937359333, "epoch": 0.05058052649729854, "frac_reward_zero_std": 0.0, "grad_norm": 0.04315592721104622, "learning_rate": 1e-05, "loss": 0.0097, "num_tokens": 42931401.0, "reward": 0.375, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.4020549058914185, "sampling/importance_sampling_ratio/mean": 0.9999202489852905, "sampling/importance_sampling_ratio/min": 0.6073699593544006, "sampling/sampling_logp_difference/max": 0.49861717224121094, "sampling/sampling_logp_difference/mean": 0.023641500622034073, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8792.0, "completions/max_terminated_length": 8792.0, "completions/mean_length": 4550.875, "completions/mean_terminated_length": 4550.875, "completions/min_length": 1789.0, "completions/min_terminated_length": 1789.0, "entropy": 0.5362890139222145, "epoch": 0.050638004368318196, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42969000.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5889239311218262, "sampling/importance_sampling_ratio/mean": 1.0003232955932617, "sampling/importance_sampling_ratio/min": 0.6032710075378418, "sampling/sampling_logp_difference/max": 0.5053887367248535, "sampling/sampling_logp_difference/mean": 0.021709280088543892, "step": 881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5658.0, "completions/max_terminated_length": 5658.0, "completions/mean_length": 2697.625, "completions/mean_terminated_length": 2697.625, "completions/min_length": 1543.0, "completions/min_terminated_length": 1543.0, "entropy": 0.3586522564291954, "epoch": 0.050695482239337855, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 42991269.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4441182613372803, "sampling/importance_sampling_ratio/mean": 1.0002607107162476, "sampling/importance_sampling_ratio/min": 0.6144427061080933, "sampling/sampling_logp_difference/max": 0.48703956604003906, "sampling/sampling_logp_difference/mean": 0.014730799943208694, "step": 882 }, { "clip_ratio/high_max": 0.0001106874724428053, "clip_ratio/high_mean": 0.0001106874724428053, "clip_ratio/low_mean": 0.000152587890625, "clip_ratio/low_min": 0.000152587890625, "clip_ratio/region_mean": 0.0002632753630678053, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 9259.0, "completions/mean_length": 8834.625, "completions/mean_terminated_length": 6318.1669921875, "completions/min_length": 4293.0, "completions/min_terminated_length": 4293.0, "entropy": 0.4619143381714821, "epoch": 0.050752960110357515, "frac_reward_zero_std": 0.0, "grad_norm": 0.011678778566420078, "learning_rate": 1e-05, "loss": 0.4616, "num_tokens": 43063018.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998613595962524, "sampling/importance_sampling_ratio/min": 0.4947577714920044, "sampling/sampling_logp_difference/max": 1.1669292449951172, "sampling/sampling_logp_difference/mean": 0.016659585759043694, "step": 883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1199.0, "completions/max_terminated_length": 1199.0, "completions/mean_length": 817.875, "completions/mean_terminated_length": 817.875, "completions/min_length": 637.0, "completions/min_terminated_length": 637.0, "entropy": 0.2739987690001726, "epoch": 0.05081043798137717, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43070609.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.32776939868927, "sampling/importance_sampling_ratio/mean": 0.9994729161262512, "sampling/importance_sampling_ratio/min": 0.6625231504440308, "sampling/sampling_logp_difference/max": 0.4116997718811035, "sampling/sampling_logp_difference/mean": 0.011875614523887634, "step": 884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3994.0, "completions/max_terminated_length": 3994.0, "completions/mean_length": 1841.375, "completions/mean_terminated_length": 1841.375, "completions/min_length": 803.0, "completions/min_terminated_length": 803.0, "entropy": 0.2501242561265826, "epoch": 0.050867915852396826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43086212.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.311762809753418, "sampling/importance_sampling_ratio/mean": 0.9999081492424011, "sampling/importance_sampling_ratio/min": 0.637551486492157, "sampling/sampling_logp_difference/max": 0.450120210647583, "sampling/sampling_logp_difference/mean": 0.01199504267424345, "step": 885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1302.0, "completions/max_terminated_length": 1302.0, "completions/mean_length": 880.0, "completions/mean_terminated_length": 880.0, "completions/min_length": 500.0, "completions/min_terminated_length": 500.0, "entropy": 0.3579131867736578, "epoch": 0.050925393723416486, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43094108.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4543228149414062, "sampling/importance_sampling_ratio/mean": 0.999966025352478, "sampling/importance_sampling_ratio/min": 0.6328878402709961, "sampling/sampling_logp_difference/max": 0.4574620723724365, "sampling/sampling_logp_difference/mean": 0.015180517919361591, "step": 886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4906.0, "completions/max_terminated_length": 4906.0, "completions/mean_length": 2888.5, "completions/mean_terminated_length": 2888.5, "completions/min_length": 1450.0, "completions/min_terminated_length": 1450.0, "entropy": 0.40141788125038147, "epoch": 0.050982871594436145, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43118336.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2983653545379639, "sampling/importance_sampling_ratio/mean": 0.9995899200439453, "sampling/importance_sampling_ratio/min": 0.6492436528205872, "sampling/sampling_logp_difference/max": 0.4319472312927246, "sampling/sampling_logp_difference/mean": 0.01596933789551258, "step": 887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9857.0, "completions/max_terminated_length": 9857.0, "completions/mean_length": 5404.5, "completions/mean_terminated_length": 5404.5, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.733769703656435, "epoch": 0.0510403494654558, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43162348.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998623728752136, "sampling/importance_sampling_ratio/min": 0.4092647433280945, "sampling/sampling_logp_difference/max": 0.8933930397033691, "sampling/sampling_logp_difference/mean": 0.02970987930893898, "step": 888 }, { "clip_ratio/high_max": 3.7158411942073144e-05, "clip_ratio/high_mean": 3.7158411942073144e-05, "clip_ratio/low_mean": 0.00015869649359956384, "clip_ratio/low_min": 0.00015869649359956384, "clip_ratio/region_mean": 0.00019585490554163698, "completions/clipped_ratio": 0.0, "completions/max_length": 12992.0, "completions/max_terminated_length": 12992.0, "completions/mean_length": 7048.375, "completions/mean_terminated_length": 7048.375, "completions/min_length": 4766.0, "completions/min_terminated_length": 4766.0, "entropy": 0.2805531397461891, "epoch": 0.05109782733647546, "frac_reward_zero_std": 0.0, "grad_norm": 0.008574248291552067, "learning_rate": 1e-05, "loss": 0.1922, "num_tokens": 43220319.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999893307685852, "sampling/importance_sampling_ratio/min": 0.2446233183145523, "sampling/sampling_logp_difference/max": 1.4080357551574707, "sampling/sampling_logp_difference/mean": 0.012378632090985775, "step": 889 }, { "clip_ratio/high_max": 8.590617289883085e-05, "clip_ratio/high_mean": 8.590617289883085e-05, "clip_ratio/low_mean": 0.000213623046875, "clip_ratio/low_min": 0.000213623046875, "clip_ratio/region_mean": 0.00029952921977383085, "completions/clipped_ratio": 0.25, "completions/max_length": 16384.0, "completions/max_terminated_length": 8073.0, "completions/mean_length": 7899.25, "completions/mean_terminated_length": 5071.0, "completions/min_length": 2504.0, "completions/min_terminated_length": 2504.0, "entropy": 0.7566853053867817, "epoch": 0.051155305207495116, "frac_reward_zero_std": 0.0, "grad_norm": 0.07121574878692627, "learning_rate": 1e-05, "loss": 0.5796, "num_tokens": 43285201.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000442266464233, "sampling/importance_sampling_ratio/min": 0.24970847368240356, "sampling/sampling_logp_difference/max": 1.4501371383666992, "sampling/sampling_logp_difference/mean": 0.029447944834828377, "step": 890 }, { "clip_ratio/high_max": 0.00012437672558007762, "clip_ratio/high_mean": 0.00012437672558007762, "clip_ratio/low_mean": 0.00033499952405691147, "clip_ratio/low_min": 0.00033499952405691147, "clip_ratio/region_mean": 0.0004593762496369891, "completions/clipped_ratio": 0.0, "completions/max_length": 15805.0, "completions/max_terminated_length": 15805.0, "completions/mean_length": 10540.625, "completions/mean_terminated_length": 10540.625, "completions/min_length": 6255.0, "completions/min_terminated_length": 6255.0, "entropy": 0.5008097812533379, "epoch": 0.05121278307851477, "frac_reward_zero_std": 0.0, "grad_norm": 0.01730172149837017, "learning_rate": 1e-05, "loss": -0.1017, "num_tokens": 43370550.0, "reward": 0.625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000239610671997, "sampling/importance_sampling_ratio/min": 0.17038124799728394, "sampling/sampling_logp_difference/max": 1.769716739654541, "sampling/sampling_logp_difference/mean": 0.019213862717151642, "step": 891 }, { "clip_ratio/high_max": 0.00012024778698105365, "clip_ratio/high_mean": 0.00012024778698105365, "clip_ratio/low_mean": 0.00019627343863248825, "clip_ratio/low_min": 0.00019627343863248825, "clip_ratio/region_mean": 0.0003165212256135419, "completions/clipped_ratio": 0.125, "completions/max_length": 16384.0, "completions/max_terminated_length": 10526.0, "completions/mean_length": 8316.0, "completions/mean_terminated_length": 7163.4287109375, "completions/min_length": 4204.0, "completions/min_terminated_length": 4204.0, "entropy": 0.44185203313827515, "epoch": 0.05127026094953443, "frac_reward_zero_std": 0.0, "grad_norm": 0.01222233846783638, "learning_rate": 1e-05, "loss": 0.278, "num_tokens": 43438062.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4629100561141968, "sampling/importance_sampling_ratio/max": 1.7989411354064941, "sampling/importance_sampling_ratio/mean": 0.9998687505722046, "sampling/importance_sampling_ratio/min": 0.06547701358795166, "sampling/sampling_logp_difference/max": 2.7260560989379883, "sampling/sampling_logp_difference/mean": 0.01698482781648636, "step": 892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 3461.0, "completions/max_terminated_length": 3461.0, "completions/mean_length": 2126.75, "completions/mean_terminated_length": 2126.75, "completions/min_length": 1404.0, "completions/min_terminated_length": 1404.0, "entropy": 0.3092490378767252, "epoch": 0.05132773882055409, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43455924.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5071698427200317, "sampling/importance_sampling_ratio/mean": 0.9997444748878479, "sampling/importance_sampling_ratio/min": 0.620227038860321, "sampling/sampling_logp_difference/max": 0.47766971588134766, "sampling/sampling_logp_difference/mean": 0.01292417198419571, "step": 893 }, { "clip_ratio/high_max": 0.00012211994908284396, "clip_ratio/high_mean": 0.00012211994908284396, "clip_ratio/low_mean": 0.0004847710224566981, "clip_ratio/low_min": 0.0004847710224566981, "clip_ratio/region_mean": 0.0006068909715395421, "completions/clipped_ratio": 0.375, "completions/max_length": 16384.0, "completions/max_terminated_length": 15210.0, "completions/mean_length": 11700.25, "completions/mean_terminated_length": 8890.0, "completions/min_length": 2649.0, "completions/min_terminated_length": 2649.0, "entropy": 0.49228333309292793, "epoch": 0.05138521669157375, "frac_reward_zero_std": 0.0, "grad_norm": 0.007313584443181753, "learning_rate": 1e-05, "loss": 0.3338, "num_tokens": 43551950.0, "reward": 0.5, "reward_std": 0.5345224738121033, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5345224738121033, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000382661819458, "sampling/importance_sampling_ratio/min": 0.24452489614486694, "sampling/sampling_logp_difference/max": 1.4084382057189941, "sampling/sampling_logp_difference/mean": 0.02175779640674591, "step": 894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 16116.0, "completions/max_terminated_length": 16061.0, "completions/mean_length": 13691.25, "completions/mean_terminated_length": 13344.857421875, "completions/min_length": 7334.0, "completions/min_terminated_length": 7334.0, "entropy": 0.8975198417901993, "epoch": 0.0514426945625934, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43667720.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000101089477539, "sampling/importance_sampling_ratio/min": 0.12812969088554382, "sampling/sampling_logp_difference/max": 2.0547122955322266, "sampling/sampling_logp_difference/mean": 0.029267266392707825, "step": 895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15390.0, "completions/max_terminated_length": 15390.0, "completions/mean_length": 9777.25, "completions/mean_terminated_length": 9777.25, "completions/min_length": 3883.0, "completions/min_terminated_length": 3883.0, "entropy": 0.5858627036213875, "epoch": 0.05150017243361306, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 43746778.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999750256538391, "sampling/importance_sampling_ratio/min": 0.31018519401550293, "sampling/sampling_logp_difference/max": 1.1705857515335083, "sampling/sampling_logp_difference/mean": 0.020111290737986565, "step": 896 } ], "logging_steps": 1, "max_steps": 1024, "num_input_tokens_seen": 43746778, "num_train_epochs": 1, "save_steps": 64, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }