diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,31778 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5481798715203426, + "eval_steps": 500, + "global_step": 1024, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 664.84375, + "completions/mean_terminated_length": 561.6875, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.48598330840468407, + "epoch": 0.0005353319057815846, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003244089661166072, + "learning_rate": 1e-05, + "loss": 0.0473, + "num_tokens": 25147.0, + "reward": 0.46875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2930827140808105, + "sampling/importance_sampling_ratio/mean": 0.9998305439949036, + "sampling/importance_sampling_ratio/min": 0.6979114413261414, + "sampling/sampling_logp_difference/max": 0.3596630096435547, + "sampling/sampling_logp_difference/mean": 0.012516415677964687, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011034102863050066, + "clip_ratio/low_min": 0.00011034102863050066, + "clip_ratio/region_mean": 0.00011034102863050066, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 615.625, + "completions/mean_terminated_length": 564.8333740234375, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "entropy": 0.4234888218343258, + "epoch": 0.0010706638115631692, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011267696507275105, + "learning_rate": 1e-05, + "loss": 0.0936, + "num_tokens": 48695.0, + "reward": 0.59375, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3123857975006104, + "sampling/importance_sampling_ratio/mean": 0.9996686577796936, + "sampling/importance_sampling_ratio/min": 0.6920886039733887, + "sampling/sampling_logp_difference/max": 0.3680412769317627, + "sampling/sampling_logp_difference/mean": 0.011322839185595512, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 525.0, + "completions/mean_terminated_length": 468.923095703125, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.450862105935812, + "epoch": 0.0016059957173447537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005509702488780022, + "learning_rate": 1e-05, + "loss": 0.0714, + "num_tokens": 68967.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2685112953186035, + "sampling/importance_sampling_ratio/mean": 0.9999655485153198, + "sampling/importance_sampling_ratio/min": 0.7146701812744141, + "sampling/sampling_logp_difference/max": 0.3359341621398926, + "sampling/sampling_logp_difference/mean": 0.012161738239228725, + "step": 3 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 599.4375, + "completions/mean_terminated_length": 560.5384521484375, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.40837138518691063, + "epoch": 0.0021413276231263384, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009048921056091785, + "learning_rate": 1e-05, + "loss": 0.0159, + "num_tokens": 92461.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3900072574615479, + "sampling/importance_sampling_ratio/mean": 0.9997454285621643, + "sampling/importance_sampling_ratio/min": 0.6160321831703186, + "sampling/sampling_logp_difference/max": 0.48445606231689453, + "sampling/sampling_logp_difference/mean": 0.011591978371143341, + "step": 4 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 671.46875, + "completions/mean_terminated_length": 596.388916015625, + "completions/min_length": 427.0, + "completions/min_terminated_length": 427.0, + "entropy": 0.41664107516407967, + "epoch": 0.0026766595289079227, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00951086264103651, + "learning_rate": 1e-05, + "loss": 0.0529, + "num_tokens": 117740.0, + "reward": 0.34375, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2843331098556519, + "sampling/importance_sampling_ratio/mean": 1.0001472234725952, + "sampling/importance_sampling_ratio/min": 0.5767552852630615, + "sampling/sampling_logp_difference/max": 0.5503373146057129, + "sampling/sampling_logp_difference/mean": 0.012003127485513687, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.263157800072804e-05, + "clip_ratio/low_min": 5.263157800072804e-05, + "clip_ratio/region_mean": 5.263157800072804e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 534.1875, + "completions/mean_terminated_length": 468.7200012207031, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.4959532096982002, + "epoch": 0.0032119914346895075, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014368094503879547, + "learning_rate": 1e-05, + "loss": 0.0858, + "num_tokens": 138738.0, + "reward": 0.625, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3477996587753296, + "sampling/importance_sampling_ratio/mean": 1.0001568794250488, + "sampling/importance_sampling_ratio/min": 0.7303528189659119, + "sampling/sampling_logp_difference/max": 0.3142275810241699, + "sampling/sampling_logp_difference/mean": 0.013215973973274231, + "step": 6 + }, + { + "clip_ratio/high_max": 5.781683648820035e-05, + "clip_ratio/high_mean": 5.781683648820035e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.781683648820035e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 557.65625, + "completions/mean_terminated_length": 535.8965454101562, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3992920517921448, + "epoch": 0.003747323340471092, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0207853764295578, + "learning_rate": 1e-05, + "loss": 0.0173, + "num_tokens": 160207.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5296844244003296, + "sampling/importance_sampling_ratio/mean": 0.9999474287033081, + "sampling/importance_sampling_ratio/min": 0.7430385947227478, + "sampling/sampling_logp_difference/max": 0.4250614643096924, + "sampling/sampling_logp_difference/mean": 0.011352009139955044, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.5126354962121695e-05, + "clip_ratio/low_min": 4.5126354962121695e-05, + "clip_ratio/region_mean": 4.5126354962121695e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 666.65625, + "completions/mean_terminated_length": 627.0, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 0.32646026462316513, + "epoch": 0.004282655246252677, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006821369286626577, + "learning_rate": 1e-05, + "loss": 0.0193, + "num_tokens": 185060.0, + "reward": 0.5, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3118391036987305, + "sampling/importance_sampling_ratio/mean": 0.99983811378479, + "sampling/importance_sampling_ratio/min": 0.7614741921424866, + "sampling/sampling_logp_difference/max": 0.27249908447265625, + "sampling/sampling_logp_difference/mean": 0.009400330483913422, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 618.90625, + "completions/mean_terminated_length": 529.4500122070312, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.35843971744179726, + "epoch": 0.004817987152034261, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02422148361802101, + "learning_rate": 1e-05, + "loss": 0.1226, + "num_tokens": 208801.0, + "reward": 0.625, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3372341394424438, + "sampling/importance_sampling_ratio/mean": 0.9997857213020325, + "sampling/importance_sampling_ratio/min": 0.6132058501243591, + "sampling/sampling_logp_difference/max": 0.48905467987060547, + "sampling/sampling_logp_difference/mean": 0.010105549357831478, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.747126488131471e-05, + "clip_ratio/low_min": 5.747126488131471e-05, + "clip_ratio/region_mean": 5.747126488131471e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 520.53125, + "completions/mean_terminated_length": 485.1785888671875, + "completions/min_length": 300.0, + "completions/min_terminated_length": 300.0, + "entropy": 0.34622709080576897, + "epoch": 0.0053533190578158455, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00984555296599865, + "learning_rate": 1e-05, + "loss": 0.0782, + "num_tokens": 228490.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2665451765060425, + "sampling/importance_sampling_ratio/mean": 0.9997326731681824, + "sampling/importance_sampling_ratio/min": 0.7133243083953857, + "sampling/sampling_logp_difference/max": 0.33781909942626953, + "sampling/sampling_logp_difference/mean": 0.009415730834007263, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012798634998034686, + "clip_ratio/low_min": 0.00012798634998034686, + "clip_ratio/region_mean": 0.00012798634998034686, + "completions/clipped_ratio": 0.6875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 724.15625, + "completions/mean_terminated_length": 627.7000122070312, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 0.6496313735842705, + "epoch": 0.005888650963597431, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005864573642611504, + "learning_rate": 1e-05, + "loss": -0.0007, + "num_tokens": 255631.0, + "reward": 0.15625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6560299396514893, + "sampling/importance_sampling_ratio/mean": 1.000302791595459, + "sampling/importance_sampling_ratio/min": 0.7081769704818726, + "sampling/sampling_logp_difference/max": 0.5044231414794922, + "sampling/sampling_logp_difference/mean": 0.01602434553205967, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 610.59375, + "completions/mean_terminated_length": 528.1428833007812, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.4252280630171299, + "epoch": 0.006423982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013786358758807182, + "learning_rate": 1e-05, + "loss": 0.0442, + "num_tokens": 278674.0, + "reward": 0.625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5340704917907715, + "sampling/importance_sampling_ratio/mean": 0.9997371435165405, + "sampling/importance_sampling_ratio/min": 0.7066288590431213, + "sampling/sampling_logp_difference/max": 0.42792463302612305, + "sampling/sampling_logp_difference/mean": 0.011518296785652637, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 553.90625, + "completions/mean_terminated_length": 470.13043212890625, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.4396296590566635, + "epoch": 0.006959314775160599, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011722812429070473, + "learning_rate": 1e-05, + "loss": -0.0626, + "num_tokens": 299823.0, + "reward": 0.53125, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.475489854812622, + "sampling/importance_sampling_ratio/mean": 0.9998274445533752, + "sampling/importance_sampling_ratio/min": 0.5904674530029297, + "sampling/sampling_logp_difference/max": 0.5268406867980957, + "sampling/sampling_logp_difference/mean": 0.012197126634418964, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016838656665640883, + "clip_ratio/low_min": 0.00016838656665640883, + "clip_ratio/region_mean": 0.00016838656665640883, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 594.40625, + "completions/mean_terminated_length": 526.478271484375, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.3696550354361534, + "epoch": 0.007494646680942184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013173367828130722, + "learning_rate": 1e-05, + "loss": 0.0506, + "num_tokens": 322396.0, + "reward": 0.65625, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2909218072891235, + "sampling/importance_sampling_ratio/mean": 1.0002373456954956, + "sampling/importance_sampling_ratio/min": 0.6608690619468689, + "sampling/sampling_logp_difference/max": 0.4141995906829834, + "sampling/sampling_logp_difference/mean": 0.010886088944971561, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 613.46875, + "completions/mean_terminated_length": 591.3928833007812, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.3582111708819866, + "epoch": 0.008029978586723769, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008435110561549664, + "learning_rate": 1e-05, + "loss": -0.0, + "num_tokens": 345475.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2862032651901245, + "sampling/importance_sampling_ratio/mean": 1.0000219345092773, + "sampling/importance_sampling_ratio/min": 0.7341545224189758, + "sampling/sampling_logp_difference/max": 0.3090357780456543, + "sampling/sampling_logp_difference/mean": 0.010146892629563808, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001517675700597465, + "clip_ratio/low_min": 0.0001517675700597465, + "clip_ratio/region_mean": 0.0001517675700597465, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 628.65625, + "completions/mean_terminated_length": 574.1304321289062, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "entropy": 0.7098877020180225, + "epoch": 0.008565310492505354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018570125102996826, + "learning_rate": 1e-05, + "loss": 0.0919, + "num_tokens": 370344.0, + "reward": 0.59375, + "reward_std": 0.512376070022583, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6732138395309448, + "sampling/importance_sampling_ratio/mean": 0.9998745322227478, + "sampling/importance_sampling_ratio/min": 0.7282438278198242, + "sampling/sampling_logp_difference/max": 0.5147461891174316, + "sampling/sampling_logp_difference/mean": 0.0150149529799819, + "step": 16 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.829984391108155e-05, + "clip_ratio/low_min": 4.829984391108155e-05, + "clip_ratio/region_mean": 4.829984391108155e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 666.34375, + "completions/mean_terminated_length": 576.6470336914062, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.5932197459042072, + "epoch": 0.009100642398286937, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00621638773009181, + "learning_rate": 1e-05, + "loss": 0.0107, + "num_tokens": 395475.0, + "reward": 0.15625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3349889516830444, + "sampling/importance_sampling_ratio/mean": 0.9999136924743652, + "sampling/importance_sampling_ratio/min": 0.6694192290306091, + "sampling/sampling_logp_difference/max": 0.40134477615356445, + "sampling/sampling_logp_difference/mean": 0.01411344762891531, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.530973430722952e-05, + "clip_ratio/low_min": 5.530973430722952e-05, + "clip_ratio/region_mean": 5.530973430722952e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 671.96875, + "completions/mean_terminated_length": 575.9375, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.47273609787225723, + "epoch": 0.009635974304068522, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016480356454849243, + "learning_rate": 1e-05, + "loss": 0.0461, + "num_tokens": 420858.0, + "reward": 0.53125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3640564680099487, + "sampling/importance_sampling_ratio/mean": 1.000091552734375, + "sampling/importance_sampling_ratio/min": 0.5932015180587769, + "sampling/sampling_logp_difference/max": 0.5222210884094238, + "sampling/sampling_logp_difference/mean": 0.012002711184322834, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.925135964294896e-05, + "clip_ratio/low_min": 9.925135964294896e-05, + "clip_ratio/region_mean": 9.925135964294896e-05, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 677.96875, + "completions/mean_terminated_length": 546.3846435546875, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.4563947506248951, + "epoch": 0.010171306209850108, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012845396064221859, + "learning_rate": 1e-05, + "loss": 0.0329, + "num_tokens": 446721.0, + "reward": 0.4375, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.48965322971344, + "sampling/importance_sampling_ratio/mean": 1.0000492334365845, + "sampling/importance_sampling_ratio/min": 0.6992079019546509, + "sampling/sampling_logp_difference/max": 0.3985433578491211, + "sampling/sampling_logp_difference/mean": 0.012940243817865849, + "step": 19 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013168069563107565, + "clip_ratio/low_min": 0.00013168069563107565, + "clip_ratio/region_mean": 0.00013168069563107565, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 529.21875, + "completions/mean_terminated_length": 449.625, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.489932082593441, + "epoch": 0.010706638115631691, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015450346283614635, + "learning_rate": 1e-05, + "loss": 0.0744, + "num_tokens": 467336.0, + "reward": 0.40625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7126078605651855, + "sampling/importance_sampling_ratio/mean": 0.9998895525932312, + "sampling/importance_sampling_ratio/min": 0.7210950255393982, + "sampling/sampling_logp_difference/max": 0.5380172729492188, + "sampling/sampling_logp_difference/mean": 0.012707538902759552, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.526083259610459e-05, + "clip_ratio/low_min": 5.526083259610459e-05, + "clip_ratio/region_mean": 5.526083259610459e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 591.96875, + "completions/mean_terminated_length": 533.2916870117188, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.4202882684767246, + "epoch": 0.011241970021413276, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01361797470599413, + "learning_rate": 1e-05, + "loss": 0.0766, + "num_tokens": 490311.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2968697547912598, + "sampling/importance_sampling_ratio/mean": 0.9999016523361206, + "sampling/importance_sampling_ratio/min": 0.7421221733093262, + "sampling/sampling_logp_difference/max": 0.29824137687683105, + "sampling/sampling_logp_difference/mean": 0.010941424407064915, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001045044045895338, + "clip_ratio/low_min": 0.0001045044045895338, + "clip_ratio/region_mean": 0.0001045044045895338, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 589.90625, + "completions/mean_terminated_length": 530.5416870117188, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.4353359416127205, + "epoch": 0.011777301927194861, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006856705527752638, + "learning_rate": 1e-05, + "loss": 0.0813, + "num_tokens": 512564.0, + "reward": 0.53125, + "reward_std": 0.5038893818855286, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4417986869812012, + "sampling/importance_sampling_ratio/mean": 1.0003200769424438, + "sampling/importance_sampling_ratio/min": 0.7698214650154114, + "sampling/sampling_logp_difference/max": 0.3658914566040039, + "sampling/sampling_logp_difference/mean": 0.012175213545560837, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 592.125, + "completions/mean_terminated_length": 551.5384521484375, + "completions/min_length": 347.0, + "completions/min_terminated_length": 347.0, + "entropy": 0.4204316474497318, + "epoch": 0.012312633832976445, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01653178781270981, + "learning_rate": 1e-05, + "loss": 0.023, + "num_tokens": 535624.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.393133521080017, + "sampling/importance_sampling_ratio/mean": 1.0003411769866943, + "sampling/importance_sampling_ratio/min": 0.7714617848396301, + "sampling/sampling_logp_difference/max": 0.3315556049346924, + "sampling/sampling_logp_difference/mean": 0.01182654406875372, + "step": 23 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.463286652229726e-05, + "clip_ratio/low_min": 5.463286652229726e-05, + "clip_ratio/region_mean": 5.463286652229726e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 532.75, + "completions/mean_terminated_length": 508.4137878417969, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "entropy": 0.41935884580016136, + "epoch": 0.01284796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00237714103423059, + "learning_rate": 1e-05, + "loss": 0.0965, + "num_tokens": 556744.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4122968912124634, + "sampling/importance_sampling_ratio/mean": 0.9998799562454224, + "sampling/importance_sampling_ratio/min": 0.7301450967788696, + "sampling/sampling_logp_difference/max": 0.3452174663543701, + "sampling/sampling_logp_difference/mean": 0.01174243539571762, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.431052730069496e-05, + "clip_ratio/low_min": 4.431052730069496e-05, + "clip_ratio/region_mean": 4.431052730069496e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 602.75, + "completions/mean_terminated_length": 564.6154174804688, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.5643983520567417, + "epoch": 0.013383297644539615, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0036444501020014286, + "learning_rate": 1e-05, + "loss": 0.0616, + "num_tokens": 579528.0, + "reward": 0.65625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3097071647644043, + "sampling/importance_sampling_ratio/mean": 1.0000592470169067, + "sampling/importance_sampling_ratio/min": 0.7228518724441528, + "sampling/sampling_logp_difference/max": 0.324550986289978, + "sampling/sampling_logp_difference/mean": 0.01465004775673151, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.006520849652588e-05, + "clip_ratio/low_min": 9.006520849652588e-05, + "clip_ratio/region_mean": 9.006520849652588e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 676.3125, + "completions/mean_terminated_length": 595.4117431640625, + "completions/min_length": 449.0, + "completions/min_terminated_length": 449.0, + "entropy": 0.455068938434124, + "epoch": 0.013918629550321198, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007197745610028505, + "learning_rate": 1e-05, + "loss": 0.0294, + "num_tokens": 605210.0, + "reward": 0.21875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.21875, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5281137228012085, + "sampling/importance_sampling_ratio/mean": 0.9997848272323608, + "sampling/importance_sampling_ratio/min": 0.6987742781639099, + "sampling/sampling_logp_difference/max": 0.42403411865234375, + "sampling/sampling_logp_difference/mean": 0.012703796848654747, + "step": 26 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013512342411559075, + "clip_ratio/low_min": 0.00013512342411559075, + "clip_ratio/region_mean": 0.00013512342411559075, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 673.3125, + "completions/mean_terminated_length": 551.5714721679688, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 0.6482994183897972, + "epoch": 0.014453961456102784, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016385406255722046, + "learning_rate": 1e-05, + "loss": 0.0969, + "num_tokens": 630612.0, + "reward": 0.40625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6067651510238647, + "sampling/importance_sampling_ratio/mean": 1.000059962272644, + "sampling/importance_sampling_ratio/min": 0.6817489266395569, + "sampling/sampling_logp_difference/max": 0.47422289848327637, + "sampling/sampling_logp_difference/mean": 0.014250493608415127, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001526857631688472, + "clip_ratio/low_min": 0.0001526857631688472, + "clip_ratio/region_mean": 0.0001526857631688472, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 693.84375, + "completions/mean_terminated_length": 619.6875, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.4106770195066929, + "epoch": 0.014989293361884369, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010193257592618465, + "learning_rate": 1e-05, + "loss": 0.0151, + "num_tokens": 657095.0, + "reward": 0.25, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3577656745910645, + "sampling/importance_sampling_ratio/mean": 1.0001087188720703, + "sampling/importance_sampling_ratio/min": 0.7057578563690186, + "sampling/sampling_logp_difference/max": 0.3484830856323242, + "sampling/sampling_logp_difference/mean": 0.011250431649386883, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 622.375, + "completions/mean_terminated_length": 588.7692260742188, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.44800711050629616, + "epoch": 0.015524625267665952, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005607192870229483, + "learning_rate": 1e-05, + "loss": 0.0299, + "num_tokens": 681251.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3808902502059937, + "sampling/importance_sampling_ratio/mean": 1.0000520944595337, + "sampling/importance_sampling_ratio/min": 0.658728837966919, + "sampling/sampling_logp_difference/max": 0.41744327545166016, + "sampling/sampling_logp_difference/mean": 0.0115430923178792, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 715.875, + "completions/mean_terminated_length": 629.0, + "completions/min_length": 494.0, + "completions/min_terminated_length": 494.0, + "entropy": 0.6344446279108524, + "epoch": 0.016059957173447537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0017212615348398685, + "learning_rate": 1e-05, + "loss": 0.0028, + "num_tokens": 708455.0, + "reward": 0.3125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3901278972625732, + "sampling/importance_sampling_ratio/mean": 0.9994648694992065, + "sampling/importance_sampling_ratio/min": 0.5709463357925415, + "sampling/sampling_logp_difference/max": 0.560460090637207, + "sampling/sampling_logp_difference/mean": 0.01577536202967167, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 613.96875, + "completions/mean_terminated_length": 521.5499877929688, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.34220656752586365, + "epoch": 0.016595289079229122, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00600267481058836, + "learning_rate": 1e-05, + "loss": 0.0678, + "num_tokens": 731782.0, + "reward": 0.53125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2601256370544434, + "sampling/importance_sampling_ratio/mean": 0.9999644160270691, + "sampling/importance_sampling_ratio/min": 0.7214158773422241, + "sampling/sampling_logp_difference/max": 0.3265395164489746, + "sampling/sampling_logp_difference/mean": 0.009856891818344593, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.158894055057317e-05, + "clip_ratio/low_min": 5.158894055057317e-05, + "clip_ratio/region_mean": 5.158894055057317e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 600.375, + "completions/mean_terminated_length": 534.7825927734375, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.38323255628347397, + "epoch": 0.017130620985010708, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012019449844956398, + "learning_rate": 1e-05, + "loss": 0.0308, + "num_tokens": 754738.0, + "reward": 0.53125, + "reward_std": 0.512376070022583, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6615517139434814, + "sampling/importance_sampling_ratio/mean": 1.0002702474594116, + "sampling/importance_sampling_ratio/min": 0.7132049798965454, + "sampling/sampling_logp_difference/max": 0.5077519416809082, + "sampling/sampling_logp_difference/mean": 0.010766430757939816, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.969436460873112e-05, + "clip_ratio/low_min": 5.969436460873112e-05, + "clip_ratio/region_mean": 5.969436460873112e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 582.875, + "completions/mean_terminated_length": 540.1538696289062, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.5091266073286533, + "epoch": 0.017665952890792293, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01115038525313139, + "learning_rate": 1e-05, + "loss": 0.0964, + "num_tokens": 777014.0, + "reward": 0.59375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.586151123046875, + "sampling/importance_sampling_ratio/mean": 0.9998085498809814, + "sampling/importance_sampling_ratio/min": 0.6728514432907104, + "sampling/sampling_logp_difference/max": 0.46131038665771484, + "sampling/sampling_logp_difference/mean": 0.013545024208724499, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.069010537932627e-05, + "clip_ratio/low_min": 4.069010537932627e-05, + "clip_ratio/region_mean": 4.069010537932627e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 685.46875, + "completions/mean_terminated_length": 629.0, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.4806034788489342, + "epoch": 0.018201284796573874, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012209653854370117, + "learning_rate": 1e-05, + "loss": 0.0312, + "num_tokens": 802837.0, + "reward": 0.40625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.41061532497406, + "sampling/importance_sampling_ratio/mean": 0.9998142719268799, + "sampling/importance_sampling_ratio/min": 0.716969907283783, + "sampling/sampling_logp_difference/max": 0.34402596950531006, + "sampling/sampling_logp_difference/mean": 0.012602795846760273, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 624.9375, + "completions/mean_terminated_length": 559.9091186523438, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.5183686465024948, + "epoch": 0.01873661670235546, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005360551178455353, + "learning_rate": 1e-05, + "loss": 0.0511, + "num_tokens": 826731.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5687141418457031, + "sampling/importance_sampling_ratio/mean": 0.9999753832817078, + "sampling/importance_sampling_ratio/min": 0.7450584173202515, + "sampling/sampling_logp_difference/max": 0.45025634765625, + "sampling/sampling_logp_difference/mean": 0.013193635269999504, + "step": 35 + }, + { + "clip_ratio/high_max": 6.695232877973467e-05, + "clip_ratio/high_mean": 6.695232877973467e-05, + "clip_ratio/low_mean": 4.41852243966423e-05, + "clip_ratio/low_min": 4.41852243966423e-05, + "clip_ratio/region_mean": 0.00011113755317637697, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 581.6875, + "completions/mean_terminated_length": 547.1851806640625, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 0.39413612335920334, + "epoch": 0.019271948608137045, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01678226888179779, + "learning_rate": 1e-05, + "loss": 0.112, + "num_tokens": 849001.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.382460594177246, + "sampling/importance_sampling_ratio/mean": 0.9995832443237305, + "sampling/importance_sampling_ratio/min": 0.6742712259292603, + "sampling/sampling_logp_difference/max": 0.394122838973999, + "sampling/sampling_logp_difference/mean": 0.01092858798801899, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 553.0, + "completions/mean_terminated_length": 530.7586059570312, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.38594045862555504, + "epoch": 0.01980728051391863, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005036050453782082, + "learning_rate": 1e-05, + "loss": 0.0155, + "num_tokens": 870657.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3651607036590576, + "sampling/importance_sampling_ratio/mean": 1.0000442266464233, + "sampling/importance_sampling_ratio/min": 0.7367088794708252, + "sampling/sampling_logp_difference/max": 0.31127214431762695, + "sampling/sampling_logp_difference/mean": 0.011241357773542404, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 622.15625, + "completions/mean_terminated_length": 476.3125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "entropy": 0.4367636665701866, + "epoch": 0.020342612419700215, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010261477902531624, + "learning_rate": 1e-05, + "loss": 0.0506, + "num_tokens": 894446.0, + "reward": 0.40625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3593658208847046, + "sampling/importance_sampling_ratio/mean": 1.0002293586730957, + "sampling/importance_sampling_ratio/min": 0.7748647332191467, + "sampling/sampling_logp_difference/max": 0.3070182800292969, + "sampling/sampling_logp_difference/mean": 0.012136059813201427, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.607445589499548e-05, + "clip_ratio/low_min": 4.607445589499548e-05, + "clip_ratio/region_mean": 4.607445589499548e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 591.90625, + "completions/mean_terminated_length": 523.0, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.3763354830443859, + "epoch": 0.0208779443254818, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007283422164618969, + "learning_rate": 1e-05, + "loss": 0.0209, + "num_tokens": 916947.0, + "reward": 0.4375, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4077885150909424, + "sampling/importance_sampling_ratio/mean": 1.000258445739746, + "sampling/importance_sampling_ratio/min": 0.6510267853736877, + "sampling/sampling_logp_difference/max": 0.42920446395874023, + "sampling/sampling_logp_difference/mean": 0.0101675596088171, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.553734106593765e-05, + "clip_ratio/low_min": 4.553734106593765e-05, + "clip_ratio/region_mean": 4.553734106593765e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 639.375, + "completions/mean_terminated_length": 525.8823852539062, + "completions/min_length": 358.0, + "completions/min_terminated_length": 358.0, + "entropy": 0.6507696583867073, + "epoch": 0.021413276231263382, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006135659758001566, + "learning_rate": 1e-05, + "loss": 0.0579, + "num_tokens": 941271.0, + "reward": 0.34375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2908605337142944, + "sampling/importance_sampling_ratio/mean": 0.9999883770942688, + "sampling/importance_sampling_ratio/min": 0.7787885665893555, + "sampling/sampling_logp_difference/max": 0.2553091049194336, + "sampling/sampling_logp_difference/mean": 0.016175610944628716, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011416822235332802, + "clip_ratio/low_min": 0.00011416822235332802, + "clip_ratio/region_mean": 0.00011416822235332802, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 580.96875, + "completions/mean_terminated_length": 495.9545593261719, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.44963598996400833, + "epoch": 0.021948608137044967, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006392901297658682, + "learning_rate": 1e-05, + "loss": 0.0577, + "num_tokens": 963374.0, + "reward": 0.40625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2743258476257324, + "sampling/importance_sampling_ratio/mean": 1.0001462697982788, + "sampling/importance_sampling_ratio/min": 0.788698673248291, + "sampling/sampling_logp_difference/max": 0.2424173355102539, + "sampling/sampling_logp_difference/mean": 0.012541564181447029, + "step": 41 + }, + { + "clip_ratio/high_max": 6.28456546110101e-05, + "clip_ratio/high_mean": 6.28456546110101e-05, + "clip_ratio/low_mean": 0.00010341246161260642, + "clip_ratio/low_min": 0.00010341246161260642, + "clip_ratio/region_mean": 0.00016625811622361653, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 553.0625, + "completions/mean_terminated_length": 522.357177734375, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3921343423426151, + "epoch": 0.022483940042826552, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008824815042316914, + "learning_rate": 1e-05, + "loss": 0.0089, + "num_tokens": 984496.0, + "reward": 0.6875, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3491756916046143, + "sampling/importance_sampling_ratio/mean": 1.0000351667404175, + "sampling/importance_sampling_ratio/min": 0.611833393573761, + "sampling/sampling_logp_difference/max": 0.49129533767700195, + "sampling/sampling_logp_difference/mean": 0.01122310571372509, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.25, + "completions/mean_terminated_length": 513.8181762695312, + "completions/min_length": 367.0, + "completions/min_terminated_length": 367.0, + "entropy": 0.372965756803751, + "epoch": 0.023019271948608137, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01738766022026539, + "learning_rate": 1e-05, + "loss": 0.009, + "num_tokens": 1007360.0, + "reward": 0.5625, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5648531913757324, + "sampling/importance_sampling_ratio/mean": 0.9999549984931946, + "sampling/importance_sampling_ratio/min": 0.5807793736457825, + "sampling/sampling_logp_difference/max": 0.543384313583374, + "sampling/sampling_logp_difference/mean": 0.010912786237895489, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.434629828319885e-05, + "clip_ratio/low_min": 9.434629828319885e-05, + "clip_ratio/region_mean": 9.434629828319885e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 613.5, + "completions/mean_terminated_length": 543.2727661132812, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3671180475503206, + "epoch": 0.023554603854389723, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011241797357797623, + "learning_rate": 1e-05, + "loss": 0.0035, + "num_tokens": 1030720.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.35966956615448, + "sampling/importance_sampling_ratio/mean": 0.9995852112770081, + "sampling/importance_sampling_ratio/min": 0.728817343711853, + "sampling/sampling_logp_difference/max": 0.3163321018218994, + "sampling/sampling_logp_difference/mean": 0.009871450252830982, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.3614792957669124e-05, + "clip_ratio/low_min": 4.3614792957669124e-05, + "clip_ratio/region_mean": 4.3614792957669124e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 643.15625, + "completions/mean_terminated_length": 586.4091186523438, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "entropy": 0.3872305490076542, + "epoch": 0.024089935760171308, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007765679154545069, + "learning_rate": 1e-05, + "loss": 0.0361, + "num_tokens": 1055429.0, + "reward": 0.5625, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.9600664377212524, + "sampling/importance_sampling_ratio/mean": 0.9999384880065918, + "sampling/importance_sampling_ratio/min": 0.6967722177505493, + "sampling/sampling_logp_difference/max": 0.672978401184082, + "sampling/sampling_logp_difference/mean": 0.011130640283226967, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 672.6875, + "completions/mean_terminated_length": 588.5882568359375, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 0.48364455066621304, + "epoch": 0.02462526766595289, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019248412922024727, + "learning_rate": 1e-05, + "loss": 0.0493, + "num_tokens": 1080971.0, + "reward": 0.5, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3275351524353027, + "sampling/importance_sampling_ratio/mean": 0.9994391798973083, + "sampling/importance_sampling_ratio/min": 0.7093425393104553, + "sampling/sampling_logp_difference/max": 0.343416690826416, + "sampling/sampling_logp_difference/mean": 0.01235866080969572, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 534.1875, + "completions/mean_terminated_length": 500.7857360839844, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.5136398896574974, + "epoch": 0.025160599571734475, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018716780468821526, + "learning_rate": 1e-05, + "loss": 0.0673, + "num_tokens": 1101825.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.299971580505371, + "sampling/importance_sampling_ratio/mean": 0.9997742772102356, + "sampling/importance_sampling_ratio/min": 0.7442049980163574, + "sampling/sampling_logp_difference/max": 0.2954387664794922, + "sampling/sampling_logp_difference/mean": 0.013368945568799973, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.494505603564903e-05, + "clip_ratio/low_min": 5.494505603564903e-05, + "clip_ratio/region_mean": 5.494505603564903e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 534.5625, + "completions/mean_terminated_length": 501.21429443359375, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.3989657089114189, + "epoch": 0.02569593147751606, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008748093619942665, + "learning_rate": 1e-05, + "loss": 0.0894, + "num_tokens": 1122067.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3007398843765259, + "sampling/importance_sampling_ratio/mean": 0.9996334910392761, + "sampling/importance_sampling_ratio/min": 0.765166699886322, + "sampling/sampling_logp_difference/max": 0.26766157150268555, + "sampling/sampling_logp_difference/mean": 0.011446605436503887, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.698820652673021e-05, + "clip_ratio/low_min": 6.698820652673021e-05, + "clip_ratio/region_mean": 6.698820652673021e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 586.5625, + "completions/mean_terminated_length": 426.4705810546875, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.6239189095795155, + "epoch": 0.026231263383297645, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01232483796775341, + "learning_rate": 1e-05, + "loss": 0.02, + "num_tokens": 1144773.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4282665252685547, + "sampling/importance_sampling_ratio/mean": 0.9999468922615051, + "sampling/importance_sampling_ratio/min": 0.7118720412254333, + "sampling/sampling_logp_difference/max": 0.3564615249633789, + "sampling/sampling_logp_difference/mean": 0.016130099073052406, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.6023564209463075e-05, + "clip_ratio/low_min": 4.6023564209463075e-05, + "clip_ratio/region_mean": 4.6023564209463075e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 605.1875, + "completions/mean_terminated_length": 541.478271484375, + "completions/min_length": 409.0, + "completions/min_terminated_length": 409.0, + "entropy": 0.43123210966587067, + "epoch": 0.02676659528907923, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01060446910560131, + "learning_rate": 1e-05, + "loss": 0.046, + "num_tokens": 1167691.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.429684042930603, + "sampling/importance_sampling_ratio/mean": 1.000015377998352, + "sampling/importance_sampling_ratio/min": 0.7132202982902527, + "sampling/sampling_logp_difference/max": 0.35745346546173096, + "sampling/sampling_logp_difference/mean": 0.011622624471783638, + "step": 50 + }, + { + "clip_ratio/high_max": 4.822530900128186e-05, + "clip_ratio/high_mean": 4.822530900128186e-05, + "clip_ratio/low_mean": 0.0001036484245560132, + "clip_ratio/low_min": 0.0001036484245560132, + "clip_ratio/region_mean": 0.00015187373355729505, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 614.28125, + "completions/mean_terminated_length": 585.8148193359375, + "completions/min_length": 358.0, + "completions/min_terminated_length": 358.0, + "entropy": 0.3837196100503206, + "epoch": 0.027301927194860815, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009614828042685986, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 1191244.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.337222933769226, + "sampling/importance_sampling_ratio/mean": 0.9998320937156677, + "sampling/importance_sampling_ratio/min": 0.7234455943107605, + "sampling/sampling_logp_difference/max": 0.3237299919128418, + "sampling/sampling_logp_difference/mean": 0.010459691286087036, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 523.34375, + "completions/mean_terminated_length": 478.03704833984375, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.3651442211121321, + "epoch": 0.027837259100642397, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01307887677103281, + "learning_rate": 1e-05, + "loss": 0.0541, + "num_tokens": 1211647.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2707027196884155, + "sampling/importance_sampling_ratio/mean": 1.0000180006027222, + "sampling/importance_sampling_ratio/min": 0.7189217209815979, + "sampling/sampling_logp_difference/max": 0.33000290393829346, + "sampling/sampling_logp_difference/mean": 0.01065903715789318, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.2130097426706925e-05, + "clip_ratio/low_min": 4.2130097426706925e-05, + "clip_ratio/region_mean": 4.2130097426706925e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 627.09375, + "completions/mean_terminated_length": 563.0454711914062, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.4228546507656574, + "epoch": 0.028372591006423982, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013921543024480343, + "learning_rate": 1e-05, + "loss": 0.0994, + "num_tokens": 1235306.0, + "reward": 0.6875, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3246186971664429, + "sampling/importance_sampling_ratio/mean": 1.0000325441360474, + "sampling/importance_sampling_ratio/min": 0.6458011269569397, + "sampling/sampling_logp_difference/max": 0.43726372718811035, + "sampling/sampling_logp_difference/mean": 0.01152092032134533, + "step": 53 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00024238396872533485, + "clip_ratio/low_min": 0.00024238396872533485, + "clip_ratio/region_mean": 0.00024238396872533485, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 643.28125, + "completions/mean_terminated_length": 568.4500122070312, + "completions/min_length": 362.0, + "completions/min_terminated_length": 362.0, + "entropy": 0.4607427045702934, + "epoch": 0.028907922912205567, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017727605998516083, + "learning_rate": 1e-05, + "loss": 0.0639, + "num_tokens": 1259675.0, + "reward": 0.46875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6147571802139282, + "sampling/importance_sampling_ratio/mean": 0.999921441078186, + "sampling/importance_sampling_ratio/min": 0.6753311157226562, + "sampling/sampling_logp_difference/max": 0.479184627532959, + "sampling/sampling_logp_difference/mean": 0.012804209254682064, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.4626918679568917e-05, + "clip_ratio/low_min": 4.4626918679568917e-05, + "clip_ratio/region_mean": 4.4626918679568917e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 616.46875, + "completions/mean_terminated_length": 537.0952758789062, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.44960571825504303, + "epoch": 0.029443254817987152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006553029175847769, + "learning_rate": 1e-05, + "loss": 0.1434, + "num_tokens": 1283146.0, + "reward": 0.625, + "reward_std": 0.48503684997558594, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2985329627990723, + "sampling/importance_sampling_ratio/mean": 1.0000635385513306, + "sampling/importance_sampling_ratio/min": 0.699820339679718, + "sampling/sampling_logp_difference/max": 0.3569316864013672, + "sampling/sampling_logp_difference/mean": 0.012360668741166592, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.5236410844372585e-05, + "clip_ratio/low_min": 5.5236410844372585e-05, + "clip_ratio/region_mean": 5.5236410844372585e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 598.34375, + "completions/mean_terminated_length": 496.5500183105469, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.6260490566492081, + "epoch": 0.029978586723768737, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034410294611006975, + "learning_rate": 1e-05, + "loss": 0.0055, + "num_tokens": 1306053.0, + "reward": 0.46875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3745598793029785, + "sampling/importance_sampling_ratio/mean": 1.0001850128173828, + "sampling/importance_sampling_ratio/min": 0.7050579190254211, + "sampling/sampling_logp_difference/max": 0.3494753837585449, + "sampling/sampling_logp_difference/mean": 0.01465124636888504, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 483.21875, + "completions/mean_terminated_length": 464.2333679199219, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.4522119015455246, + "epoch": 0.030513918629550323, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0114, + "num_tokens": 1324756.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4202548265457153, + "sampling/importance_sampling_ratio/mean": 1.0000139474868774, + "sampling/importance_sampling_ratio/min": 0.6987296342849731, + "sampling/sampling_logp_difference/max": 0.3584914207458496, + "sampling/sampling_logp_difference/mean": 0.012116029858589172, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.6957175072748214e-05, + "clip_ratio/low_min": 4.6957175072748214e-05, + "clip_ratio/region_mean": 4.6957175072748214e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 646.40625, + "completions/mean_terminated_length": 582.7142944335938, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "entropy": 0.4879292882978916, + "epoch": 0.031049250535331904, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008696580305695534, + "learning_rate": 1e-05, + "loss": -0.0096, + "num_tokens": 1349505.0, + "reward": 0.5625, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5511821508407593, + "sampling/importance_sampling_ratio/mean": 0.9999637007713318, + "sampling/importance_sampling_ratio/min": 0.6396264433860779, + "sampling/sampling_logp_difference/max": 0.4468710422515869, + "sampling/sampling_logp_difference/mean": 0.012860474176704884, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 520.75, + "completions/mean_terminated_length": 463.69232177734375, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.40757014974951744, + "epoch": 0.03158458244111349, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009540090337395668, + "learning_rate": 1e-05, + "loss": 0.0508, + "num_tokens": 1370033.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4402263164520264, + "sampling/importance_sampling_ratio/mean": 1.000462293624878, + "sampling/importance_sampling_ratio/min": 0.5509796738624573, + "sampling/sampling_logp_difference/max": 0.5960574150085449, + "sampling/sampling_logp_difference/mean": 0.01166108250617981, + "step": 59 + }, + { + "clip_ratio/high_max": 4.752851600642316e-05, + "clip_ratio/high_mean": 4.752851600642316e-05, + "clip_ratio/low_mean": 4.07298794016242e-05, + "clip_ratio/low_min": 4.07298794016242e-05, + "clip_ratio/region_mean": 8.825839540804736e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 623.84375, + "completions/mean_terminated_length": 537.3500366210938, + "completions/min_length": 397.0, + "completions/min_terminated_length": 397.0, + "entropy": 0.5110281445086002, + "epoch": 0.032119914346895075, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008212021552026272, + "learning_rate": 1e-05, + "loss": 0.0801, + "num_tokens": 1394372.0, + "reward": 0.59375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3844656944274902, + "sampling/importance_sampling_ratio/mean": 1.0003094673156738, + "sampling/importance_sampling_ratio/min": 0.70673006772995, + "sampling/sampling_logp_difference/max": 0.3471064567565918, + "sampling/sampling_logp_difference/mean": 0.014087519608438015, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 563.875, + "completions/mean_terminated_length": 484.0, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.42480451986193657, + "epoch": 0.032655246252676656, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004322219640016556, + "learning_rate": 1e-05, + "loss": -0.0194, + "num_tokens": 1416256.0, + "reward": 0.53125, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3357380628585815, + "sampling/importance_sampling_ratio/mean": 0.9996961951255798, + "sampling/importance_sampling_ratio/min": 0.7725263833999634, + "sampling/sampling_logp_difference/max": 0.28948402404785156, + "sampling/sampling_logp_difference/mean": 0.01185667235404253, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.028157829656266e-05, + "clip_ratio/low_min": 5.028157829656266e-05, + "clip_ratio/region_mean": 5.028157829656266e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 512.8125, + "completions/mean_terminated_length": 453.923095703125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "entropy": 0.5509180948138237, + "epoch": 0.033190578158458245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009267719462513924, + "learning_rate": 1e-05, + "loss": 0.0678, + "num_tokens": 1436114.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.281874418258667, + "sampling/importance_sampling_ratio/mean": 0.9997726678848267, + "sampling/importance_sampling_ratio/min": 0.434792697429657, + "sampling/sampling_logp_difference/max": 0.8328859806060791, + "sampling/sampling_logp_difference/mean": 0.014345284551382065, + "step": 62 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011029374581994489, + "clip_ratio/low_min": 0.00011029374581994489, + "clip_ratio/region_mean": 0.00011029374581994489, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 588.0625, + "completions/mean_terminated_length": 493.8095397949219, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.5057032033801079, + "epoch": 0.03372591006423983, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005740536376833916, + "learning_rate": 1e-05, + "loss": 0.0254, + "num_tokens": 1458724.0, + "reward": 0.40625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5643742084503174, + "sampling/importance_sampling_ratio/mean": 1.0002732276916504, + "sampling/importance_sampling_ratio/min": 0.5843526124954224, + "sampling/sampling_logp_difference/max": 0.5372506380081177, + "sampling/sampling_logp_difference/mean": 0.013789980672299862, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019218432862544432, + "clip_ratio/low_min": 0.00019218432862544432, + "clip_ratio/region_mean": 0.00019218432862544432, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 618.875, + "completions/mean_terminated_length": 540.7619018554688, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.5076218545436859, + "epoch": 0.034261241970021415, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008714552037417889, + "learning_rate": 1e-05, + "loss": 0.015, + "num_tokens": 1482432.0, + "reward": 0.46875, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4101232290267944, + "sampling/importance_sampling_ratio/mean": 1.0002740621566772, + "sampling/importance_sampling_ratio/min": 0.5970253348350525, + "sampling/sampling_logp_difference/max": 0.5157957077026367, + "sampling/sampling_logp_difference/mean": 0.013612430542707443, + "step": 64 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.223867891705595e-05, + "clip_ratio/low_min": 9.223867891705595e-05, + "clip_ratio/region_mean": 9.223867891705595e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 632.71875, + "completions/mean_terminated_length": 579.7825927734375, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.38875191286206245, + "epoch": 0.034796573875803, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009033304639160633, + "learning_rate": 1e-05, + "loss": 0.0495, + "num_tokens": 1506431.0, + "reward": 0.6875, + "reward_std": 0.3924052119255066, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4655523300170898, + "sampling/importance_sampling_ratio/mean": 1.000010371208191, + "sampling/importance_sampling_ratio/min": 0.682938814163208, + "sampling/sampling_logp_difference/max": 0.3822321891784668, + "sampling/sampling_logp_difference/mean": 0.011259174905717373, + "step": 65 + }, + { + "clip_ratio/high_max": 4.552075915853493e-05, + "clip_ratio/high_mean": 4.552075915853493e-05, + "clip_ratio/low_mean": 5.34645005245693e-05, + "clip_ratio/low_min": 5.34645005245693e-05, + "clip_ratio/region_mean": 9.898525968310423e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 606.21875, + "completions/mean_terminated_length": 532.6818237304688, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.42484213784337044, + "epoch": 0.035331905781584586, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02042962983250618, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 1529622.0, + "reward": 0.75, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4566490650177002, + "sampling/importance_sampling_ratio/mean": 1.0000078678131104, + "sampling/importance_sampling_ratio/min": 0.5184912085533142, + "sampling/sampling_logp_difference/max": 0.656832218170166, + "sampling/sampling_logp_difference/mean": 0.011515411548316479, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.318500865949318e-05, + "clip_ratio/low_min": 7.318500865949318e-05, + "clip_ratio/region_mean": 7.318500865949318e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 590.625, + "completions/mean_terminated_length": 452.6666564941406, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.5239551886916161, + "epoch": 0.03586723768736617, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003302964847534895, + "learning_rate": 1e-05, + "loss": 0.0211, + "num_tokens": 1552554.0, + "reward": 0.15625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4207777976989746, + "sampling/importance_sampling_ratio/mean": 0.9995875954627991, + "sampling/importance_sampling_ratio/min": 0.3964710533618927, + "sampling/sampling_logp_difference/max": 0.9251523017883301, + "sampling/sampling_logp_difference/mean": 0.013529600575566292, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 539.6875, + "completions/mean_terminated_length": 475.7599792480469, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.5009765848517418, + "epoch": 0.03640256959314775, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0059877983294427395, + "learning_rate": 1e-05, + "loss": 0.0903, + "num_tokens": 1573376.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4059853553771973, + "sampling/importance_sampling_ratio/mean": 1.000280499458313, + "sampling/importance_sampling_ratio/min": 0.581082820892334, + "sampling/sampling_logp_difference/max": 0.5428619384765625, + "sampling/sampling_logp_difference/mean": 0.013365201652050018, + "step": 68 + }, + { + "clip_ratio/high_max": 4.892368087894283e-05, + "clip_ratio/high_mean": 4.892368087894283e-05, + "clip_ratio/low_mean": 6.33874224149622e-05, + "clip_ratio/low_min": 6.33874224149622e-05, + "clip_ratio/region_mean": 0.00011231110329390503, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 551.25, + "completions/mean_terminated_length": 511.1111145019531, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.3620293475687504, + "epoch": 0.03693790149892934, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0056429170072078705, + "learning_rate": 1e-05, + "loss": 0.0131, + "num_tokens": 1594616.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4040403366088867, + "sampling/importance_sampling_ratio/mean": 1.000301718711853, + "sampling/importance_sampling_ratio/min": 0.7337977290153503, + "sampling/sampling_logp_difference/max": 0.3393540382385254, + "sampling/sampling_logp_difference/mean": 0.010295838117599487, + "step": 69 + }, + { + "clip_ratio/high_max": 6.225099787116051e-05, + "clip_ratio/high_mean": 6.225099787116051e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.225099787116051e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 522.4375, + "completions/mean_terminated_length": 514.51611328125, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 0.28916994854807854, + "epoch": 0.03747323340471092, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002398902550339699, + "learning_rate": 1e-05, + "loss": 0.0202, + "num_tokens": 1614702.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4625073671340942, + "sampling/importance_sampling_ratio/mean": 1.0001654624938965, + "sampling/importance_sampling_ratio/min": 0.7339086532592773, + "sampling/sampling_logp_difference/max": 0.3801523447036743, + "sampling/sampling_logp_difference/mean": 0.008633803576231003, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 519.28125, + "completions/mean_terminated_length": 502.70001220703125, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.4071941450238228, + "epoch": 0.03800856531049251, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017797451466321945, + "learning_rate": 1e-05, + "loss": 0.0142, + "num_tokens": 1634791.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.303678274154663, + "sampling/importance_sampling_ratio/mean": 0.9998772740364075, + "sampling/importance_sampling_ratio/min": 0.7757813334465027, + "sampling/sampling_logp_difference/max": 0.26518964767456055, + "sampling/sampling_logp_difference/mean": 0.011396494694054127, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.0792361435014755e-05, + "clip_ratio/low_min": 5.0792361435014755e-05, + "clip_ratio/region_mean": 5.0792361435014755e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 524.65625, + "completions/mean_terminated_length": 468.5000305175781, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.47270843386650085, + "epoch": 0.03854389721627409, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006420954596251249, + "learning_rate": 1e-05, + "loss": 0.0073, + "num_tokens": 1655068.0, + "reward": 0.40625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2891714572906494, + "sampling/importance_sampling_ratio/mean": 0.999759316444397, + "sampling/importance_sampling_ratio/min": 0.5663257241249084, + "sampling/sampling_logp_difference/max": 0.5685858726501465, + "sampling/sampling_logp_difference/mean": 0.012472325935959816, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.17812768521253e-05, + "clip_ratio/low_min": 5.17812768521253e-05, + "clip_ratio/region_mean": 5.17812768521253e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 615.8125, + "completions/mean_terminated_length": 511.6842041015625, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "entropy": 0.4182238429784775, + "epoch": 0.03907922912205567, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012820947915315628, + "learning_rate": 1e-05, + "loss": 0.038, + "num_tokens": 1678062.0, + "reward": 0.46875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.8784372806549072, + "sampling/importance_sampling_ratio/mean": 0.9999639391899109, + "sampling/importance_sampling_ratio/min": 0.7614421844482422, + "sampling/sampling_logp_difference/max": 0.6304402351379395, + "sampling/sampling_logp_difference/mean": 0.012042717076838017, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017125568774645217, + "clip_ratio/low_min": 0.00017125568774645217, + "clip_ratio/region_mean": 0.00017125568774645217, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 505.3125, + "completions/mean_terminated_length": 478.137939453125, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.43436403200030327, + "epoch": 0.03961456102783726, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003982620779424906, + "learning_rate": 1e-05, + "loss": 0.0692, + "num_tokens": 1698096.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.486291527748108, + "sampling/importance_sampling_ratio/mean": 0.9996992349624634, + "sampling/importance_sampling_ratio/min": 0.5546742081642151, + "sampling/sampling_logp_difference/max": 0.589374303817749, + "sampling/sampling_logp_difference/mean": 0.011237709783017635, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9058086005970836e-05, + "clip_ratio/low_min": 4.9058086005970836e-05, + "clip_ratio/region_mean": 4.9058086005970836e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 560.59375, + "completions/mean_terminated_length": 522.1851806640625, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.41883276030421257, + "epoch": 0.04014989293361884, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010470947250723839, + "learning_rate": 1e-05, + "loss": 0.0191, + "num_tokens": 1719403.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.940041184425354, + "sampling/importance_sampling_ratio/mean": 0.9999339580535889, + "sampling/importance_sampling_ratio/min": 0.7019378542900085, + "sampling/sampling_logp_difference/max": 0.6627092361450195, + "sampling/sampling_logp_difference/mean": 0.010889444500207901, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.069010537932627e-05, + "clip_ratio/low_min": 4.069010537932627e-05, + "clip_ratio/region_mean": 4.069010537932627e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 581.96875, + "completions/mean_terminated_length": 484.5238037109375, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.4762725979089737, + "epoch": 0.04068522483940043, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014987333677709103, + "learning_rate": 1e-05, + "loss": 0.0228, + "num_tokens": 1742426.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3577488660812378, + "sampling/importance_sampling_ratio/mean": 0.9997811913490295, + "sampling/importance_sampling_ratio/min": 0.6076112389564514, + "sampling/sampling_logp_difference/max": 0.49821996688842773, + "sampling/sampling_logp_difference/mean": 0.013569815084338188, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014673568512080237, + "clip_ratio/low_min": 0.00014673568512080237, + "clip_ratio/region_mean": 0.00014673568512080237, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 648.84375, + "completions/mean_terminated_length": 556.1666870117188, + "completions/min_length": 380.0, + "completions/min_terminated_length": 380.0, + "entropy": 0.5342629104852676, + "epoch": 0.04122055674518201, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013919983990490437, + "learning_rate": 1e-05, + "loss": 0.0549, + "num_tokens": 1767157.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.386790156364441, + "sampling/importance_sampling_ratio/mean": 0.9998939037322998, + "sampling/importance_sampling_ratio/min": 0.5932039618492126, + "sampling/sampling_logp_difference/max": 0.5222170352935791, + "sampling/sampling_logp_difference/mean": 0.013663308694958687, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.046427031629719e-05, + "clip_ratio/low_min": 5.046427031629719e-05, + "clip_ratio/region_mean": 5.046427031629719e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 528.90625, + "completions/mean_terminated_length": 435.34783935546875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.5320987552404404, + "epoch": 0.0417558886509636, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0025, + "num_tokens": 1788234.0, + "reward": 0.53125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.416995644569397, + "sampling/importance_sampling_ratio/mean": 0.9996212720870972, + "sampling/importance_sampling_ratio/min": 0.5916980504989624, + "sampling/sampling_logp_difference/max": 0.5247588157653809, + "sampling/sampling_logp_difference/mean": 0.014332974329590797, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019893924036296085, + "clip_ratio/low_min": 0.00019893924036296085, + "clip_ratio/region_mean": 0.00019893924036296085, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 624.8125, + "completions/mean_terminated_length": 577.0833740234375, + "completions/min_length": 398.0, + "completions/min_terminated_length": 398.0, + "entropy": 0.3578733876347542, + "epoch": 0.04229122055674518, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006287885829806328, + "learning_rate": 1e-05, + "loss": 0.0442, + "num_tokens": 1811620.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2843456268310547, + "sampling/importance_sampling_ratio/mean": 0.9999666810035706, + "sampling/importance_sampling_ratio/min": 0.775160014629364, + "sampling/sampling_logp_difference/max": 0.2546858787536621, + "sampling/sampling_logp_difference/mean": 0.010287775658071041, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 622.03125, + "completions/mean_terminated_length": 555.6818237304688, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 0.5975942797958851, + "epoch": 0.042826552462526764, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011447020806372166, + "learning_rate": 1e-05, + "loss": -0.0076, + "num_tokens": 1835797.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3627763986587524, + "sampling/importance_sampling_ratio/mean": 1.0000898838043213, + "sampling/importance_sampling_ratio/min": 0.4534623622894287, + "sampling/sampling_logp_difference/max": 0.7908430099487305, + "sampling/sampling_logp_difference/mean": 0.013825246132910252, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010936133185168728, + "clip_ratio/low_min": 0.00010936133185168728, + "clip_ratio/region_mean": 0.00010936133185168728, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 543.21875, + "completions/mean_terminated_length": 491.3461608886719, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.4437571093440056, + "epoch": 0.04336188436830835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010027516633272171, + "learning_rate": 1e-05, + "loss": 0.1184, + "num_tokens": 1857156.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3923063278198242, + "sampling/importance_sampling_ratio/mean": 1.000060796737671, + "sampling/importance_sampling_ratio/min": 0.6629504561424255, + "sampling/sampling_logp_difference/max": 0.4110550880432129, + "sampling/sampling_logp_difference/mean": 0.011959006078541279, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019894931028829888, + "clip_ratio/low_min": 0.00019894931028829888, + "clip_ratio/region_mean": 0.00019894931028829888, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 516.21875, + "completions/mean_terminated_length": 458.1153869628906, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.378199927508831, + "epoch": 0.043897216274089934, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011769967153668404, + "learning_rate": 1e-05, + "loss": -0.0326, + "num_tokens": 1877275.0, + "reward": 0.5625, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4055572748184204, + "sampling/importance_sampling_ratio/mean": 1.0005459785461426, + "sampling/importance_sampling_ratio/min": 0.7398213744163513, + "sampling/sampling_logp_difference/max": 0.34043383598327637, + "sampling/sampling_logp_difference/mean": 0.010453764349222183, + "step": 82 + }, + { + "clip_ratio/high_max": 4.6125460357870907e-05, + "clip_ratio/high_mean": 4.6125460357870907e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.6125460357870907e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 648.78125, + "completions/mean_terminated_length": 615.3999633789062, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.3781026266515255, + "epoch": 0.04443254817987152, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010206066071987152, + "learning_rate": 1e-05, + "loss": 0.0189, + "num_tokens": 1901892.0, + "reward": 0.59375, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3806746006011963, + "sampling/importance_sampling_ratio/mean": 1.000369668006897, + "sampling/importance_sampling_ratio/min": 0.6630758047103882, + "sampling/sampling_logp_difference/max": 0.41086602210998535, + "sampling/sampling_logp_difference/mean": 0.01089530996978283, + "step": 83 + }, + { + "clip_ratio/high_max": 5.763024455518462e-05, + "clip_ratio/high_mean": 5.763024455518462e-05, + "clip_ratio/low_mean": 4.48028658865951e-05, + "clip_ratio/low_min": 4.48028658865951e-05, + "clip_ratio/region_mean": 0.00010243311044177972, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 649.71875, + "completions/mean_terminated_length": 578.75, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.5005024299025536, + "epoch": 0.044967880085653104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008646410889923573, + "learning_rate": 1e-05, + "loss": 0.0632, + "num_tokens": 1926651.0, + "reward": 0.4375, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4742728471755981, + "sampling/importance_sampling_ratio/mean": 0.9998708367347717, + "sampling/importance_sampling_ratio/min": 0.5484101176261902, + "sampling/sampling_logp_difference/max": 0.6007318496704102, + "sampling/sampling_logp_difference/mean": 0.012708599679172039, + "step": 84 + }, + { + "clip_ratio/high_max": 6.589351687580347e-05, + "clip_ratio/high_mean": 6.589351687580347e-05, + "clip_ratio/low_mean": 6.713211769238114e-05, + "clip_ratio/low_min": 6.713211769238114e-05, + "clip_ratio/region_mean": 0.00013302563456818461, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 524.90625, + "completions/mean_terminated_length": 499.75860595703125, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.35518455505371094, + "epoch": 0.045503211991434686, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003262939862906933, + "learning_rate": 1e-05, + "loss": 0.0258, + "num_tokens": 1947264.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3010053634643555, + "sampling/importance_sampling_ratio/mean": 0.9998657703399658, + "sampling/importance_sampling_ratio/min": 0.7846524715423584, + "sampling/sampling_logp_difference/max": 0.2631373405456543, + "sampling/sampling_logp_difference/mean": 0.00970697682350874, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011831519077531993, + "clip_ratio/low_min": 0.00011831519077531993, + "clip_ratio/region_mean": 0.00011831519077531993, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 515.53125, + "completions/mean_terminated_length": 498.70001220703125, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.41699234768748283, + "epoch": 0.046038543897216275, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01781095378100872, + "learning_rate": 1e-05, + "loss": 0.023, + "num_tokens": 1967001.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3767662048339844, + "sampling/importance_sampling_ratio/mean": 1.0002977848052979, + "sampling/importance_sampling_ratio/min": 0.6516874432563782, + "sampling/sampling_logp_difference/max": 0.4281902313232422, + "sampling/sampling_logp_difference/mean": 0.012085186317563057, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014681524044135585, + "clip_ratio/low_min": 0.00014681524044135585, + "clip_ratio/region_mean": 0.00014681524044135585, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 597.625, + "completions/mean_terminated_length": 540.8333740234375, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.3625175543129444, + "epoch": 0.046573875802997856, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0166816096752882, + "learning_rate": 1e-05, + "loss": 0.0258, + "num_tokens": 1990149.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3373417854309082, + "sampling/importance_sampling_ratio/mean": 1.0002481937408447, + "sampling/importance_sampling_ratio/min": 0.7186197638511658, + "sampling/sampling_logp_difference/max": 0.3304229974746704, + "sampling/sampling_logp_difference/mean": 0.010592046193778515, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016476913879159838, + "clip_ratio/low_min": 0.00016476913879159838, + "clip_ratio/region_mean": 0.00016476913879159838, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 716.65625, + "completions/mean_terminated_length": 671.3529663085938, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "entropy": 0.4938277080655098, + "epoch": 0.047109207708779445, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011431061662733555, + "learning_rate": 1e-05, + "loss": 0.028, + "num_tokens": 2017066.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6422104835510254, + "sampling/importance_sampling_ratio/mean": 1.0000323057174683, + "sampling/importance_sampling_ratio/min": 0.7556983828544617, + "sampling/sampling_logp_difference/max": 0.49604320526123047, + "sampling/sampling_logp_difference/mean": 0.012017695233225822, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.4404972868505865e-05, + "clip_ratio/low_min": 4.4404972868505865e-05, + "clip_ratio/region_mean": 4.4404972868505865e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 667.71875, + "completions/mean_terminated_length": 589.7222290039062, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 0.5489422045648098, + "epoch": 0.04764453961456103, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014813964255154133, + "learning_rate": 1e-05, + "loss": 0.0558, + "num_tokens": 2042441.0, + "reward": 0.5, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3826590776443481, + "sampling/importance_sampling_ratio/mean": 1.000160574913025, + "sampling/importance_sampling_ratio/min": 0.14336563646793365, + "sampling/sampling_logp_difference/max": 1.942357063293457, + "sampling/sampling_logp_difference/mean": 0.014316913671791553, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.70278527145274e-05, + "clip_ratio/low_min": 8.70278527145274e-05, + "clip_ratio/region_mean": 8.70278527145274e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 651.3125, + "completions/mean_terminated_length": 534.625, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.5815541483461857, + "epoch": 0.048179871520342615, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00825833436101675, + "learning_rate": 1e-05, + "loss": 0.0972, + "num_tokens": 2067611.0, + "reward": 0.375, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7591688632965088, + "sampling/importance_sampling_ratio/mean": 1.0003857612609863, + "sampling/importance_sampling_ratio/min": 0.6370046734809875, + "sampling/sampling_logp_difference/max": 0.5648415088653564, + "sampling/sampling_logp_difference/mean": 0.013535372912883759, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 558.21875, + "completions/mean_terminated_length": 476.13043212890625, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "entropy": 0.40279893949627876, + "epoch": 0.0487152034261242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01722664013504982, + "learning_rate": 1e-05, + "loss": -0.0101, + "num_tokens": 2089202.0, + "reward": 0.5625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4850578308105469, + "sampling/importance_sampling_ratio/mean": 0.9998269081115723, + "sampling/importance_sampling_ratio/min": 0.607376754283905, + "sampling/sampling_logp_difference/max": 0.49860596656799316, + "sampling/sampling_logp_difference/mean": 0.011388967745006084, + "step": 91 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00024040970674832352, + "clip_ratio/low_min": 0.00024040970674832352, + "clip_ratio/region_mean": 0.00024040970674832352, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 526.375, + "completions/mean_terminated_length": 491.857177734375, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.513542901724577, + "epoch": 0.04925053533190578, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00837272685021162, + "learning_rate": 1e-05, + "loss": 0.0441, + "num_tokens": 2109454.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.8132972717285156, + "sampling/importance_sampling_ratio/mean": 0.9997177720069885, + "sampling/importance_sampling_ratio/min": 0.5944162607192993, + "sampling/sampling_logp_difference/max": 0.595146894454956, + "sampling/sampling_logp_difference/mean": 0.013096220791339874, + "step": 92 + }, + { + "clip_ratio/high_max": 9.506726928520948e-05, + "clip_ratio/high_mean": 9.506726928520948e-05, + "clip_ratio/low_mean": 4.1145489376503974e-05, + "clip_ratio/low_min": 4.1145489376503974e-05, + "clip_ratio/region_mean": 0.00013621275866171345, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 634.53125, + "completions/mean_terminated_length": 573.8636474609375, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.4839530698955059, + "epoch": 0.04978586723768737, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014939776621758938, + "learning_rate": 1e-05, + "loss": 0.0643, + "num_tokens": 2133583.0, + "reward": 0.53125, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6251133680343628, + "sampling/importance_sampling_ratio/mean": 0.9999118447303772, + "sampling/importance_sampling_ratio/min": 0.7049896121025085, + "sampling/sampling_logp_difference/max": 0.4855775833129883, + "sampling/sampling_logp_difference/mean": 0.012979834340512753, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 678.375, + "completions/mean_terminated_length": 588.75, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.5444022342562675, + "epoch": 0.05032119914346895, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005601354409009218, + "learning_rate": 1e-05, + "loss": 0.0001, + "num_tokens": 2159131.0, + "reward": 0.46875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.460326075553894, + "sampling/importance_sampling_ratio/mean": 1.0002961158752441, + "sampling/importance_sampling_ratio/min": 0.7336817383766174, + "sampling/sampling_logp_difference/max": 0.378659725189209, + "sampling/sampling_logp_difference/mean": 0.01323297806084156, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 547.9375, + "completions/mean_terminated_length": 525.1724243164062, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.35648392140865326, + "epoch": 0.05085653104925054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005154947284609079, + "learning_rate": 1e-05, + "loss": 0.1319, + "num_tokens": 2179857.0, + "reward": 0.8125, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.26478111743927, + "sampling/importance_sampling_ratio/mean": 0.9999009966850281, + "sampling/importance_sampling_ratio/min": 0.61456698179245, + "sampling/sampling_logp_difference/max": 0.48683738708496094, + "sampling/sampling_logp_difference/mean": 0.009781748987734318, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.002801430644467e-05, + "clip_ratio/low_min": 7.002801430644467e-05, + "clip_ratio/region_mean": 7.002801430644467e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 534.8125, + "completions/mean_terminated_length": 491.629638671875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.5257068872451782, + "epoch": 0.05139186295503212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02136640064418316, + "learning_rate": 1e-05, + "loss": 0.1841, + "num_tokens": 2200491.0, + "reward": 0.71875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2795007228851318, + "sampling/importance_sampling_ratio/mean": 1.0003798007965088, + "sampling/importance_sampling_ratio/min": 0.725347101688385, + "sampling/sampling_logp_difference/max": 0.3211050033569336, + "sampling/sampling_logp_difference/mean": 0.01373940147459507, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 617.6875, + "completions/mean_terminated_length": 567.5833740234375, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.3262052573263645, + "epoch": 0.05192719486081371, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006677660625427961, + "learning_rate": 1e-05, + "loss": 0.0107, + "num_tokens": 2223961.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4547582864761353, + "sampling/importance_sampling_ratio/mean": 0.9997563362121582, + "sampling/importance_sampling_ratio/min": 0.6356496214866638, + "sampling/sampling_logp_difference/max": 0.4531078338623047, + "sampling/sampling_logp_difference/mean": 0.009383311495184898, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 547.15625, + "completions/mean_terminated_length": 532.433349609375, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.32807236537337303, + "epoch": 0.05246252676659529, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0381, + "num_tokens": 2245238.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3241428136825562, + "sampling/importance_sampling_ratio/mean": 0.9998565316200256, + "sampling/importance_sampling_ratio/min": 0.7468260526657104, + "sampling/sampling_logp_difference/max": 0.29192304611206055, + "sampling/sampling_logp_difference/mean": 0.009362924844026566, + "step": 98 + }, + { + "clip_ratio/high_max": 5.383290408644825e-05, + "clip_ratio/high_mean": 5.383290408644825e-05, + "clip_ratio/low_mean": 9.745195711730048e-05, + "clip_ratio/low_min": 9.745195711730048e-05, + "clip_ratio/region_mean": 0.00015128486120374873, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 604.59375, + "completions/mean_terminated_length": 558.8399658203125, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.42305028066039085, + "epoch": 0.05299785867237687, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026652174070477486, + "learning_rate": 1e-05, + "loss": 0.0419, + "num_tokens": 2268145.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3046342134475708, + "sampling/importance_sampling_ratio/mean": 0.9999422430992126, + "sampling/importance_sampling_ratio/min": 0.7187533378601074, + "sampling/sampling_logp_difference/max": 0.33023715019226074, + "sampling/sampling_logp_difference/mean": 0.011149080470204353, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 679.34375, + "completions/mean_terminated_length": 578.86669921875, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.5958646275103092, + "epoch": 0.05353319057815846, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018127331510186195, + "learning_rate": 1e-05, + "loss": 0.0501, + "num_tokens": 2294052.0, + "reward": 0.375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.6077208518981934, + "sampling/importance_sampling_ratio/mean": 1.0001519918441772, + "sampling/importance_sampling_ratio/min": 0.6073815226554871, + "sampling/sampling_logp_difference/max": 0.4985980987548828, + "sampling/sampling_logp_difference/mean": 0.014581737108528614, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.415401053847745e-05, + "clip_ratio/low_min": 4.415401053847745e-05, + "clip_ratio/region_mean": 4.415401053847745e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 654.96875, + "completions/mean_terminated_length": 541.9375, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.7058289349079132, + "epoch": 0.05406852248394004, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01279167179018259, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 2319051.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3844224214553833, + "sampling/importance_sampling_ratio/mean": 0.9999025464057922, + "sampling/importance_sampling_ratio/min": 0.6449640989303589, + "sampling/sampling_logp_difference/max": 0.43856072425842285, + "sampling/sampling_logp_difference/mean": 0.015389555133879185, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 528.40625, + "completions/mean_terminated_length": 503.6206970214844, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.5093299299478531, + "epoch": 0.05460385438972163, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01063461508601904, + "learning_rate": 1e-05, + "loss": 0.0487, + "num_tokens": 2339472.0, + "reward": 0.78125, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4027093648910522, + "sampling/importance_sampling_ratio/mean": 0.999841570854187, + "sampling/importance_sampling_ratio/min": 0.4838002026081085, + "sampling/sampling_logp_difference/max": 0.7260832786560059, + "sampling/sampling_logp_difference/mean": 0.012134391814470291, + "step": 102 + }, + { + "clip_ratio/high_max": 6.55479816487059e-05, + "clip_ratio/high_mean": 6.55479816487059e-05, + "clip_ratio/low_mean": 0.00010203138299402781, + "clip_ratio/low_min": 0.00010203138299402781, + "clip_ratio/region_mean": 0.0001675793646427337, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 639.84375, + "completions/mean_terminated_length": 572.7142944335938, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.3798535577952862, + "epoch": 0.05513918629550321, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01762148179113865, + "learning_rate": 1e-05, + "loss": 0.1169, + "num_tokens": 2363603.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3716785907745361, + "sampling/importance_sampling_ratio/mean": 0.9998103380203247, + "sampling/importance_sampling_ratio/min": 0.35052594542503357, + "sampling/sampling_logp_difference/max": 1.0483205318450928, + "sampling/sampling_logp_difference/mean": 0.010568685829639435, + "step": 103 + }, + { + "clip_ratio/high_max": 7.208765600807965e-05, + "clip_ratio/high_mean": 7.208765600807965e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.208765600807965e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 533.4375, + "completions/mean_terminated_length": 509.17242431640625, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3971620574593544, + "epoch": 0.055674518201284794, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004956406075507402, + "learning_rate": 1e-05, + "loss": 0.0906, + "num_tokens": 2384033.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3015110492706299, + "sampling/importance_sampling_ratio/mean": 0.9997437000274658, + "sampling/importance_sampling_ratio/min": 0.6486752033233643, + "sampling/sampling_logp_difference/max": 0.43282318115234375, + "sampling/sampling_logp_difference/mean": 0.011015697382390499, + "step": 104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 550.4375, + "completions/mean_terminated_length": 535.933349609375, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.4596591927111149, + "epoch": 0.05620985010706638, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007774466648697853, + "learning_rate": 1e-05, + "loss": -0.0272, + "num_tokens": 2405175.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4565011262893677, + "sampling/importance_sampling_ratio/mean": 0.9999442100524902, + "sampling/importance_sampling_ratio/min": 0.718629777431488, + "sampling/sampling_logp_difference/max": 0.3760371208190918, + "sampling/sampling_logp_difference/mean": 0.011729179881513119, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 598.25, + "completions/mean_terminated_length": 559.0769653320312, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.37072158232331276, + "epoch": 0.056745182012847964, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014647644944489002, + "learning_rate": 1e-05, + "loss": 0.0362, + "num_tokens": 2427967.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2708412408828735, + "sampling/importance_sampling_ratio/mean": 0.99993896484375, + "sampling/importance_sampling_ratio/min": 0.7111266851425171, + "sampling/sampling_logp_difference/max": 0.34090471267700195, + "sampling/sampling_logp_difference/mean": 0.010582929477095604, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010191806359216571, + "clip_ratio/low_min": 0.00010191806359216571, + "clip_ratio/region_mean": 0.00010191806359216571, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 586.09375, + "completions/mean_terminated_length": 525.4583740234375, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.42987873405218124, + "epoch": 0.05728051391862955, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006501646712422371, + "learning_rate": 1e-05, + "loss": 0.0117, + "num_tokens": 2450434.0, + "reward": 0.65625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3616158962249756, + "sampling/importance_sampling_ratio/mean": 1.0002968311309814, + "sampling/importance_sampling_ratio/min": 0.7154538631439209, + "sampling/sampling_logp_difference/max": 0.3348381519317627, + "sampling/sampling_logp_difference/mean": 0.01188938319683075, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.2272575228707865e-05, + "clip_ratio/low_min": 4.2272575228707865e-05, + "clip_ratio/region_mean": 4.2272575228707865e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 699.96875, + "completions/mean_terminated_length": 639.941162109375, + "completions/min_length": 488.0, + "completions/min_terminated_length": 488.0, + "entropy": 0.4379051923751831, + "epoch": 0.057815845824411134, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005879446864128113, + "learning_rate": 1e-05, + "loss": 0.0104, + "num_tokens": 2476785.0, + "reward": 0.34375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2980092763900757, + "sampling/importance_sampling_ratio/mean": 0.9998241066932678, + "sampling/importance_sampling_ratio/min": 0.6074737906455994, + "sampling/sampling_logp_difference/max": 0.4984462261199951, + "sampling/sampling_logp_difference/mean": 0.01169486902654171, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014120041669229977, + "clip_ratio/low_min": 0.00014120041669229977, + "clip_ratio/region_mean": 0.00014120041669229977, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 685.3125, + "completions/mean_terminated_length": 621.0, + "completions/min_length": 365.0, + "completions/min_terminated_length": 365.0, + "entropy": 0.478887639939785, + "epoch": 0.05835117773019272, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008670409210026264, + "learning_rate": 1e-05, + "loss": 0.0565, + "num_tokens": 2503187.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2992057800292969, + "sampling/importance_sampling_ratio/mean": 1.000105381011963, + "sampling/importance_sampling_ratio/min": 0.7309198975563049, + "sampling/sampling_logp_difference/max": 0.3134514093399048, + "sampling/sampling_logp_difference/mean": 0.011437108740210533, + "step": 109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023890918237157166, + "clip_ratio/low_min": 0.00023890918237157166, + "clip_ratio/region_mean": 0.00023890918237157166, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 514.5625, + "completions/mean_terminated_length": 488.3448181152344, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.5211943462491035, + "epoch": 0.058886509635974305, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01529600378125906, + "learning_rate": 1e-05, + "loss": 0.0501, + "num_tokens": 2523613.0, + "reward": 0.6875, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4445958137512207, + "sampling/importance_sampling_ratio/mean": 0.9999364018440247, + "sampling/importance_sampling_ratio/min": 0.7228748798370361, + "sampling/sampling_logp_difference/max": 0.3678295612335205, + "sampling/sampling_logp_difference/mean": 0.013533909805119038, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021671498325304128, + "clip_ratio/low_min": 0.00021671498325304128, + "clip_ratio/region_mean": 0.00021671498325304128, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 577.59375, + "completions/mean_terminated_length": 533.6538696289062, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.4334462657570839, + "epoch": 0.059421841541755886, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01121017150580883, + "learning_rate": 1e-05, + "loss": 0.0219, + "num_tokens": 2545800.0, + "reward": 0.25, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5534660816192627, + "sampling/importance_sampling_ratio/mean": 0.9999406933784485, + "sampling/importance_sampling_ratio/min": 0.6609777808189392, + "sampling/sampling_logp_difference/max": 0.4404885768890381, + "sampling/sampling_logp_difference/mean": 0.012062346562743187, + "step": 111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.560498175327666e-05, + "clip_ratio/low_min": 5.560498175327666e-05, + "clip_ratio/region_mean": 5.560498175327666e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 618.6875, + "completions/mean_terminated_length": 568.9166870117188, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.5460376776754856, + "epoch": 0.059957173447537475, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010709904134273529, + "learning_rate": 1e-05, + "loss": 0.0829, + "num_tokens": 2569158.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3855860233306885, + "sampling/importance_sampling_ratio/mean": 0.9998103976249695, + "sampling/importance_sampling_ratio/min": 0.6119871139526367, + "sampling/sampling_logp_difference/max": 0.4910440444946289, + "sampling/sampling_logp_difference/mean": 0.013752535916864872, + "step": 112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9135218432638794e-05, + "clip_ratio/low_min": 4.9135218432638794e-05, + "clip_ratio/region_mean": 4.9135218432638794e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 631.875, + "completions/mean_terminated_length": 550.2000122070312, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.4064355418086052, + "epoch": 0.06049250535331906, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01075573731213808, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 2593002.0, + "reward": 0.5, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.610756754875183, + "sampling/importance_sampling_ratio/mean": 0.9999549388885498, + "sampling/importance_sampling_ratio/min": 0.4840478301048279, + "sampling/sampling_logp_difference/max": 0.7255716323852539, + "sampling/sampling_logp_difference/mean": 0.011302068829536438, + "step": 113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.758279465022497e-05, + "clip_ratio/low_min": 4.758279465022497e-05, + "clip_ratio/region_mean": 4.758279465022497e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 654.53125, + "completions/mean_terminated_length": 602.95458984375, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.3643207363784313, + "epoch": 0.061027837259100645, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018865488469600677, + "learning_rate": 1e-05, + "loss": 0.0161, + "num_tokens": 2617595.0, + "reward": 0.6875, + "reward_std": 0.3535533845424652, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.421895980834961, + "sampling/importance_sampling_ratio/mean": 0.9998403787612915, + "sampling/importance_sampling_ratio/min": 0.5850104093551636, + "sampling/sampling_logp_difference/max": 0.536125659942627, + "sampling/sampling_logp_difference/mean": 0.009876951575279236, + "step": 114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 544.9375, + "completions/mean_terminated_length": 521.862060546875, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 0.4699717238545418, + "epoch": 0.06156316916488223, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007969534024596214, + "learning_rate": 1e-05, + "loss": 0.0742, + "num_tokens": 2638785.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.360924243927002, + "sampling/importance_sampling_ratio/mean": 1.000280499458313, + "sampling/importance_sampling_ratio/min": 0.6759506464004517, + "sampling/sampling_logp_difference/max": 0.3916351795196533, + "sampling/sampling_logp_difference/mean": 0.011387757956981659, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.80437034741044e-05, + "clip_ratio/low_min": 9.80437034741044e-05, + "clip_ratio/region_mean": 9.80437034741044e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 621.0625, + "completions/mean_terminated_length": 572.0833740234375, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "entropy": 0.35027093812823296, + "epoch": 0.06209850107066381, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006722511723637581, + "learning_rate": 1e-05, + "loss": -0.0039, + "num_tokens": 2662747.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4416756629943848, + "sampling/importance_sampling_ratio/mean": 1.0001320838928223, + "sampling/importance_sampling_ratio/min": 0.7622732520103455, + "sampling/sampling_logp_difference/max": 0.36580610275268555, + "sampling/sampling_logp_difference/mean": 0.010345006361603737, + "step": 116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 597.65625, + "completions/mean_terminated_length": 566.1111450195312, + "completions/min_length": 374.0, + "completions/min_terminated_length": 374.0, + "entropy": 0.3591281324625015, + "epoch": 0.0626338329764454, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015098914504051208, + "learning_rate": 1e-05, + "loss": 0.0252, + "num_tokens": 2685552.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4866523742675781, + "sampling/importance_sampling_ratio/mean": 1.0001282691955566, + "sampling/importance_sampling_ratio/min": 0.6318407654762268, + "sampling/sampling_logp_difference/max": 0.4591178894042969, + "sampling/sampling_logp_difference/mean": 0.010378815233707428, + "step": 117 + }, + { + "clip_ratio/high_max": 0.00013217430387157947, + "clip_ratio/high_mean": 0.00013217430387157947, + "clip_ratio/low_mean": 6.645401299465448e-05, + "clip_ratio/low_min": 6.645401299465448e-05, + "clip_ratio/region_mean": 0.00019862831686623394, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 562.8125, + "completions/mean_terminated_length": 494.41668701171875, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.31836752966046333, + "epoch": 0.06316916488222699, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010840936563909054, + "learning_rate": 1e-05, + "loss": 0.0123, + "num_tokens": 2706810.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2777732610702515, + "sampling/importance_sampling_ratio/mean": 0.9999618530273438, + "sampling/importance_sampling_ratio/min": 0.607105016708374, + "sampling/sampling_logp_difference/max": 0.4990534782409668, + "sampling/sampling_logp_difference/mean": 0.00928731169551611, + "step": 118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.456327951629646e-05, + "clip_ratio/low_min": 4.456327951629646e-05, + "clip_ratio/region_mean": 4.456327951629646e-05, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 716.5, + "completions/mean_terminated_length": 630.6666870117188, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.45794132724404335, + "epoch": 0.06370449678800856, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015280363149940968, + "learning_rate": 1e-05, + "loss": 0.0099, + "num_tokens": 2734178.0, + "reward": 0.125, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3790987730026245, + "sampling/importance_sampling_ratio/mean": 0.9996334910392761, + "sampling/importance_sampling_ratio/min": 0.4426460266113281, + "sampling/sampling_logp_difference/max": 0.814984917640686, + "sampling/sampling_logp_difference/mean": 0.012173281982541084, + "step": 119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.367110497900285e-05, + "clip_ratio/low_min": 5.367110497900285e-05, + "clip_ratio/region_mean": 5.367110497900285e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 549.90625, + "completions/mean_terminated_length": 535.36669921875, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.42989468201994896, + "epoch": 0.06423982869379015, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034308782778680325, + "learning_rate": 1e-05, + "loss": 0.039, + "num_tokens": 2755271.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3252981901168823, + "sampling/importance_sampling_ratio/mean": 1.0002095699310303, + "sampling/importance_sampling_ratio/min": 0.7031346559524536, + "sampling/sampling_logp_difference/max": 0.3522069454193115, + "sampling/sampling_logp_difference/mean": 0.011786284856498241, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001159017119789496, + "clip_ratio/low_min": 0.0001159017119789496, + "clip_ratio/region_mean": 0.0001159017119789496, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 604.25, + "completions/mean_terminated_length": 529.8181762695312, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.39081184566020966, + "epoch": 0.06477516059957174, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0095, + "num_tokens": 2778375.0, + "reward": 0.5, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2685933113098145, + "sampling/importance_sampling_ratio/mean": 1.0003443956375122, + "sampling/importance_sampling_ratio/min": 0.713601291179657, + "sampling/sampling_logp_difference/max": 0.3374309539794922, + "sampling/sampling_logp_difference/mean": 0.011368668638169765, + "step": 121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 557.34375, + "completions/mean_terminated_length": 535.5516967773438, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.4360150769352913, + "epoch": 0.06531049250535331, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012217814102768898, + "learning_rate": 1e-05, + "loss": 0.0288, + "num_tokens": 2800706.0, + "reward": 0.75, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3406745195388794, + "sampling/importance_sampling_ratio/mean": 0.9999719858169556, + "sampling/importance_sampling_ratio/min": 0.490824818611145, + "sampling/sampling_logp_difference/max": 0.7116680145263672, + "sampling/sampling_logp_difference/mean": 0.012638597749173641, + "step": 122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013946122999186628, + "clip_ratio/low_min": 0.00013946122999186628, + "clip_ratio/region_mean": 0.00013946122999186628, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 671.625, + "completions/mean_terminated_length": 575.25, + "completions/min_length": 367.0, + "completions/min_terminated_length": 367.0, + "entropy": 0.34236453101038933, + "epoch": 0.0658458244111349, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015692198649048805, + "learning_rate": 1e-05, + "loss": 0.0391, + "num_tokens": 2826134.0, + "reward": 0.46875, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4008420705795288, + "sampling/importance_sampling_ratio/mean": 0.999833881855011, + "sampling/importance_sampling_ratio/min": 0.7842716574668884, + "sampling/sampling_logp_difference/max": 0.33707356452941895, + "sampling/sampling_logp_difference/mean": 0.009405428543686867, + "step": 123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.1845705456798896e-05, + "clip_ratio/low_min": 5.1845705456798896e-05, + "clip_ratio/region_mean": 5.1845705456798896e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 491.8125, + "completions/mean_terminated_length": 473.4000244140625, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.4624374695122242, + "epoch": 0.06638115631691649, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008339719846844673, + "learning_rate": 1e-05, + "loss": 0.0588, + "num_tokens": 2845304.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3198801279067993, + "sampling/importance_sampling_ratio/mean": 1.0002994537353516, + "sampling/importance_sampling_ratio/min": 0.5299164056777954, + "sampling/sampling_logp_difference/max": 0.6350359916687012, + "sampling/sampling_logp_difference/mean": 0.011694015003740788, + "step": 124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 611.5625, + "completions/mean_terminated_length": 529.6190795898438, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "entropy": 0.5198213867843151, + "epoch": 0.06691648822269808, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.022294864058494568, + "learning_rate": 1e-05, + "loss": 0.0512, + "num_tokens": 2868842.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4437618255615234, + "sampling/importance_sampling_ratio/mean": 1.0000042915344238, + "sampling/importance_sampling_ratio/min": 0.6878994703292847, + "sampling/sampling_logp_difference/max": 0.374112606048584, + "sampling/sampling_logp_difference/mean": 0.01330228615552187, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014667538198409602, + "clip_ratio/low_min": 0.00014667538198409602, + "clip_ratio/region_mean": 0.00014667538198409602, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 595.6875, + "completions/mean_terminated_length": 555.923095703125, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.4224700592458248, + "epoch": 0.06745182012847965, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01445845514535904, + "learning_rate": 1e-05, + "loss": -0.0236, + "num_tokens": 2891416.0, + "reward": 0.46875, + "reward_std": 0.521792471408844, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3168127536773682, + "sampling/importance_sampling_ratio/mean": 0.9996733665466309, + "sampling/importance_sampling_ratio/min": 0.6416264772415161, + "sampling/sampling_logp_difference/max": 0.44374895095825195, + "sampling/sampling_logp_difference/mean": 0.01204050425440073, + "step": 126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.645889905281365e-05, + "clip_ratio/low_min": 5.645889905281365e-05, + "clip_ratio/region_mean": 5.645889905281365e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 511.65625, + "completions/mean_terminated_length": 464.1851806640625, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "entropy": 0.7288839221000671, + "epoch": 0.06798715203426124, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026353413239121437, + "learning_rate": 1e-05, + "loss": -0.0769, + "num_tokens": 2912117.0, + "reward": 0.34375, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4234583377838135, + "sampling/importance_sampling_ratio/mean": 1.0000054836273193, + "sampling/importance_sampling_ratio/min": 0.739001452922821, + "sampling/sampling_logp_difference/max": 0.3530893325805664, + "sampling/sampling_logp_difference/mean": 0.01712021417915821, + "step": 127 + }, + { + "clip_ratio/high_max": 4.716981129604392e-05, + "clip_ratio/high_mean": 4.716981129604392e-05, + "clip_ratio/low_mean": 9.705970660434105e-05, + "clip_ratio/low_min": 9.705970660434105e-05, + "clip_ratio/region_mean": 0.00014422951790038496, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 606.90625, + "completions/mean_terminated_length": 553.2083740234375, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.5988429039716721, + "epoch": 0.06852248394004283, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012898923829197884, + "learning_rate": 1e-05, + "loss": 0.076, + "num_tokens": 2935474.0, + "reward": 0.5, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4216952323913574, + "sampling/importance_sampling_ratio/mean": 1.000023365020752, + "sampling/importance_sampling_ratio/min": 0.7354314923286438, + "sampling/sampling_logp_difference/max": 0.3518500328063965, + "sampling/sampling_logp_difference/mean": 0.015339154750108719, + "step": 128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001535204501124099, + "clip_ratio/low_min": 0.0001535204501124099, + "clip_ratio/region_mean": 0.0001535204501124099, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 597.375, + "completions/mean_terminated_length": 558.0, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "entropy": 0.5921585410833359, + "epoch": 0.0690578158458244, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004030481446534395, + "learning_rate": 1e-05, + "loss": 0.0072, + "num_tokens": 2959342.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4255763292312622, + "sampling/importance_sampling_ratio/mean": 0.999934196472168, + "sampling/importance_sampling_ratio/min": 0.7412431836128235, + "sampling/sampling_logp_difference/max": 0.35457611083984375, + "sampling/sampling_logp_difference/mean": 0.013684538193047047, + "step": 129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001262442019651644, + "clip_ratio/low_min": 0.0001262442019651644, + "clip_ratio/region_mean": 0.0001262442019651644, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 524.875, + "completions/mean_terminated_length": 490.14288330078125, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.38221365958452225, + "epoch": 0.069593147751606, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007477280683815479, + "learning_rate": 1e-05, + "loss": 0.0205, + "num_tokens": 2980474.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3646857738494873, + "sampling/importance_sampling_ratio/mean": 1.0000596046447754, + "sampling/importance_sampling_ratio/min": 0.6582945585250854, + "sampling/sampling_logp_difference/max": 0.4181027412414551, + "sampling/sampling_logp_difference/mean": 0.011347775347530842, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.609833432477899e-05, + "clip_ratio/low_min": 9.609833432477899e-05, + "clip_ratio/region_mean": 9.609833432477899e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 605.46875, + "completions/mean_terminated_length": 567.9615478515625, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.4094332605600357, + "epoch": 0.07012847965738758, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011829032562673092, + "learning_rate": 1e-05, + "loss": 0.0585, + "num_tokens": 3003297.0, + "reward": 0.65625, + "reward_std": 0.47137710452079773, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2982062101364136, + "sampling/importance_sampling_ratio/mean": 1.0001816749572754, + "sampling/importance_sampling_ratio/min": 0.6821297407150269, + "sampling/sampling_logp_difference/max": 0.382535457611084, + "sampling/sampling_logp_difference/mean": 0.01232686173170805, + "step": 131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.6766577472444624e-05, + "clip_ratio/low_min": 5.6766577472444624e-05, + "clip_ratio/region_mean": 5.6766577472444624e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 590.65625, + "completions/mean_terminated_length": 521.2608642578125, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.3864295594394207, + "epoch": 0.07066381156316917, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016091959550976753, + "learning_rate": 1e-05, + "loss": 0.048, + "num_tokens": 3026230.0, + "reward": 0.71875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4359915256500244, + "sampling/importance_sampling_ratio/mean": 0.9996742010116577, + "sampling/importance_sampling_ratio/min": 0.7111673355102539, + "sampling/sampling_logp_difference/max": 0.36185550689697266, + "sampling/sampling_logp_difference/mean": 0.011034548282623291, + "step": 132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.292125206324272e-05, + "clip_ratio/low_min": 5.292125206324272e-05, + "clip_ratio/region_mean": 5.292125206324272e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 542.40625, + "completions/mean_terminated_length": 527.36669921875, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.3996155969798565, + "epoch": 0.07119914346895075, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015324263833463192, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 3047355.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3368474245071411, + "sampling/importance_sampling_ratio/mean": 1.000064730644226, + "sampling/importance_sampling_ratio/min": 0.6329601407051086, + "sampling/sampling_logp_difference/max": 0.4573478698730469, + "sampling/sampling_logp_difference/mean": 0.011135010048747063, + "step": 133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 624.1875, + "completions/mean_terminated_length": 576.25, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.3773885704576969, + "epoch": 0.07173447537473233, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002062029903754592, + "learning_rate": 1e-05, + "loss": 0.0356, + "num_tokens": 3071337.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3878531455993652, + "sampling/importance_sampling_ratio/mean": 0.9996427297592163, + "sampling/importance_sampling_ratio/min": 0.6468920111656189, + "sampling/sampling_logp_difference/max": 0.4355759620666504, + "sampling/sampling_logp_difference/mean": 0.01128737535327673, + "step": 134 + }, + { + "clip_ratio/high_max": 6.262525130296126e-05, + "clip_ratio/high_mean": 6.262525130296126e-05, + "clip_ratio/low_mean": 5.910165418754332e-05, + "clip_ratio/low_min": 5.910165418754332e-05, + "clip_ratio/region_mean": 0.00012172690549050458, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 509.5625, + "completions/mean_terminated_length": 472.64288330078125, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.4508117511868477, + "epoch": 0.07226980728051392, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007427507545799017, + "learning_rate": 1e-05, + "loss": 0.0781, + "num_tokens": 3091315.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4224412441253662, + "sampling/importance_sampling_ratio/mean": 0.9999260902404785, + "sampling/importance_sampling_ratio/min": 0.23657158017158508, + "sampling/sampling_logp_difference/max": 1.4415044784545898, + "sampling/sampling_logp_difference/mean": 0.01295729074627161, + "step": 135 + }, + { + "clip_ratio/high_max": 4.328254726715386e-05, + "clip_ratio/high_mean": 4.328254726715386e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.328254726715386e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 612.5, + "completions/mean_terminated_length": 551.6521606445312, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "entropy": 0.3710810951888561, + "epoch": 0.0728051391862955, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0049619609490036964, + "learning_rate": 1e-05, + "loss": 0.0789, + "num_tokens": 3114939.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3776192665100098, + "sampling/importance_sampling_ratio/mean": 0.9997196793556213, + "sampling/importance_sampling_ratio/min": 0.6667206883430481, + "sampling/sampling_logp_difference/max": 0.4053840637207031, + "sampling/sampling_logp_difference/mean": 0.00999332033097744, + "step": 136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 539.71875, + "completions/mean_terminated_length": 497.4444580078125, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.37066930159926414, + "epoch": 0.07334047109207709, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0033103583846241236, + "learning_rate": 1e-05, + "loss": 0.0609, + "num_tokens": 3135642.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3367377519607544, + "sampling/importance_sampling_ratio/mean": 0.9998750686645508, + "sampling/importance_sampling_ratio/min": 0.46006274223327637, + "sampling/sampling_logp_difference/max": 0.7763924598693848, + "sampling/sampling_logp_difference/mean": 0.010519558563828468, + "step": 137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00025644271227065474, + "clip_ratio/low_min": 0.00025644271227065474, + "clip_ratio/region_mean": 0.00025644271227065474, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 602.09375, + "completions/mean_terminated_length": 571.370361328125, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.4131225012242794, + "epoch": 0.07387580299785867, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015307110734283924, + "learning_rate": 1e-05, + "loss": 0.0993, + "num_tokens": 3158629.0, + "reward": 0.59375, + "reward_std": 0.5123760104179382, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4257789850234985, + "sampling/importance_sampling_ratio/mean": 0.9999570250511169, + "sampling/importance_sampling_ratio/min": 0.7596944570541382, + "sampling/sampling_logp_difference/max": 0.3547182083129883, + "sampling/sampling_logp_difference/mean": 0.011196279898285866, + "step": 138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.036261063651182e-05, + "clip_ratio/low_min": 5.036261063651182e-05, + "clip_ratio/region_mean": 5.036261063651182e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 525.875, + "completions/mean_terminated_length": 491.2857360839844, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.3909878507256508, + "epoch": 0.07441113490364026, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017996681854128838, + "learning_rate": 1e-05, + "loss": 0.0453, + "num_tokens": 3178657.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3117836713790894, + "sampling/importance_sampling_ratio/mean": 1.0001606941223145, + "sampling/importance_sampling_ratio/min": 0.6962023973464966, + "sampling/sampling_logp_difference/max": 0.36211490631103516, + "sampling/sampling_logp_difference/mean": 0.011155658401548862, + "step": 139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.1597337258281186e-05, + "clip_ratio/low_min": 4.1597337258281186e-05, + "clip_ratio/region_mean": 4.1597337258281186e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 605.21875, + "completions/mean_terminated_length": 550.9583740234375, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.4558464288711548, + "epoch": 0.07494646680942184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011749648489058018, + "learning_rate": 1e-05, + "loss": 0.0399, + "num_tokens": 3201344.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.336584210395813, + "sampling/importance_sampling_ratio/mean": 1.0002506971359253, + "sampling/importance_sampling_ratio/min": 0.61990886926651, + "sampling/sampling_logp_difference/max": 0.4781827926635742, + "sampling/sampling_logp_difference/mean": 0.01124854851514101, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 675.0, + "completions/mean_terminated_length": 626.2857055664062, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "entropy": 0.409841600805521, + "epoch": 0.07548179871520343, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009168663993477821, + "learning_rate": 1e-05, + "loss": 0.018, + "num_tokens": 3226832.0, + "reward": 0.59375, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.588563323020935, + "sampling/importance_sampling_ratio/mean": 1.000203251838684, + "sampling/importance_sampling_ratio/min": 0.686457633972168, + "sampling/sampling_logp_difference/max": 0.4628300666809082, + "sampling/sampling_logp_difference/mean": 0.01134434062987566, + "step": 141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 478.53125, + "completions/mean_terminated_length": 469.19354248046875, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.36848144605755806, + "epoch": 0.07601713062098502, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007296881638467312, + "learning_rate": 1e-05, + "loss": -0.023, + "num_tokens": 3245497.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2988835573196411, + "sampling/importance_sampling_ratio/mean": 0.9998937249183655, + "sampling/importance_sampling_ratio/min": 0.5236707925796509, + "sampling/sampling_logp_difference/max": 0.6468920707702637, + "sampling/sampling_logp_difference/mean": 0.010770690627396107, + "step": 142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001506620683358051, + "clip_ratio/low_min": 0.0001506620683358051, + "clip_ratio/region_mean": 0.0001506620683358051, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 659.5, + "completions/mean_terminated_length": 602.6666870117188, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.6066175773739815, + "epoch": 0.07655246252676659, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009867552667856216, + "learning_rate": 1e-05, + "loss": 0.0742, + "num_tokens": 3270265.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5600217580795288, + "sampling/importance_sampling_ratio/mean": 1.0000202655792236, + "sampling/importance_sampling_ratio/min": 0.5449547171592712, + "sampling/sampling_logp_difference/max": 0.6070525646209717, + "sampling/sampling_logp_difference/mean": 0.013590005226433277, + "step": 143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 515.34375, + "completions/mean_terminated_length": 468.5555725097656, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.39527034014463425, + "epoch": 0.07708779443254818, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00386427016928792, + "learning_rate": 1e-05, + "loss": 0.0279, + "num_tokens": 3290372.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3550678491592407, + "sampling/importance_sampling_ratio/mean": 0.9998969435691833, + "sampling/importance_sampling_ratio/min": 0.7136199474334717, + "sampling/sampling_logp_difference/max": 0.337404727935791, + "sampling/sampling_logp_difference/mean": 0.011627251282334328, + "step": 144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 556.0625, + "completions/mean_terminated_length": 534.137939453125, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "entropy": 0.4369659088551998, + "epoch": 0.07762312633832977, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012809324078261852, + "learning_rate": 1e-05, + "loss": -0.0447, + "num_tokens": 3311822.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4632304906845093, + "sampling/importance_sampling_ratio/mean": 1.0001205205917358, + "sampling/importance_sampling_ratio/min": 0.7598057389259338, + "sampling/sampling_logp_difference/max": 0.3806467056274414, + "sampling/sampling_logp_difference/mean": 0.011080690659582615, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.567928747041151e-05, + "clip_ratio/low_min": 5.567928747041151e-05, + "clip_ratio/region_mean": 5.567928747041151e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 537.28125, + "completions/mean_terminated_length": 460.375, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.4744526818394661, + "epoch": 0.07815845824411134, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011317940428853035, + "learning_rate": 1e-05, + "loss": 0.0471, + "num_tokens": 3332503.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2816593647003174, + "sampling/importance_sampling_ratio/mean": 0.999502420425415, + "sampling/importance_sampling_ratio/min": 0.6133771538734436, + "sampling/sampling_logp_difference/max": 0.48877525329589844, + "sampling/sampling_logp_difference/mean": 0.012451513670384884, + "step": 146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017116693561547436, + "clip_ratio/low_min": 0.00017116693561547436, + "clip_ratio/region_mean": 0.00017116693561547436, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 593.5, + "completions/mean_terminated_length": 502.0952453613281, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.5474798791110516, + "epoch": 0.07869379014989293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008388844318687916, + "learning_rate": 1e-05, + "loss": 0.0919, + "num_tokens": 3355151.0, + "reward": 0.5, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2929469347000122, + "sampling/importance_sampling_ratio/mean": 0.9996303915977478, + "sampling/importance_sampling_ratio/min": 0.5192931294441223, + "sampling/sampling_logp_difference/max": 0.6552867889404297, + "sampling/sampling_logp_difference/mean": 0.013475755229592323, + "step": 147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.58534411538858e-05, + "clip_ratio/low_min": 5.58534411538858e-05, + "clip_ratio/region_mean": 5.58534411538858e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 539.625, + "completions/mean_terminated_length": 497.3333435058594, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3991433121263981, + "epoch": 0.07922912205567452, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005428406875580549, + "learning_rate": 1e-05, + "loss": 0.0806, + "num_tokens": 3376067.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2868996858596802, + "sampling/importance_sampling_ratio/mean": 0.9997435808181763, + "sampling/importance_sampling_ratio/min": 0.5954288244247437, + "sampling/sampling_logp_difference/max": 0.5184733867645264, + "sampling/sampling_logp_difference/mean": 0.011121694929897785, + "step": 148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 573.21875, + "completions/mean_terminated_length": 537.1481323242188, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.36447297781705856, + "epoch": 0.07976445396145611, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009165140800178051, + "learning_rate": 1e-05, + "loss": 0.0768, + "num_tokens": 3397874.0, + "reward": 0.71875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3582088947296143, + "sampling/importance_sampling_ratio/mean": 0.9998113512992859, + "sampling/importance_sampling_ratio/min": 0.623322069644928, + "sampling/sampling_logp_difference/max": 0.4726918935775757, + "sampling/sampling_logp_difference/mean": 0.010827361606061459, + "step": 149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.735202183946967e-05, + "clip_ratio/low_min": 9.735202183946967e-05, + "clip_ratio/region_mean": 9.735202183946967e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 515.3125, + "completions/mean_terminated_length": 507.1612854003906, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.3126232773065567, + "epoch": 0.08029978586723768, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0048127430491149426, + "learning_rate": 1e-05, + "loss": -0.0217, + "num_tokens": 3417988.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3360776901245117, + "sampling/importance_sampling_ratio/mean": 0.9999316930770874, + "sampling/importance_sampling_ratio/min": 0.7416387796401978, + "sampling/sampling_logp_difference/max": 0.2988929748535156, + "sampling/sampling_logp_difference/mean": 0.009871783666312695, + "step": 150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017024551198119298, + "clip_ratio/low_min": 0.00017024551198119298, + "clip_ratio/region_mean": 0.00017024551198119298, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 608.75, + "completions/mean_terminated_length": 586.0, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3984130211174488, + "epoch": 0.08083511777301927, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003835368202999234, + "learning_rate": 1e-05, + "loss": 0.0299, + "num_tokens": 3441460.0, + "reward": 0.5, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0003395080566406, + "sampling/importance_sampling_ratio/min": 0.7250773906707764, + "sampling/sampling_logp_difference/max": 0.8758184909820557, + "sampling/sampling_logp_difference/mean": 0.011509026400744915, + "step": 151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.724044942529872e-05, + "clip_ratio/low_min": 6.724044942529872e-05, + "clip_ratio/region_mean": 6.724044942529872e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 606.125, + "completions/mean_terminated_length": 521.3333129882812, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "entropy": 0.4458345100283623, + "epoch": 0.08137044967880086, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008848967961966991, + "learning_rate": 1e-05, + "loss": 0.0694, + "num_tokens": 3464696.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4253044128417969, + "sampling/importance_sampling_ratio/mean": 0.9997562766075134, + "sampling/importance_sampling_ratio/min": 0.4247634708881378, + "sampling/sampling_logp_difference/max": 0.8562228679656982, + "sampling/sampling_logp_difference/mean": 0.012223445810377598, + "step": 152 + }, + { + "clip_ratio/high_max": 5.5704098485875875e-05, + "clip_ratio/high_mean": 5.5704098485875875e-05, + "clip_ratio/low_mean": 9.4307182735065e-05, + "clip_ratio/low_min": 9.4307182735065e-05, + "clip_ratio/region_mean": 0.00015001128122094087, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 619.46875, + "completions/mean_terminated_length": 569.9583740234375, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.4194677323102951, + "epoch": 0.08190578158458243, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007004813756793737, + "learning_rate": 1e-05, + "loss": 0.0624, + "num_tokens": 3488143.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4491379261016846, + "sampling/importance_sampling_ratio/mean": 1.000097393989563, + "sampling/importance_sampling_ratio/min": 0.3960610032081604, + "sampling/sampling_logp_difference/max": 0.9261870384216309, + "sampling/sampling_logp_difference/mean": 0.012135032564401627, + "step": 153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010301547808921896, + "clip_ratio/low_min": 0.00010301547808921896, + "clip_ratio/region_mean": 0.00010301547808921896, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 577.875, + "completions/mean_terminated_length": 542.6666870117188, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.5109613388776779, + "epoch": 0.08244111349036402, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0164, + "num_tokens": 3510731.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3617922067642212, + "sampling/importance_sampling_ratio/mean": 0.9996497631072998, + "sampling/importance_sampling_ratio/min": 0.635783314704895, + "sampling/sampling_logp_difference/max": 0.4528975486755371, + "sampling/sampling_logp_difference/mean": 0.013292813673615456, + "step": 154 + }, + { + "clip_ratio/high_max": 4.8885412979871035e-05, + "clip_ratio/high_mean": 4.8885412979871035e-05, + "clip_ratio/low_mean": 4.6921923058107495e-05, + "clip_ratio/low_min": 4.6921923058107495e-05, + "clip_ratio/region_mean": 9.580733603797853e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 582.4375, + "completions/mean_terminated_length": 498.0909118652344, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.3889825753867626, + "epoch": 0.08297644539614561, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022144809365272522, + "learning_rate": 1e-05, + "loss": 0.0725, + "num_tokens": 3533225.0, + "reward": 0.65625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.335777997970581, + "sampling/importance_sampling_ratio/mean": 1.000206470489502, + "sampling/importance_sampling_ratio/min": 0.5145502090454102, + "sampling/sampling_logp_difference/max": 0.6644620895385742, + "sampling/sampling_logp_difference/mean": 0.010203310288488865, + "step": 155 + }, + { + "clip_ratio/high_max": 0.00015236240142257884, + "clip_ratio/high_mean": 0.00015236240142257884, + "clip_ratio/low_mean": 4.633061689673923e-05, + "clip_ratio/low_min": 4.633061689673923e-05, + "clip_ratio/region_mean": 0.00019869301831931807, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 554.9375, + "completions/mean_terminated_length": 471.5652160644531, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3928482383489609, + "epoch": 0.0835117773019272, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01015439536422491, + "learning_rate": 1e-05, + "loss": 0.0289, + "num_tokens": 3554879.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4893429279327393, + "sampling/importance_sampling_ratio/mean": 1.0001931190490723, + "sampling/importance_sampling_ratio/min": 0.6360498070716858, + "sampling/sampling_logp_difference/max": 0.45247840881347656, + "sampling/sampling_logp_difference/mean": 0.011713199317455292, + "step": 156 + }, + { + "clip_ratio/high_max": 0.00012404915469232947, + "clip_ratio/high_mean": 0.00012404915469232947, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012404915469232947, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 481.15625, + "completions/mean_terminated_length": 451.4827575683594, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.5006781779229641, + "epoch": 0.08404710920770878, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01573370024561882, + "learning_rate": 1e-05, + "loss": 0.0707, + "num_tokens": 3573804.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4681521654129028, + "sampling/importance_sampling_ratio/mean": 0.9999622106552124, + "sampling/importance_sampling_ratio/min": 0.5331178307533264, + "sampling/sampling_logp_difference/max": 0.6290128231048584, + "sampling/sampling_logp_difference/mean": 0.01235072873532772, + "step": 157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 535.75, + "completions/mean_terminated_length": 444.86956787109375, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "entropy": 0.4573538564145565, + "epoch": 0.08458244111349036, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0254, + "num_tokens": 3594844.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3168195486068726, + "sampling/importance_sampling_ratio/mean": 1.000217080116272, + "sampling/importance_sampling_ratio/min": 0.672410786151886, + "sampling/sampling_logp_difference/max": 0.39688587188720703, + "sampling/sampling_logp_difference/mean": 0.012935231439769268, + "step": 158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.543802424450405e-05, + "clip_ratio/low_min": 4.543802424450405e-05, + "clip_ratio/region_mean": 4.543802424450405e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 594.0, + "completions/mean_terminated_length": 553.84619140625, + "completions/min_length": 371.0, + "completions/min_terminated_length": 371.0, + "entropy": 0.43147435039281845, + "epoch": 0.08511777301927195, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014614534564316273, + "learning_rate": 1e-05, + "loss": 0.0512, + "num_tokens": 3617548.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6154052019119263, + "sampling/importance_sampling_ratio/mean": 0.9999131560325623, + "sampling/importance_sampling_ratio/min": 0.665272057056427, + "sampling/sampling_logp_difference/max": 0.4795858860015869, + "sampling/sampling_logp_difference/mean": 0.01227882131934166, + "step": 159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.910868539009243e-05, + "clip_ratio/low_min": 9.910868539009243e-05, + "clip_ratio/region_mean": 9.910868539009243e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 587.875, + "completions/mean_terminated_length": 517.3912963867188, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.6415975391864777, + "epoch": 0.08565310492505353, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014024967327713966, + "learning_rate": 1e-05, + "loss": 0.0666, + "num_tokens": 3639960.0, + "reward": 0.71875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.328179121017456, + "sampling/importance_sampling_ratio/mean": 0.9996159076690674, + "sampling/importance_sampling_ratio/min": 0.4720625877380371, + "sampling/sampling_logp_difference/max": 0.7506437301635742, + "sampling/sampling_logp_difference/mean": 0.01451694406569004, + "step": 160 + }, + { + "clip_ratio/high_max": 0.00011413964602979831, + "clip_ratio/high_mean": 0.00011413964602979831, + "clip_ratio/low_mean": 0.0001784025298547931, + "clip_ratio/low_min": 0.0001784025298547931, + "clip_ratio/region_mean": 0.0002925421758845914, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 624.96875, + "completions/mean_terminated_length": 527.1052856445312, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.610698401927948, + "epoch": 0.08618843683083512, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011172808706760406, + "learning_rate": 1e-05, + "loss": 0.1149, + "num_tokens": 3663719.0, + "reward": 0.46875, + "reward_std": 0.47137707471847534, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3557729721069336, + "sampling/importance_sampling_ratio/mean": 1.0003107786178589, + "sampling/importance_sampling_ratio/min": 0.7412463426589966, + "sampling/sampling_logp_difference/max": 0.30437183380126953, + "sampling/sampling_logp_difference/mean": 0.01587352715432644, + "step": 161 + }, + { + "clip_ratio/high_max": 5.6382497859885916e-05, + "clip_ratio/high_mean": 5.6382497859885916e-05, + "clip_ratio/low_mean": 0.00017631493028602563, + "clip_ratio/low_min": 0.00017631493028602563, + "clip_ratio/region_mean": 0.00023269742814591154, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 575.0, + "completions/mean_terminated_length": 530.4615478515625, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.4971841536462307, + "epoch": 0.0867237687366167, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004410617519170046, + "learning_rate": 1e-05, + "loss": 0.0459, + "num_tokens": 3685967.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4432697296142578, + "sampling/importance_sampling_ratio/mean": 1.0001212358474731, + "sampling/importance_sampling_ratio/min": 0.7172502875328064, + "sampling/sampling_logp_difference/max": 0.3669111728668213, + "sampling/sampling_logp_difference/mean": 0.013134903274476528, + "step": 162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 644.84375, + "completions/mean_terminated_length": 464.8461608886719, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.662838339805603, + "epoch": 0.0872591006423983, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004100402817130089, + "learning_rate": 1e-05, + "loss": -0.0033, + "num_tokens": 3710938.0, + "reward": 0.28125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4349920749664307, + "sampling/importance_sampling_ratio/mean": 1.0004535913467407, + "sampling/importance_sampling_ratio/min": 0.5462856292724609, + "sampling/sampling_logp_difference/max": 0.6046133041381836, + "sampling/sampling_logp_difference/mean": 0.017001347616314888, + "step": 163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013185653369873762, + "clip_ratio/low_min": 0.00013185653369873762, + "clip_ratio/region_mean": 0.00013185653369873762, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 583.84375, + "completions/mean_terminated_length": 522.4583740234375, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "entropy": 0.514828659594059, + "epoch": 0.08779443254817987, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010722745209932327, + "learning_rate": 1e-05, + "loss": 0.0237, + "num_tokens": 3733253.0, + "reward": 0.53125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4557102918624878, + "sampling/importance_sampling_ratio/mean": 1.0003795623779297, + "sampling/importance_sampling_ratio/min": 0.7550866603851318, + "sampling/sampling_logp_difference/max": 0.37549400329589844, + "sampling/sampling_logp_difference/mean": 0.0130677605047822, + "step": 164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010053625737782568, + "clip_ratio/low_min": 0.00010053625737782568, + "clip_ratio/region_mean": 0.00010053625737782568, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 592.4375, + "completions/mean_terminated_length": 543.2799682617188, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "entropy": 0.39805489405989647, + "epoch": 0.08832976445396146, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010290173813700676, + "learning_rate": 1e-05, + "loss": -0.0248, + "num_tokens": 3755411.0, + "reward": 0.40625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2697464227676392, + "sampling/importance_sampling_ratio/mean": 1.0000461339950562, + "sampling/importance_sampling_ratio/min": 0.5553558468818665, + "sampling/sampling_logp_difference/max": 0.5881462097167969, + "sampling/sampling_logp_difference/mean": 0.010934079997241497, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 574.5625, + "completions/mean_terminated_length": 538.74072265625, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.3548167943954468, + "epoch": 0.08886509635974305, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020712891593575478, + "learning_rate": 1e-05, + "loss": 0.0697, + "num_tokens": 3777453.0, + "reward": 0.6875, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4247281551361084, + "sampling/importance_sampling_ratio/mean": 0.9998741149902344, + "sampling/importance_sampling_ratio/min": 0.7023288607597351, + "sampling/sampling_logp_difference/max": 0.35398101806640625, + "sampling/sampling_logp_difference/mean": 0.010358977131545544, + "step": 166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 573.28125, + "completions/mean_terminated_length": 553.137939453125, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.39695925265550613, + "epoch": 0.08940042826552462, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009602807462215424, + "learning_rate": 1e-05, + "loss": 0.0377, + "num_tokens": 3799270.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2755472660064697, + "sampling/importance_sampling_ratio/mean": 0.999658465385437, + "sampling/importance_sampling_ratio/min": 0.5595287680625916, + "sampling/sampling_logp_difference/max": 0.580660343170166, + "sampling/sampling_logp_difference/mean": 0.011128809303045273, + "step": 167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.047411807230674e-05, + "clip_ratio/low_min": 6.047411807230674e-05, + "clip_ratio/region_mean": 6.047411807230674e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 504.21875, + "completions/mean_terminated_length": 466.5357360839844, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "entropy": 0.45196371898055077, + "epoch": 0.08993576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004683290142565966, + "learning_rate": 1e-05, + "loss": 0.0917, + "num_tokens": 3818909.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.46566903591156, + "sampling/importance_sampling_ratio/mean": 0.999906063079834, + "sampling/importance_sampling_ratio/min": 0.6109153628349304, + "sampling/sampling_logp_difference/max": 0.4927968978881836, + "sampling/sampling_logp_difference/mean": 0.011523962952196598, + "step": 168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 552.4375, + "completions/mean_terminated_length": 439.5238037109375, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.5149292200803757, + "epoch": 0.0904710920770878, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0017130245687440038, + "learning_rate": 1e-05, + "loss": 0.0647, + "num_tokens": 3840803.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.572034478187561, + "sampling/importance_sampling_ratio/mean": 1.0001252889633179, + "sampling/importance_sampling_ratio/min": 0.7152678370475769, + "sampling/sampling_logp_difference/max": 0.45237064361572266, + "sampling/sampling_logp_difference/mean": 0.013675541616976261, + "step": 169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.792944673681632e-05, + "clip_ratio/low_min": 4.792944673681632e-05, + "clip_ratio/region_mean": 4.792944673681632e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 511.6875, + "completions/mean_terminated_length": 475.0714416503906, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.47603290528059006, + "epoch": 0.09100642398286937, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0046, + "num_tokens": 3861041.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.43668794631958, + "sampling/importance_sampling_ratio/mean": 1.000077247619629, + "sampling/importance_sampling_ratio/min": 0.6418737769126892, + "sampling/sampling_logp_difference/max": 0.44336366653442383, + "sampling/sampling_logp_difference/mean": 0.013669274747371674, + "step": 170 + }, + { + "clip_ratio/high_max": 4.765535777551122e-05, + "clip_ratio/high_mean": 4.765535777551122e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.765535777551122e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 594.71875, + "completions/mean_terminated_length": 503.952392578125, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "entropy": 0.35366255417466164, + "epoch": 0.09154175588865096, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0049919020384550095, + "learning_rate": 1e-05, + "loss": -0.0555, + "num_tokens": 3884000.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2704286575317383, + "sampling/importance_sampling_ratio/mean": 1.000166654586792, + "sampling/importance_sampling_ratio/min": 0.6700748801231384, + "sampling/sampling_logp_difference/max": 0.40036582946777344, + "sampling/sampling_logp_difference/mean": 0.010553712956607342, + "step": 171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 537.9375, + "completions/mean_terminated_length": 522.6000366210938, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.4217514730989933, + "epoch": 0.09207708779443255, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004006472881883383, + "learning_rate": 1e-05, + "loss": 0.0315, + "num_tokens": 3904670.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4209245443344116, + "sampling/importance_sampling_ratio/mean": 1.0002920627593994, + "sampling/importance_sampling_ratio/min": 0.5328462719917297, + "sampling/sampling_logp_difference/max": 0.6295223236083984, + "sampling/sampling_logp_difference/mean": 0.01134710293263197, + "step": 172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 570.5625, + "completions/mean_terminated_length": 550.137939453125, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "entropy": 0.4218416325747967, + "epoch": 0.09261241970021414, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002963853068649769, + "learning_rate": 1e-05, + "loss": -0.0135, + "num_tokens": 3927008.0, + "reward": 0.4375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4399760961532593, + "sampling/importance_sampling_ratio/mean": 1.0001033544540405, + "sampling/importance_sampling_ratio/min": 0.590807318687439, + "sampling/sampling_logp_difference/max": 0.5262653827667236, + "sampling/sampling_logp_difference/mean": 0.012068185023963451, + "step": 173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 457.46875, + "completions/mean_terminated_length": 436.7666931152344, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.36681827902793884, + "epoch": 0.09314775160599571, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009849708527326584, + "learning_rate": 1e-05, + "loss": 0.0329, + "num_tokens": 3945191.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5823709964752197, + "sampling/importance_sampling_ratio/mean": 1.0002021789550781, + "sampling/importance_sampling_ratio/min": 0.7909921407699585, + "sampling/sampling_logp_difference/max": 0.4589242935180664, + "sampling/sampling_logp_difference/mean": 0.010868269018828869, + "step": 174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.520795482676476e-05, + "clip_ratio/low_min": 4.520795482676476e-05, + "clip_ratio/region_mean": 4.520795482676476e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 608.03125, + "completions/mean_terminated_length": 571.1154174804688, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.41545870900154114, + "epoch": 0.0936830835117773, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006024455651640892, + "learning_rate": 1e-05, + "loss": 0.026, + "num_tokens": 3968568.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.771843433380127, + "sampling/importance_sampling_ratio/mean": 1.0000227689743042, + "sampling/importance_sampling_ratio/min": 0.6450721621513367, + "sampling/sampling_logp_difference/max": 0.5720205307006836, + "sampling/sampling_logp_difference/mean": 0.011757463216781616, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 555.34375, + "completions/mean_terminated_length": 484.4583435058594, + "completions/min_length": 347.0, + "completions/min_terminated_length": 347.0, + "entropy": 0.4451611191034317, + "epoch": 0.09421841541755889, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007888618856668472, + "learning_rate": 1e-05, + "loss": 0.0464, + "num_tokens": 3990555.0, + "reward": 0.375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0000252723693848, + "sampling/importance_sampling_ratio/min": 0.6822840571403503, + "sampling/sampling_logp_difference/max": 1.145115852355957, + "sampling/sampling_logp_difference/mean": 0.01246458850800991, + "step": 176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 648.59375, + "completions/mean_terminated_length": 543.2352905273438, + "completions/min_length": 357.0, + "completions/min_terminated_length": 357.0, + "entropy": 0.5301221944391727, + "epoch": 0.09475374732334046, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006469637155532837, + "learning_rate": 1e-05, + "loss": 0.0262, + "num_tokens": 4015790.0, + "reward": 0.25, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4269806146621704, + "sampling/importance_sampling_ratio/mean": 1.0002111196517944, + "sampling/importance_sampling_ratio/min": 0.7875787615776062, + "sampling/sampling_logp_difference/max": 0.3555607795715332, + "sampling/sampling_logp_difference/mean": 0.012512173503637314, + "step": 177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 489.28125, + "completions/mean_terminated_length": 396.375, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.5177386812865734, + "epoch": 0.09528907922912205, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006932341493666172, + "learning_rate": 1e-05, + "loss": -0.0428, + "num_tokens": 4034903.0, + "reward": 0.53125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3761708736419678, + "sampling/importance_sampling_ratio/mean": 1.0001049041748047, + "sampling/importance_sampling_ratio/min": 0.6697771549224854, + "sampling/sampling_logp_difference/max": 0.40081024169921875, + "sampling/sampling_logp_difference/mean": 0.013819603249430656, + "step": 178 + }, + { + "clip_ratio/high_max": 7.982119859661907e-05, + "clip_ratio/high_mean": 7.982119859661907e-05, + "clip_ratio/low_mean": 4.5306271204026416e-05, + "clip_ratio/low_min": 4.5306271204026416e-05, + "clip_ratio/region_mean": 0.00012512746980064549, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 535.03125, + "completions/mean_terminated_length": 501.7500305175781, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.5154776051640511, + "epoch": 0.09582441113490364, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009936594404280186, + "learning_rate": 1e-05, + "loss": 0.0506, + "num_tokens": 4055600.0, + "reward": 0.4375, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3805251121520996, + "sampling/importance_sampling_ratio/mean": 1.0003159046173096, + "sampling/importance_sampling_ratio/min": 0.5657485127449036, + "sampling/sampling_logp_difference/max": 0.5696055889129639, + "sampling/sampling_logp_difference/mean": 0.013102860189974308, + "step": 179 + }, + { + "clip_ratio/high_max": 5.9495479945326224e-05, + "clip_ratio/high_mean": 5.9495479945326224e-05, + "clip_ratio/low_mean": 4.829984391108155e-05, + "clip_ratio/low_min": 4.829984391108155e-05, + "clip_ratio/region_mean": 0.00010779532385640778, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 586.6875, + "completions/mean_terminated_length": 544.84619140625, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.3384445421397686, + "epoch": 0.09635974304068523, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0049512009136378765, + "learning_rate": 1e-05, + "loss": 0.0639, + "num_tokens": 4078358.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2987746000289917, + "sampling/importance_sampling_ratio/mean": 0.999905526638031, + "sampling/importance_sampling_ratio/min": 0.3770841360092163, + "sampling/sampling_logp_difference/max": 0.9752869606018066, + "sampling/sampling_logp_difference/mean": 0.009852582588791847, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001063377276295796, + "clip_ratio/low_min": 0.0001063377276295796, + "clip_ratio/region_mean": 0.0001063377276295796, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 500.96875, + "completions/mean_terminated_length": 462.8214416503906, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.5007325038313866, + "epoch": 0.0968950749464668, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013404779136180878, + "learning_rate": 1e-05, + "loss": 0.1112, + "num_tokens": 4097981.0, + "reward": 0.5625, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3490186929702759, + "sampling/importance_sampling_ratio/mean": 1.0000747442245483, + "sampling/importance_sampling_ratio/min": 0.6561200618743896, + "sampling/sampling_logp_difference/max": 0.42141151428222656, + "sampling/sampling_logp_difference/mean": 0.013804241083562374, + "step": 181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.427913518156856e-05, + "clip_ratio/low_min": 4.427913518156856e-05, + "clip_ratio/region_mean": 4.427913518156856e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 629.25, + "completions/mean_terminated_length": 574.95654296875, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 0.3962312899529934, + "epoch": 0.0974304068522484, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0059975991025567055, + "learning_rate": 1e-05, + "loss": 0.0009, + "num_tokens": 4121725.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3976343870162964, + "sampling/importance_sampling_ratio/mean": 1.0000478029251099, + "sampling/importance_sampling_ratio/min": 0.7750535607337952, + "sampling/sampling_logp_difference/max": 0.3347811698913574, + "sampling/sampling_logp_difference/mean": 0.011098440736532211, + "step": 182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 650.84375, + "completions/mean_terminated_length": 597.5909423828125, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.4029821865260601, + "epoch": 0.09796573875802998, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004008599556982517, + "learning_rate": 1e-05, + "loss": 0.0272, + "num_tokens": 4146952.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2983002662658691, + "sampling/importance_sampling_ratio/mean": 1.000109314918518, + "sampling/importance_sampling_ratio/min": 0.6508774757385254, + "sampling/sampling_logp_difference/max": 0.42943382263183594, + "sampling/sampling_logp_difference/mean": 0.011147662065923214, + "step": 183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011048150190617889, + "clip_ratio/low_min": 0.00011048150190617889, + "clip_ratio/region_mean": 0.00011048150190617889, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 556.96875, + "completions/mean_terminated_length": 486.625, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.4544137492775917, + "epoch": 0.09850107066381156, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011129366233944893, + "learning_rate": 1e-05, + "loss": 0.0533, + "num_tokens": 4169095.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3407338857650757, + "sampling/importance_sampling_ratio/mean": 0.999880313873291, + "sampling/importance_sampling_ratio/min": 0.4061211049556732, + "sampling/sampling_logp_difference/max": 0.9011039137840271, + "sampling/sampling_logp_difference/mean": 0.011544800363481045, + "step": 184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 546.28125, + "completions/mean_terminated_length": 531.5, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.35225995630025864, + "epoch": 0.09903640256959315, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01049327664077282, + "learning_rate": 1e-05, + "loss": 0.0165, + "num_tokens": 4190336.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3624271154403687, + "sampling/importance_sampling_ratio/mean": 0.9999856948852539, + "sampling/importance_sampling_ratio/min": 0.6186681389808655, + "sampling/sampling_logp_difference/max": 0.48018622398376465, + "sampling/sampling_logp_difference/mean": 0.010624032467603683, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.007704647956416e-05, + "clip_ratio/low_min": 9.007704647956416e-05, + "clip_ratio/region_mean": 9.007704647956416e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 653.5625, + "completions/mean_terminated_length": 601.5454711914062, + "completions/min_length": 409.0, + "completions/min_terminated_length": 409.0, + "entropy": 0.4254562631249428, + "epoch": 0.09957173447537473, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010685713030397892, + "learning_rate": 1e-05, + "loss": 0.0386, + "num_tokens": 4215210.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.816921591758728, + "sampling/importance_sampling_ratio/mean": 0.999805212020874, + "sampling/importance_sampling_ratio/min": 0.012927633710205555, + "sampling/sampling_logp_difference/max": 4.348388195037842, + "sampling/sampling_logp_difference/mean": 0.012169590219855309, + "step": 186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019159849398420192, + "clip_ratio/low_min": 0.00019159849398420192, + "clip_ratio/region_mean": 0.00019159849398420192, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 476.03125, + "completions/mean_terminated_length": 434.3214416503906, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.4416114613413811, + "epoch": 0.10010706638115632, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005831408780068159, + "learning_rate": 1e-05, + "loss": 0.0415, + "num_tokens": 4233715.0, + "reward": 0.09375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.2900828123092651, + "sampling/importance_sampling_ratio/mean": 1.0000747442245483, + "sampling/importance_sampling_ratio/min": 0.7412513494491577, + "sampling/sampling_logp_difference/max": 0.29941558837890625, + "sampling/sampling_logp_difference/mean": 0.012264476157724857, + "step": 187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.5553937525255606e-05, + "clip_ratio/low_min": 4.5553937525255606e-05, + "clip_ratio/region_mean": 4.5553937525255606e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 588.5, + "completions/mean_terminated_length": 555.25927734375, + "completions/min_length": 355.0, + "completions/min_terminated_length": 355.0, + "entropy": 0.3835572823882103, + "epoch": 0.1006423982869379, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013091735541820526, + "learning_rate": 1e-05, + "loss": 0.079, + "num_tokens": 4256451.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5063872337341309, + "sampling/importance_sampling_ratio/mean": 0.9999738335609436, + "sampling/importance_sampling_ratio/min": 0.37463682889938354, + "sampling/sampling_logp_difference/max": 0.9817981719970703, + "sampling/sampling_logp_difference/mean": 0.0106464559212327, + "step": 188 + }, + { + "clip_ratio/high_max": 5.089576370664872e-05, + "clip_ratio/high_mean": 5.089576370664872e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.089576370664872e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 588.03125, + "completions/mean_terminated_length": 554.7037353515625, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.4354232996702194, + "epoch": 0.10117773019271949, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012282553128898144, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 4279044.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3277099132537842, + "sampling/importance_sampling_ratio/mean": 1.0001565217971802, + "sampling/importance_sampling_ratio/min": 0.6818786859512329, + "sampling/sampling_logp_difference/max": 0.3829035758972168, + "sampling/sampling_logp_difference/mean": 0.012432866729795933, + "step": 189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018578031085780822, + "clip_ratio/low_min": 0.00018578031085780822, + "clip_ratio/region_mean": 0.00018578031085780822, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 684.96875, + "completions/mean_terminated_length": 546.5833740234375, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "entropy": 0.5379181317985058, + "epoch": 0.10171306209850108, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0023843999952077866, + "learning_rate": 1e-05, + "loss": 0.0123, + "num_tokens": 4305499.0, + "reward": 0.03125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3420454263687134, + "sampling/importance_sampling_ratio/mean": 1.0002247095108032, + "sampling/importance_sampling_ratio/min": 0.7700905799865723, + "sampling/sampling_logp_difference/max": 0.29419493675231934, + "sampling/sampling_logp_difference/mean": 0.01415625587105751, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.1736228013178334e-05, + "clip_ratio/low_min": 4.1736228013178334e-05, + "clip_ratio/region_mean": 4.1736228013178334e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 645.375, + "completions/mean_terminated_length": 604.5, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.3155921474099159, + "epoch": 0.10224839400428265, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006586519535630941, + "learning_rate": 1e-05, + "loss": 0.0194, + "num_tokens": 4330479.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3585346937179565, + "sampling/importance_sampling_ratio/mean": 1.0000219345092773, + "sampling/importance_sampling_ratio/min": 0.6594923138618469, + "sampling/sampling_logp_difference/max": 0.41628503799438477, + "sampling/sampling_logp_difference/mean": 0.009408274665474892, + "step": 191 + }, + { + "clip_ratio/high_max": 0.00010500150165171362, + "clip_ratio/high_mean": 0.00010500150165171362, + "clip_ratio/low_mean": 4.41852243966423e-05, + "clip_ratio/low_min": 4.41852243966423e-05, + "clip_ratio/region_mean": 0.00014918672604835592, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 671.40625, + "completions/mean_terminated_length": 596.2777709960938, + "completions/min_length": 446.0, + "completions/min_terminated_length": 446.0, + "entropy": 0.4409182704985142, + "epoch": 0.10278372591006424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013342789374291897, + "learning_rate": 1e-05, + "loss": 0.0426, + "num_tokens": 4355868.0, + "reward": 0.34375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4797009229660034, + "sampling/importance_sampling_ratio/mean": 1.000070333480835, + "sampling/importance_sampling_ratio/min": 0.7045159935951233, + "sampling/sampling_logp_difference/max": 0.39183998107910156, + "sampling/sampling_logp_difference/mean": 0.012391703203320503, + "step": 192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014794354865443893, + "clip_ratio/low_min": 0.00014794354865443893, + "clip_ratio/region_mean": 0.00014794354865443893, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 622.5, + "completions/mean_terminated_length": 522.9473876953125, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 0.49924905225634575, + "epoch": 0.10331905781584583, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008667836897075176, + "learning_rate": 1e-05, + "loss": 0.0616, + "num_tokens": 4379716.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4387460947036743, + "sampling/importance_sampling_ratio/mean": 0.9998687505722046, + "sampling/importance_sampling_ratio/min": 0.7021949887275696, + "sampling/sampling_logp_difference/max": 0.363771915435791, + "sampling/sampling_logp_difference/mean": 0.013727799989283085, + "step": 193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.475474270293489e-05, + "clip_ratio/low_min": 4.475474270293489e-05, + "clip_ratio/region_mean": 4.475474270293489e-05, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 702.53125, + "completions/mean_terminated_length": 593.4166870117188, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "entropy": 0.55833300948143, + "epoch": 0.10385438972162742, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01016563642770052, + "learning_rate": 1e-05, + "loss": 0.0434, + "num_tokens": 4406357.0, + "reward": 0.25, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7262457609176636, + "sampling/importance_sampling_ratio/mean": 0.9997605681419373, + "sampling/importance_sampling_ratio/min": 0.7002395391464233, + "sampling/sampling_logp_difference/max": 0.5459489822387695, + "sampling/sampling_logp_difference/mean": 0.014348062686622143, + "step": 194 + }, + { + "clip_ratio/high_max": 0.00011837556303362362, + "clip_ratio/high_mean": 0.00011837556303362362, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00011837556303362362, + "completions/clipped_ratio": 0.0, + "completions/max_length": 761.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 560.84375, + "completions/mean_terminated_length": 560.84375, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.3299122229218483, + "epoch": 0.10438972162740899, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01046715397387743, + "learning_rate": 1e-05, + "loss": 0.0722, + "num_tokens": 4427864.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4348326921463013, + "sampling/importance_sampling_ratio/mean": 0.9998531937599182, + "sampling/importance_sampling_ratio/min": 0.7042420506477356, + "sampling/sampling_logp_difference/max": 0.36104822158813477, + "sampling/sampling_logp_difference/mean": 0.00976661778986454, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.271951860981062e-05, + "clip_ratio/low_min": 6.271951860981062e-05, + "clip_ratio/region_mean": 6.271951860981062e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 525.90625, + "completions/mean_terminated_length": 518.0967407226562, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.30321745201945305, + "epoch": 0.10492505353319058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011381514370441437, + "learning_rate": 1e-05, + "loss": 0.0069, + "num_tokens": 4448381.0, + "reward": 0.6875, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3842442035675049, + "sampling/importance_sampling_ratio/mean": 1.0000147819519043, + "sampling/importance_sampling_ratio/min": 0.47558218240737915, + "sampling/sampling_logp_difference/max": 0.7432155609130859, + "sampling/sampling_logp_difference/mean": 0.009583337232470512, + "step": 196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 612.84375, + "completions/mean_terminated_length": 506.6842041015625, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.5010893531143665, + "epoch": 0.10546038543897217, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015702135860919952, + "learning_rate": 1e-05, + "loss": 0.0208, + "num_tokens": 4471688.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4714659452438354, + "sampling/importance_sampling_ratio/mean": 1.0001922845840454, + "sampling/importance_sampling_ratio/min": 0.6426209807395935, + "sampling/sampling_logp_difference/max": 0.4422001838684082, + "sampling/sampling_logp_difference/mean": 0.013482176698744297, + "step": 197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010368593939347193, + "clip_ratio/low_min": 0.00010368593939347193, + "clip_ratio/region_mean": 0.00010368593939347193, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 586.375, + "completions/mean_terminated_length": 515.3043823242188, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.37887583673000336, + "epoch": 0.10599571734475374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019367409870028496, + "learning_rate": 1e-05, + "loss": 0.0901, + "num_tokens": 4494548.0, + "reward": 0.5, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6417173147201538, + "sampling/importance_sampling_ratio/mean": 0.9996522665023804, + "sampling/importance_sampling_ratio/min": 0.5451266169548035, + "sampling/sampling_logp_difference/max": 0.6067371368408203, + "sampling/sampling_logp_difference/mean": 0.010913314297795296, + "step": 198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.2244002543156967e-05, + "clip_ratio/low_min": 4.2244002543156967e-05, + "clip_ratio/region_mean": 4.2244002543156967e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 574.96875, + "completions/mean_terminated_length": 473.8571472167969, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.3855096586048603, + "epoch": 0.10653104925053533, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008093578740954399, + "learning_rate": 1e-05, + "loss": 0.1202, + "num_tokens": 4516411.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2982853651046753, + "sampling/importance_sampling_ratio/mean": 0.9998065829277039, + "sampling/importance_sampling_ratio/min": 0.6771559119224548, + "sampling/sampling_logp_difference/max": 0.38985371589660645, + "sampling/sampling_logp_difference/mean": 0.011096972040832043, + "step": 199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 471.25, + "completions/mean_terminated_length": 451.4667053222656, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.43591704219579697, + "epoch": 0.10706638115631692, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0033567268401384354, + "learning_rate": 1e-05, + "loss": 0.0702, + "num_tokens": 4535019.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3899041414260864, + "sampling/importance_sampling_ratio/mean": 1.0000931024551392, + "sampling/importance_sampling_ratio/min": 0.6976861953735352, + "sampling/sampling_logp_difference/max": 0.3599858283996582, + "sampling/sampling_logp_difference/mean": 0.013180355541408062, + "step": 200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 519.90625, + "completions/mean_terminated_length": 494.2413635253906, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3362443335354328, + "epoch": 0.1076017130620985, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0033777381759136915, + "learning_rate": 1e-05, + "loss": -0.0141, + "num_tokens": 4554952.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.2886570692062378, + "sampling/importance_sampling_ratio/mean": 1.0002273321151733, + "sampling/importance_sampling_ratio/min": 0.6547046303749084, + "sampling/sampling_logp_difference/max": 0.4235711097717285, + "sampling/sampling_logp_difference/mean": 0.010121806524693966, + "step": 201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 598.71875, + "completions/mean_terminated_length": 551.3200073242188, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 0.31316785886883736, + "epoch": 0.10813704496788008, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0072125643491744995, + "learning_rate": 1e-05, + "loss": 0.0626, + "num_tokens": 4577591.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4741240739822388, + "sampling/importance_sampling_ratio/mean": 1.0000097751617432, + "sampling/importance_sampling_ratio/min": 0.739193856716156, + "sampling/sampling_logp_difference/max": 0.388063907623291, + "sampling/sampling_logp_difference/mean": 0.009544799104332924, + "step": 202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 551.375, + "completions/mean_terminated_length": 528.9655151367188, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.3635198548436165, + "epoch": 0.10867237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007378813344985247, + "learning_rate": 1e-05, + "loss": 0.0863, + "num_tokens": 4598667.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4234294891357422, + "sampling/importance_sampling_ratio/mean": 0.9999359250068665, + "sampling/importance_sampling_ratio/min": 0.713753342628479, + "sampling/sampling_logp_difference/max": 0.3530690670013428, + "sampling/sampling_logp_difference/mean": 0.010574338026344776, + "step": 203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015298444486688823, + "clip_ratio/low_min": 0.00015298444486688823, + "clip_ratio/region_mean": 0.00015298444486688823, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 660.9375, + "completions/mean_terminated_length": 596.7000122070312, + "completions/min_length": 395.0, + "completions/min_terminated_length": 395.0, + "entropy": 0.4041886292397976, + "epoch": 0.10920770877944326, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013808957301080227, + "learning_rate": 1e-05, + "loss": 0.0327, + "num_tokens": 4623601.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3408790826797485, + "sampling/importance_sampling_ratio/mean": 1.0000574588775635, + "sampling/importance_sampling_ratio/min": 0.7060325741767883, + "sampling/sampling_logp_difference/max": 0.34809398651123047, + "sampling/sampling_logp_difference/mean": 0.011319300159811974, + "step": 204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 515.15625, + "completions/mean_terminated_length": 444.3599853515625, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.6250455193221569, + "epoch": 0.10974304068522484, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007350800558924675, + "learning_rate": 1e-05, + "loss": -0.0154, + "num_tokens": 4644134.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.457682728767395, + "sampling/importance_sampling_ratio/mean": 1.0000706911087036, + "sampling/importance_sampling_ratio/min": 0.7095704078674316, + "sampling/sampling_logp_difference/max": 0.3768479824066162, + "sampling/sampling_logp_difference/mean": 0.015043867751955986, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 549.1875, + "completions/mean_terminated_length": 508.6666564941406, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "entropy": 0.39418603852391243, + "epoch": 0.11027837259100642, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0042163603939116, + "learning_rate": 1e-05, + "loss": 0.0581, + "num_tokens": 4665684.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5864545106887817, + "sampling/importance_sampling_ratio/mean": 0.9994666576385498, + "sampling/importance_sampling_ratio/min": 0.5278363823890686, + "sampling/sampling_logp_difference/max": 0.6389689445495605, + "sampling/sampling_logp_difference/mean": 0.011330260895192623, + "step": 206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.976114723831415e-05, + "clip_ratio/low_min": 4.976114723831415e-05, + "clip_ratio/region_mean": 4.976114723831415e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 571.78125, + "completions/mean_terminated_length": 543.75, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.4335530735552311, + "epoch": 0.11081370449678801, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.002859686966985464, + "learning_rate": 1e-05, + "loss": 0.0697, + "num_tokens": 4687573.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.550080418586731, + "sampling/importance_sampling_ratio/mean": 0.9997480511665344, + "sampling/importance_sampling_ratio/min": 0.7509428858757019, + "sampling/sampling_logp_difference/max": 0.4383068084716797, + "sampling/sampling_logp_difference/mean": 0.011670460924506187, + "step": 207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00022257391174207442, + "clip_ratio/low_min": 0.00022257391174207442, + "clip_ratio/region_mean": 0.00022257391174207442, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 597.96875, + "completions/mean_terminated_length": 541.2916870117188, + "completions/min_length": 370.0, + "completions/min_terminated_length": 370.0, + "entropy": 0.4528747610747814, + "epoch": 0.11134903640256959, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012036731466650963, + "learning_rate": 1e-05, + "loss": 0.0409, + "num_tokens": 4710412.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001055002212524, + "sampling/importance_sampling_ratio/min": 0.7277035713195801, + "sampling/sampling_logp_difference/max": 0.757115364074707, + "sampling/sampling_logp_difference/mean": 0.012383118271827698, + "step": 208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.002000762033276e-05, + "clip_ratio/low_min": 5.002000762033276e-05, + "clip_ratio/region_mean": 5.002000762033276e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 553.34375, + "completions/mean_terminated_length": 539.0333862304688, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.40291527658700943, + "epoch": 0.11188436830835118, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.004115377552807331, + "learning_rate": 1e-05, + "loss": 0.0368, + "num_tokens": 4731567.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3464263677597046, + "sampling/importance_sampling_ratio/mean": 0.9997650384902954, + "sampling/importance_sampling_ratio/min": 0.7798070907592773, + "sampling/sampling_logp_difference/max": 0.2974538803100586, + "sampling/sampling_logp_difference/mean": 0.011762114241719246, + "step": 209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 526.6875, + "completions/mean_terminated_length": 446.25, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.44322478026151657, + "epoch": 0.11241970021413276, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009065919555723667, + "learning_rate": 1e-05, + "loss": 0.0643, + "num_tokens": 4751861.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4833462238311768, + "sampling/importance_sampling_ratio/mean": 0.9995315670967102, + "sampling/importance_sampling_ratio/min": 0.765781581401825, + "sampling/sampling_logp_difference/max": 0.3943004608154297, + "sampling/sampling_logp_difference/mean": 0.012334013357758522, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 584.90625, + "completions/mean_terminated_length": 523.875, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.3720376640558243, + "epoch": 0.11295503211991435, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009011111222207546, + "learning_rate": 1e-05, + "loss": 0.0261, + "num_tokens": 4774082.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.502943992614746, + "sampling/importance_sampling_ratio/mean": 1.0001670122146606, + "sampling/importance_sampling_ratio/min": 0.6831524968147278, + "sampling/sampling_logp_difference/max": 0.4074258804321289, + "sampling/sampling_logp_difference/mean": 0.010632229968905449, + "step": 211 + }, + { + "clip_ratio/high_max": 5.6637971283635125e-05, + "clip_ratio/high_mean": 5.6637971283635125e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.6637971283635125e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 536.0, + "completions/mean_terminated_length": 520.5333862304688, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.3791799619793892, + "epoch": 0.11349036402569593, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009205985814332962, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 4794986.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4290452003479004, + "sampling/importance_sampling_ratio/mean": 0.9997783899307251, + "sampling/importance_sampling_ratio/min": 0.7603805065155029, + "sampling/sampling_logp_difference/max": 0.3570065498352051, + "sampling/sampling_logp_difference/mean": 0.010945064947009087, + "step": 212 + }, + { + "clip_ratio/high_max": 4.2866940930252895e-05, + "clip_ratio/high_mean": 4.2866940930252895e-05, + "clip_ratio/low_mean": 0.00022432948389905505, + "clip_ratio/low_min": 0.00022432948389905505, + "clip_ratio/region_mean": 0.00026719642482930794, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 679.5, + "completions/mean_terminated_length": 610.6666870117188, + "completions/min_length": 401.0, + "completions/min_terminated_length": 401.0, + "entropy": 0.4458453692495823, + "epoch": 0.11402569593147752, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0072832051664590836, + "learning_rate": 1e-05, + "loss": 0.0266, + "num_tokens": 4820618.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4859117269515991, + "sampling/importance_sampling_ratio/mean": 0.9998586177825928, + "sampling/importance_sampling_ratio/min": 0.6292915940284729, + "sampling/sampling_logp_difference/max": 0.46316051483154297, + "sampling/sampling_logp_difference/mean": 0.01230671163648367, + "step": 213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.283178325043991e-05, + "clip_ratio/low_min": 5.283178325043991e-05, + "clip_ratio/region_mean": 5.283178325043991e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 630.25, + "completions/mean_terminated_length": 547.6000366210938, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.4581412896513939, + "epoch": 0.1145610278372591, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0157, + "num_tokens": 4844802.0, + "reward": 0.46875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.635581612586975, + "sampling/importance_sampling_ratio/mean": 0.9998361468315125, + "sampling/importance_sampling_ratio/min": 0.7378471493721008, + "sampling/sampling_logp_difference/max": 0.49199843406677246, + "sampling/sampling_logp_difference/mean": 0.01316896453499794, + "step": 214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 536.5625, + "completions/mean_terminated_length": 521.1333618164062, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.35761559009552, + "epoch": 0.11509635974304068, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019765743985772133, + "learning_rate": 1e-05, + "loss": -0.0135, + "num_tokens": 4865948.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4213097095489502, + "sampling/importance_sampling_ratio/mean": 0.9999829530715942, + "sampling/importance_sampling_ratio/min": 0.5343505144119263, + "sampling/sampling_logp_difference/max": 0.6267032623291016, + "sampling/sampling_logp_difference/mean": 0.010485834442079067, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 560.0625, + "completions/mean_terminated_length": 490.75, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.4157865084707737, + "epoch": 0.11563169164882227, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0412, + "num_tokens": 4887566.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.556046724319458, + "sampling/importance_sampling_ratio/mean": 1.0002392530441284, + "sampling/importance_sampling_ratio/min": 0.7258667945861816, + "sampling/sampling_logp_difference/max": 0.44214844703674316, + "sampling/sampling_logp_difference/mean": 0.011593695729970932, + "step": 216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 549.65625, + "completions/mean_terminated_length": 518.4642944335938, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.34925256110727787, + "epoch": 0.11616702355460386, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01225263997912407, + "learning_rate": 1e-05, + "loss": 0.0054, + "num_tokens": 4908755.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4094576835632324, + "sampling/importance_sampling_ratio/mean": 1.0001062154769897, + "sampling/importance_sampling_ratio/min": 0.6197565793991089, + "sampling/sampling_logp_difference/max": 0.47842860221862793, + "sampling/sampling_logp_difference/mean": 0.011083544231951237, + "step": 217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013465177471516654, + "clip_ratio/low_min": 0.00013465177471516654, + "clip_ratio/region_mean": 0.00013465177471516654, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 694.5625, + "completions/mean_terminated_length": 587.2307739257812, + "completions/min_length": 426.0, + "completions/min_terminated_length": 426.0, + "entropy": 0.5617510974407196, + "epoch": 0.11670235546038545, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008800877258181572, + "learning_rate": 1e-05, + "loss": 0.0277, + "num_tokens": 4935429.0, + "reward": 0.25, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3903852701187134, + "sampling/importance_sampling_ratio/mean": 0.9998555779457092, + "sampling/importance_sampling_ratio/min": 0.4617181420326233, + "sampling/sampling_logp_difference/max": 0.7728006839752197, + "sampling/sampling_logp_difference/mean": 0.014596041291952133, + "step": 218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.682675252202898e-05, + "clip_ratio/low_min": 8.682675252202898e-05, + "clip_ratio/region_mean": 8.682675252202898e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 663.375, + "completions/mean_terminated_length": 582.0, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.48347389698028564, + "epoch": 0.11723768736616702, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010706407949328423, + "learning_rate": 1e-05, + "loss": 0.0167, + "num_tokens": 4960833.0, + "reward": 0.40625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5666464567184448, + "sampling/importance_sampling_ratio/mean": 0.9998987913131714, + "sampling/importance_sampling_ratio/min": 0.5342758893966675, + "sampling/sampling_logp_difference/max": 0.6268428564071655, + "sampling/sampling_logp_difference/mean": 0.013395486399531364, + "step": 219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015082780009834096, + "clip_ratio/low_min": 0.00015082780009834096, + "clip_ratio/region_mean": 0.00015082780009834096, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 566.78125, + "completions/mean_terminated_length": 488.0434875488281, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.5746861696243286, + "epoch": 0.11777301927194861, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021935919299721718, + "learning_rate": 1e-05, + "loss": 0.1448, + "num_tokens": 4982834.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.405480146408081, + "sampling/importance_sampling_ratio/mean": 1.0001972913742065, + "sampling/importance_sampling_ratio/min": 0.6488431692123413, + "sampling/sampling_logp_difference/max": 0.43256425857543945, + "sampling/sampling_logp_difference/mean": 0.013339702971279621, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 574.125, + "completions/mean_terminated_length": 538.2222290039062, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.3626738339662552, + "epoch": 0.1183083511777302, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011936082504689693, + "learning_rate": 1e-05, + "loss": 0.0502, + "num_tokens": 5004766.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3696194887161255, + "sampling/importance_sampling_ratio/mean": 1.0002532005310059, + "sampling/importance_sampling_ratio/min": 0.6588507294654846, + "sampling/sampling_logp_difference/max": 0.41725826263427734, + "sampling/sampling_logp_difference/mean": 0.01117282547056675, + "step": 221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001569602572999429, + "clip_ratio/low_min": 0.0001569602572999429, + "clip_ratio/region_mean": 0.0001569602572999429, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 584.3125, + "completions/mean_terminated_length": 512.434814453125, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.42847940325737, + "epoch": 0.11884368308351177, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010854063555598259, + "learning_rate": 1e-05, + "loss": -0.0077, + "num_tokens": 5026904.0, + "reward": 0.40625, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.36565363407135, + "sampling/importance_sampling_ratio/mean": 0.9997027516365051, + "sampling/importance_sampling_ratio/min": 0.6231113076210022, + "sampling/sampling_logp_difference/max": 0.47303009033203125, + "sampling/sampling_logp_difference/mean": 0.012134634889662266, + "step": 222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.999999873689376e-05, + "clip_ratio/low_min": 4.999999873689376e-05, + "clip_ratio/region_mean": 4.999999873689376e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 606.3125, + "completions/mean_terminated_length": 521.6190795898438, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.44984953850507736, + "epoch": 0.11937901498929336, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009545247070491314, + "learning_rate": 1e-05, + "loss": 0.0274, + "num_tokens": 5050130.0, + "reward": 0.5625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.385037899017334, + "sampling/importance_sampling_ratio/mean": 1.0000183582305908, + "sampling/importance_sampling_ratio/min": 0.7200703024864197, + "sampling/sampling_logp_difference/max": 0.3284064531326294, + "sampling/sampling_logp_difference/mean": 0.012084768153727055, + "step": 223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 602.3125, + "completions/mean_terminated_length": 564.0769653320312, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "entropy": 0.3796752355992794, + "epoch": 0.11991434689507495, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017084071412682533, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 5072548.0, + "reward": 0.8125, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.297909140586853, + "sampling/importance_sampling_ratio/mean": 0.9999545812606812, + "sampling/importance_sampling_ratio/min": 0.7055865526199341, + "sampling/sampling_logp_difference/max": 0.3487257957458496, + "sampling/sampling_logp_difference/mean": 0.010714119300246239, + "step": 224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 662.53125, + "completions/mean_terminated_length": 557.0625, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "entropy": 0.48830728977918625, + "epoch": 0.12044967880085652, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.02, + "num_tokens": 5097821.0, + "reward": 0.46875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4616483449935913, + "sampling/importance_sampling_ratio/mean": 1.0001503229141235, + "sampling/importance_sampling_ratio/min": 0.564860999584198, + "sampling/sampling_logp_difference/max": 0.5711755752563477, + "sampling/sampling_logp_difference/mean": 0.013598200865089893, + "step": 225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.84872312983498e-05, + "clip_ratio/low_min": 9.84872312983498e-05, + "clip_ratio/region_mean": 9.84872312983498e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 578.59375, + "completions/mean_terminated_length": 515.4583740234375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.49579084664583206, + "epoch": 0.12098501070663811, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01154306624084711, + "learning_rate": 1e-05, + "loss": 0.0863, + "num_tokens": 5120392.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6366358995437622, + "sampling/importance_sampling_ratio/mean": 0.9999333620071411, + "sampling/importance_sampling_ratio/min": 0.5065307021141052, + "sampling/sampling_logp_difference/max": 0.6801702976226807, + "sampling/sampling_logp_difference/mean": 0.01242026686668396, + "step": 226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002624518565426115, + "clip_ratio/low_min": 0.0002624518565426115, + "clip_ratio/region_mean": 0.0002624518565426115, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 711.53125, + "completions/mean_terminated_length": 617.4166870117188, + "completions/min_length": 494.0, + "completions/min_terminated_length": 494.0, + "entropy": 0.6983655318617821, + "epoch": 0.1215203426124197, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020195450633764267, + "learning_rate": 1e-05, + "loss": 0.0289, + "num_tokens": 5147929.0, + "reward": 0.1875, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6050238609313965, + "sampling/importance_sampling_ratio/mean": 0.9996456503868103, + "sampling/importance_sampling_ratio/min": 0.625224769115448, + "sampling/sampling_logp_difference/max": 0.47313857078552246, + "sampling/sampling_logp_difference/mean": 0.017196593806147575, + "step": 227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.796623034053482e-05, + "clip_ratio/low_min": 4.796623034053482e-05, + "clip_ratio/region_mean": 4.796623034053482e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 587.59375, + "completions/mean_terminated_length": 554.1851806640625, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "entropy": 0.32752997800707817, + "epoch": 0.12205567451820129, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003851040732115507, + "learning_rate": 1e-05, + "loss": 0.0239, + "num_tokens": 5170444.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4324769973754883, + "sampling/importance_sampling_ratio/mean": 1.0000749826431274, + "sampling/importance_sampling_ratio/min": 0.7732476592063904, + "sampling/sampling_logp_difference/max": 0.3594050407409668, + "sampling/sampling_logp_difference/mean": 0.009868286550045013, + "step": 228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017453929103794508, + "clip_ratio/low_min": 0.00017453929103794508, + "clip_ratio/region_mean": 0.00017453929103794508, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 646.53125, + "completions/mean_terminated_length": 591.3181762695312, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "entropy": 0.3754481337964535, + "epoch": 0.12259100642398287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00590737909078598, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 5194869.0, + "reward": 0.5625, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3619716167449951, + "sampling/importance_sampling_ratio/mean": 1.0001739263534546, + "sampling/importance_sampling_ratio/min": 0.6845538020133972, + "sampling/sampling_logp_difference/max": 0.37898802757263184, + "sampling/sampling_logp_difference/mean": 0.010325351729989052, + "step": 229 + }, + { + "clip_ratio/high_max": 7.318500865949318e-05, + "clip_ratio/high_mean": 7.318500865949318e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.318500865949318e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 504.125, + "completions/mean_terminated_length": 486.5333557128906, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3812452107667923, + "epoch": 0.12312633832976445, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0037694680504500866, + "learning_rate": 1e-05, + "loss": -0.0002, + "num_tokens": 5214745.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5416697263717651, + "sampling/importance_sampling_ratio/mean": 0.9997934699058533, + "sampling/importance_sampling_ratio/min": 0.6923429369926453, + "sampling/sampling_logp_difference/max": 0.43286609649658203, + "sampling/sampling_logp_difference/mean": 0.011613366194069386, + "step": 230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.278716344037093e-05, + "clip_ratio/low_min": 5.278716344037093e-05, + "clip_ratio/region_mean": 5.278716344037093e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 534.96875, + "completions/mean_terminated_length": 491.8148193359375, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.49364667758345604, + "epoch": 0.12366167023554604, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0083, + "num_tokens": 5235528.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5320346355438232, + "sampling/importance_sampling_ratio/mean": 1.0000654458999634, + "sampling/importance_sampling_ratio/min": 0.6256884932518005, + "sampling/sampling_logp_difference/max": 0.46890270709991455, + "sampling/sampling_logp_difference/mean": 0.012900633737444878, + "step": 231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.555555617320351e-05, + "clip_ratio/low_min": 5.555555617320351e-05, + "clip_ratio/region_mean": 5.555555617320351e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 478.3199768066406, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.3573584407567978, + "epoch": 0.12419700214132762, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002308719092980027, + "learning_rate": 1e-05, + "loss": 0.0239, + "num_tokens": 5256486.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.526574730873108, + "sampling/importance_sampling_ratio/mean": 1.0002806186676025, + "sampling/importance_sampling_ratio/min": 0.7009093165397644, + "sampling/sampling_logp_difference/max": 0.423026442527771, + "sampling/sampling_logp_difference/mean": 0.011031332425773144, + "step": 232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018253981397720054, + "clip_ratio/low_min": 0.00018253981397720054, + "clip_ratio/region_mean": 0.00018253981397720054, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 677.65625, + "completions/mean_terminated_length": 575.2667236328125, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "entropy": 0.433471143245697, + "epoch": 0.1247323340471092, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005887295119464397, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 5282179.0, + "reward": 0.34375, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.9891812801361084, + "sampling/importance_sampling_ratio/mean": 1.0000905990600586, + "sampling/importance_sampling_ratio/min": 0.6790370941162109, + "sampling/sampling_logp_difference/max": 0.6877231597900391, + "sampling/sampling_logp_difference/mean": 0.011292919516563416, + "step": 233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015975481073837727, + "clip_ratio/low_min": 0.00015975481073837727, + "clip_ratio/region_mean": 0.00015975481073837727, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 599.15625, + "completions/mean_terminated_length": 533.0869750976562, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 0.4720841348171234, + "epoch": 0.1252676659528908, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0028473229613155127, + "learning_rate": 1e-05, + "loss": 0.0933, + "num_tokens": 5304928.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5715802907943726, + "sampling/importance_sampling_ratio/mean": 1.0001165866851807, + "sampling/importance_sampling_ratio/min": 0.670651376247406, + "sampling/sampling_logp_difference/max": 0.45208168029785156, + "sampling/sampling_logp_difference/mean": 0.013936175964772701, + "step": 234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 517.65625, + "completions/mean_terminated_length": 491.75860595703125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.45977578312158585, + "epoch": 0.12580299785867238, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004914202727377415, + "learning_rate": 1e-05, + "loss": 0.0165, + "num_tokens": 5325693.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3612115383148193, + "sampling/importance_sampling_ratio/mean": 0.9997780919075012, + "sampling/importance_sampling_ratio/min": 0.7317842841148376, + "sampling/sampling_logp_difference/max": 0.3122694492340088, + "sampling/sampling_logp_difference/mean": 0.012379621155560017, + "step": 235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 559.6875, + "completions/mean_terminated_length": 501.3599853515625, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.48104002699255943, + "epoch": 0.12633832976445397, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0393, + "num_tokens": 5348611.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3752455711364746, + "sampling/importance_sampling_ratio/mean": 0.9995938539505005, + "sampling/importance_sampling_ratio/min": 0.6394771337509155, + "sampling/sampling_logp_difference/max": 0.44710445404052734, + "sampling/sampling_logp_difference/mean": 0.013101525604724884, + "step": 236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001430235934094526, + "clip_ratio/low_min": 0.0001430235934094526, + "clip_ratio/region_mean": 0.0001430235934094526, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 540.59375, + "completions/mean_terminated_length": 488.11541748046875, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.3864423558115959, + "epoch": 0.12687366167023553, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0048812697641551495, + "learning_rate": 1e-05, + "loss": -0.0265, + "num_tokens": 5370014.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.299454689025879, + "sampling/importance_sampling_ratio/mean": 1.0000145435333252, + "sampling/importance_sampling_ratio/min": 0.7125881910324097, + "sampling/sampling_logp_difference/max": 0.3388516902923584, + "sampling/sampling_logp_difference/mean": 0.01179390400648117, + "step": 237 + }, + { + "clip_ratio/high_max": 4.8449612222611904e-05, + "clip_ratio/high_mean": 4.8449612222611904e-05, + "clip_ratio/low_mean": 9.447983029531315e-05, + "clip_ratio/low_min": 9.447983029531315e-05, + "clip_ratio/region_mean": 0.00014292944251792505, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 603.9375, + "completions/mean_terminated_length": 558.0, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.3614657334983349, + "epoch": 0.12740899357601712, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01957891508936882, + "learning_rate": 1e-05, + "loss": 0.0009, + "num_tokens": 5392988.0, + "reward": 0.71875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6607438325881958, + "sampling/importance_sampling_ratio/mean": 1.0000667572021484, + "sampling/importance_sampling_ratio/min": 0.7405574321746826, + "sampling/sampling_logp_difference/max": 0.507265567779541, + "sampling/sampling_logp_difference/mean": 0.01022407691925764, + "step": 238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 594.71875, + "completions/mean_terminated_length": 515.95458984375, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.44415390864014626, + "epoch": 0.1279443254817987, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0031295267399400473, + "learning_rate": 1e-05, + "loss": 0.0267, + "num_tokens": 5416299.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4592589139938354, + "sampling/importance_sampling_ratio/mean": 1.0006099939346313, + "sampling/importance_sampling_ratio/min": 0.6672098636627197, + "sampling/sampling_logp_difference/max": 0.4046506881713867, + "sampling/sampling_logp_difference/mean": 0.012880331836640835, + "step": 239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7313158322358504e-05, + "clip_ratio/low_min": 5.7313158322358504e-05, + "clip_ratio/region_mean": 5.7313158322358504e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 535.25, + "completions/mean_terminated_length": 481.5384826660156, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.5112757124006748, + "epoch": 0.1284796573875803, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015150925144553185, + "learning_rate": 1e-05, + "loss": 0.0354, + "num_tokens": 5437403.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4094347953796387, + "sampling/importance_sampling_ratio/mean": 1.0003074407577515, + "sampling/importance_sampling_ratio/min": 0.7242313027381897, + "sampling/sampling_logp_difference/max": 0.3431887626647949, + "sampling/sampling_logp_difference/mean": 0.014134557917714119, + "step": 240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011728248864528723, + "clip_ratio/low_min": 0.00011728248864528723, + "clip_ratio/region_mean": 0.00011728248864528723, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 583.1875, + "completions/mean_terminated_length": 521.5833740234375, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "entropy": 0.4333116486668587, + "epoch": 0.1290149892933619, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011063863523304462, + "learning_rate": 1e-05, + "loss": 0.0412, + "num_tokens": 5459801.0, + "reward": 0.625, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.333984613418579, + "sampling/importance_sampling_ratio/mean": 0.9999924302101135, + "sampling/importance_sampling_ratio/min": 0.698438823223114, + "sampling/sampling_logp_difference/max": 0.35890769958496094, + "sampling/sampling_logp_difference/mean": 0.012637640349566936, + "step": 241 + }, + { + "clip_ratio/high_max": 4.197447924525477e-05, + "clip_ratio/high_mean": 4.197447924525477e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.197447924525477e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 615.28125, + "completions/mean_terminated_length": 545.8636474609375, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.40572981908917427, + "epoch": 0.12955032119914348, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0034186160191893578, + "learning_rate": 1e-05, + "loss": 0.0382, + "num_tokens": 5483442.0, + "reward": 0.65625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2892894744873047, + "sampling/importance_sampling_ratio/mean": 1.0002504587173462, + "sampling/importance_sampling_ratio/min": 0.5395433902740479, + "sampling/sampling_logp_difference/max": 0.6170320510864258, + "sampling/sampling_logp_difference/mean": 0.011737329885363579, + "step": 242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.49688154226169e-05, + "clip_ratio/low_min": 6.49688154226169e-05, + "clip_ratio/region_mean": 6.49688154226169e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 581.15625, + "completions/mean_terminated_length": 561.8275756835938, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.34565189108252525, + "epoch": 0.13008565310492506, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006013086065649986, + "learning_rate": 1e-05, + "loss": 0.0372, + "num_tokens": 5505615.0, + "reward": 0.65625, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3997101783752441, + "sampling/importance_sampling_ratio/mean": 0.999925971031189, + "sampling/importance_sampling_ratio/min": 0.6597829461097717, + "sampling/sampling_logp_difference/max": 0.4158444404602051, + "sampling/sampling_logp_difference/mean": 0.010271354578435421, + "step": 243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 543.125, + "completions/mean_terminated_length": 468.16668701171875, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.40996165573596954, + "epoch": 0.13062098501070663, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013304386287927628, + "learning_rate": 1e-05, + "loss": 0.0606, + "num_tokens": 5526739.0, + "reward": 0.75, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2847847938537598, + "sampling/importance_sampling_ratio/mean": 0.9998285174369812, + "sampling/importance_sampling_ratio/min": 0.6456636786460876, + "sampling/sampling_logp_difference/max": 0.43747663497924805, + "sampling/sampling_logp_difference/mean": 0.011566098779439926, + "step": 244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013087704064673744, + "clip_ratio/low_min": 0.00013087704064673744, + "clip_ratio/region_mean": 0.00013087704064673744, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 675.0625, + "completions/mean_terminated_length": 611.4736938476562, + "completions/min_length": 350.0, + "completions/min_terminated_length": 350.0, + "entropy": 0.4671286754310131, + "epoch": 0.1311563169164882, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007557358127087355, + "learning_rate": 1e-05, + "loss": 0.0517, + "num_tokens": 5552573.0, + "reward": 0.46875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6242812871932983, + "sampling/importance_sampling_ratio/mean": 0.9999816417694092, + "sampling/importance_sampling_ratio/min": 0.7106167078018188, + "sampling/sampling_logp_difference/max": 0.4850654602050781, + "sampling/sampling_logp_difference/mean": 0.013589474372565746, + "step": 245 + }, + { + "clip_ratio/high_max": 0.0001058401758200489, + "clip_ratio/high_mean": 0.0001058401758200489, + "clip_ratio/low_mean": 6.003842281643301e-05, + "clip_ratio/low_min": 6.003842281643301e-05, + "clip_ratio/region_mean": 0.0001658785986364819, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 547.65625, + "completions/mean_terminated_length": 532.9666748046875, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.40443161129951477, + "epoch": 0.1316916488222698, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011984323151409626, + "learning_rate": 1e-05, + "loss": 0.0347, + "num_tokens": 5573706.0, + "reward": 0.53125, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5090829133987427, + "sampling/importance_sampling_ratio/mean": 0.9999943375587463, + "sampling/importance_sampling_ratio/min": 0.6009371280670166, + "sampling/sampling_logp_difference/max": 0.5092649459838867, + "sampling/sampling_logp_difference/mean": 0.011626939289271832, + "step": 246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018206869208370335, + "clip_ratio/low_min": 0.00018206869208370335, + "clip_ratio/region_mean": 0.00018206869208370335, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 538.15625, + "completions/mean_terminated_length": 495.59259033203125, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.37508508935570717, + "epoch": 0.1322269807280514, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016642436385154724, + "learning_rate": 1e-05, + "loss": 0.0751, + "num_tokens": 5594767.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4443391561508179, + "sampling/importance_sampling_ratio/mean": 0.9999256730079651, + "sampling/importance_sampling_ratio/min": 0.7800746560096741, + "sampling/sampling_logp_difference/max": 0.36765193939208984, + "sampling/sampling_logp_difference/mean": 0.011149761267006397, + "step": 247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.3995681810192764e-05, + "clip_ratio/low_min": 5.3995681810192764e-05, + "clip_ratio/region_mean": 5.3995681810192764e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 525.21875, + "completions/mean_terminated_length": 517.3870849609375, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.3842773735523224, + "epoch": 0.13276231263383298, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013428544625639915, + "learning_rate": 1e-05, + "loss": 0.0636, + "num_tokens": 5614830.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4436107873916626, + "sampling/importance_sampling_ratio/mean": 0.999886691570282, + "sampling/importance_sampling_ratio/min": 0.7112228274345398, + "sampling/sampling_logp_difference/max": 0.36714744567871094, + "sampling/sampling_logp_difference/mean": 0.011435474269092083, + "step": 248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011107330647064373, + "clip_ratio/low_min": 0.00011107330647064373, + "clip_ratio/region_mean": 0.00011107330647064373, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 566.65625, + "completions/mean_terminated_length": 520.1923217773438, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.4407459795475006, + "epoch": 0.13329764453961457, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008654451929032803, + "learning_rate": 1e-05, + "loss": 0.06, + "num_tokens": 5636955.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3829621076583862, + "sampling/importance_sampling_ratio/mean": 1.0000816583633423, + "sampling/importance_sampling_ratio/min": 0.7467939257621765, + "sampling/sampling_logp_difference/max": 0.32422757148742676, + "sampling/sampling_logp_difference/mean": 0.011981659568846226, + "step": 249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.693954178947024e-05, + "clip_ratio/low_min": 4.693954178947024e-05, + "clip_ratio/region_mean": 4.693954178947024e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 530.25, + "completions/mean_terminated_length": 463.67999267578125, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.5331684872508049, + "epoch": 0.13383297644539616, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0049696266651153564, + "learning_rate": 1e-05, + "loss": 0.0073, + "num_tokens": 5658067.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.44119393825531, + "sampling/importance_sampling_ratio/mean": 1.0000510215759277, + "sampling/importance_sampling_ratio/min": 0.5510550737380981, + "sampling/sampling_logp_difference/max": 0.5959205627441406, + "sampling/sampling_logp_difference/mean": 0.013961504213511944, + "step": 250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.08543525938876e-05, + "clip_ratio/low_min": 5.08543525938876e-05, + "clip_ratio/region_mean": 5.08543525938876e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 586.03125, + "completions/mean_terminated_length": 560.0357666015625, + "completions/min_length": 362.0, + "completions/min_terminated_length": 362.0, + "entropy": 0.3641898073256016, + "epoch": 0.13436830835117772, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01703214831650257, + "learning_rate": 1e-05, + "loss": 0.0301, + "num_tokens": 5680404.0, + "reward": 0.65625, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4004987478256226, + "sampling/importance_sampling_ratio/mean": 1.0002009868621826, + "sampling/importance_sampling_ratio/min": 0.7036284804344177, + "sampling/sampling_logp_difference/max": 0.3515048027038574, + "sampling/sampling_logp_difference/mean": 0.01081143319606781, + "step": 251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016276042151730508, + "clip_ratio/low_min": 0.00016276042151730508, + "clip_ratio/region_mean": 0.00016276042151730508, + "completions/clipped_ratio": 0.71875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 706.90625, + "completions/mean_terminated_length": 550.7777709960938, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.6074881665408611, + "epoch": 0.1349036402569593, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009656330570578575, + "learning_rate": 1e-05, + "loss": 0.076, + "num_tokens": 5707537.0, + "reward": 0.34375, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4766384363174438, + "sampling/importance_sampling_ratio/mean": 1.000126838684082, + "sampling/importance_sampling_ratio/min": 0.6533249616622925, + "sampling/sampling_logp_difference/max": 0.42568063735961914, + "sampling/sampling_logp_difference/mean": 0.01621743105351925, + "step": 252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 504.90625, + "completions/mean_terminated_length": 496.4193420410156, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.28246027417480946, + "epoch": 0.1354389721627409, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0446, + "num_tokens": 5727046.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.2752832174301147, + "sampling/importance_sampling_ratio/mean": 1.0000879764556885, + "sampling/importance_sampling_ratio/min": 0.7149717807769775, + "sampling/sampling_logp_difference/max": 0.3355121612548828, + "sampling/sampling_logp_difference/mean": 0.008951162919402122, + "step": 253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.770992381963879e-05, + "clip_ratio/low_min": 4.770992381963879e-05, + "clip_ratio/region_mean": 4.770992381963879e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 587.34375, + "completions/mean_terminated_length": 536.760009765625, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.46078526228666306, + "epoch": 0.13597430406852248, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008114302530884743, + "learning_rate": 1e-05, + "loss": 0.0234, + "num_tokens": 5749641.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4352549314498901, + "sampling/importance_sampling_ratio/mean": 1.0001859664916992, + "sampling/importance_sampling_ratio/min": 0.7173982262611389, + "sampling/sampling_logp_difference/max": 0.3613424301147461, + "sampling/sampling_logp_difference/mean": 0.013140284456312656, + "step": 254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012068824435118586, + "clip_ratio/low_min": 0.00012068824435118586, + "clip_ratio/region_mean": 0.00012068824435118586, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 495.90625, + "completions/mean_terminated_length": 487.1290283203125, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.32981937006115913, + "epoch": 0.13650963597430407, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008741078898310661, + "learning_rate": 1e-05, + "loss": 0.0194, + "num_tokens": 5768766.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2965061664581299, + "sampling/importance_sampling_ratio/mean": 1.0001507997512817, + "sampling/importance_sampling_ratio/min": 0.6830660104751587, + "sampling/sampling_logp_difference/max": 0.3811638355255127, + "sampling/sampling_logp_difference/mean": 0.010534428060054779, + "step": 255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 559.0625, + "completions/mean_terminated_length": 510.8461608886719, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.47280626744031906, + "epoch": 0.13704496788008566, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008221897296607494, + "learning_rate": 1e-05, + "loss": 0.0166, + "num_tokens": 5790224.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.355219841003418, + "sampling/importance_sampling_ratio/mean": 1.0001143217086792, + "sampling/importance_sampling_ratio/min": 0.782821774482727, + "sampling/sampling_logp_difference/max": 0.30396366119384766, + "sampling/sampling_logp_difference/mean": 0.013127387501299381, + "step": 256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 552.1875, + "completions/mean_terminated_length": 537.800048828125, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.39103712514042854, + "epoch": 0.13758029978586725, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0028837770223617554, + "learning_rate": 1e-05, + "loss": -0.005, + "num_tokens": 5811094.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2821139097213745, + "sampling/importance_sampling_ratio/mean": 1.0000741481781006, + "sampling/importance_sampling_ratio/min": 0.6260427832603455, + "sampling/sampling_logp_difference/max": 0.4683365821838379, + "sampling/sampling_logp_difference/mean": 0.011365697719156742, + "step": 257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 533.90625, + "completions/mean_terminated_length": 518.300048828125, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.3572545275092125, + "epoch": 0.1381156316916488, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0038978864904493093, + "learning_rate": 1e-05, + "loss": -0.0356, + "num_tokens": 5831635.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5498930215835571, + "sampling/importance_sampling_ratio/mean": 0.9995436072349548, + "sampling/importance_sampling_ratio/min": 0.5019902586936951, + "sampling/sampling_logp_difference/max": 0.6891746520996094, + "sampling/sampling_logp_difference/mean": 0.011141622439026833, + "step": 258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001326857745880261, + "clip_ratio/low_min": 0.0001326857745880261, + "clip_ratio/region_mean": 0.0001326857745880261, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 669.0625, + "completions/mean_terminated_length": 592.1111450195312, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.444234162569046, + "epoch": 0.1386509635974304, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003991715610027313, + "learning_rate": 1e-05, + "loss": -0.0024, + "num_tokens": 5856781.0, + "reward": 0.1875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.308322787284851, + "sampling/importance_sampling_ratio/mean": 0.9999985098838806, + "sampling/importance_sampling_ratio/min": 0.6211650967597961, + "sampling/sampling_logp_difference/max": 0.47615838050842285, + "sampling/sampling_logp_difference/mean": 0.011888940818607807, + "step": 259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 556.0, + "completions/mean_terminated_length": 534.0689697265625, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.3420996032655239, + "epoch": 0.139186295503212, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014006612822413445, + "learning_rate": 1e-05, + "loss": 0.0753, + "num_tokens": 5877957.0, + "reward": 0.53125, + "reward_std": 0.5038893818855286, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2864056825637817, + "sampling/importance_sampling_ratio/mean": 0.9997958540916443, + "sampling/importance_sampling_ratio/min": 0.6887884736061096, + "sampling/sampling_logp_difference/max": 0.3728210926055908, + "sampling/sampling_logp_difference/mean": 0.010547701269388199, + "step": 260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000139106712595094, + "clip_ratio/low_min": 0.000139106712595094, + "clip_ratio/region_mean": 0.000139106712595094, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 629.28125, + "completions/mean_terminated_length": 546.0499877929688, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 0.5140561163425446, + "epoch": 0.13972162740899358, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008071399293839931, + "learning_rate": 1e-05, + "loss": -0.0269, + "num_tokens": 5902118.0, + "reward": 0.40625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4044896364212036, + "sampling/importance_sampling_ratio/mean": 1.0003046989440918, + "sampling/importance_sampling_ratio/min": 0.7481744289398193, + "sampling/sampling_logp_difference/max": 0.3396739959716797, + "sampling/sampling_logp_difference/mean": 0.01442759856581688, + "step": 261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.36329597607255e-05, + "clip_ratio/low_min": 9.36329597607255e-05, + "clip_ratio/region_mean": 9.36329597607255e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 509.875, + "completions/mean_terminated_length": 437.5999755859375, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.4288115091621876, + "epoch": 0.14025695931477516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018146095797419548, + "learning_rate": 1e-05, + "loss": 0.0464, + "num_tokens": 5921810.0, + "reward": 0.53125, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3087602853775024, + "sampling/importance_sampling_ratio/mean": 0.9999146461486816, + "sampling/importance_sampling_ratio/min": 0.732586145401001, + "sampling/sampling_logp_difference/max": 0.3111743927001953, + "sampling/sampling_logp_difference/mean": 0.01181797869503498, + "step": 262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010246776582789607, + "clip_ratio/low_min": 0.00010246776582789607, + "clip_ratio/region_mean": 0.00010246776582789607, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 600.59375, + "completions/mean_terminated_length": 524.5, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.43451959267258644, + "epoch": 0.14079229122055675, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011997496709227562, + "learning_rate": 1e-05, + "loss": 0.0171, + "num_tokens": 5944581.0, + "reward": 0.40625, + "reward_std": 0.48928019404411316, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4284863471984863, + "sampling/importance_sampling_ratio/mean": 0.9999670386314392, + "sampling/importance_sampling_ratio/min": 0.5742226839065552, + "sampling/sampling_logp_difference/max": 0.5547380447387695, + "sampling/sampling_logp_difference/mean": 0.012722854502499104, + "step": 263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 522.625, + "completions/mean_terminated_length": 466.0000305175781, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.4376721791923046, + "epoch": 0.14132762312633834, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.009, + "num_tokens": 5965257.0, + "reward": 0.1875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.312913179397583, + "sampling/importance_sampling_ratio/mean": 1.0000436305999756, + "sampling/importance_sampling_ratio/min": 0.7160667181015015, + "sampling/sampling_logp_difference/max": 0.333981990814209, + "sampling/sampling_logp_difference/mean": 0.013143196702003479, + "step": 264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 632.34375, + "completions/mean_terminated_length": 570.6818237304688, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "entropy": 0.4544507935643196, + "epoch": 0.1418629550321199, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013826858252286911, + "learning_rate": 1e-05, + "loss": 0.0514, + "num_tokens": 5989164.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4265916347503662, + "sampling/importance_sampling_ratio/mean": 1.0002870559692383, + "sampling/importance_sampling_ratio/min": 0.6972841024398804, + "sampling/sampling_logp_difference/max": 0.3605623245239258, + "sampling/sampling_logp_difference/mean": 0.012940780259668827, + "step": 265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.469979416579008e-05, + "clip_ratio/low_min": 6.469979416579008e-05, + "clip_ratio/region_mean": 6.469979416579008e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 592.375, + "completions/mean_terminated_length": 543.2000122070312, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.5141049586236477, + "epoch": 0.1423982869379015, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015743238851428032, + "learning_rate": 1e-05, + "loss": 0.0378, + "num_tokens": 6012272.0, + "reward": 0.65625, + "reward_std": 0.47137710452079773, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.759840965270996, + "sampling/importance_sampling_ratio/mean": 0.9995988011360168, + "sampling/importance_sampling_ratio/min": 0.7500654458999634, + "sampling/sampling_logp_difference/max": 0.5652234554290771, + "sampling/sampling_logp_difference/mean": 0.014157882891595364, + "step": 266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 744.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 504.5625, + "completions/mean_terminated_length": 504.5625, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.38440496288239956, + "epoch": 0.14293361884368308, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017426220700144768, + "learning_rate": 1e-05, + "loss": -0.0089, + "num_tokens": 6031738.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4726731777191162, + "sampling/importance_sampling_ratio/mean": 0.9999735951423645, + "sampling/importance_sampling_ratio/min": 0.6526588201522827, + "sampling/sampling_logp_difference/max": 0.4267008304595947, + "sampling/sampling_logp_difference/mean": 0.011597482487559319, + "step": 267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.858142347075045e-05, + "clip_ratio/low_min": 4.858142347075045e-05, + "clip_ratio/region_mean": 4.858142347075045e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 560.78125, + "completions/mean_terminated_length": 502.7599792480469, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.35142942145466805, + "epoch": 0.14346895074946467, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007161566521972418, + "learning_rate": 1e-05, + "loss": -0.0032, + "num_tokens": 6053307.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.400130271911621, + "sampling/importance_sampling_ratio/mean": 1.000180721282959, + "sampling/importance_sampling_ratio/min": 0.7326385378837585, + "sampling/sampling_logp_difference/max": 0.3365652561187744, + "sampling/sampling_logp_difference/mean": 0.009951233863830566, + "step": 268 + }, + { + "clip_ratio/high_max": 6.887052586534992e-05, + "clip_ratio/high_mean": 6.887052586534992e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.887052586534992e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 484.5625, + "completions/mean_terminated_length": 455.2413635253906, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.41471974551677704, + "epoch": 0.14400428265524626, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013710001483559608, + "learning_rate": 1e-05, + "loss": -0.0051, + "num_tokens": 6072165.0, + "reward": 0.5, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4713863134384155, + "sampling/importance_sampling_ratio/mean": 0.9999328255653381, + "sampling/importance_sampling_ratio/min": 0.6127001643180847, + "sampling/sampling_logp_difference/max": 0.4898796081542969, + "sampling/sampling_logp_difference/mean": 0.012783356010913849, + "step": 269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 495.59375, + "completions/mean_terminated_length": 456.6785888671875, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.4830066040158272, + "epoch": 0.14453961456102785, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0023124227300286293, + "learning_rate": 1e-05, + "loss": 0.0654, + "num_tokens": 6091312.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.2866265773773193, + "sampling/importance_sampling_ratio/mean": 1.0001729726791382, + "sampling/importance_sampling_ratio/min": 0.6610352993011475, + "sampling/sampling_logp_difference/max": 0.41394805908203125, + "sampling/sampling_logp_difference/mean": 0.012617088854312897, + "step": 270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.790906915441155e-05, + "clip_ratio/low_min": 9.790906915441155e-05, + "clip_ratio/region_mean": 9.790906915441155e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 613.28125, + "completions/mean_terminated_length": 542.95458984375, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.413370955735445, + "epoch": 0.14507494646680943, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009229264222085476, + "learning_rate": 1e-05, + "loss": 0.0575, + "num_tokens": 6114313.0, + "reward": 0.65625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.373965859413147, + "sampling/importance_sampling_ratio/mean": 1.0002484321594238, + "sampling/importance_sampling_ratio/min": 0.7165717482566833, + "sampling/sampling_logp_difference/max": 0.3332768678665161, + "sampling/sampling_logp_difference/mean": 0.012102281674742699, + "step": 271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 651.0, + "completions/mean_length": 550.3125, + "completions/mean_terminated_length": 519.2142944335938, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.41443566605448723, + "epoch": 0.145610278372591, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00584491016343236, + "learning_rate": 1e-05, + "loss": -0.0001, + "num_tokens": 6135643.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6588126420974731, + "sampling/importance_sampling_ratio/mean": 1.000101923942566, + "sampling/importance_sampling_ratio/min": 0.39590126276016235, + "sampling/sampling_logp_difference/max": 0.9265904426574707, + "sampling/sampling_logp_difference/mean": 0.012218380346894264, + "step": 272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.796623034053482e-05, + "clip_ratio/low_min": 4.796623034053482e-05, + "clip_ratio/region_mean": 4.796623034053482e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 479.875, + "completions/mean_terminated_length": 426.5185241699219, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.43369118496775627, + "epoch": 0.14614561027837258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020795246586203575, + "learning_rate": 1e-05, + "loss": 0.016, + "num_tokens": 6154295.0, + "reward": 0.46875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3095465898513794, + "sampling/importance_sampling_ratio/mean": 0.9997921586036682, + "sampling/importance_sampling_ratio/min": 0.6530885696411133, + "sampling/sampling_logp_difference/max": 0.4260425567626953, + "sampling/sampling_logp_difference/mean": 0.013379311189055443, + "step": 273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.2105042414041236e-05, + "clip_ratio/low_min": 5.2105042414041236e-05, + "clip_ratio/region_mean": 5.2105042414041236e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 578.4375, + "completions/mean_terminated_length": 525.3599853515625, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.3660472631454468, + "epoch": 0.14668094218415417, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0219, + "num_tokens": 6176293.0, + "reward": 0.40625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4787331819534302, + "sampling/importance_sampling_ratio/mean": 1.0000271797180176, + "sampling/importance_sampling_ratio/min": 0.6978476047515869, + "sampling/sampling_logp_difference/max": 0.3911857604980469, + "sampling/sampling_logp_difference/mean": 0.010998491197824478, + "step": 274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 542.4375, + "completions/mean_terminated_length": 490.3846435546875, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.5499711148440838, + "epoch": 0.14721627408993576, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015120045281946659, + "learning_rate": 1e-05, + "loss": 0.1106, + "num_tokens": 6197379.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3861445188522339, + "sampling/importance_sampling_ratio/mean": 0.9997429847717285, + "sampling/importance_sampling_ratio/min": 0.6772403717041016, + "sampling/sampling_logp_difference/max": 0.38972902297973633, + "sampling/sampling_logp_difference/mean": 0.015370858833193779, + "step": 275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010428234600112773, + "clip_ratio/low_min": 0.00010428234600112773, + "clip_ratio/region_mean": 0.00010428234600112773, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 542.15625, + "completions/mean_terminated_length": 490.0384826660156, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.4484858773648739, + "epoch": 0.14775160599571735, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002846313640475273, + "learning_rate": 1e-05, + "loss": 0.0359, + "num_tokens": 6218568.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3625099658966064, + "sampling/importance_sampling_ratio/mean": 1.00043523311615, + "sampling/importance_sampling_ratio/min": 0.6556280851364136, + "sampling/sampling_logp_difference/max": 0.4221615791320801, + "sampling/sampling_logp_difference/mean": 0.012717328034341335, + "step": 276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015897070625214837, + "clip_ratio/low_min": 0.00015897070625214837, + "clip_ratio/region_mean": 0.00015897070625214837, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 588.28125, + "completions/mean_terminated_length": 546.8077392578125, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.3977391868829727, + "epoch": 0.14828693790149894, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008605397306382656, + "learning_rate": 1e-05, + "loss": 0.0472, + "num_tokens": 6241449.0, + "reward": 0.65625, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5247626304626465, + "sampling/importance_sampling_ratio/mean": 0.9999570250511169, + "sampling/importance_sampling_ratio/min": 0.7116842269897461, + "sampling/sampling_logp_difference/max": 0.42183876037597656, + "sampling/sampling_logp_difference/mean": 0.011903348378837109, + "step": 277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 569.8125, + "completions/mean_terminated_length": 541.5, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.33303431794047356, + "epoch": 0.14882226980728053, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 6263451.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.506096363067627, + "sampling/importance_sampling_ratio/mean": 1.0001150369644165, + "sampling/importance_sampling_ratio/min": 0.7046422958374023, + "sampling/sampling_logp_difference/max": 0.40952110290527344, + "sampling/sampling_logp_difference/mean": 0.010882987640798092, + "step": 278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00025589248616597615, + "clip_ratio/low_min": 0.00025589248616597615, + "clip_ratio/region_mean": 0.00025589248616597615, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 597.09375, + "completions/mean_terminated_length": 507.5714416503906, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.7360854744911194, + "epoch": 0.1493576017130621, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0050915502943098545, + "learning_rate": 1e-05, + "loss": 0.0607, + "num_tokens": 6287118.0, + "reward": 0.40625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002696514129639, + "sampling/importance_sampling_ratio/min": 0.6256908178329468, + "sampling/sampling_logp_difference/max": 0.928502082824707, + "sampling/sampling_logp_difference/mean": 0.01567136123776436, + "step": 279 + }, + { + "clip_ratio/high_max": 6.834335363237187e-05, + "clip_ratio/high_mean": 6.834335363237187e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.834335363237187e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 431.71875, + "completions/mean_terminated_length": 431.71875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.2850475553423166, + "epoch": 0.14989293361884368, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011554716154932976, + "learning_rate": 1e-05, + "loss": 0.006, + "num_tokens": 6304021.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3164055347442627, + "sampling/importance_sampling_ratio/mean": 1.0000574588775635, + "sampling/importance_sampling_ratio/min": 0.5672076940536499, + "sampling/sampling_logp_difference/max": 0.5670297145843506, + "sampling/sampling_logp_difference/mean": 0.00954667292535305, + "step": 280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 574.96875, + "completions/mean_terminated_length": 562.1000366210938, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.4140516445040703, + "epoch": 0.15042826552462527, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002030004281550646, + "learning_rate": 1e-05, + "loss": -0.0161, + "num_tokens": 6326292.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.403630018234253, + "sampling/importance_sampling_ratio/mean": 0.9998246431350708, + "sampling/importance_sampling_ratio/min": 0.6674271821975708, + "sampling/sampling_logp_difference/max": 0.404325008392334, + "sampling/sampling_logp_difference/mean": 0.012476157397031784, + "step": 281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010199918324360624, + "clip_ratio/low_min": 0.00010199918324360624, + "clip_ratio/region_mean": 0.00010199918324360624, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 551.28125, + "completions/mean_terminated_length": 544.290283203125, + "completions/min_length": 346.0, + "completions/min_terminated_length": 346.0, + "entropy": 0.3446253836154938, + "epoch": 0.15096359743040685, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007887735031545162, + "learning_rate": 1e-05, + "loss": -0.0259, + "num_tokens": 6347365.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.43865966796875, + "sampling/importance_sampling_ratio/mean": 1.000063180923462, + "sampling/importance_sampling_ratio/min": 0.5418812036514282, + "sampling/sampling_logp_difference/max": 0.612708568572998, + "sampling/sampling_logp_difference/mean": 0.01101731602102518, + "step": 282 + }, + { + "clip_ratio/high_max": 5.5580258049303666e-05, + "clip_ratio/high_mean": 5.5580258049303666e-05, + "clip_ratio/low_mean": 4.4326239731162786e-05, + "clip_ratio/low_min": 4.4326239731162786e-05, + "clip_ratio/region_mean": 9.990649778046645e-05, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 690.375, + "completions/mean_terminated_length": 542.1818237304688, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.4889049455523491, + "epoch": 0.15149892933618844, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014394853264093399, + "learning_rate": 1e-05, + "loss": 0.0643, + "num_tokens": 6373537.0, + "reward": 0.375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.585191249847412, + "sampling/importance_sampling_ratio/mean": 1.0000473260879517, + "sampling/importance_sampling_ratio/min": 0.6346446871757507, + "sampling/sampling_logp_difference/max": 0.460705041885376, + "sampling/sampling_logp_difference/mean": 0.012962386943399906, + "step": 283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012900454385089688, + "clip_ratio/low_min": 0.00012900454385089688, + "clip_ratio/region_mean": 0.00012900454385089688, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 562.46875, + "completions/mean_terminated_length": 515.0384521484375, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.4238313138484955, + "epoch": 0.15203426124197003, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015825411304831505, + "learning_rate": 1e-05, + "loss": 0.0603, + "num_tokens": 6395344.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6557939052581787, + "sampling/importance_sampling_ratio/mean": 1.0001264810562134, + "sampling/importance_sampling_ratio/min": 0.42597201466560364, + "sampling/sampling_logp_difference/max": 0.8533816337585449, + "sampling/sampling_logp_difference/mean": 0.011711214669048786, + "step": 284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012601517300936393, + "clip_ratio/low_min": 0.00012601517300936393, + "clip_ratio/region_mean": 0.00012601517300936393, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 535.90625, + "completions/mean_terminated_length": 470.91998291015625, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.4247622564435005, + "epoch": 0.15256959314775162, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0074, + "num_tokens": 6416453.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3460848331451416, + "sampling/importance_sampling_ratio/mean": 0.9999608397483826, + "sampling/importance_sampling_ratio/min": 0.6614370942115784, + "sampling/sampling_logp_difference/max": 0.4133404493331909, + "sampling/sampling_logp_difference/mean": 0.012951512821018696, + "step": 285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.879586024093442e-05, + "clip_ratio/low_min": 5.879586024093442e-05, + "clip_ratio/region_mean": 5.879586024093442e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 520.21875, + "completions/mean_terminated_length": 503.70001220703125, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.39539410546422005, + "epoch": 0.15310492505353318, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0048318887129426, + "learning_rate": 1e-05, + "loss": 0.0713, + "num_tokens": 6436668.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.354333519935608, + "sampling/importance_sampling_ratio/mean": 0.9997705221176147, + "sampling/importance_sampling_ratio/min": 0.6209386587142944, + "sampling/sampling_logp_difference/max": 0.47652292251586914, + "sampling/sampling_logp_difference/mean": 0.011598610319197178, + "step": 286 + }, + { + "clip_ratio/high_max": 6.319514795904979e-05, + "clip_ratio/high_mean": 6.319514795904979e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.319514795904979e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 584.9375, + "completions/mean_terminated_length": 513.3043823242188, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.5881256833672523, + "epoch": 0.15364025695931477, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00954087171703577, + "learning_rate": 1e-05, + "loss": 0.0026, + "num_tokens": 6459010.0, + "reward": 0.46875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3682405948638916, + "sampling/importance_sampling_ratio/mean": 0.999691367149353, + "sampling/importance_sampling_ratio/min": 0.16219860315322876, + "sampling/sampling_logp_difference/max": 1.8189337253570557, + "sampling/sampling_logp_difference/mean": 0.01596952974796295, + "step": 287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.627915768651292e-05, + "clip_ratio/low_min": 4.627915768651292e-05, + "clip_ratio/region_mean": 4.627915768651292e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 632.25, + "completions/mean_terminated_length": 579.1304321289062, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "entropy": 0.3997486066073179, + "epoch": 0.15417558886509636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004742585588246584, + "learning_rate": 1e-05, + "loss": 0.0729, + "num_tokens": 6483682.0, + "reward": 0.625, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3171533346176147, + "sampling/importance_sampling_ratio/mean": 0.9998674988746643, + "sampling/importance_sampling_ratio/min": 0.6995875835418701, + "sampling/sampling_logp_difference/max": 0.35726428031921387, + "sampling/sampling_logp_difference/mean": 0.011724493466317654, + "step": 288 + }, + { + "clip_ratio/high_max": 4.958349745720625e-05, + "clip_ratio/high_mean": 4.958349745720625e-05, + "clip_ratio/low_mean": 0.00020616228721337393, + "clip_ratio/low_min": 0.00020616228721337393, + "clip_ratio/region_mean": 0.0002557457846705802, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 605.5625, + "completions/mean_terminated_length": 542.0, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.41044608503580093, + "epoch": 0.15471092077087795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012034800834953785, + "learning_rate": 1e-05, + "loss": -0.031, + "num_tokens": 6506732.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4353028535842896, + "sampling/importance_sampling_ratio/mean": 1.0002548694610596, + "sampling/importance_sampling_ratio/min": 0.5381591320037842, + "sampling/sampling_logp_difference/max": 0.6196010112762451, + "sampling/sampling_logp_difference/mean": 0.012869988568127155, + "step": 289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.4216485548531637e-05, + "clip_ratio/low_min": 4.4216485548531637e-05, + "clip_ratio/region_mean": 4.4216485548531637e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 630.03125, + "completions/mean_terminated_length": 557.7619018554688, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.3749997690320015, + "epoch": 0.15524625267665954, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01595156081020832, + "learning_rate": 1e-05, + "loss": 0.0419, + "num_tokens": 6530765.0, + "reward": 0.46875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4583345651626587, + "sampling/importance_sampling_ratio/mean": 1.0001922845840454, + "sampling/importance_sampling_ratio/min": 0.6439215540885925, + "sampling/sampling_logp_difference/max": 0.44017839431762695, + "sampling/sampling_logp_difference/mean": 0.011338340118527412, + "step": 290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 576.5625, + "completions/mean_terminated_length": 532.3846435546875, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.30455345287919044, + "epoch": 0.15578158458244112, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0046155196614563465, + "learning_rate": 1e-05, + "loss": 0.0296, + "num_tokens": 6552679.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3491917848587036, + "sampling/importance_sampling_ratio/mean": 0.9999334216117859, + "sampling/importance_sampling_ratio/min": 0.6191225051879883, + "sampling/sampling_logp_difference/max": 0.47945213317871094, + "sampling/sampling_logp_difference/mean": 0.009973663836717606, + "step": 291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7339449995197356e-05, + "clip_ratio/low_min": 5.7339449995197356e-05, + "clip_ratio/region_mean": 5.7339449995197356e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 566.09375, + "completions/mean_terminated_length": 528.7037353515625, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.3746066950261593, + "epoch": 0.15631691648822268, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010570799931883812, + "learning_rate": 1e-05, + "loss": 0.0421, + "num_tokens": 6574674.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6160459518432617, + "sampling/importance_sampling_ratio/mean": 1.000115990638733, + "sampling/importance_sampling_ratio/min": 0.663177490234375, + "sampling/sampling_logp_difference/max": 0.4799823760986328, + "sampling/sampling_logp_difference/mean": 0.012090645730495453, + "step": 292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015084210826898925, + "clip_ratio/low_min": 0.00015084210826898925, + "clip_ratio/region_mean": 0.00015084210826898925, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 642.15625, + "completions/mean_terminated_length": 600.2083740234375, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "entropy": 0.46960141137242317, + "epoch": 0.15685224839400427, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006962858606129885, + "learning_rate": 1e-05, + "loss": 0.0248, + "num_tokens": 6599495.0, + "reward": 0.5, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.350098729133606, + "sampling/importance_sampling_ratio/mean": 1.0000478029251099, + "sampling/importance_sampling_ratio/min": 0.7264196872711182, + "sampling/sampling_logp_difference/max": 0.3196272850036621, + "sampling/sampling_logp_difference/mean": 0.013042032718658447, + "step": 293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 535.21875, + "completions/mean_terminated_length": 457.625, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.47649625316262245, + "epoch": 0.15738758029978586, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005913050379604101, + "learning_rate": 1e-05, + "loss": 0.1513, + "num_tokens": 6620262.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.8259203433990479, + "sampling/importance_sampling_ratio/mean": 0.9998753666877747, + "sampling/importance_sampling_ratio/min": 0.7434327602386475, + "sampling/sampling_logp_difference/max": 0.6020841598510742, + "sampling/sampling_logp_difference/mean": 0.014077939093112946, + "step": 294 + }, + { + "clip_ratio/high_max": 6.351625779643655e-05, + "clip_ratio/high_mean": 6.351625779643655e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.351625779643655e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 494.46875, + "completions/mean_terminated_length": 455.39288330078125, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.42062101513147354, + "epoch": 0.15792291220556745, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010757889598608017, + "learning_rate": 1e-05, + "loss": 0.0413, + "num_tokens": 6639845.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4439904689788818, + "sampling/importance_sampling_ratio/mean": 1.0000611543655396, + "sampling/importance_sampling_ratio/min": 0.6982812881469727, + "sampling/sampling_logp_difference/max": 0.36741042137145996, + "sampling/sampling_logp_difference/mean": 0.012994526885449886, + "step": 295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 561.5625, + "completions/mean_terminated_length": 532.0714721679688, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.45113179460167885, + "epoch": 0.15845824411134904, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014833190478384495, + "learning_rate": 1e-05, + "loss": 0.0159, + "num_tokens": 6661119.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3035452365875244, + "sampling/importance_sampling_ratio/mean": 0.999748170375824, + "sampling/importance_sampling_ratio/min": 0.7063039541244507, + "sampling/sampling_logp_difference/max": 0.34770965576171875, + "sampling/sampling_logp_difference/mean": 0.012418637052178383, + "step": 296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017841537919593975, + "clip_ratio/low_min": 0.00017841537919593975, + "clip_ratio/region_mean": 0.00017841537919593975, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 562.3125, + "completions/mean_terminated_length": 481.82611083984375, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.4274291656911373, + "epoch": 0.15899357601713063, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007896609604358673, + "learning_rate": 1e-05, + "loss": 0.0588, + "num_tokens": 6682857.0, + "reward": 0.53125, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6551094055175781, + "sampling/importance_sampling_ratio/mean": 0.9993698000907898, + "sampling/importance_sampling_ratio/min": 0.6031726598739624, + "sampling/sampling_logp_difference/max": 0.505551815032959, + "sampling/sampling_logp_difference/mean": 0.012979673221707344, + "step": 297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 447.40625, + "completions/mean_terminated_length": 437.06451416015625, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.4299395754933357, + "epoch": 0.15952890792291222, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004124860744923353, + "learning_rate": 1e-05, + "loss": 0.0293, + "num_tokens": 6700470.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3150358200073242, + "sampling/importance_sampling_ratio/mean": 0.9992318749427795, + "sampling/importance_sampling_ratio/min": 0.5699240565299988, + "sampling/sampling_logp_difference/max": 0.5622521638870239, + "sampling/sampling_logp_difference/mean": 0.01316915825009346, + "step": 298 + }, + { + "clip_ratio/high_max": 5.173841054784134e-05, + "clip_ratio/high_mean": 5.173841054784134e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.173841054784134e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 612.28125, + "completions/mean_terminated_length": 491.1666564941406, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.4086042698472738, + "epoch": 0.16006423982869378, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0149, + "num_tokens": 6724039.0, + "reward": 0.46875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.500494360923767, + "sampling/importance_sampling_ratio/mean": 1.0000535249710083, + "sampling/importance_sampling_ratio/min": 0.6633061766624451, + "sampling/sampling_logp_difference/max": 0.4105186462402344, + "sampling/sampling_logp_difference/mean": 0.011918322183191776, + "step": 299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 589.71875, + "completions/mean_terminated_length": 548.5769653320312, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.3584157060831785, + "epoch": 0.16059957173447537, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004774761851876974, + "learning_rate": 1e-05, + "loss": 0.0465, + "num_tokens": 6746910.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.545341968536377, + "sampling/importance_sampling_ratio/mean": 0.9997391104698181, + "sampling/importance_sampling_ratio/min": 0.7122886776924133, + "sampling/sampling_logp_difference/max": 0.4352452754974365, + "sampling/sampling_logp_difference/mean": 0.011193247511982918, + "step": 300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.413545634131879e-05, + "clip_ratio/low_min": 6.413545634131879e-05, + "clip_ratio/region_mean": 6.413545634131879e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 490.46875, + "completions/mean_terminated_length": 439.0740661621094, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.4165109172463417, + "epoch": 0.16113490364025695, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009167240932583809, + "learning_rate": 1e-05, + "loss": 0.0491, + "num_tokens": 6766317.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4356251955032349, + "sampling/importance_sampling_ratio/mean": 1.000047206878662, + "sampling/importance_sampling_ratio/min": 0.6076681017875671, + "sampling/sampling_logp_difference/max": 0.4981265068054199, + "sampling/sampling_logp_difference/mean": 0.013111790642142296, + "step": 301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 549.5, + "completions/mean_terminated_length": 518.2857666015625, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.40456661209464073, + "epoch": 0.16167023554603854, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01440720446407795, + "learning_rate": 1e-05, + "loss": 0.0694, + "num_tokens": 6787725.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3568141460418701, + "sampling/importance_sampling_ratio/mean": 1.0003018379211426, + "sampling/importance_sampling_ratio/min": 0.6405941247940063, + "sampling/sampling_logp_difference/max": 0.4453592300415039, + "sampling/sampling_logp_difference/mean": 0.012022528797388077, + "step": 302 + }, + { + "clip_ratio/high_max": 4.461099160835147e-05, + "clip_ratio/high_mean": 4.461099160835147e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.461099160835147e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 657.0, + "completions/mean_terminated_length": 546.0, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "entropy": 0.5198293589055538, + "epoch": 0.16220556745182013, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012838945724070072, + "learning_rate": 1e-05, + "loss": 0.0428, + "num_tokens": 6812285.0, + "reward": 0.5625, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3716440200805664, + "sampling/importance_sampling_ratio/mean": 1.000239372253418, + "sampling/importance_sampling_ratio/min": 0.7117252349853516, + "sampling/sampling_logp_difference/max": 0.3400633931159973, + "sampling/sampling_logp_difference/mean": 0.014528676867485046, + "step": 303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 561.84375, + "completions/mean_terminated_length": 468.1363830566406, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.607463214546442, + "epoch": 0.16274089935760172, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005798875819891691, + "learning_rate": 1e-05, + "loss": 0.068, + "num_tokens": 6834048.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998785853385925, + "sampling/importance_sampling_ratio/min": 0.6077396273612976, + "sampling/sampling_logp_difference/max": 0.7001485824584961, + "sampling/sampling_logp_difference/mean": 0.015073039568960667, + "step": 304 + }, + { + "clip_ratio/high_max": 5.195345147512853e-05, + "clip_ratio/high_mean": 5.195345147512853e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.195345147512853e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 576.84375, + "completions/mean_terminated_length": 476.71429443359375, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.6039410643279552, + "epoch": 0.1632762312633833, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0079, + "num_tokens": 6856163.0, + "reward": 0.4375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5125415325164795, + "sampling/importance_sampling_ratio/mean": 0.9999698996543884, + "sampling/importance_sampling_ratio/min": 0.45265570282936096, + "sampling/sampling_logp_difference/max": 0.7926235198974609, + "sampling/sampling_logp_difference/mean": 0.017051981762051582, + "step": 305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010205216676695272, + "clip_ratio/low_min": 0.00010205216676695272, + "clip_ratio/region_mean": 0.00010205216676695272, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 575.625, + "completions/mean_terminated_length": 460.20001220703125, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.4296734184026718, + "epoch": 0.16381156316916487, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014403407461941242, + "learning_rate": 1e-05, + "loss": 0.1056, + "num_tokens": 6878343.0, + "reward": 0.59375, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999287128448486, + "sampling/importance_sampling_ratio/min": 0.7375364899635315, + "sampling/sampling_logp_difference/max": 0.6974811553955078, + "sampling/sampling_logp_difference/mean": 0.01293996162712574, + "step": 306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.992012691218406e-05, + "clip_ratio/low_min": 4.992012691218406e-05, + "clip_ratio/region_mean": 4.992012691218406e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 607.78125, + "completions/mean_terminated_length": 545.0869750976562, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.3507284037768841, + "epoch": 0.16434689507494646, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012407874688506126, + "learning_rate": 1e-05, + "loss": 0.0623, + "num_tokens": 6902232.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5367474555969238, + "sampling/importance_sampling_ratio/mean": 0.9999766945838928, + "sampling/importance_sampling_ratio/min": 0.6973090171813965, + "sampling/sampling_logp_difference/max": 0.4296681880950928, + "sampling/sampling_logp_difference/mean": 0.010848877020180225, + "step": 307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 701.8125, + "completions/mean_terminated_length": 591.5, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 0.5943136811256409, + "epoch": 0.16488222698072805, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007644558325409889, + "learning_rate": 1e-05, + "loss": 0.0287, + "num_tokens": 6929282.0, + "reward": 0.1875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4155625104904175, + "sampling/importance_sampling_ratio/mean": 1.0000989437103271, + "sampling/importance_sampling_ratio/min": 0.630094051361084, + "sampling/sampling_logp_difference/max": 0.4618861675262451, + "sampling/sampling_logp_difference/mean": 0.014622432179749012, + "step": 308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.484563270234503e-05, + "clip_ratio/low_min": 9.484563270234503e-05, + "clip_ratio/region_mean": 9.484563270234503e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 571.03125, + "completions/mean_terminated_length": 525.5769653320312, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.35181776620447636, + "epoch": 0.16541755888650964, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005052079446613789, + "learning_rate": 1e-05, + "loss": 0.0434, + "num_tokens": 6950915.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5886921882629395, + "sampling/importance_sampling_ratio/mean": 1.0000771284103394, + "sampling/importance_sampling_ratio/min": 0.700106680393219, + "sampling/sampling_logp_difference/max": 0.46291112899780273, + "sampling/sampling_logp_difference/mean": 0.010631774552166462, + "step": 309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.8809059080667794e-05, + "clip_ratio/low_min": 4.8809059080667794e-05, + "clip_ratio/region_mean": 4.8809059080667794e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 636.96875, + "completions/mean_terminated_length": 521.3529663085938, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.37192361056804657, + "epoch": 0.16595289079229122, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019923802465200424, + "learning_rate": 1e-05, + "loss": 0.0388, + "num_tokens": 6975178.0, + "reward": 0.53125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.371717929840088, + "sampling/importance_sampling_ratio/mean": 0.9998781085014343, + "sampling/importance_sampling_ratio/min": 0.7024648785591125, + "sampling/sampling_logp_difference/max": 0.35315990447998047, + "sampling/sampling_logp_difference/mean": 0.010792315006256104, + "step": 310 + }, + { + "clip_ratio/high_max": 5.715592124033719e-05, + "clip_ratio/high_mean": 5.715592124033719e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.715592124033719e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 561.84375, + "completions/mean_terminated_length": 453.8571472167969, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.466085322201252, + "epoch": 0.1664882226980728, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009683496318757534, + "learning_rate": 1e-05, + "loss": 0.0455, + "num_tokens": 6997693.0, + "reward": 0.5, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.596548318862915, + "sampling/importance_sampling_ratio/mean": 1.0000803470611572, + "sampling/importance_sampling_ratio/min": 0.5866580605506897, + "sampling/sampling_logp_difference/max": 0.5333130955696106, + "sampling/sampling_logp_difference/mean": 0.013436157256364822, + "step": 311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.88146280683577e-05, + "clip_ratio/low_min": 7.88146280683577e-05, + "clip_ratio/region_mean": 7.88146280683577e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 679.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 415.3125, + "completions/mean_terminated_length": 415.3125, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.41137731820344925, + "epoch": 0.1670235546038544, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014312022365629673, + "learning_rate": 1e-05, + "loss": -0.0326, + "num_tokens": 7013967.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.7482023239135742, + "sampling/importance_sampling_ratio/mean": 1.0004349946975708, + "sampling/importance_sampling_ratio/min": 0.7703766822814941, + "sampling/sampling_logp_difference/max": 0.5585880279541016, + "sampling/sampling_logp_difference/mean": 0.012294778600335121, + "step": 312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 513.59375, + "completions/mean_terminated_length": 477.2500305175781, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.34391969814896584, + "epoch": 0.16755888650963596, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003923412878066301, + "learning_rate": 1e-05, + "loss": 0.0187, + "num_tokens": 7034434.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.322226881980896, + "sampling/importance_sampling_ratio/mean": 1.0004199743270874, + "sampling/importance_sampling_ratio/min": 0.65497225522995, + "sampling/sampling_logp_difference/max": 0.42316246032714844, + "sampling/sampling_logp_difference/mean": 0.010589324869215488, + "step": 313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 432.9375, + "completions/mean_terminated_length": 422.1290283203125, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.35445887967944145, + "epoch": 0.16809421841541755, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0608, + "num_tokens": 7051712.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3308967351913452, + "sampling/importance_sampling_ratio/mean": 0.9997856616973877, + "sampling/importance_sampling_ratio/min": 0.7060892581939697, + "sampling/sampling_logp_difference/max": 0.34801363945007324, + "sampling/sampling_logp_difference/mean": 0.011282389983534813, + "step": 314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 540.09375, + "completions/mean_terminated_length": 524.9000244140625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.4350382909178734, + "epoch": 0.16862955032119914, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006640596780925989, + "learning_rate": 1e-05, + "loss": 0.0339, + "num_tokens": 7072467.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.2996902465820312, + "sampling/importance_sampling_ratio/mean": 1.0000028610229492, + "sampling/importance_sampling_ratio/min": 0.7655131816864014, + "sampling/sampling_logp_difference/max": 0.2672088146209717, + "sampling/sampling_logp_difference/mean": 0.012654009275138378, + "step": 315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 614.0, + "completions/mean_terminated_length": 494.22222900390625, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.6355114802718163, + "epoch": 0.16916488222698073, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007677148096263409, + "learning_rate": 1e-05, + "loss": 0.0147, + "num_tokens": 7095763.0, + "reward": 0.40625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4618357419967651, + "sampling/importance_sampling_ratio/mean": 0.9997237920761108, + "sampling/importance_sampling_ratio/min": 0.6855196952819824, + "sampling/sampling_logp_difference/max": 0.37969303131103516, + "sampling/sampling_logp_difference/mean": 0.01625332050025463, + "step": 316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.626638737041503e-05, + "clip_ratio/low_min": 8.626638737041503e-05, + "clip_ratio/region_mean": 8.626638737041503e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 558.28125, + "completions/mean_terminated_length": 499.55999755859375, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.3198474645614624, + "epoch": 0.16970021413276232, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0033788979053497314, + "learning_rate": 1e-05, + "loss": 0.0412, + "num_tokens": 7117252.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.270493745803833, + "sampling/importance_sampling_ratio/mean": 0.9998461008071899, + "sampling/importance_sampling_ratio/min": 0.6651964783668518, + "sampling/sampling_logp_difference/max": 0.4076728820800781, + "sampling/sampling_logp_difference/mean": 0.01024382933974266, + "step": 317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 499.53125, + "completions/mean_terminated_length": 490.8709411621094, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.3893941566348076, + "epoch": 0.1702355460385439, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015175970271229744, + "learning_rate": 1e-05, + "loss": 0.0093, + "num_tokens": 7136957.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3951607942581177, + "sampling/importance_sampling_ratio/mean": 1.0000625848770142, + "sampling/importance_sampling_ratio/min": 0.6843892931938171, + "sampling/sampling_logp_difference/max": 0.3792283535003662, + "sampling/sampling_logp_difference/mean": 0.011749176308512688, + "step": 318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 702.3125, + "completions/mean_terminated_length": 606.3077392578125, + "completions/min_length": 412.0, + "completions/min_terminated_length": 412.0, + "entropy": 0.6505813598632812, + "epoch": 0.1707708779443255, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 7163991.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3685336112976074, + "sampling/importance_sampling_ratio/mean": 0.999991774559021, + "sampling/importance_sampling_ratio/min": 0.5428902506828308, + "sampling/sampling_logp_difference/max": 0.6108481884002686, + "sampling/sampling_logp_difference/mean": 0.016265524551272392, + "step": 319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.48028658865951e-05, + "clip_ratio/low_min": 4.48028658865951e-05, + "clip_ratio/region_mean": 4.48028658865951e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 602.53125, + "completions/mean_terminated_length": 571.888916015625, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.31928472593426704, + "epoch": 0.17130620985010706, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0050224303267896175, + "learning_rate": 1e-05, + "loss": 0.0228, + "num_tokens": 7187520.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4198600053787231, + "sampling/importance_sampling_ratio/mean": 0.9996873736381531, + "sampling/importance_sampling_ratio/min": 0.4986302852630615, + "sampling/sampling_logp_difference/max": 0.6958904266357422, + "sampling/sampling_logp_difference/mean": 0.010240714997053146, + "step": 320 + }, + { + "clip_ratio/high_max": 5.560498175327666e-05, + "clip_ratio/high_mean": 5.560498175327666e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.560498175327666e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 581.75, + "completions/mean_terminated_length": 529.5999755859375, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.3794487714767456, + "epoch": 0.17184154175588864, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00835055485367775, + "learning_rate": 1e-05, + "loss": 0.0692, + "num_tokens": 7209624.0, + "reward": 0.59375, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3938608169555664, + "sampling/importance_sampling_ratio/mean": 1.0001498460769653, + "sampling/importance_sampling_ratio/min": 0.744274914264679, + "sampling/sampling_logp_difference/max": 0.3320775032043457, + "sampling/sampling_logp_difference/mean": 0.011007760651409626, + "step": 321 + }, + { + "clip_ratio/high_max": 5.026135841035284e-05, + "clip_ratio/high_mean": 5.026135841035284e-05, + "clip_ratio/low_mean": 9.896593837765977e-05, + "clip_ratio/low_min": 9.896593837765977e-05, + "clip_ratio/region_mean": 0.0001492272967880126, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 581.34375, + "completions/mean_terminated_length": 508.3043518066406, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "entropy": 0.4647734649479389, + "epoch": 0.17237687366167023, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016229046508669853, + "learning_rate": 1e-05, + "loss": 0.1117, + "num_tokens": 7232283.0, + "reward": 0.625, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.799541711807251, + "sampling/importance_sampling_ratio/mean": 1.0000022649765015, + "sampling/importance_sampling_ratio/min": 0.6365535259246826, + "sampling/sampling_logp_difference/max": 0.5875320434570312, + "sampling/sampling_logp_difference/mean": 0.013627829030156136, + "step": 322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.191029777051881e-05, + "clip_ratio/low_min": 5.191029777051881e-05, + "clip_ratio/region_mean": 5.191029777051881e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 577.5625, + "completions/mean_terminated_length": 514.0833740234375, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.4767385721206665, + "epoch": 0.17291220556745182, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0296, + "num_tokens": 7254709.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4214529991149902, + "sampling/importance_sampling_ratio/mean": 1.0003039836883545, + "sampling/importance_sampling_ratio/min": 0.7111378908157349, + "sampling/sampling_logp_difference/max": 0.35167956352233887, + "sampling/sampling_logp_difference/mean": 0.013442853465676308, + "step": 323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 623.46875, + "completions/mean_terminated_length": 583.0, + "completions/min_length": 333.0, + "completions/min_terminated_length": 333.0, + "entropy": 0.49009233340620995, + "epoch": 0.1734475374732334, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016870608553290367, + "learning_rate": 1e-05, + "loss": 0.0539, + "num_tokens": 7278428.0, + "reward": 0.65625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.7307504415512085, + "sampling/importance_sampling_ratio/mean": 1.0006048679351807, + "sampling/importance_sampling_ratio/min": 0.7399675846099854, + "sampling/sampling_logp_difference/max": 0.5485551357269287, + "sampling/sampling_logp_difference/mean": 0.013142632320523262, + "step": 324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.1440329116303474e-05, + "clip_ratio/low_min": 5.1440329116303474e-05, + "clip_ratio/region_mean": 5.1440329116303474e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 607.15625, + "completions/mean_terminated_length": 570.0385131835938, + "completions/min_length": 370.0, + "completions/min_terminated_length": 370.0, + "entropy": 0.36767593026161194, + "epoch": 0.173982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0196528360247612, + "learning_rate": 1e-05, + "loss": 0.07, + "num_tokens": 7301817.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3494623899459839, + "sampling/importance_sampling_ratio/mean": 0.9995817542076111, + "sampling/importance_sampling_ratio/min": 0.6885505318641663, + "sampling/sampling_logp_difference/max": 0.373166561126709, + "sampling/sampling_logp_difference/mean": 0.011270838789641857, + "step": 325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.83746116515249e-05, + "clip_ratio/low_min": 4.83746116515249e-05, + "clip_ratio/region_mean": 4.83746116515249e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 574.25, + "completions/mean_terminated_length": 509.66668701171875, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4425664134323597, + "epoch": 0.1745182012847966, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01783912628889084, + "learning_rate": 1e-05, + "loss": 0.0823, + "num_tokens": 7324025.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3928987979888916, + "sampling/importance_sampling_ratio/mean": 0.9998793601989746, + "sampling/importance_sampling_ratio/min": 0.7173065543174744, + "sampling/sampling_logp_difference/max": 0.33225202560424805, + "sampling/sampling_logp_difference/mean": 0.012686881236732006, + "step": 326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012377842358546332, + "clip_ratio/low_min": 0.00012377842358546332, + "clip_ratio/region_mean": 0.00012377842358546332, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 547.28125, + "completions/mean_terminated_length": 460.9130554199219, + "completions/min_length": 300.0, + "completions/min_terminated_length": 300.0, + "entropy": 0.43768591433763504, + "epoch": 0.17505353319057815, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01271110214293003, + "learning_rate": 1e-05, + "loss": 0.1081, + "num_tokens": 7345146.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.404424786567688, + "sampling/importance_sampling_ratio/mean": 1.0002367496490479, + "sampling/importance_sampling_ratio/min": 0.775164008140564, + "sampling/sampling_logp_difference/max": 0.339627742767334, + "sampling/sampling_logp_difference/mean": 0.013005612418055534, + "step": 327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 572.34375, + "completions/mean_terminated_length": 527.1923217773438, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.3581581190228462, + "epoch": 0.17558886509635974, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008894316852092743, + "learning_rate": 1e-05, + "loss": 0.0195, + "num_tokens": 7367581.0, + "reward": 0.6875, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4476288557052612, + "sampling/importance_sampling_ratio/mean": 1.000088095664978, + "sampling/importance_sampling_ratio/min": 0.7586356401443481, + "sampling/sampling_logp_difference/max": 0.36992692947387695, + "sampling/sampling_logp_difference/mean": 0.011610949411988258, + "step": 328 + }, + { + "clip_ratio/high_max": 7.697044202359393e-05, + "clip_ratio/high_mean": 7.697044202359393e-05, + "clip_ratio/low_mean": 6.047411807230674e-05, + "clip_ratio/low_min": 6.047411807230674e-05, + "clip_ratio/region_mean": 0.00013744456009590067, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 533.0625, + "completions/mean_terminated_length": 525.4838256835938, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.38617009297013283, + "epoch": 0.17612419700214133, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008866837248206139, + "learning_rate": 1e-05, + "loss": -0.036, + "num_tokens": 7388487.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3624176979064941, + "sampling/importance_sampling_ratio/mean": 0.9998299479484558, + "sampling/importance_sampling_ratio/min": 0.6679694056510925, + "sampling/sampling_logp_difference/max": 0.40351295471191406, + "sampling/sampling_logp_difference/mean": 0.01204304676502943, + "step": 329 + }, + { + "clip_ratio/high_max": 4.8281188355758786e-05, + "clip_ratio/high_mean": 4.8281188355758786e-05, + "clip_ratio/low_mean": 0.00010299221685272641, + "clip_ratio/low_min": 0.00010299221685272641, + "clip_ratio/region_mean": 0.0001512734052084852, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 588.75, + "completions/mean_terminated_length": 555.5555419921875, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.5742395892739296, + "epoch": 0.1766595289079229, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011959652416408062, + "learning_rate": 1e-05, + "loss": -0.0274, + "num_tokens": 7411607.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3552334308624268, + "sampling/importance_sampling_ratio/mean": 1.0000379085540771, + "sampling/importance_sampling_ratio/min": 0.733167827129364, + "sampling/sampling_logp_difference/max": 0.3103806972503662, + "sampling/sampling_logp_difference/mean": 0.01542259193956852, + "step": 330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 466.03125, + "completions/mean_terminated_length": 456.2903137207031, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.4616716504096985, + "epoch": 0.1771948608137045, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01152541022747755, + "learning_rate": 1e-05, + "loss": 0.007, + "num_tokens": 7429936.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3555200099945068, + "sampling/importance_sampling_ratio/mean": 0.9999625086784363, + "sampling/importance_sampling_ratio/min": 0.6500133872032166, + "sampling/sampling_logp_difference/max": 0.43076229095458984, + "sampling/sampling_logp_difference/mean": 0.013224722817540169, + "step": 331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 552.71875, + "completions/mean_terminated_length": 492.44000244140625, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "entropy": 0.4250369407236576, + "epoch": 0.1777301927194861, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01536703109741211, + "learning_rate": 1e-05, + "loss": 0.0528, + "num_tokens": 7451031.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3783841133117676, + "sampling/importance_sampling_ratio/mean": 0.9997681379318237, + "sampling/importance_sampling_ratio/min": 0.6973655819892883, + "sampling/sampling_logp_difference/max": 0.360445499420166, + "sampling/sampling_logp_difference/mean": 0.012299950234591961, + "step": 332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 608.8125, + "completions/mean_terminated_length": 513.2999877929688, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.44707604870200157, + "epoch": 0.17826552462526768, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008990665897727013, + "learning_rate": 1e-05, + "loss": 0.0209, + "num_tokens": 7474097.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3583874702453613, + "sampling/importance_sampling_ratio/mean": 0.9996063113212585, + "sampling/importance_sampling_ratio/min": 0.7477813363075256, + "sampling/sampling_logp_difference/max": 0.30629831552505493, + "sampling/sampling_logp_difference/mean": 0.011986833065748215, + "step": 333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011857796562253498, + "clip_ratio/low_min": 0.00011857796562253498, + "clip_ratio/region_mean": 0.00011857796562253498, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 540.34375, + "completions/mean_terminated_length": 476.5999755859375, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.6022821851074696, + "epoch": 0.17880085653104924, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013709074817597866, + "learning_rate": 1e-05, + "loss": 0.0002, + "num_tokens": 7495308.0, + "reward": 0.59375, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5634510517120361, + "sampling/importance_sampling_ratio/mean": 1.0003116130828857, + "sampling/importance_sampling_ratio/min": 0.563000500202179, + "sampling/sampling_logp_difference/max": 0.5744748115539551, + "sampling/sampling_logp_difference/mean": 0.015677832067012787, + "step": 334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 590.90625, + "completions/mean_terminated_length": 510.40911865234375, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.4364216811954975, + "epoch": 0.17933618843683083, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011650744825601578, + "learning_rate": 1e-05, + "loss": 0.0991, + "num_tokens": 7518073.0, + "reward": 0.6875, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4041688442230225, + "sampling/importance_sampling_ratio/mean": 1.000081181526184, + "sampling/importance_sampling_ratio/min": 0.7033185362815857, + "sampling/sampling_logp_difference/max": 0.3519454002380371, + "sampling/sampling_logp_difference/mean": 0.012732066214084625, + "step": 335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 607.53125, + "completions/mean_terminated_length": 570.5, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 0.31602367386221886, + "epoch": 0.17987152034261242, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005973147228360176, + "learning_rate": 1e-05, + "loss": 0.097, + "num_tokens": 7541450.0, + "reward": 0.75, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4350091218948364, + "sampling/importance_sampling_ratio/mean": 0.9999140501022339, + "sampling/importance_sampling_ratio/min": 0.7572264671325684, + "sampling/sampling_logp_difference/max": 0.36117124557495117, + "sampling/sampling_logp_difference/mean": 0.01013131719082594, + "step": 336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.1888750022044405e-05, + "clip_ratio/low_min": 5.1888750022044405e-05, + "clip_ratio/region_mean": 5.1888750022044405e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 615.0, + "completions/mean_terminated_length": 564.0, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3818574286997318, + "epoch": 0.180406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007271267473697662, + "learning_rate": 1e-05, + "loss": 0.0051, + "num_tokens": 7565898.0, + "reward": 0.40625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.476676344871521, + "sampling/importance_sampling_ratio/mean": 0.9995765089988708, + "sampling/importance_sampling_ratio/min": 0.6412937641143799, + "sampling/sampling_logp_difference/max": 0.4442676603794098, + "sampling/sampling_logp_difference/mean": 0.011861482635140419, + "step": 337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 517.53125, + "completions/mean_terminated_length": 500.8333740234375, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "entropy": 0.35317888110876083, + "epoch": 0.1809421841541756, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008780697360634804, + "learning_rate": 1e-05, + "loss": -0.0228, + "num_tokens": 7586003.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.449013590812683, + "sampling/importance_sampling_ratio/mean": 1.000103235244751, + "sampling/importance_sampling_ratio/min": 0.7200444936752319, + "sampling/sampling_logp_difference/max": 0.37088310718536377, + "sampling/sampling_logp_difference/mean": 0.01070118136703968, + "step": 338 + }, + { + "clip_ratio/high_max": 4.9662296078167856e-05, + "clip_ratio/high_mean": 4.9662296078167856e-05, + "clip_ratio/low_mean": 4.9662296078167856e-05, + "clip_ratio/low_min": 4.9662296078167856e-05, + "clip_ratio/region_mean": 9.932459215633571e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 502.5, + "completions/mean_terminated_length": 475.03448486328125, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.3665863908827305, + "epoch": 0.18147751605995718, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00488157058134675, + "learning_rate": 1e-05, + "loss": 0.0782, + "num_tokens": 7605659.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3858939409255981, + "sampling/importance_sampling_ratio/mean": 0.9999417662620544, + "sampling/importance_sampling_ratio/min": 0.68963623046875, + "sampling/sampling_logp_difference/max": 0.37159109115600586, + "sampling/sampling_logp_difference/mean": 0.011676940135657787, + "step": 339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 567.03125, + "completions/mean_terminated_length": 529.8148193359375, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.3395811803638935, + "epoch": 0.18201284796573874, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010610250756144524, + "learning_rate": 1e-05, + "loss": 0.1187, + "num_tokens": 7627124.0, + "reward": 0.8125, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4310650825500488, + "sampling/importance_sampling_ratio/mean": 1.0000576972961426, + "sampling/importance_sampling_ratio/min": 0.6813254952430725, + "sampling/sampling_logp_difference/max": 0.3837151527404785, + "sampling/sampling_logp_difference/mean": 0.010930436663329601, + "step": 340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011768460171879269, + "clip_ratio/low_min": 0.00011768460171879269, + "clip_ratio/region_mean": 0.00011768460171879269, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 529.84375, + "completions/mean_terminated_length": 463.1600036621094, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.4085201546549797, + "epoch": 0.18254817987152033, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008732098154723644, + "learning_rate": 1e-05, + "loss": 0.0371, + "num_tokens": 7648263.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4668234586715698, + "sampling/importance_sampling_ratio/mean": 1.0001531839370728, + "sampling/importance_sampling_ratio/min": 0.6989305019378662, + "sampling/sampling_logp_difference/max": 0.3830990791320801, + "sampling/sampling_logp_difference/mean": 0.012125181034207344, + "step": 341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 511.15625, + "completions/mean_terminated_length": 484.5862121582031, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.37089499831199646, + "epoch": 0.18308351177730192, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006230442319065332, + "learning_rate": 1e-05, + "loss": 0.0112, + "num_tokens": 7668484.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4687354564666748, + "sampling/importance_sampling_ratio/mean": 1.0003365278244019, + "sampling/importance_sampling_ratio/min": 0.729246199131012, + "sampling/sampling_logp_difference/max": 0.384401798248291, + "sampling/sampling_logp_difference/mean": 0.01226845383644104, + "step": 342 + }, + { + "clip_ratio/high_max": 6.541077891597524e-05, + "clip_ratio/high_mean": 6.541077891597524e-05, + "clip_ratio/low_mean": 6.620762724196538e-05, + "clip_ratio/low_min": 6.620762724196538e-05, + "clip_ratio/region_mean": 0.00013161840615794063, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 469.40625, + "completions/mean_terminated_length": 459.774169921875, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.3941527418792248, + "epoch": 0.1836188436830835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00416077533736825, + "learning_rate": 1e-05, + "loss": 0.0712, + "num_tokens": 7686961.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4136728048324585, + "sampling/importance_sampling_ratio/mean": 0.9997018575668335, + "sampling/importance_sampling_ratio/min": 0.7000539302825928, + "sampling/sampling_logp_difference/max": 0.356597900390625, + "sampling/sampling_logp_difference/mean": 0.01219549123197794, + "step": 343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 597.0, + "completions/mean_terminated_length": 549.1199951171875, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.43536716513335705, + "epoch": 0.1841541755888651, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0173, + "num_tokens": 7710473.0, + "reward": 0.46875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.511583685874939, + "sampling/importance_sampling_ratio/mean": 1.0004948377609253, + "sampling/importance_sampling_ratio/min": 0.4704502820968628, + "sampling/sampling_logp_difference/max": 0.7540650367736816, + "sampling/sampling_logp_difference/mean": 0.012780013494193554, + "step": 344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 560.875, + "completions/mean_terminated_length": 531.2857666015625, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.37558479234576225, + "epoch": 0.1846895074946467, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029174676164984703, + "learning_rate": 1e-05, + "loss": 0.0137, + "num_tokens": 7732181.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4214953184127808, + "sampling/importance_sampling_ratio/mean": 1.000383734703064, + "sampling/importance_sampling_ratio/min": 0.7225403785705566, + "sampling/sampling_logp_difference/max": 0.35170936584472656, + "sampling/sampling_logp_difference/mean": 0.011788973584771156, + "step": 345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 527.15625, + "completions/mean_terminated_length": 502.2413635253906, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.4374905489385128, + "epoch": 0.18522483940042828, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0033343981485813856, + "learning_rate": 1e-05, + "loss": 0.0643, + "num_tokens": 7752810.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.954554796218872, + "sampling/importance_sampling_ratio/mean": 1.0001956224441528, + "sampling/importance_sampling_ratio/min": 0.6426865458488464, + "sampling/sampling_logp_difference/max": 0.6701624393463135, + "sampling/sampling_logp_difference/mean": 0.012586500495672226, + "step": 346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.587155854096636e-05, + "clip_ratio/low_min": 4.587155854096636e-05, + "clip_ratio/region_mean": 4.587155854096636e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 584.15625, + "completions/mean_terminated_length": 550.1111450195312, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.3952989913523197, + "epoch": 0.18576017130620984, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008386149071156979, + "learning_rate": 1e-05, + "loss": -0.009, + "num_tokens": 7775431.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.317029356956482, + "sampling/importance_sampling_ratio/mean": 0.9999770522117615, + "sampling/importance_sampling_ratio/min": 0.677424430847168, + "sampling/sampling_logp_difference/max": 0.38945722579956055, + "sampling/sampling_logp_difference/mean": 0.011926273815333843, + "step": 347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 465.84375, + "completions/mean_terminated_length": 445.70001220703125, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.3603689447045326, + "epoch": 0.18629550321199143, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008089978247880936, + "learning_rate": 1e-05, + "loss": -0.0137, + "num_tokens": 7793946.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3611254692077637, + "sampling/importance_sampling_ratio/mean": 0.9999449253082275, + "sampling/importance_sampling_ratio/min": 0.7004105448722839, + "sampling/sampling_logp_difference/max": 0.35608863830566406, + "sampling/sampling_logp_difference/mean": 0.011440886184573174, + "step": 348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.131362922838889e-05, + "clip_ratio/low_min": 5.131362922838889e-05, + "clip_ratio/region_mean": 5.131362922838889e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 495.28125, + "completions/mean_terminated_length": 404.375, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.45432041212916374, + "epoch": 0.18683083511777301, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018666157498955727, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 7813187.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3087438344955444, + "sampling/importance_sampling_ratio/mean": 0.9999655485153198, + "sampling/importance_sampling_ratio/min": 0.6450303196907043, + "sampling/sampling_logp_difference/max": 0.4384579658508301, + "sampling/sampling_logp_difference/mean": 0.013790863566100597, + "step": 349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 535.8125, + "completions/mean_terminated_length": 511.7930908203125, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.3715438283979893, + "epoch": 0.1873661670235546, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0037117162719368935, + "learning_rate": 1e-05, + "loss": 0.0082, + "num_tokens": 7834165.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4204705953598022, + "sampling/importance_sampling_ratio/mean": 1.0002492666244507, + "sampling/importance_sampling_ratio/min": 0.6965051293373108, + "sampling/sampling_logp_difference/max": 0.3616800308227539, + "sampling/sampling_logp_difference/mean": 0.011881536804139614, + "step": 350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.0797665355494246e-05, + "clip_ratio/low_min": 6.0797665355494246e-05, + "clip_ratio/region_mean": 6.0797665355494246e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 502.1875, + "completions/mean_terminated_length": 427.7599792480469, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.43836427107453346, + "epoch": 0.1879014989293362, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0015, + "num_tokens": 7854003.0, + "reward": 0.53125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2850538492202759, + "sampling/importance_sampling_ratio/mean": 0.9995579123497009, + "sampling/importance_sampling_ratio/min": 0.59810471534729, + "sampling/sampling_logp_difference/max": 0.5139894485473633, + "sampling/sampling_logp_difference/mean": 0.012770895846188068, + "step": 351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010048757030745037, + "clip_ratio/low_min": 0.00010048757030745037, + "clip_ratio/region_mean": 0.00010048757030745037, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 584.96875, + "completions/mean_terminated_length": 533.719970703125, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.41844386607408524, + "epoch": 0.18843683083511778, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015738243237137794, + "learning_rate": 1e-05, + "loss": 0.0174, + "num_tokens": 7876474.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.760148525238037, + "sampling/importance_sampling_ratio/mean": 1.0003736019134521, + "sampling/importance_sampling_ratio/min": 0.7047797441482544, + "sampling/sampling_logp_difference/max": 0.5653982162475586, + "sampling/sampling_logp_difference/mean": 0.012314577586948872, + "step": 352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 553.9375, + "completions/mean_terminated_length": 470.1739196777344, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.5313730835914612, + "epoch": 0.18897216274089937, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0038939369842410088, + "learning_rate": 1e-05, + "loss": 0.0235, + "num_tokens": 7897976.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5618528127670288, + "sampling/importance_sampling_ratio/mean": 1.0003228187561035, + "sampling/importance_sampling_ratio/min": 0.6975950598716736, + "sampling/sampling_logp_difference/max": 0.44587278366088867, + "sampling/sampling_logp_difference/mean": 0.015048760920763016, + "step": 353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 583.375, + "completions/mean_terminated_length": 564.27587890625, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3671898692846298, + "epoch": 0.18950749464668093, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007814828306436539, + "learning_rate": 1e-05, + "loss": -0.0143, + "num_tokens": 7920100.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4202724695205688, + "sampling/importance_sampling_ratio/mean": 0.9999342560768127, + "sampling/importance_sampling_ratio/min": 0.6841111779212952, + "sampling/sampling_logp_difference/max": 0.3796348571777344, + "sampling/sampling_logp_difference/mean": 0.011666848324239254, + "step": 354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.367804598994553e-05, + "clip_ratio/low_min": 6.367804598994553e-05, + "clip_ratio/region_mean": 6.367804598994553e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 622.34375, + "completions/mean_terminated_length": 588.7307739257812, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.3561982773244381, + "epoch": 0.19004282655246252, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010745654813945293, + "learning_rate": 1e-05, + "loss": 0.0504, + "num_tokens": 7943879.0, + "reward": 0.4375, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6341825723648071, + "sampling/importance_sampling_ratio/mean": 0.9998425841331482, + "sampling/importance_sampling_ratio/min": 0.7280601859092712, + "sampling/sampling_logp_difference/max": 0.49114274978637695, + "sampling/sampling_logp_difference/mean": 0.01099233515560627, + "step": 355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 679.3125, + "completions/mean_terminated_length": 565.2857666015625, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "entropy": 0.5292008481919765, + "epoch": 0.1905781584582441, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003873254870995879, + "learning_rate": 1e-05, + "loss": 0.0581, + "num_tokens": 7970537.0, + "reward": 0.4375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4053484201431274, + "sampling/importance_sampling_ratio/mean": 0.9999608993530273, + "sampling/importance_sampling_ratio/min": 0.6079300045967102, + "sampling/sampling_logp_difference/max": 0.4976954460144043, + "sampling/sampling_logp_difference/mean": 0.014417771250009537, + "step": 356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.172839493956417e-05, + "clip_ratio/low_min": 6.172839493956417e-05, + "clip_ratio/region_mean": 6.172839493956417e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 497.34375, + "completions/mean_terminated_length": 458.6785888671875, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "entropy": 0.33522066473960876, + "epoch": 0.1911134903640257, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005561012774705887, + "learning_rate": 1e-05, + "loss": 0.0407, + "num_tokens": 7989564.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3128960132598877, + "sampling/importance_sampling_ratio/mean": 0.9998707175254822, + "sampling/importance_sampling_ratio/min": 0.5142340660095215, + "sampling/sampling_logp_difference/max": 0.665076732635498, + "sampling/sampling_logp_difference/mean": 0.010443082079291344, + "step": 357 + }, + { + "clip_ratio/high_max": 0.00010860121983569115, + "clip_ratio/high_mean": 0.00010860121983569115, + "clip_ratio/low_mean": 5.1355793402763084e-05, + "clip_ratio/low_min": 5.1355793402763084e-05, + "clip_ratio/region_mean": 0.00015995701323845424, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 607.53125, + "completions/mean_terminated_length": 554.0416870117188, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.40050872415304184, + "epoch": 0.19164882226980728, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013117474503815174, + "learning_rate": 1e-05, + "loss": -0.0281, + "num_tokens": 8012533.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3459420204162598, + "sampling/importance_sampling_ratio/mean": 1.0003983974456787, + "sampling/importance_sampling_ratio/min": 0.5469598174095154, + "sampling/sampling_logp_difference/max": 0.6033799648284912, + "sampling/sampling_logp_difference/mean": 0.011664224788546562, + "step": 358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.289885666570626e-05, + "clip_ratio/low_min": 5.289885666570626e-05, + "clip_ratio/region_mean": 5.289885666570626e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 501.4375, + "completions/mean_terminated_length": 439.923095703125, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "entropy": 0.5156456530094147, + "epoch": 0.19218415417558887, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018113363534212112, + "learning_rate": 1e-05, + "loss": 0.0591, + "num_tokens": 8032603.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4613863229751587, + "sampling/importance_sampling_ratio/mean": 0.9998681545257568, + "sampling/importance_sampling_ratio/min": 0.700824499130249, + "sampling/sampling_logp_difference/max": 0.37938547134399414, + "sampling/sampling_logp_difference/mean": 0.015054013580083847, + "step": 359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010434056457597762, + "clip_ratio/low_min": 0.00010434056457597762, + "clip_ratio/region_mean": 0.00010434056457597762, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 528.1875, + "completions/mean_terminated_length": 461.03997802734375, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.3673683628439903, + "epoch": 0.19271948608137046, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006402954924851656, + "learning_rate": 1e-05, + "loss": 0.0682, + "num_tokens": 8053081.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.633599042892456, + "sampling/importance_sampling_ratio/mean": 0.9999885559082031, + "sampling/importance_sampling_ratio/min": 0.7214100360870361, + "sampling/sampling_logp_difference/max": 0.4907855987548828, + "sampling/sampling_logp_difference/mean": 0.012097165919840336, + "step": 360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.439979188144207e-05, + "clip_ratio/low_min": 6.439979188144207e-05, + "clip_ratio/region_mean": 6.439979188144207e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 549.0, + "completions/mean_terminated_length": 517.7142944335938, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3545941151678562, + "epoch": 0.19325481798715202, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0052088904194533825, + "learning_rate": 1e-05, + "loss": -0.006, + "num_tokens": 8074649.0, + "reward": 0.5625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2932671308517456, + "sampling/importance_sampling_ratio/mean": 0.999854564666748, + "sampling/importance_sampling_ratio/min": 0.6198790669441223, + "sampling/sampling_logp_difference/max": 0.47823095321655273, + "sampling/sampling_logp_difference/mean": 0.01117474865168333, + "step": 361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 508.90625, + "completions/mean_terminated_length": 449.1153869628906, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.37205641344189644, + "epoch": 0.1937901498929336, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01416859868913889, + "learning_rate": 1e-05, + "loss": 0.0557, + "num_tokens": 8095174.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4567490816116333, + "sampling/importance_sampling_ratio/mean": 1.0000224113464355, + "sampling/importance_sampling_ratio/min": 0.6502732038497925, + "sampling/sampling_logp_difference/max": 0.4303627014160156, + "sampling/sampling_logp_difference/mean": 0.012502220459282398, + "step": 362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 625.03125, + "completions/mean_terminated_length": 560.0454711914062, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "entropy": 0.3963019587099552, + "epoch": 0.1943254817987152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013624940067529678, + "learning_rate": 1e-05, + "loss": 0.0821, + "num_tokens": 8119263.0, + "reward": 0.46875, + "reward_std": 0.47137707471847534, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.8045307397842407, + "sampling/importance_sampling_ratio/mean": 0.9999462962150574, + "sampling/importance_sampling_ratio/min": 0.6579004526138306, + "sampling/sampling_logp_difference/max": 0.5903005599975586, + "sampling/sampling_logp_difference/mean": 0.012125816196203232, + "step": 363 + }, + { + "clip_ratio/high_max": 4.8904537834459916e-05, + "clip_ratio/high_mean": 4.8904537834459916e-05, + "clip_ratio/low_mean": 5.8220772189088166e-05, + "clip_ratio/low_min": 5.8220772189088166e-05, + "clip_ratio/region_mean": 0.00010712531002354808, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 557.875, + "completions/mean_terminated_length": 487.8333435058594, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.386314794421196, + "epoch": 0.1948608137044968, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014079393818974495, + "learning_rate": 1e-05, + "loss": -0.0407, + "num_tokens": 8141307.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2817705869674683, + "sampling/importance_sampling_ratio/mean": 1.0008752346038818, + "sampling/importance_sampling_ratio/min": 0.6971936821937561, + "sampling/sampling_logp_difference/max": 0.36069202423095703, + "sampling/sampling_logp_difference/mean": 0.012224792502820492, + "step": 364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 579.59375, + "completions/mean_terminated_length": 573.51611328125, + "completions/min_length": 393.0, + "completions/min_terminated_length": 393.0, + "entropy": 0.31728189811110497, + "epoch": 0.19539614561027838, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01950717344880104, + "learning_rate": 1e-05, + "loss": -0.0291, + "num_tokens": 8163582.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4351892471313477, + "sampling/importance_sampling_ratio/mean": 0.9998736381530762, + "sampling/importance_sampling_ratio/min": 0.697848916053772, + "sampling/sampling_logp_difference/max": 0.3612966537475586, + "sampling/sampling_logp_difference/mean": 0.009971254505217075, + "step": 365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 488.0625, + "completions/mean_terminated_length": 459.10345458984375, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.39941294491291046, + "epoch": 0.19593147751605997, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006344099994748831, + "learning_rate": 1e-05, + "loss": 0.0189, + "num_tokens": 8182824.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3463642597198486, + "sampling/importance_sampling_ratio/mean": 0.9999861717224121, + "sampling/importance_sampling_ratio/min": 0.6290184855461121, + "sampling/sampling_logp_difference/max": 0.4635946750640869, + "sampling/sampling_logp_difference/mean": 0.011870177462697029, + "step": 366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.427702853921801e-05, + "clip_ratio/low_min": 5.427702853921801e-05, + "clip_ratio/region_mean": 5.427702853921801e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 555.65625, + "completions/mean_terminated_length": 444.4285888671875, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.4908212870359421, + "epoch": 0.19646680942184155, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02309938333928585, + "learning_rate": 1e-05, + "loss": 0.1003, + "num_tokens": 8204773.0, + "reward": 0.53125, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4570693969726562, + "sampling/importance_sampling_ratio/mean": 0.9994863271713257, + "sampling/importance_sampling_ratio/min": 0.4955500364303589, + "sampling/sampling_logp_difference/max": 0.7020869255065918, + "sampling/sampling_logp_difference/mean": 0.014599011279642582, + "step": 367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 476.03125, + "completions/mean_terminated_length": 476.03125, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.4153149016201496, + "epoch": 0.19700214132762311, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002889768686145544, + "learning_rate": 1e-05, + "loss": -0.0296, + "num_tokens": 8223782.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.7374821901321411, + "sampling/importance_sampling_ratio/mean": 1.0005125999450684, + "sampling/importance_sampling_ratio/min": 0.6276167631149292, + "sampling/sampling_logp_difference/max": 0.5524370670318604, + "sampling/sampling_logp_difference/mean": 0.012754712253808975, + "step": 368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001386214207741432, + "clip_ratio/low_min": 0.0001386214207741432, + "clip_ratio/region_mean": 0.0001386214207741432, + "completions/clipped_ratio": 0.0, + "completions/max_length": 737.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 468.25, + "completions/mean_terminated_length": 468.25, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "entropy": 0.3890383690595627, + "epoch": 0.1975374732334047, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.022693229839205742, + "learning_rate": 1e-05, + "loss": -0.0174, + "num_tokens": 8242150.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.439625859260559, + "sampling/importance_sampling_ratio/mean": 1.000091791152954, + "sampling/importance_sampling_ratio/min": 0.6898723244667053, + "sampling/sampling_logp_difference/max": 0.371248722076416, + "sampling/sampling_logp_difference/mean": 0.011646188795566559, + "step": 369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.345177826005965e-05, + "clip_ratio/low_min": 6.345177826005965e-05, + "clip_ratio/region_mean": 6.345177826005965e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 495.125, + "completions/mean_terminated_length": 466.89654541015625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.42222118377685547, + "epoch": 0.1980728051391863, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011758032254874706, + "learning_rate": 1e-05, + "loss": 0.0572, + "num_tokens": 8261770.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3863540887832642, + "sampling/importance_sampling_ratio/mean": 0.9994974136352539, + "sampling/importance_sampling_ratio/min": 0.7322672605514526, + "sampling/sampling_logp_difference/max": 0.3266773223876953, + "sampling/sampling_logp_difference/mean": 0.01309940405189991, + "step": 370 + }, + { + "clip_ratio/high_max": 0.00011445986456237733, + "clip_ratio/high_mean": 0.00011445986456237733, + "clip_ratio/low_mean": 0.00013022391067352146, + "clip_ratio/low_min": 0.00013022391067352146, + "clip_ratio/region_mean": 0.0002446837752358988, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 537.46875, + "completions/mean_terminated_length": 494.77777099609375, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.4241586662828922, + "epoch": 0.19860813704496788, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005418703425675631, + "learning_rate": 1e-05, + "loss": 0.0116, + "num_tokens": 8283161.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5823909044265747, + "sampling/importance_sampling_ratio/mean": 1.0000547170639038, + "sampling/importance_sampling_ratio/min": 0.579826831817627, + "sampling/sampling_logp_difference/max": 0.5450257658958435, + "sampling/sampling_logp_difference/mean": 0.012620266526937485, + "step": 371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.798464578925632e-05, + "clip_ratio/low_min": 4.798464578925632e-05, + "clip_ratio/region_mean": 4.798464578925632e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 558.625, + "completions/mean_terminated_length": 544.6666870117188, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3315769210457802, + "epoch": 0.19914346895074947, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011637786403298378, + "learning_rate": 1e-05, + "loss": 0.0771, + "num_tokens": 8304453.0, + "reward": 0.65625, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.36954927444458, + "sampling/importance_sampling_ratio/mean": 1.000084638595581, + "sampling/importance_sampling_ratio/min": 0.6938039660453796, + "sampling/sampling_logp_difference/max": 0.36556577682495117, + "sampling/sampling_logp_difference/mean": 0.010713249444961548, + "step": 372 + }, + { + "clip_ratio/high_max": 5.236698780208826e-05, + "clip_ratio/high_mean": 5.236698780208826e-05, + "clip_ratio/low_mean": 4.74743646918796e-05, + "clip_ratio/low_min": 4.74743646918796e-05, + "clip_ratio/region_mean": 9.984135249396786e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 603.15625, + "completions/mean_terminated_length": 548.2083740234375, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 0.4163140803575516, + "epoch": 0.19967880085653106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01129836030304432, + "learning_rate": 1e-05, + "loss": 0.0385, + "num_tokens": 8328066.0, + "reward": 0.53125, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4257516860961914, + "sampling/importance_sampling_ratio/mean": 1.0003459453582764, + "sampling/importance_sampling_ratio/min": 0.6285473108291626, + "sampling/sampling_logp_difference/max": 0.4643440246582031, + "sampling/sampling_logp_difference/mean": 0.012463225051760674, + "step": 373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.58043250546325e-05, + "clip_ratio/low_min": 4.58043250546325e-05, + "clip_ratio/region_mean": 4.58043250546325e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 551.28125, + "completions/mean_terminated_length": 520.3214721679688, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3488573506474495, + "epoch": 0.20021413276231265, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014986801892518997, + "learning_rate": 1e-05, + "loss": 0.0967, + "num_tokens": 8349131.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3101072311401367, + "sampling/importance_sampling_ratio/mean": 0.9999088644981384, + "sampling/importance_sampling_ratio/min": 0.6880736351013184, + "sampling/sampling_logp_difference/max": 0.3738594055175781, + "sampling/sampling_logp_difference/mean": 0.010670335032045841, + "step": 374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 545.75, + "completions/mean_terminated_length": 444.727294921875, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4944803100079298, + "epoch": 0.2007494646680942, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005338540766388178, + "learning_rate": 1e-05, + "loss": 0.0431, + "num_tokens": 8370531.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5178382396697998, + "sampling/importance_sampling_ratio/mean": 0.999999463558197, + "sampling/importance_sampling_ratio/min": 0.7099547386169434, + "sampling/sampling_logp_difference/max": 0.41728711128234863, + "sampling/sampling_logp_difference/mean": 0.014006479643285275, + "step": 375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.638218888430856e-05, + "clip_ratio/low_min": 4.638218888430856e-05, + "clip_ratio/region_mean": 4.638218888430856e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 569.46875, + "completions/mean_terminated_length": 532.7037353515625, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.3828435130417347, + "epoch": 0.2012847965738758, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020134570077061653, + "learning_rate": 1e-05, + "loss": 0.0551, + "num_tokens": 8392642.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4648139476776123, + "sampling/importance_sampling_ratio/mean": 1.000060796737671, + "sampling/importance_sampling_ratio/min": 0.7272341847419739, + "sampling/sampling_logp_difference/max": 0.3817281723022461, + "sampling/sampling_logp_difference/mean": 0.011920085176825523, + "step": 376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011117736357846297, + "clip_ratio/low_min": 0.00011117736357846297, + "clip_ratio/region_mean": 0.00011117736357846297, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 619.90625, + "completions/mean_terminated_length": 570.5416870117188, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.39548028632998466, + "epoch": 0.20182012847965738, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006651423405855894, + "learning_rate": 1e-05, + "loss": 0.0123, + "num_tokens": 8416855.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.2932239770889282, + "sampling/importance_sampling_ratio/mean": 0.9998322129249573, + "sampling/importance_sampling_ratio/min": 0.7805500030517578, + "sampling/sampling_logp_difference/max": 0.2571382522583008, + "sampling/sampling_logp_difference/mean": 0.011906609870493412, + "step": 377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.669406189350411e-05, + "clip_ratio/low_min": 4.669406189350411e-05, + "clip_ratio/region_mean": 4.669406189350411e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 580.0625, + "completions/mean_terminated_length": 527.4400024414062, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.4088635481894016, + "epoch": 0.20235546038543897, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011806381866335869, + "learning_rate": 1e-05, + "loss": 0.0068, + "num_tokens": 8438897.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3545053005218506, + "sampling/importance_sampling_ratio/mean": 0.9998217225074768, + "sampling/importance_sampling_ratio/min": 0.7168411612510681, + "sampling/sampling_logp_difference/max": 0.3329010009765625, + "sampling/sampling_logp_difference/mean": 0.012179611250758171, + "step": 378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 560.8125, + "completions/mean_terminated_length": 491.75, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.38923313096165657, + "epoch": 0.20289079229122056, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010750584304332733, + "learning_rate": 1e-05, + "loss": 0.0021, + "num_tokens": 8460651.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.387181282043457, + "sampling/importance_sampling_ratio/mean": 0.9999170899391174, + "sampling/importance_sampling_ratio/min": 0.7010316252708435, + "sampling/sampling_logp_difference/max": 0.355202317237854, + "sampling/sampling_logp_difference/mean": 0.01274655107408762, + "step": 379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.636845788918436e-05, + "clip_ratio/low_min": 9.636845788918436e-05, + "clip_ratio/region_mean": 9.636845788918436e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 537.59375, + "completions/mean_terminated_length": 484.423095703125, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.43413735553622246, + "epoch": 0.20342612419700215, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011824137531220913, + "learning_rate": 1e-05, + "loss": 0.0937, + "num_tokens": 8481398.0, + "reward": 0.65625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3749160766601562, + "sampling/importance_sampling_ratio/mean": 1.000190258026123, + "sampling/importance_sampling_ratio/min": 0.699332594871521, + "sampling/sampling_logp_difference/max": 0.35762882232666016, + "sampling/sampling_logp_difference/mean": 0.013350420631468296, + "step": 380 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 536.71875, + "completions/mean_terminated_length": 521.300048828125, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.3783344514667988, + "epoch": 0.20396145610278374, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0184, + "num_tokens": 8502261.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.347518801689148, + "sampling/importance_sampling_ratio/mean": 0.9999199509620667, + "sampling/importance_sampling_ratio/min": 0.17710205912590027, + "sampling/sampling_logp_difference/max": 1.7310290336608887, + "sampling/sampling_logp_difference/mean": 0.011571547947824001, + "step": 381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 490.9375, + "completions/mean_terminated_length": 451.357177734375, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.3456154465675354, + "epoch": 0.2044967880085653, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013170991092920303, + "learning_rate": 1e-05, + "loss": 0.0837, + "num_tokens": 8521483.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3907924890518188, + "sampling/importance_sampling_ratio/mean": 1.0000041723251343, + "sampling/importance_sampling_ratio/min": 0.605514645576477, + "sampling/sampling_logp_difference/max": 0.5016765594482422, + "sampling/sampling_logp_difference/mean": 0.011147287674248219, + "step": 382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.594167982460931e-05, + "clip_ratio/low_min": 7.594167982460931e-05, + "clip_ratio/region_mean": 7.594167982460931e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 536.40625, + "completions/mean_terminated_length": 503.3214416503906, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.38958995789289474, + "epoch": 0.2050321199143469, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01350939180701971, + "learning_rate": 1e-05, + "loss": 0.0677, + "num_tokens": 8542160.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4676110744476318, + "sampling/importance_sampling_ratio/mean": 1.000357747077942, + "sampling/importance_sampling_ratio/min": 0.7181212902069092, + "sampling/sampling_logp_difference/max": 0.3836359977722168, + "sampling/sampling_logp_difference/mean": 0.012127463705837727, + "step": 383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 583.9375, + "completions/mean_terminated_length": 522.5833740234375, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.46484004333615303, + "epoch": 0.20556745182012848, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009122509509325027, + "learning_rate": 1e-05, + "loss": 0.035, + "num_tokens": 8564878.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6090890169143677, + "sampling/importance_sampling_ratio/mean": 0.9999677538871765, + "sampling/importance_sampling_ratio/min": 0.7001003623008728, + "sampling/sampling_logp_difference/max": 0.47566819190979004, + "sampling/sampling_logp_difference/mean": 0.013996811583638191, + "step": 384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.758871601661667e-05, + "clip_ratio/low_min": 9.758871601661667e-05, + "clip_ratio/region_mean": 9.758871601661667e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 615.625, + "completions/mean_terminated_length": 572.9599609375, + "completions/min_length": 387.0, + "completions/min_terminated_length": 387.0, + "entropy": 0.39049726352095604, + "epoch": 0.20610278372591007, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.025379296392202377, + "learning_rate": 1e-05, + "loss": 0.0438, + "num_tokens": 8588426.0, + "reward": 0.53125, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4643515348434448, + "sampling/importance_sampling_ratio/mean": 1.000016689300537, + "sampling/importance_sampling_ratio/min": 0.607111930847168, + "sampling/sampling_logp_difference/max": 0.4990421533584595, + "sampling/sampling_logp_difference/mean": 0.01193297654390335, + "step": 385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011785374226747081, + "clip_ratio/low_min": 0.00011785374226747081, + "clip_ratio/region_mean": 0.00011785374226747081, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 566.40625, + "completions/mean_terminated_length": 537.607177734375, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.4207429699599743, + "epoch": 0.20663811563169165, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010531638748943806, + "learning_rate": 1e-05, + "loss": 0.0627, + "num_tokens": 8610159.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4233890771865845, + "sampling/importance_sampling_ratio/mean": 1.000326156616211, + "sampling/importance_sampling_ratio/min": 0.7141737937927246, + "sampling/sampling_logp_difference/max": 0.3530406951904297, + "sampling/sampling_logp_difference/mean": 0.012914708815515041, + "step": 386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011987982361461036, + "clip_ratio/low_min": 0.00011987982361461036, + "clip_ratio/region_mean": 0.00011987982361461036, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 560.9375, + "completions/mean_terminated_length": 491.91668701171875, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.29077574610710144, + "epoch": 0.20717344753747324, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012365398928523064, + "learning_rate": 1e-05, + "loss": 0.0157, + "num_tokens": 8631797.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.422987937927246, + "sampling/importance_sampling_ratio/mean": 0.9994892477989197, + "sampling/importance_sampling_ratio/min": 0.6514736413955688, + "sampling/sampling_logp_difference/max": 0.42851829528808594, + "sampling/sampling_logp_difference/mean": 0.01023923885077238, + "step": 387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 541.78125, + "completions/mean_terminated_length": 509.46429443359375, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.3683685436844826, + "epoch": 0.20770877944325483, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0029659071005880833, + "learning_rate": 1e-05, + "loss": -0.0355, + "num_tokens": 8652998.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2972773313522339, + "sampling/importance_sampling_ratio/mean": 1.000082015991211, + "sampling/importance_sampling_ratio/min": 0.6745245456695557, + "sampling/sampling_logp_difference/max": 0.39374732971191406, + "sampling/sampling_logp_difference/mean": 0.011581224389374256, + "step": 388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012244734898558818, + "clip_ratio/low_min": 0.00012244734898558818, + "clip_ratio/region_mean": 0.00012244734898558818, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 541.875, + "completions/mean_terminated_length": 500.0, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.3554263301193714, + "epoch": 0.2082441113490364, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007479190360754728, + "learning_rate": 1e-05, + "loss": 0.0059, + "num_tokens": 8673658.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6044725179672241, + "sampling/importance_sampling_ratio/mean": 0.9996060132980347, + "sampling/importance_sampling_ratio/min": 0.7043182849884033, + "sampling/sampling_logp_difference/max": 0.4727950096130371, + "sampling/sampling_logp_difference/mean": 0.011445174925029278, + "step": 389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 540.65625, + "completions/mean_terminated_length": 464.875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.42104338482022285, + "epoch": 0.20877944325481798, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034305029548704624, + "learning_rate": 1e-05, + "loss": -0.0099, + "num_tokens": 8694759.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5624512434005737, + "sampling/importance_sampling_ratio/mean": 0.9999898672103882, + "sampling/importance_sampling_ratio/min": 0.7069579362869263, + "sampling/sampling_logp_difference/max": 0.4462559223175049, + "sampling/sampling_logp_difference/mean": 0.013074529357254505, + "step": 390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 472.09375, + "completions/mean_terminated_length": 441.4827575683594, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.339879784733057, + "epoch": 0.20931477516059957, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009654682129621506, + "learning_rate": 1e-05, + "loss": 0.0432, + "num_tokens": 8713314.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6273754835128784, + "sampling/importance_sampling_ratio/mean": 1.0000404119491577, + "sampling/importance_sampling_ratio/min": 0.7747105956077576, + "sampling/sampling_logp_difference/max": 0.4869685173034668, + "sampling/sampling_logp_difference/mean": 0.011338535696268082, + "step": 391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 449.28125, + "completions/mean_terminated_length": 439.0, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.39829106256365776, + "epoch": 0.20985010706638116, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015813199803233147, + "learning_rate": 1e-05, + "loss": -0.0001, + "num_tokens": 8731259.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3294726610183716, + "sampling/importance_sampling_ratio/mean": 0.9998857975006104, + "sampling/importance_sampling_ratio/min": 0.7047446370124817, + "sampling/sampling_logp_difference/max": 0.34991979598999023, + "sampling/sampling_logp_difference/mean": 0.012459760531783104, + "step": 392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.607445589499548e-05, + "clip_ratio/low_min": 4.607445589499548e-05, + "clip_ratio/region_mean": 4.607445589499548e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 579.40625, + "completions/mean_terminated_length": 516.5416870117188, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.4902245067059994, + "epoch": 0.21038543897216275, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0068397908471524715, + "learning_rate": 1e-05, + "loss": 0.0321, + "num_tokens": 8753632.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2944300174713135, + "sampling/importance_sampling_ratio/mean": 0.9996551275253296, + "sampling/importance_sampling_ratio/min": 0.6621697545051575, + "sampling/sampling_logp_difference/max": 0.4122333526611328, + "sampling/sampling_logp_difference/mean": 0.01450817659497261, + "step": 393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011655086200335063, + "clip_ratio/low_min": 0.00011655086200335063, + "clip_ratio/region_mean": 0.00011655086200335063, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 546.84375, + "completions/mean_terminated_length": 484.91998291015625, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.5028059035539627, + "epoch": 0.21092077087794434, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020201072096824646, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 8775035.0, + "reward": 0.59375, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3976861238479614, + "sampling/importance_sampling_ratio/mean": 1.0001035928726196, + "sampling/importance_sampling_ratio/min": 0.5971508622169495, + "sampling/sampling_logp_difference/max": 0.5155854821205139, + "sampling/sampling_logp_difference/mean": 0.014604815281927586, + "step": 394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 561.9375, + "completions/mean_terminated_length": 548.2000122070312, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.31315233558416367, + "epoch": 0.2114561027837259, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010948101989924908, + "learning_rate": 1e-05, + "loss": 0.0285, + "num_tokens": 8797289.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6328351497650146, + "sampling/importance_sampling_ratio/mean": 0.9999956488609314, + "sampling/importance_sampling_ratio/min": 0.7200225591659546, + "sampling/sampling_logp_difference/max": 0.49031782150268555, + "sampling/sampling_logp_difference/mean": 0.010241299867630005, + "step": 395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 565.84375, + "completions/mean_terminated_length": 536.9642944335938, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.28643761575222015, + "epoch": 0.21199143468950749, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006162581965327263, + "learning_rate": 1e-05, + "loss": -0.0147, + "num_tokens": 8819196.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4234240055084229, + "sampling/importance_sampling_ratio/mean": 0.9998257160186768, + "sampling/importance_sampling_ratio/min": 0.5781893134117126, + "sampling/sampling_logp_difference/max": 0.547853946685791, + "sampling/sampling_logp_difference/mean": 0.009983896277844906, + "step": 396 + }, + { + "clip_ratio/high_max": 5.835667616338469e-05, + "clip_ratio/high_mean": 5.835667616338469e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.835667616338469e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 506.25, + "completions/mean_terminated_length": 468.857177734375, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.36724481359124184, + "epoch": 0.21252676659528907, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004155013710260391, + "learning_rate": 1e-05, + "loss": 0.0533, + "num_tokens": 8839356.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.431915521621704, + "sampling/importance_sampling_ratio/mean": 0.9999504685401917, + "sampling/importance_sampling_ratio/min": 0.6292864680290222, + "sampling/sampling_logp_difference/max": 0.463168740272522, + "sampling/sampling_logp_difference/mean": 0.01224219985306263, + "step": 397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 464.53125, + "completions/mean_terminated_length": 444.3000183105469, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.456684835255146, + "epoch": 0.21306209850107066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013800724409520626, + "learning_rate": 1e-05, + "loss": -0.014, + "num_tokens": 8858837.0, + "reward": 0.5625, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.399673342704773, + "sampling/importance_sampling_ratio/mean": 1.0001552104949951, + "sampling/importance_sampling_ratio/min": 0.7700387835502625, + "sampling/sampling_logp_difference/max": 0.3362388610839844, + "sampling/sampling_logp_difference/mean": 0.014229970052838326, + "step": 398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.33275563409552e-05, + "clip_ratio/low_min": 4.33275563409552e-05, + "clip_ratio/region_mean": 4.33275563409552e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 630.5, + "completions/mean_terminated_length": 576.6956787109375, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.43897998705506325, + "epoch": 0.21359743040685225, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005376460030674934, + "learning_rate": 1e-05, + "loss": 0.0637, + "num_tokens": 8882869.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4658955335617065, + "sampling/importance_sampling_ratio/mean": 1.000138759613037, + "sampling/importance_sampling_ratio/min": 0.7277007102966309, + "sampling/sampling_logp_difference/max": 0.38246631622314453, + "sampling/sampling_logp_difference/mean": 0.012856343761086464, + "step": 399 + }, + { + "clip_ratio/high_max": 5.468066592584364e-05, + "clip_ratio/high_mean": 5.468066592584364e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.468066592584364e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 526.0, + "completions/mean_terminated_length": 509.86669921875, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.30568647384643555, + "epoch": 0.21413276231263384, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020914584398269653, + "learning_rate": 1e-05, + "loss": 0.0765, + "num_tokens": 8903293.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4682308435440063, + "sampling/importance_sampling_ratio/mean": 1.000411868095398, + "sampling/importance_sampling_ratio/min": 0.6125555634498596, + "sampling/sampling_logp_difference/max": 0.4901156425476074, + "sampling/sampling_logp_difference/mean": 0.01073786523193121, + "step": 400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.2105042414041236e-05, + "clip_ratio/low_min": 5.2105042414041236e-05, + "clip_ratio/region_mean": 5.2105042414041236e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 582.59375, + "completions/mean_terminated_length": 548.25927734375, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.2904437258839607, + "epoch": 0.21466809421841543, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0036875430960208178, + "learning_rate": 1e-05, + "loss": 0.0899, + "num_tokens": 8925400.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4000310897827148, + "sampling/importance_sampling_ratio/mean": 1.0001499652862549, + "sampling/importance_sampling_ratio/min": 0.605435848236084, + "sampling/sampling_logp_difference/max": 0.5018067359924316, + "sampling/sampling_logp_difference/mean": 0.009563757106661797, + "step": 401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 448.03125, + "completions/mean_terminated_length": 437.70965576171875, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.3770170584321022, + "epoch": 0.215203426124197, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011369570158421993, + "learning_rate": 1e-05, + "loss": -0.0049, + "num_tokens": 8942953.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4061670303344727, + "sampling/importance_sampling_ratio/mean": 1.000148892402649, + "sampling/importance_sampling_ratio/min": 0.7544295787811279, + "sampling/sampling_logp_difference/max": 0.3408675193786621, + "sampling/sampling_logp_difference/mean": 0.0119005823507905, + "step": 402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 570.15625, + "completions/mean_terminated_length": 514.760009765625, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.3198831807821989, + "epoch": 0.21573875802997858, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005257037468254566, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 8964846.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4231312274932861, + "sampling/importance_sampling_ratio/mean": 0.9996529817581177, + "sampling/importance_sampling_ratio/min": 0.6967759132385254, + "sampling/sampling_logp_difference/max": 0.36129140853881836, + "sampling/sampling_logp_difference/mean": 0.010347031056880951, + "step": 403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011215791892027482, + "clip_ratio/low_min": 0.00011215791892027482, + "clip_ratio/region_mean": 0.00011215791892027482, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 474.5625, + "completions/mean_terminated_length": 465.0967712402344, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.38425299152731895, + "epoch": 0.21627408993576017, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010644332505762577, + "learning_rate": 1e-05, + "loss": -0.0006, + "num_tokens": 8983448.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.445139765739441, + "sampling/importance_sampling_ratio/mean": 0.9999208450317383, + "sampling/importance_sampling_ratio/min": 0.6692987084388733, + "sampling/sampling_logp_difference/max": 0.40152478218078613, + "sampling/sampling_logp_difference/mean": 0.012317796237766743, + "step": 404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.32821832166519e-05, + "clip_ratio/low_min": 5.32821832166519e-05, + "clip_ratio/region_mean": 5.32821832166519e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 479.5, + "completions/mean_terminated_length": 470.19354248046875, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.33991294354200363, + "epoch": 0.21680942184154176, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00593644380569458, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 9002560.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4745886325836182, + "sampling/importance_sampling_ratio/mean": 1.0000718832015991, + "sampling/importance_sampling_ratio/min": 0.700054943561554, + "sampling/sampling_logp_difference/max": 0.3883790969848633, + "sampling/sampling_logp_difference/mean": 0.011626236140727997, + "step": 405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 511.21875, + "completions/mean_terminated_length": 494.10003662109375, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.3885650038719177, + "epoch": 0.21734475374732334, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011330134235322475, + "learning_rate": 1e-05, + "loss": 0.0533, + "num_tokens": 9022535.0, + "reward": 0.5625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5095738172531128, + "sampling/importance_sampling_ratio/mean": 1.0001286268234253, + "sampling/importance_sampling_ratio/min": 0.6953168511390686, + "sampling/sampling_logp_difference/max": 0.41182732582092285, + "sampling/sampling_logp_difference/mean": 0.012502442114055157, + "step": 406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.3533189202426e-05, + "clip_ratio/low_min": 5.3533189202426e-05, + "clip_ratio/region_mean": 5.3533189202426e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 537.75, + "completions/mean_terminated_length": 522.4000244140625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.3448605425655842, + "epoch": 0.21788008565310493, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01386034395545721, + "learning_rate": 1e-05, + "loss": -0.058, + "num_tokens": 9043839.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3660922050476074, + "sampling/importance_sampling_ratio/mean": 1.000069499015808, + "sampling/importance_sampling_ratio/min": 0.6906020045280457, + "sampling/sampling_logp_difference/max": 0.3701915740966797, + "sampling/sampling_logp_difference/mean": 0.011480391025543213, + "step": 407 + }, + { + "clip_ratio/high_max": 5.5285272537730634e-05, + "clip_ratio/high_mean": 5.5285272537730634e-05, + "clip_ratio/low_mean": 5.912961205467582e-05, + "clip_ratio/low_min": 5.912961205467582e-05, + "clip_ratio/region_mean": 0.00011441488459240645, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 569.53125, + "completions/mean_terminated_length": 532.7777709960938, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "entropy": 0.2892577201128006, + "epoch": 0.21841541755888652, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0128361526876688, + "learning_rate": 1e-05, + "loss": 0.0604, + "num_tokens": 9065408.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5042604207992554, + "sampling/importance_sampling_ratio/mean": 0.9997498989105225, + "sampling/importance_sampling_ratio/min": 0.7064053416252136, + "sampling/sampling_logp_difference/max": 0.40830135345458984, + "sampling/sampling_logp_difference/mean": 0.009226069785654545, + "step": 408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.069010537932627e-05, + "clip_ratio/low_min": 4.069010537932627e-05, + "clip_ratio/region_mean": 4.069010537932627e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 615.875, + "completions/mean_terminated_length": 511.78948974609375, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.6065098270773888, + "epoch": 0.21895074946466808, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0186065174639225, + "learning_rate": 1e-05, + "loss": 0.0543, + "num_tokens": 9088772.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3956139087677002, + "sampling/importance_sampling_ratio/mean": 1.0001424551010132, + "sampling/importance_sampling_ratio/min": 0.6796916723251343, + "sampling/sampling_logp_difference/max": 0.38611602783203125, + "sampling/sampling_logp_difference/mean": 0.016564738005399704, + "step": 409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.758587111718953e-05, + "clip_ratio/low_min": 9.758587111718953e-05, + "clip_ratio/region_mean": 9.758587111718953e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 578.53125, + "completions/mean_terminated_length": 492.40911865234375, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.5033793821930885, + "epoch": 0.21948608137044967, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0057432218454778194, + "learning_rate": 1e-05, + "loss": 0.1145, + "num_tokens": 9111373.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4896433353424072, + "sampling/importance_sampling_ratio/mean": 1.0002572536468506, + "sampling/importance_sampling_ratio/min": 0.7555832266807556, + "sampling/sampling_logp_difference/max": 0.39853668212890625, + "sampling/sampling_logp_difference/mean": 0.013849738985300064, + "step": 410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 597.09375, + "completions/mean_terminated_length": 507.5714416503906, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.6196390092372894, + "epoch": 0.22002141327623126, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00944578181952238, + "learning_rate": 1e-05, + "loss": 0.0325, + "num_tokens": 9134904.0, + "reward": 0.5, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.372896671295166, + "sampling/importance_sampling_ratio/mean": 0.9999515414237976, + "sampling/importance_sampling_ratio/min": 0.7232800126075745, + "sampling/sampling_logp_difference/max": 0.3239588737487793, + "sampling/sampling_logp_difference/mean": 0.016830801963806152, + "step": 411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 591.0, + "completions/mean_terminated_length": 541.4400024414062, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.41069863736629486, + "epoch": 0.22055674518201285, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011540129780769348, + "learning_rate": 1e-05, + "loss": 0.0398, + "num_tokens": 9157360.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3424558639526367, + "sampling/importance_sampling_ratio/mean": 0.999802827835083, + "sampling/importance_sampling_ratio/min": 0.6996087431907654, + "sampling/sampling_logp_difference/max": 0.35723400115966797, + "sampling/sampling_logp_difference/mean": 0.012154701165854931, + "step": 412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.80769231216982e-05, + "clip_ratio/low_min": 4.80769231216982e-05, + "clip_ratio/region_mean": 4.80769231216982e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 529.6363525390625, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.4063192680478096, + "epoch": 0.22109207708779444, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013522643595933914, + "learning_rate": 1e-05, + "loss": 0.0541, + "num_tokens": 9181020.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.8388729095458984, + "sampling/importance_sampling_ratio/mean": 0.9999910593032837, + "sampling/importance_sampling_ratio/min": 0.6064245104789734, + "sampling/sampling_logp_difference/max": 0.6091527938842773, + "sampling/sampling_logp_difference/mean": 0.012002038769423962, + "step": 413 + }, + { + "clip_ratio/high_max": 7.237985118990764e-05, + "clip_ratio/high_mean": 7.237985118990764e-05, + "clip_ratio/low_mean": 0.00012242898810654879, + "clip_ratio/low_min": 0.00012242898810654879, + "clip_ratio/region_mean": 0.00019480883929645643, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 521.4375, + "completions/mean_terminated_length": 439.25, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.4026436246931553, + "epoch": 0.22162740899357602, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004440732765942812, + "learning_rate": 1e-05, + "loss": 0.0381, + "num_tokens": 9201306.0, + "reward": 0.5, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3815584182739258, + "sampling/importance_sampling_ratio/mean": 1.0002615451812744, + "sampling/importance_sampling_ratio/min": 0.40709826350212097, + "sampling/sampling_logp_difference/max": 0.8987007141113281, + "sampling/sampling_logp_difference/mean": 0.012605084106326103, + "step": 414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.0698194981087e-05, + "clip_ratio/low_min": 9.0698194981087e-05, + "clip_ratio/region_mean": 9.0698194981087e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 655.09375, + "completions/mean_terminated_length": 577.8421020507812, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 0.44373518601059914, + "epoch": 0.2221627408993576, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012777360156178474, + "learning_rate": 1e-05, + "loss": -0.0018, + "num_tokens": 9226389.0, + "reward": 0.40625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6369410753250122, + "sampling/importance_sampling_ratio/mean": 1.0006061792373657, + "sampling/importance_sampling_ratio/min": 0.6410353779792786, + "sampling/sampling_logp_difference/max": 0.4928293228149414, + "sampling/sampling_logp_difference/mean": 0.012791387736797333, + "step": 415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 622.15625, + "completions/mean_terminated_length": 545.7619018554688, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4507865346968174, + "epoch": 0.22269807280513917, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015010213479399681, + "learning_rate": 1e-05, + "loss": 0.0346, + "num_tokens": 9249730.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2977815866470337, + "sampling/importance_sampling_ratio/mean": 1.0001089572906494, + "sampling/importance_sampling_ratio/min": 0.6073641777038574, + "sampling/sampling_logp_difference/max": 0.498626708984375, + "sampling/sampling_logp_difference/mean": 0.01323573850095272, + "step": 416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019604443150456063, + "clip_ratio/low_min": 0.00019604443150456063, + "clip_ratio/region_mean": 0.00019604443150456063, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 587.65625, + "completions/mean_terminated_length": 493.19049072265625, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.4715174324810505, + "epoch": 0.22323340471092076, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013832558877766132, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 9272495.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2830474376678467, + "sampling/importance_sampling_ratio/mean": 0.999737024307251, + "sampling/importance_sampling_ratio/min": 0.5005362629890442, + "sampling/sampling_logp_difference/max": 0.692075252532959, + "sampling/sampling_logp_difference/mean": 0.013608349487185478, + "step": 417 + }, + { + "clip_ratio/high_max": 5.6947606935864314e-05, + "clip_ratio/high_mean": 5.6947606935864314e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.6947606935864314e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 587.0, + "completions/mean_terminated_length": 504.727294921875, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.6519546769559383, + "epoch": 0.22376873661670235, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021211620420217514, + "learning_rate": 1e-05, + "loss": 0.0725, + "num_tokens": 9295959.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4707138538360596, + "sampling/importance_sampling_ratio/mean": 1.0002268552780151, + "sampling/importance_sampling_ratio/min": 0.5587266683578491, + "sampling/sampling_logp_difference/max": 0.5820949077606201, + "sampling/sampling_logp_difference/mean": 0.017742017284035683, + "step": 418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.4324205848388374e-05, + "clip_ratio/low_min": 5.4324205848388374e-05, + "clip_ratio/region_mean": 5.4324205848388374e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 615.46875, + "completions/mean_terminated_length": 535.5714111328125, + "completions/min_length": 362.0, + "completions/min_terminated_length": 362.0, + "entropy": 0.3769516982138157, + "epoch": 0.22430406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0035179273691028357, + "learning_rate": 1e-05, + "loss": 0.0939, + "num_tokens": 9319446.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3247960805892944, + "sampling/importance_sampling_ratio/mean": 0.9999805688858032, + "sampling/importance_sampling_ratio/min": 0.6951610445976257, + "sampling/sampling_logp_difference/max": 0.36361169815063477, + "sampling/sampling_logp_difference/mean": 0.010986430570483208, + "step": 419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 630.78125, + "completions/mean_terminated_length": 585.0416870117188, + "completions/min_length": 413.0, + "completions/min_terminated_length": 413.0, + "entropy": 0.31577062606811523, + "epoch": 0.22483940042826553, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0051013389602303505, + "learning_rate": 1e-05, + "loss": 0.0481, + "num_tokens": 9343263.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4925427436828613, + "sampling/importance_sampling_ratio/mean": 1.0003926753997803, + "sampling/importance_sampling_ratio/min": 0.7386350035667419, + "sampling/sampling_logp_difference/max": 0.4004812240600586, + "sampling/sampling_logp_difference/mean": 0.010470103472471237, + "step": 420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 505.5625, + "completions/mean_terminated_length": 497.0967712402344, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.32444397918879986, + "epoch": 0.22537473233404712, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006141118239611387, + "learning_rate": 1e-05, + "loss": 0.0871, + "num_tokens": 9363265.0, + "reward": 0.5, + "reward_std": 0.3535533845424652, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.416572093963623, + "sampling/importance_sampling_ratio/mean": 0.9998906254768372, + "sampling/importance_sampling_ratio/min": 0.7291104197502136, + "sampling/sampling_logp_difference/max": 0.3482398986816406, + "sampling/sampling_logp_difference/mean": 0.011126390658318996, + "step": 421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023380395214189775, + "clip_ratio/low_min": 0.00023380395214189775, + "clip_ratio/region_mean": 0.00023380395214189775, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 563.3125, + "completions/mean_terminated_length": 516.0769653320312, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.33970316872000694, + "epoch": 0.2259100642398287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01292883139103651, + "learning_rate": 1e-05, + "loss": 0.0555, + "num_tokens": 9385555.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5158705711364746, + "sampling/importance_sampling_ratio/mean": 0.9997826814651489, + "sampling/importance_sampling_ratio/min": 0.6873456239700317, + "sampling/sampling_logp_difference/max": 0.41598987579345703, + "sampling/sampling_logp_difference/mean": 0.011197828687727451, + "step": 422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.094211468938738e-05, + "clip_ratio/low_min": 7.094211468938738e-05, + "clip_ratio/region_mean": 7.094211468938738e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 706.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 460.3125, + "completions/mean_terminated_length": 460.3125, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.3652568534016609, + "epoch": 0.22644539614561027, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.04, + "num_tokens": 9403701.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.310981035232544, + "sampling/importance_sampling_ratio/mean": 0.9997658729553223, + "sampling/importance_sampling_ratio/min": 0.6754453182220459, + "sampling/sampling_logp_difference/max": 0.3923830986022949, + "sampling/sampling_logp_difference/mean": 0.011771125718951225, + "step": 423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.493170490604825e-05, + "clip_ratio/low_min": 4.493170490604825e-05, + "clip_ratio/region_mean": 4.493170490604825e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 558.9375, + "completions/mean_terminated_length": 477.13043212890625, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "entropy": 0.44727589562535286, + "epoch": 0.22698072805139186, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024435359984636307, + "learning_rate": 1e-05, + "loss": 0.0439, + "num_tokens": 9425763.0, + "reward": 0.6875, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.9711543321609497, + "sampling/importance_sampling_ratio/mean": 1.0003437995910645, + "sampling/importance_sampling_ratio/min": 0.6911482214927673, + "sampling/sampling_logp_difference/max": 0.678619384765625, + "sampling/sampling_logp_difference/mean": 0.013055948540568352, + "step": 424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 533.40625, + "completions/mean_terminated_length": 479.2692565917969, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.36951202899217606, + "epoch": 0.22751605995717344, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027764614671468735, + "learning_rate": 1e-05, + "loss": 0.1327, + "num_tokens": 9446264.0, + "reward": 0.625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2995736598968506, + "sampling/importance_sampling_ratio/mean": 0.9997979402542114, + "sampling/importance_sampling_ratio/min": 0.6881005167961121, + "sampling/sampling_logp_difference/max": 0.37382030487060547, + "sampling/sampling_logp_difference/mean": 0.011245404370129108, + "step": 425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.946717283222824e-05, + "clip_ratio/low_min": 5.946717283222824e-05, + "clip_ratio/region_mean": 5.946717283222824e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 584.40625, + "completions/mean_terminated_length": 533.0, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.31000653095543385, + "epoch": 0.22805139186295503, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011753677390515804, + "learning_rate": 1e-05, + "loss": 0.0523, + "num_tokens": 9468717.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3142720460891724, + "sampling/importance_sampling_ratio/mean": 1.0002434253692627, + "sampling/importance_sampling_ratio/min": 0.6745606660842896, + "sampling/sampling_logp_difference/max": 0.3936936855316162, + "sampling/sampling_logp_difference/mean": 0.0103093720972538, + "step": 426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 606.65625, + "completions/mean_terminated_length": 533.3181762695312, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.34468335285782814, + "epoch": 0.22858672376873662, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014401816762983799, + "learning_rate": 1e-05, + "loss": 0.0063, + "num_tokens": 9492026.0, + "reward": 0.5625, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000317931175232, + "sampling/importance_sampling_ratio/min": 0.5973264575004578, + "sampling/sampling_logp_difference/max": 0.7384324073791504, + "sampling/sampling_logp_difference/mean": 0.011003500781953335, + "step": 427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 460.96875, + "completions/mean_terminated_length": 460.96875, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.3451710008084774, + "epoch": 0.2291220556745182, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019438588991761208, + "learning_rate": 1e-05, + "loss": 0.0016, + "num_tokens": 9510321.0, + "reward": 0.65625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6240769624710083, + "sampling/importance_sampling_ratio/mean": 0.9998361468315125, + "sampling/importance_sampling_ratio/min": 0.7779816389083862, + "sampling/sampling_logp_difference/max": 0.4849395751953125, + "sampling/sampling_logp_difference/mean": 0.01068849116563797, + "step": 428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 565.96875, + "completions/mean_terminated_length": 537.107177734375, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 0.31969908252358437, + "epoch": 0.2296573875802998, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011303309351205826, + "learning_rate": 1e-05, + "loss": 0.0354, + "num_tokens": 9532320.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4453920125961304, + "sampling/importance_sampling_ratio/mean": 1.000016450881958, + "sampling/importance_sampling_ratio/min": 0.697854220867157, + "sampling/sampling_logp_difference/max": 0.3683805465698242, + "sampling/sampling_logp_difference/mean": 0.010260508395731449, + "step": 429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 445.9375, + "completions/mean_terminated_length": 424.4666748046875, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.3787587955594063, + "epoch": 0.23019271948608136, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018236199393868446, + "learning_rate": 1e-05, + "loss": 0.1, + "num_tokens": 9550118.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.40244722366333, + "sampling/importance_sampling_ratio/mean": 0.9997003078460693, + "sampling/importance_sampling_ratio/min": 0.6140540242195129, + "sampling/sampling_logp_difference/max": 0.4876723289489746, + "sampling/sampling_logp_difference/mean": 0.011893510818481445, + "step": 430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011145786993438378, + "clip_ratio/low_min": 0.00011145786993438378, + "clip_ratio/region_mean": 0.00011145786993438378, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 478.375, + "completions/mean_terminated_length": 424.7407531738281, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.373266514390707, + "epoch": 0.23072805139186295, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014188689179718494, + "learning_rate": 1e-05, + "loss": 0.0714, + "num_tokens": 9569058.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001626014709473, + "sampling/importance_sampling_ratio/min": 0.7322058081626892, + "sampling/sampling_logp_difference/max": 1.0407233238220215, + "sampling/sampling_logp_difference/mean": 0.011204676702618599, + "step": 431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.506608067662455e-05, + "clip_ratio/low_min": 5.506608067662455e-05, + "clip_ratio/region_mean": 5.506608067662455e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 544.84375, + "completions/mean_terminated_length": 512.9642944335938, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.35175613686442375, + "epoch": 0.23126338329764454, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015107778832316399, + "learning_rate": 1e-05, + "loss": -0.0242, + "num_tokens": 9590421.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.393925666809082, + "sampling/importance_sampling_ratio/mean": 1.0000779628753662, + "sampling/importance_sampling_ratio/min": 0.722366988658905, + "sampling/sampling_logp_difference/max": 0.3321239948272705, + "sampling/sampling_logp_difference/mean": 0.011217285878956318, + "step": 432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013936093091615476, + "clip_ratio/low_min": 0.00013936093091615476, + "clip_ratio/region_mean": 0.00013936093091615476, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 671.75, + "completions/mean_terminated_length": 586.8235473632812, + "completions/min_length": 380.0, + "completions/min_terminated_length": 380.0, + "entropy": 0.3009270764887333, + "epoch": 0.23179871520342613, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009971894323825836, + "learning_rate": 1e-05, + "loss": -0.0092, + "num_tokens": 9615749.0, + "reward": 0.40625, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.557236909866333, + "sampling/importance_sampling_ratio/mean": 0.9998192191123962, + "sampling/importance_sampling_ratio/min": 0.6994993090629578, + "sampling/sampling_logp_difference/max": 0.4429130554199219, + "sampling/sampling_logp_difference/mean": 0.009629164822399616, + "step": 433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.527345299720764e-05, + "clip_ratio/low_min": 4.527345299720764e-05, + "clip_ratio/region_mean": 4.527345299720764e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 645.1875, + "completions/mean_terminated_length": 610.7999877929688, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "entropy": 0.4690527394413948, + "epoch": 0.23233404710920771, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0030221513006836176, + "learning_rate": 1e-05, + "loss": 0.0371, + "num_tokens": 9640035.0, + "reward": 0.40625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4208089113235474, + "sampling/importance_sampling_ratio/mean": 1.0002892017364502, + "sampling/importance_sampling_ratio/min": 0.6786139011383057, + "sampling/sampling_logp_difference/max": 0.38770294189453125, + "sampling/sampling_logp_difference/mean": 0.013197680935263634, + "step": 434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.898454739712179e-05, + "clip_ratio/low_min": 6.898454739712179e-05, + "clip_ratio/region_mean": 6.898454739712179e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 742.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 469.875, + "completions/mean_terminated_length": 469.875, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.38235819712281227, + "epoch": 0.2328693790149893, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0011, + "num_tokens": 9658831.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4322915077209473, + "sampling/importance_sampling_ratio/mean": 1.0000020265579224, + "sampling/importance_sampling_ratio/min": 0.7452959418296814, + "sampling/sampling_logp_difference/max": 0.35927557945251465, + "sampling/sampling_logp_difference/mean": 0.012142562307417393, + "step": 435 + }, + { + "clip_ratio/high_max": 9.854770905803889e-05, + "clip_ratio/high_mean": 9.854770905803889e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.854770905803889e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 597.34375, + "completions/mean_terminated_length": 540.4583740234375, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.34139036759734154, + "epoch": 0.2334047109207709, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012364211492240429, + "learning_rate": 1e-05, + "loss": 0.0444, + "num_tokens": 9681674.0, + "reward": 0.71875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4381734132766724, + "sampling/importance_sampling_ratio/mean": 1.0000934600830078, + "sampling/importance_sampling_ratio/min": 0.6522260308265686, + "sampling/sampling_logp_difference/max": 0.4273641109466553, + "sampling/sampling_logp_difference/mean": 0.010575716383755207, + "step": 436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.3146257414482534e-05, + "clip_ratio/low_min": 5.3146257414482534e-05, + "clip_ratio/region_mean": 5.3146257414482534e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 528.375, + "completions/mean_terminated_length": 503.5862121582031, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.3023227211087942, + "epoch": 0.23394004282655245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017348093912005424, + "learning_rate": 1e-05, + "loss": 0.062, + "num_tokens": 9701814.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999216794967651, + "sampling/importance_sampling_ratio/min": 0.2950405180454254, + "sampling/sampling_logp_difference/max": 1.2206425666809082, + "sampling/sampling_logp_difference/mean": 0.010344784706830978, + "step": 437 + }, + { + "clip_ratio/high_max": 6.243756070034578e-05, + "clip_ratio/high_mean": 6.243756070034578e-05, + "clip_ratio/low_mean": 0.000227421351155499, + "clip_ratio/low_min": 0.000227421351155499, + "clip_ratio/region_mean": 0.0002898589191318024, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 540.125, + "completions/mean_terminated_length": 464.16668701171875, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.33811743929982185, + "epoch": 0.23447537473233404, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.004054592922329903, + "learning_rate": 1e-05, + "loss": 0.0526, + "num_tokens": 9723362.0, + "reward": 0.6875, + "reward_std": 0.3535533845424652, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4524321556091309, + "sampling/importance_sampling_ratio/mean": 0.9998065233230591, + "sampling/importance_sampling_ratio/min": 0.6455051302909851, + "sampling/sampling_logp_difference/max": 0.43772220611572266, + "sampling/sampling_logp_difference/mean": 0.01056714728474617, + "step": 438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016832120309118181, + "clip_ratio/low_min": 0.00016832120309118181, + "clip_ratio/region_mean": 0.00016832120309118181, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 528.59375, + "completions/mean_terminated_length": 503.82757568359375, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "entropy": 0.4121296629309654, + "epoch": 0.23501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014934531413018703, + "learning_rate": 1e-05, + "loss": -0.0677, + "num_tokens": 9743797.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4214543104171753, + "sampling/importance_sampling_ratio/mean": 1.0002490282058716, + "sampling/importance_sampling_ratio/min": 0.6828449964523315, + "sampling/sampling_logp_difference/max": 0.3814873695373535, + "sampling/sampling_logp_difference/mean": 0.012028890661895275, + "step": 439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010313531674910337, + "clip_ratio/low_min": 0.00010313531674910337, + "clip_ratio/region_mean": 0.00010313531674910337, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 545.46875, + "completions/mean_terminated_length": 494.11541748046875, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.45557525381445885, + "epoch": 0.23554603854389722, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0137118399143219, + "learning_rate": 1e-05, + "loss": 0.0342, + "num_tokens": 9765124.0, + "reward": 0.46875, + "reward_std": 0.5038893818855286, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4215961694717407, + "sampling/importance_sampling_ratio/mean": 1.0003676414489746, + "sampling/importance_sampling_ratio/min": 0.6576674580574036, + "sampling/sampling_logp_difference/max": 0.4190559387207031, + "sampling/sampling_logp_difference/mean": 0.013410435989499092, + "step": 440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 469.59375, + "completions/mean_terminated_length": 449.70001220703125, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.4070568270981312, + "epoch": 0.2360813704496788, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 9783551.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4098912477493286, + "sampling/importance_sampling_ratio/mean": 0.9998842477798462, + "sampling/importance_sampling_ratio/min": 0.720041036605835, + "sampling/sampling_logp_difference/max": 0.34351253509521484, + "sampling/sampling_logp_difference/mean": 0.012938451953232288, + "step": 441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017385257524438202, + "clip_ratio/low_min": 0.00017385257524438202, + "clip_ratio/region_mean": 0.00017385257524438202, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 557.40625, + "completions/mean_terminated_length": 527.3214721679688, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.40482622012495995, + "epoch": 0.2366167023554604, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011935973539948463, + "learning_rate": 1e-05, + "loss": 0.0967, + "num_tokens": 9805308.0, + "reward": 0.65625, + "reward_std": 0.47137710452079773, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3014415502548218, + "sampling/importance_sampling_ratio/mean": 1.0003654956817627, + "sampling/importance_sampling_ratio/min": 0.6488705277442932, + "sampling/sampling_logp_difference/max": 0.4325220584869385, + "sampling/sampling_logp_difference/mean": 0.012709065340459347, + "step": 442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 538.21875, + "completions/mean_terminated_length": 514.4483032226562, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 0.324844416230917, + "epoch": 0.23715203426124196, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008872278966009617, + "learning_rate": 1e-05, + "loss": 0.048, + "num_tokens": 9825891.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4155662059783936, + "sampling/importance_sampling_ratio/mean": 1.0002615451812744, + "sampling/importance_sampling_ratio/min": 0.7099429965019226, + "sampling/sampling_logp_difference/max": 0.34752964973449707, + "sampling/sampling_logp_difference/mean": 0.010144418105483055, + "step": 443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.351625779643655e-05, + "clip_ratio/low_min": 6.351625779643655e-05, + "clip_ratio/region_mean": 6.351625779643655e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 719.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 459.34375, + "completions/mean_terminated_length": 459.34375, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.31106376461684704, + "epoch": 0.23768736616702354, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0047, + "num_tokens": 9844110.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3937691450119019, + "sampling/importance_sampling_ratio/mean": 0.9998219609260559, + "sampling/importance_sampling_ratio/min": 0.7058107256889343, + "sampling/sampling_logp_difference/max": 0.34840822219848633, + "sampling/sampling_logp_difference/mean": 0.010355335660278797, + "step": 444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 569.78125, + "completions/mean_terminated_length": 524.0384521484375, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.3032274544239044, + "epoch": 0.23822269807280513, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006632843986153603, + "learning_rate": 1e-05, + "loss": 0.0451, + "num_tokens": 9865911.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3431744575500488, + "sampling/importance_sampling_ratio/mean": 0.9997745156288147, + "sampling/importance_sampling_ratio/min": 0.6964196562767029, + "sampling/sampling_logp_difference/max": 0.3618028163909912, + "sampling/sampling_logp_difference/mean": 0.00975789874792099, + "step": 445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.367804598994553e-05, + "clip_ratio/low_min": 6.367804598994553e-05, + "clip_ratio/region_mean": 6.367804598994553e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 559.53125, + "completions/mean_terminated_length": 545.6333618164062, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.2855161540210247, + "epoch": 0.23875802997858672, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017041988670825958, + "learning_rate": 1e-05, + "loss": 0.0306, + "num_tokens": 9887672.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2953450679779053, + "sampling/importance_sampling_ratio/mean": 1.0003676414489746, + "sampling/importance_sampling_ratio/min": 0.7011907696723938, + "sampling/sampling_logp_difference/max": 0.35497522354125977, + "sampling/sampling_logp_difference/mean": 0.009561488404870033, + "step": 446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015837948740227148, + "clip_ratio/low_min": 0.00015837948740227148, + "clip_ratio/region_mean": 0.00015837948740227148, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 575.53125, + "completions/mean_terminated_length": 531.1154174804688, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3524639941751957, + "epoch": 0.2392933618843683, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.002969164866954088, + "learning_rate": 1e-05, + "loss": 0.0423, + "num_tokens": 9909937.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.363702654838562, + "sampling/importance_sampling_ratio/mean": 1.000001072883606, + "sampling/importance_sampling_ratio/min": 0.6369519829750061, + "sampling/sampling_logp_difference/max": 0.4510610103607178, + "sampling/sampling_logp_difference/mean": 0.011127698235213757, + "step": 447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 525.375, + "completions/mean_terminated_length": 490.71429443359375, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.3839997462928295, + "epoch": 0.2398286937901499, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004479656461626291, + "learning_rate": 1e-05, + "loss": 0.0359, + "num_tokens": 9930773.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.308794379234314, + "sampling/importance_sampling_ratio/mean": 1.0000007152557373, + "sampling/importance_sampling_ratio/min": 0.6819316148757935, + "sampling/sampling_logp_difference/max": 0.3828258514404297, + "sampling/sampling_logp_difference/mean": 0.011892806738615036, + "step": 448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 545.84375, + "completions/mean_terminated_length": 522.862060546875, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "entropy": 0.37527740001678467, + "epoch": 0.2403640256959315, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018485892564058304, + "learning_rate": 1e-05, + "loss": 0.0424, + "num_tokens": 9951880.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6329386234283447, + "sampling/importance_sampling_ratio/mean": 0.9999679923057556, + "sampling/importance_sampling_ratio/min": 0.660495400428772, + "sampling/sampling_logp_difference/max": 0.49038124084472656, + "sampling/sampling_logp_difference/mean": 0.012055926024913788, + "step": 449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.0040031055686995e-05, + "clip_ratio/low_min": 5.0040031055686995e-05, + "clip_ratio/region_mean": 5.0040031055686995e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 531.21875, + "completions/mean_terminated_length": 506.72412109375, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.35922387056052685, + "epoch": 0.24089935760171305, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004421040415763855, + "learning_rate": 1e-05, + "loss": 0.0502, + "num_tokens": 9972255.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3125956058502197, + "sampling/importance_sampling_ratio/mean": 0.9998888969421387, + "sampling/importance_sampling_ratio/min": 0.5535402297973633, + "sampling/sampling_logp_difference/max": 0.5914208292961121, + "sampling/sampling_logp_difference/mean": 0.011364748701453209, + "step": 450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011941327466047369, + "clip_ratio/low_min": 0.00011941327466047369, + "clip_ratio/region_mean": 0.00011941327466047369, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 552.625, + "completions/mean_terminated_length": 502.923095703125, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.33651861920952797, + "epoch": 0.24143468950749464, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01763063855469227, + "learning_rate": 1e-05, + "loss": 0.0031, + "num_tokens": 9993547.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3175828456878662, + "sampling/importance_sampling_ratio/mean": 0.9998105764389038, + "sampling/importance_sampling_ratio/min": 0.598853588104248, + "sampling/sampling_logp_difference/max": 0.5127382278442383, + "sampling/sampling_logp_difference/mean": 0.010966668836772442, + "step": 451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 470.875, + "completions/mean_terminated_length": 461.2903137207031, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.32031749933958054, + "epoch": 0.24197002141327623, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015869755297899246, + "learning_rate": 1e-05, + "loss": -0.015, + "num_tokens": 10011935.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4097594022750854, + "sampling/importance_sampling_ratio/mean": 0.9998683333396912, + "sampling/importance_sampling_ratio/min": 0.710909366607666, + "sampling/sampling_logp_difference/max": 0.34341907501220703, + "sampling/sampling_logp_difference/mean": 0.011071456596255302, + "step": 452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010638788808137178, + "clip_ratio/low_min": 0.00010638788808137178, + "clip_ratio/region_mean": 0.00010638788808137178, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 532.09375, + "completions/mean_terminated_length": 488.40740966796875, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.41738782078027725, + "epoch": 0.24250535331905781, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002433771500363946, + "learning_rate": 1e-05, + "loss": 0.0167, + "num_tokens": 10032554.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.7308326959609985, + "sampling/importance_sampling_ratio/mean": 1.0001813173294067, + "sampling/importance_sampling_ratio/min": 0.7279193997383118, + "sampling/sampling_logp_difference/max": 0.5486025810241699, + "sampling/sampling_logp_difference/mean": 0.012861635535955429, + "step": 453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015337190779973753, + "clip_ratio/low_min": 0.00015337190779973753, + "clip_ratio/region_mean": 0.00015337190779973753, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 663.4375, + "completions/mean_terminated_length": 600.7000122070312, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.396393358707428, + "epoch": 0.2430406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011197500862181187, + "learning_rate": 1e-05, + "loss": 0.0347, + "num_tokens": 10057896.0, + "reward": 0.34375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.810591697692871, + "sampling/importance_sampling_ratio/mean": 1.0000510215759277, + "sampling/importance_sampling_ratio/min": 0.5973223447799683, + "sampling/sampling_logp_difference/max": 0.593653678894043, + "sampling/sampling_logp_difference/mean": 0.012164346873760223, + "step": 454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014273404303821735, + "clip_ratio/low_min": 0.00014273404303821735, + "clip_ratio/region_mean": 0.00014273404303821735, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 644.5, + "completions/mean_terminated_length": 579.8095092773438, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.46060651168227196, + "epoch": 0.243576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01803814060986042, + "learning_rate": 1e-05, + "loss": 0.0575, + "num_tokens": 10082760.0, + "reward": 0.53125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4235470294952393, + "sampling/importance_sampling_ratio/mean": 1.0002251863479614, + "sampling/importance_sampling_ratio/min": 0.5989532470703125, + "sampling/sampling_logp_difference/max": 0.5125718116760254, + "sampling/sampling_logp_difference/mean": 0.013906026259064674, + "step": 455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019422921104705893, + "clip_ratio/low_min": 0.00019422921104705893, + "clip_ratio/region_mean": 0.00019422921104705893, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 629.53125, + "completions/mean_terminated_length": 534.7894897460938, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.5165114663541317, + "epoch": 0.24411134903640258, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014529377222061157, + "learning_rate": 1e-05, + "loss": 0.041, + "num_tokens": 10106761.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.389373779296875, + "sampling/importance_sampling_ratio/mean": 1.0002028942108154, + "sampling/importance_sampling_ratio/min": 0.7022377848625183, + "sampling/sampling_logp_difference/max": 0.3534832000732422, + "sampling/sampling_logp_difference/mean": 0.015013284981250763, + "step": 456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 526.875, + "completions/mean_terminated_length": 501.9310302734375, + "completions/min_length": 371.0, + "completions/min_terminated_length": 371.0, + "entropy": 0.3432654179632664, + "epoch": 0.24464668094218414, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016591932624578476, + "learning_rate": 1e-05, + "loss": 0.1204, + "num_tokens": 10127565.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4345364570617676, + "sampling/importance_sampling_ratio/mean": 1.0000282526016235, + "sampling/importance_sampling_ratio/min": 0.6134641766548157, + "sampling/sampling_logp_difference/max": 0.488633394241333, + "sampling/sampling_logp_difference/mean": 0.011462590657174587, + "step": 457 + }, + { + "clip_ratio/high_max": 6.641870277235284e-05, + "clip_ratio/high_mean": 6.641870277235284e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.641870277235284e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 522.75, + "completions/mean_terminated_length": 466.15386962890625, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.3245689757168293, + "epoch": 0.24518201284796573, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008053712546825409, + "learning_rate": 1e-05, + "loss": 0.0316, + "num_tokens": 10147765.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.339956283569336, + "sampling/importance_sampling_ratio/mean": 1.0000364780426025, + "sampling/importance_sampling_ratio/min": 0.6956804990768433, + "sampling/sampling_logp_difference/max": 0.36286473274230957, + "sampling/sampling_logp_difference/mean": 0.010648135095834732, + "step": 458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 474.65625, + "completions/mean_terminated_length": 474.65625, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "entropy": 0.31016377732157707, + "epoch": 0.24571734475374732, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022604985162615776, + "learning_rate": 1e-05, + "loss": -0.0694, + "num_tokens": 10166602.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3568627834320068, + "sampling/importance_sampling_ratio/mean": 0.9997751712799072, + "sampling/importance_sampling_ratio/min": 0.7631262540817261, + "sampling/sampling_logp_difference/max": 0.3051753044128418, + "sampling/sampling_logp_difference/mean": 0.009892992675304413, + "step": 459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.808278703829274e-05, + "clip_ratio/low_min": 6.808278703829274e-05, + "clip_ratio/region_mean": 6.808278703829274e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 480.4375, + "completions/mean_terminated_length": 471.1612854003906, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.3669208362698555, + "epoch": 0.2462526766595289, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0223, + "num_tokens": 10185200.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.9818788766860962, + "sampling/importance_sampling_ratio/mean": 0.9997610449790955, + "sampling/importance_sampling_ratio/min": 0.702907145023346, + "sampling/sampling_logp_difference/max": 0.6840453147888184, + "sampling/sampling_logp_difference/mean": 0.01118472870439291, + "step": 460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 545.25, + "completions/mean_terminated_length": 522.2069091796875, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 0.32584521919488907, + "epoch": 0.2467880085653105, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019112931564450264, + "learning_rate": 1e-05, + "loss": 0.0704, + "num_tokens": 10206328.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4550303220748901, + "sampling/importance_sampling_ratio/mean": 0.9998010396957397, + "sampling/importance_sampling_ratio/min": 0.6699081063270569, + "sampling/sampling_logp_difference/max": 0.40061473846435547, + "sampling/sampling_logp_difference/mean": 0.010370478965342045, + "step": 461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 543.25, + "completions/mean_terminated_length": 491.3846435546875, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.3115219175815582, + "epoch": 0.24732334047109208, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005989633966237307, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 10227560.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4669171571731567, + "sampling/importance_sampling_ratio/mean": 0.9996733069419861, + "sampling/importance_sampling_ratio/min": 0.7713218927383423, + "sampling/sampling_logp_difference/max": 0.3831629753112793, + "sampling/sampling_logp_difference/mean": 0.010220806114375591, + "step": 462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002042240921582561, + "clip_ratio/low_min": 0.0002042240921582561, + "clip_ratio/region_mean": 0.0002042240921582561, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 607.125, + "completions/mean_terminated_length": 522.857177734375, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "entropy": 0.4051716774702072, + "epoch": 0.24785867237687367, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006537931971251965, + "learning_rate": 1e-05, + "loss": 0.0595, + "num_tokens": 10250988.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6197837591171265, + "sampling/importance_sampling_ratio/mean": 1.0001596212387085, + "sampling/importance_sampling_ratio/min": 0.7747717499732971, + "sampling/sampling_logp_difference/max": 0.48229265213012695, + "sampling/sampling_logp_difference/mean": 0.012219919823110104, + "step": 463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.491555955610238e-05, + "clip_ratio/low_min": 4.491555955610238e-05, + "clip_ratio/region_mean": 4.491555955610238e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 655.90625, + "completions/mean_terminated_length": 557.0, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.3715316690504551, + "epoch": 0.24839400428265523, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007977494969964027, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 10276401.0, + "reward": 0.5, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4375637769699097, + "sampling/importance_sampling_ratio/mean": 0.9999771118164062, + "sampling/importance_sampling_ratio/min": 0.6834230422973633, + "sampling/sampling_logp_difference/max": 0.38064122200012207, + "sampling/sampling_logp_difference/mean": 0.011099442839622498, + "step": 464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 510.5, + "completions/mean_terminated_length": 483.862060546875, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.3095720000565052, + "epoch": 0.24892933618843682, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018161436542868614, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 10296377.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4600300788879395, + "sampling/importance_sampling_ratio/mean": 0.9999551177024841, + "sampling/importance_sampling_ratio/min": 0.7093867063522339, + "sampling/sampling_logp_difference/max": 0.37845706939697266, + "sampling/sampling_logp_difference/mean": 0.011000224389135838, + "step": 465 + }, + { + "clip_ratio/high_max": 6.568575918208808e-05, + "clip_ratio/high_mean": 6.568575918208808e-05, + "clip_ratio/low_mean": 0.000293447490548715, + "clip_ratio/low_min": 0.000293447490548715, + "clip_ratio/region_mean": 0.00035913324973080307, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 475.70001220703125, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.4775240123271942, + "epoch": 0.2494646680942184, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007165522780269384, + "learning_rate": 1e-05, + "loss": 0.0388, + "num_tokens": 10318947.0, + "reward": 0.28125, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4121737480163574, + "sampling/importance_sampling_ratio/mean": 1.0000252723693848, + "sampling/importance_sampling_ratio/min": 0.5510292649269104, + "sampling/sampling_logp_difference/max": 0.5959672927856445, + "sampling/sampling_logp_difference/mean": 0.013995653949677944, + "step": 466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015253391029546037, + "clip_ratio/low_min": 0.00015253391029546037, + "clip_ratio/region_mean": 0.00015253391029546037, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 594.6875, + "completions/mean_terminated_length": 554.6923217773438, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.3064855579286814, + "epoch": 0.25, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007180414628237486, + "learning_rate": 1e-05, + "loss": 0.04, + "num_tokens": 10341449.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.8535993099212646, + "sampling/importance_sampling_ratio/mean": 0.9999573826789856, + "sampling/importance_sampling_ratio/min": 0.6286686062812805, + "sampling/sampling_logp_difference/max": 0.6171293258666992, + "sampling/sampling_logp_difference/mean": 0.009904849343001842, + "step": 467 + }, + { + "clip_ratio/high_max": 5.4561325669055805e-05, + "clip_ratio/high_mean": 5.4561325669055805e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.4561325669055805e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 618.09375, + "completions/mean_terminated_length": 559.434814453125, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.43297940120100975, + "epoch": 0.2505353319057816, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003917201422154903, + "learning_rate": 1e-05, + "loss": 0.0601, + "num_tokens": 10365084.0, + "reward": 0.4375, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3909175395965576, + "sampling/importance_sampling_ratio/mean": 0.9995114803314209, + "sampling/importance_sampling_ratio/min": 0.7199997305870056, + "sampling/sampling_logp_difference/max": 0.32996368408203125, + "sampling/sampling_logp_difference/mean": 0.012611831538379192, + "step": 468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012666822294704616, + "clip_ratio/low_min": 0.00012666822294704616, + "clip_ratio/region_mean": 0.00012666822294704616, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 449.65625, + "completions/mean_terminated_length": 439.3870849609375, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.41355736553668976, + "epoch": 0.2510706638115632, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00916921067982912, + "learning_rate": 1e-05, + "loss": 0.0521, + "num_tokens": 10382809.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.333982229232788, + "sampling/importance_sampling_ratio/mean": 0.9997520446777344, + "sampling/importance_sampling_ratio/min": 0.7351874113082886, + "sampling/sampling_logp_difference/max": 0.3076298236846924, + "sampling/sampling_logp_difference/mean": 0.013193922117352486, + "step": 469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012437811528798193, + "clip_ratio/low_min": 0.00012437811528798193, + "clip_ratio/region_mean": 0.00012437811528798193, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 591.0625, + "completions/mean_terminated_length": 510.6363830566406, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.4005119614303112, + "epoch": 0.25160599571734477, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010969983413815498, + "learning_rate": 1e-05, + "loss": 0.0567, + "num_tokens": 10405739.0, + "reward": 0.5, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2928237915039062, + "sampling/importance_sampling_ratio/mean": 0.9999876618385315, + "sampling/importance_sampling_ratio/min": 0.46985867619514465, + "sampling/sampling_logp_difference/max": 0.7553234100341797, + "sampling/sampling_logp_difference/mean": 0.012378847226500511, + "step": 470 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.969436460873112e-05, + "clip_ratio/low_min": 5.969436460873112e-05, + "clip_ratio/region_mean": 5.969436460873112e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 551.875, + "completions/mean_terminated_length": 479.8333435058594, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.36107574589550495, + "epoch": 0.25214132762312635, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013910401612520218, + "learning_rate": 1e-05, + "loss": 0.1243, + "num_tokens": 10426871.0, + "reward": 0.65625, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.309815764427185, + "sampling/importance_sampling_ratio/mean": 1.000069499015808, + "sampling/importance_sampling_ratio/min": 0.6900006532669067, + "sampling/sampling_logp_difference/max": 0.3710627555847168, + "sampling/sampling_logp_difference/mean": 0.011647488921880722, + "step": 471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.833719867747277e-05, + "clip_ratio/low_min": 4.833719867747277e-05, + "clip_ratio/region_mean": 4.833719867747277e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 625.0625, + "completions/mean_terminated_length": 560.0909423828125, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 0.40370364487171173, + "epoch": 0.25267665952890794, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004989312961697578, + "learning_rate": 1e-05, + "loss": 0.0385, + "num_tokens": 10450521.0, + "reward": 0.53125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.394562840461731, + "sampling/importance_sampling_ratio/mean": 0.9999212622642517, + "sampling/importance_sampling_ratio/min": 0.6457639932632446, + "sampling/sampling_logp_difference/max": 0.43732118606567383, + "sampling/sampling_logp_difference/mean": 0.011998952366411686, + "step": 472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019895387231372297, + "clip_ratio/low_min": 0.00019895387231372297, + "clip_ratio/region_mean": 0.00019895387231372297, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 442.3125, + "completions/mean_terminated_length": 431.8064270019531, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.37263400107622147, + "epoch": 0.25321199143468953, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02191942185163498, + "learning_rate": 1e-05, + "loss": 0.0142, + "num_tokens": 10468083.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4055616855621338, + "sampling/importance_sampling_ratio/mean": 0.9999365210533142, + "sampling/importance_sampling_ratio/min": 0.7259685397148132, + "sampling/sampling_logp_difference/max": 0.3404369354248047, + "sampling/sampling_logp_difference/mean": 0.01224359218031168, + "step": 473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 488.46875, + "completions/mean_terminated_length": 469.8333435058594, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.34486325457692146, + "epoch": 0.25374732334047106, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017580291256308556, + "learning_rate": 1e-05, + "loss": 0.0035, + "num_tokens": 10487210.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000373363494873, + "sampling/importance_sampling_ratio/min": 0.6997325420379639, + "sampling/sampling_logp_difference/max": 0.7289609909057617, + "sampling/sampling_logp_difference/mean": 0.011850179173052311, + "step": 474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000192368224816164, + "clip_ratio/low_min": 0.000192368224816164, + "clip_ratio/region_mean": 0.000192368224816164, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 588.28125, + "completions/mean_terminated_length": 494.1428527832031, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.4914706163108349, + "epoch": 0.25428265524625265, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020851751789450645, + "learning_rate": 1e-05, + "loss": -0.0075, + "num_tokens": 10510019.0, + "reward": 0.65625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.425860047340393, + "sampling/importance_sampling_ratio/mean": 1.0002589225769043, + "sampling/importance_sampling_ratio/min": 0.47174105048179626, + "sampling/sampling_logp_difference/max": 0.7513251304626465, + "sampling/sampling_logp_difference/mean": 0.014382414519786835, + "step": 475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010526915866648778, + "clip_ratio/low_min": 0.00010526915866648778, + "clip_ratio/region_mean": 0.00010526915866648778, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 530.8125, + "completions/mean_terminated_length": 506.2758483886719, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.34860437735915184, + "epoch": 0.25481798715203424, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016587065532803535, + "learning_rate": 1e-05, + "loss": 0.0155, + "num_tokens": 10530765.0, + "reward": 0.625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4301687479019165, + "sampling/importance_sampling_ratio/mean": 0.9998611211776733, + "sampling/importance_sampling_ratio/min": 0.7063822150230408, + "sampling/sampling_logp_difference/max": 0.35779237747192383, + "sampling/sampling_logp_difference/mean": 0.011494183912873268, + "step": 476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 484.84375, + "completions/mean_terminated_length": 475.70965576171875, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.3333367947489023, + "epoch": 0.25535331905781583, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005053653847426176, + "learning_rate": 1e-05, + "loss": 0.0722, + "num_tokens": 10550344.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3609040975570679, + "sampling/importance_sampling_ratio/mean": 0.9998964667320251, + "sampling/importance_sampling_ratio/min": 0.6754359602928162, + "sampling/sampling_logp_difference/max": 0.3923969268798828, + "sampling/sampling_logp_difference/mean": 0.011056541465222836, + "step": 477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011185682524228469, + "clip_ratio/low_min": 0.00011185682524228469, + "clip_ratio/region_mean": 0.00011185682524228469, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 488.0625, + "completions/mean_terminated_length": 479.0322570800781, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.5512354504317045, + "epoch": 0.2558886509635974, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01098668109625578, + "learning_rate": 1e-05, + "loss": 0.0606, + "num_tokens": 10570282.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3350861072540283, + "sampling/importance_sampling_ratio/mean": 0.9997745752334595, + "sampling/importance_sampling_ratio/min": 0.62984699010849, + "sampling/sampling_logp_difference/max": 0.4622783660888672, + "sampling/sampling_logp_difference/mean": 0.015619696117937565, + "step": 478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.6921675422927365e-05, + "clip_ratio/low_min": 5.6921675422927365e-05, + "clip_ratio/region_mean": 5.6921675422927365e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 532.0, + "completions/mean_terminated_length": 465.91998291015625, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.4183240532875061, + "epoch": 0.256423982869379, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006963755935430527, + "learning_rate": 1e-05, + "loss": 0.066, + "num_tokens": 10591626.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4004451036453247, + "sampling/importance_sampling_ratio/mean": 0.9999240636825562, + "sampling/importance_sampling_ratio/min": 0.7040327191352844, + "sampling/sampling_logp_difference/max": 0.35093045234680176, + "sampling/sampling_logp_difference/mean": 0.013495264574885368, + "step": 479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.527345299720764e-05, + "clip_ratio/low_min": 4.527345299720764e-05, + "clip_ratio/region_mean": 4.527345299720764e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 628.75, + "completions/mean_terminated_length": 574.2608642578125, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.41216954216361046, + "epoch": 0.2569593147751606, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007541249040514231, + "learning_rate": 1e-05, + "loss": 0.0468, + "num_tokens": 10615450.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4162741899490356, + "sampling/importance_sampling_ratio/mean": 1.0002094507217407, + "sampling/importance_sampling_ratio/min": 0.5041298270225525, + "sampling/sampling_logp_difference/max": 0.6849215626716614, + "sampling/sampling_logp_difference/mean": 0.01230060774832964, + "step": 480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 512.9375, + "completions/mean_terminated_length": 495.933349609375, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.3297930024564266, + "epoch": 0.2574946466809422, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0026471305172890425, + "learning_rate": 1e-05, + "loss": 0.0516, + "num_tokens": 10635208.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4261765480041504, + "sampling/importance_sampling_ratio/mean": 1.000159740447998, + "sampling/importance_sampling_ratio/min": 0.6802457571029663, + "sampling/sampling_logp_difference/max": 0.3853011131286621, + "sampling/sampling_logp_difference/mean": 0.010556701570749283, + "step": 481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 511.46875, + "completions/mean_terminated_length": 494.36669921875, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.30446585454046726, + "epoch": 0.2580299785867238, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0293, + "num_tokens": 10655143.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.2877490520477295, + "sampling/importance_sampling_ratio/mean": 1.0001307725906372, + "sampling/importance_sampling_ratio/min": 0.7307831048965454, + "sampling/sampling_logp_difference/max": 0.3136385679244995, + "sampling/sampling_logp_difference/mean": 0.00997633021324873, + "step": 482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010371941607445478, + "clip_ratio/low_min": 0.00010371941607445478, + "clip_ratio/region_mean": 0.00010371941607445478, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 545.59375, + "completions/mean_terminated_length": 471.4583435058594, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.4034724347293377, + "epoch": 0.25856531049250536, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013043330982327461, + "learning_rate": 1e-05, + "loss": 0.1117, + "num_tokens": 10676866.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4366579055786133, + "sampling/importance_sampling_ratio/mean": 0.9998902082443237, + "sampling/importance_sampling_ratio/min": 0.7108510732650757, + "sampling/sampling_logp_difference/max": 0.3623194694519043, + "sampling/sampling_logp_difference/mean": 0.013162062503397465, + "step": 483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 546.71875, + "completions/mean_terminated_length": 495.65386962890625, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.30012526363134384, + "epoch": 0.25910064239828695, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008093496784567833, + "learning_rate": 1e-05, + "loss": 0.0377, + "num_tokens": 10698009.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4611619710922241, + "sampling/importance_sampling_ratio/mean": 1.0001821517944336, + "sampling/importance_sampling_ratio/min": 0.6850831508636475, + "sampling/sampling_logp_difference/max": 0.37923192977905273, + "sampling/sampling_logp_difference/mean": 0.009886610321700573, + "step": 484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 538.09375, + "completions/mean_terminated_length": 495.5185241699219, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.3680847994983196, + "epoch": 0.25963597430406854, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010817212052643299, + "learning_rate": 1e-05, + "loss": 0.0603, + "num_tokens": 10719172.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5157670974731445, + "sampling/importance_sampling_ratio/mean": 0.999823272228241, + "sampling/importance_sampling_ratio/min": 0.7624718546867371, + "sampling/sampling_logp_difference/max": 0.415921688079834, + "sampling/sampling_logp_difference/mean": 0.011580967344343662, + "step": 485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014238310541259125, + "clip_ratio/low_min": 0.00014238310541259125, + "clip_ratio/region_mean": 0.00014238310541259125, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 474.375, + "completions/mean_terminated_length": 454.8000183105469, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.3717295974493027, + "epoch": 0.26017130620985013, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00501126516610384, + "learning_rate": 1e-05, + "loss": 0.0412, + "num_tokens": 10737856.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.2876932621002197, + "sampling/importance_sampling_ratio/mean": 1.000070571899414, + "sampling/importance_sampling_ratio/min": 0.6117763519287109, + "sampling/sampling_logp_difference/max": 0.49138855934143066, + "sampling/sampling_logp_difference/mean": 0.011895447038114071, + "step": 486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 581.53125, + "completions/mean_terminated_length": 554.8928833007812, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.34515034779906273, + "epoch": 0.2607066381156317, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024670716375112534, + "learning_rate": 1e-05, + "loss": 0.0453, + "num_tokens": 10760713.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4213733673095703, + "sampling/importance_sampling_ratio/mean": 1.000146508216858, + "sampling/importance_sampling_ratio/min": 0.7005010843276978, + "sampling/sampling_logp_difference/max": 0.355959415435791, + "sampling/sampling_logp_difference/mean": 0.011290302500128746, + "step": 487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.137690095580183e-05, + "clip_ratio/low_min": 5.137690095580183e-05, + "clip_ratio/region_mean": 5.137690095580183e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 504.875, + "completions/mean_terminated_length": 467.2857360839844, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.4183342345058918, + "epoch": 0.26124197002141325, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011635100468993187, + "learning_rate": 1e-05, + "loss": 0.0309, + "num_tokens": 10780861.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4351632595062256, + "sampling/importance_sampling_ratio/mean": 1.0002400875091553, + "sampling/importance_sampling_ratio/min": 0.702856719493866, + "sampling/sampling_logp_difference/max": 0.3612785339355469, + "sampling/sampling_logp_difference/mean": 0.012863479554653168, + "step": 488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.423212885740213e-05, + "clip_ratio/low_min": 4.423212885740213e-05, + "clip_ratio/region_mean": 4.423212885740213e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 621.4375, + "completions/mean_terminated_length": 554.8181762695312, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.4198906682431698, + "epoch": 0.26177730192719484, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017947958782315254, + "learning_rate": 1e-05, + "loss": 0.0406, + "num_tokens": 10804907.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.442814588546753, + "sampling/importance_sampling_ratio/mean": 1.0000669956207275, + "sampling/importance_sampling_ratio/min": 0.694574236869812, + "sampling/sampling_logp_difference/max": 0.3665957450866699, + "sampling/sampling_logp_difference/mean": 0.012978503480553627, + "step": 489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.940588355064392e-05, + "clip_ratio/low_min": 6.940588355064392e-05, + "clip_ratio/region_mean": 6.940588355064392e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 487.125, + "completions/mean_terminated_length": 468.4000244140625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.38713568449020386, + "epoch": 0.2623126338329764, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017323443666100502, + "learning_rate": 1e-05, + "loss": -0.0415, + "num_tokens": 10824367.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4198147058486938, + "sampling/importance_sampling_ratio/mean": 1.0001780986785889, + "sampling/importance_sampling_ratio/min": 0.7587830424308777, + "sampling/sampling_logp_difference/max": 0.3505263328552246, + "sampling/sampling_logp_difference/mean": 0.012243088334798813, + "step": 490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 617.375, + "completions/mean_terminated_length": 558.434814453125, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "entropy": 0.43042561784386635, + "epoch": 0.262847965738758, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005080129019916058, + "learning_rate": 1e-05, + "loss": 0.0135, + "num_tokens": 10848371.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.375198483467102, + "sampling/importance_sampling_ratio/mean": 0.9998195171356201, + "sampling/importance_sampling_ratio/min": 0.7337667346000671, + "sampling/sampling_logp_difference/max": 0.31859803199768066, + "sampling/sampling_logp_difference/mean": 0.012578554451465607, + "step": 491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.3998592445859686e-05, + "clip_ratio/low_min": 4.3998592445859686e-05, + "clip_ratio/region_mean": 4.3998592445859686e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 652.65625, + "completions/mean_terminated_length": 562.9444580078125, + "completions/min_length": 380.0, + "completions/min_terminated_length": 380.0, + "entropy": 0.3964984081685543, + "epoch": 0.2633832976445396, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0028810864314436913, + "learning_rate": 1e-05, + "loss": 0.0256, + "num_tokens": 10873160.0, + "reward": 0.34375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4145331382751465, + "sampling/importance_sampling_ratio/mean": 1.0005985498428345, + "sampling/importance_sampling_ratio/min": 0.7429461479187012, + "sampling/sampling_logp_difference/max": 0.34679949283599854, + "sampling/sampling_logp_difference/mean": 0.011351502500474453, + "step": 492 + }, + { + "clip_ratio/high_max": 5.099959162180312e-05, + "clip_ratio/high_mean": 5.099959162180312e-05, + "clip_ratio/low_mean": 5.0709939387161285e-05, + "clip_ratio/low_min": 5.0709939387161285e-05, + "clip_ratio/region_mean": 0.0001017095310089644, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 588.4375, + "completions/mean_terminated_length": 562.7857666015625, + "completions/min_length": 396.0, + "completions/min_terminated_length": 396.0, + "entropy": 0.43003494292497635, + "epoch": 0.2639186295503212, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01873714290559292, + "learning_rate": 1e-05, + "loss": 0.0293, + "num_tokens": 10895526.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.961113452911377, + "sampling/importance_sampling_ratio/mean": 1.0002150535583496, + "sampling/importance_sampling_ratio/min": 0.677449643611908, + "sampling/sampling_logp_difference/max": 0.6735124588012695, + "sampling/sampling_logp_difference/mean": 0.013516178354620934, + "step": 493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 541.6875, + "completions/mean_terminated_length": 499.77777099609375, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.4351819381117821, + "epoch": 0.2644539614561028, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01836119033396244, + "learning_rate": 1e-05, + "loss": 0.029, + "num_tokens": 10916580.0, + "reward": 0.625, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4574007987976074, + "sampling/importance_sampling_ratio/mean": 1.0003912448883057, + "sampling/importance_sampling_ratio/min": 0.7238929271697998, + "sampling/sampling_logp_difference/max": 0.37665462493896484, + "sampling/sampling_logp_difference/mean": 0.013308979570865631, + "step": 494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001161811851488892, + "clip_ratio/low_min": 0.0001161811851488892, + "clip_ratio/region_mean": 0.0001161811851488892, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 596.5, + "completions/mean_terminated_length": 539.3333740234375, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.4781779535114765, + "epoch": 0.26498929336188437, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020715584978461266, + "learning_rate": 1e-05, + "loss": 0.0468, + "num_tokens": 10939844.0, + "reward": 0.59375, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4176584482192993, + "sampling/importance_sampling_ratio/mean": 1.000109314918518, + "sampling/importance_sampling_ratio/min": 0.7280387878417969, + "sampling/sampling_logp_difference/max": 0.3490065336227417, + "sampling/sampling_logp_difference/mean": 0.013495479710400105, + "step": 495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.254788215504959e-05, + "clip_ratio/low_min": 7.254788215504959e-05, + "clip_ratio/region_mean": 7.254788215504959e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 760.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 419.96875, + "completions/mean_terminated_length": 419.96875, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.39975521713495255, + "epoch": 0.26552462526766596, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005138919688761234, + "learning_rate": 1e-05, + "loss": -0.1093, + "num_tokens": 10956547.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4020583629608154, + "sampling/importance_sampling_ratio/mean": 0.9998289942741394, + "sampling/importance_sampling_ratio/min": 0.5549389123916626, + "sampling/sampling_logp_difference/max": 0.5888972282409668, + "sampling/sampling_logp_difference/mean": 0.011359790340065956, + "step": 496 + }, + { + "clip_ratio/high_max": 4.470672502066009e-05, + "clip_ratio/high_mean": 4.470672502066009e-05, + "clip_ratio/low_mean": 0.00015187276585493237, + "clip_ratio/low_min": 0.00015187276585493237, + "clip_ratio/region_mean": 0.00019657949087559246, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 618.40625, + "completions/mean_terminated_length": 590.7037353515625, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.29839450120925903, + "epoch": 0.26605995717344755, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005227820016443729, + "learning_rate": 1e-05, + "loss": -0.0015, + "num_tokens": 10980064.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3167110681533813, + "sampling/importance_sampling_ratio/mean": 0.9995431900024414, + "sampling/importance_sampling_ratio/min": 0.6599874496459961, + "sampling/sampling_logp_difference/max": 0.41553449630737305, + "sampling/sampling_logp_difference/mean": 0.009761122055351734, + "step": 497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 545.90625, + "completions/mean_terminated_length": 514.1785888671875, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.3104923889040947, + "epoch": 0.26659528907922914, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006149497348815203, + "learning_rate": 1e-05, + "loss": 0.0875, + "num_tokens": 11001053.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3990099430084229, + "sampling/importance_sampling_ratio/mean": 0.9997010231018066, + "sampling/importance_sampling_ratio/min": 0.7086121439933777, + "sampling/sampling_logp_difference/max": 0.34444689750671387, + "sampling/sampling_logp_difference/mean": 0.010131840594112873, + "step": 498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 518.09375, + "completions/mean_terminated_length": 492.2413635253906, + "completions/min_length": 357.0, + "completions/min_terminated_length": 357.0, + "entropy": 0.39401593804359436, + "epoch": 0.2671306209850107, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013658261857926846, + "learning_rate": 1e-05, + "loss": 0.0287, + "num_tokens": 11021152.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4280586242675781, + "sampling/importance_sampling_ratio/mean": 1.0001189708709717, + "sampling/importance_sampling_ratio/min": 0.6740455627441406, + "sampling/sampling_logp_difference/max": 0.3944575786590576, + "sampling/sampling_logp_difference/mean": 0.012333094142377377, + "step": 499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 582.5625, + "completions/mean_terminated_length": 438.3333435058594, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.522261418402195, + "epoch": 0.2676659528907923, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010906667448580265, + "learning_rate": 1e-05, + "loss": 0.012, + "num_tokens": 11044378.0, + "reward": 0.5, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.493876576423645, + "sampling/importance_sampling_ratio/mean": 0.99949049949646, + "sampling/importance_sampling_ratio/min": 0.6382625699043274, + "sampling/sampling_logp_difference/max": 0.4490056037902832, + "sampling/sampling_logp_difference/mean": 0.015240837819874287, + "step": 500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.271669892361388e-05, + "clip_ratio/low_min": 7.271669892361388e-05, + "clip_ratio/region_mean": 7.271669892361388e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 470.40625, + "completions/mean_terminated_length": 439.6206970214844, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.45451122894883156, + "epoch": 0.2682012847965739, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013978679664433002, + "learning_rate": 1e-05, + "loss": 0.03, + "num_tokens": 11063463.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4506685733795166, + "sampling/importance_sampling_ratio/mean": 0.9997245669364929, + "sampling/importance_sampling_ratio/min": 0.7564635872840881, + "sampling/sampling_logp_difference/max": 0.3720245361328125, + "sampling/sampling_logp_difference/mean": 0.014118613675236702, + "step": 501 + }, + { + "clip_ratio/high_max": 6.627784023294225e-05, + "clip_ratio/high_mean": 6.627784023294225e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.627784023294225e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 752.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 454.90625, + "completions/mean_terminated_length": 454.90625, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.34238502010703087, + "epoch": 0.26873661670235544, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007140727713704109, + "learning_rate": 1e-05, + "loss": -0.0429, + "num_tokens": 11081892.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3643701076507568, + "sampling/importance_sampling_ratio/mean": 0.9997614622116089, + "sampling/importance_sampling_ratio/min": 0.6850076913833618, + "sampling/sampling_logp_difference/max": 0.3783252239227295, + "sampling/sampling_logp_difference/mean": 0.011558032594621181, + "step": 502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 480.15625, + "completions/mean_terminated_length": 470.8709411621094, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.36768776178359985, + "epoch": 0.269271948608137, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013215147890150547, + "learning_rate": 1e-05, + "loss": 0.0173, + "num_tokens": 11101393.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5229955911636353, + "sampling/importance_sampling_ratio/mean": 1.0001130104064941, + "sampling/importance_sampling_ratio/min": 0.6991862058639526, + "sampling/sampling_logp_difference/max": 0.42067909240722656, + "sampling/sampling_logp_difference/mean": 0.012127074413001537, + "step": 503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 541.75, + "completions/mean_terminated_length": 489.5384826660156, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.39395223557949066, + "epoch": 0.2698072805139186, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013775346800684929, + "learning_rate": 1e-05, + "loss": 0.0805, + "num_tokens": 11122289.0, + "reward": 0.71875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5974918603897095, + "sampling/importance_sampling_ratio/mean": 0.9998372793197632, + "sampling/importance_sampling_ratio/min": 0.7151423096656799, + "sampling/sampling_logp_difference/max": 0.46843481063842773, + "sampling/sampling_logp_difference/mean": 0.012173264287412167, + "step": 504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001201498816953972, + "clip_ratio/low_min": 0.0001201498816953972, + "clip_ratio/region_mean": 0.0001201498816953972, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 592.78125, + "completions/mean_terminated_length": 501.0, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.43938862904906273, + "epoch": 0.2703426124197002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016774991527199745, + "learning_rate": 1e-05, + "loss": 0.0461, + "num_tokens": 11145298.0, + "reward": 0.4375, + "reward_std": 0.5260357856750488, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5081984996795654, + "sampling/importance_sampling_ratio/mean": 1.0002576112747192, + "sampling/importance_sampling_ratio/min": 0.7404820919036865, + "sampling/sampling_logp_difference/max": 0.4109158515930176, + "sampling/sampling_logp_difference/mean": 0.013073504902422428, + "step": 505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012589900143211707, + "clip_ratio/low_min": 0.00012589900143211707, + "clip_ratio/region_mean": 0.00012589900143211707, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 484.125, + "completions/mean_terminated_length": 443.5714416503906, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.40190256014466286, + "epoch": 0.2708779443254818, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01707610860466957, + "learning_rate": 1e-05, + "loss": 0.032, + "num_tokens": 11164822.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6597273349761963, + "sampling/importance_sampling_ratio/mean": 0.9995678067207336, + "sampling/importance_sampling_ratio/min": 0.7377364039421082, + "sampling/sampling_logp_difference/max": 0.5066533088684082, + "sampling/sampling_logp_difference/mean": 0.012648704461753368, + "step": 506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 525.5, + "completions/mean_terminated_length": 490.857177734375, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.37242839112877846, + "epoch": 0.2714132762312634, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011503325775265694, + "learning_rate": 1e-05, + "loss": 0.0556, + "num_tokens": 11185726.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2950011491775513, + "sampling/importance_sampling_ratio/mean": 0.9997808337211609, + "sampling/importance_sampling_ratio/min": 0.5552176237106323, + "sampling/sampling_logp_difference/max": 0.5883951187133789, + "sampling/sampling_logp_difference/mean": 0.011658416129648685, + "step": 507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00028234537603566423, + "clip_ratio/low_min": 0.00028234537603566423, + "clip_ratio/region_mean": 0.00028234537603566423, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 565.84375, + "completions/mean_terminated_length": 498.4583435058594, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.4553045593202114, + "epoch": 0.27194860813704497, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015420237556099892, + "learning_rate": 1e-05, + "loss": 0.0217, + "num_tokens": 11207377.0, + "reward": 0.46875, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6266615390777588, + "sampling/importance_sampling_ratio/mean": 0.9997661709785461, + "sampling/importance_sampling_ratio/min": 0.6946967840194702, + "sampling/sampling_logp_difference/max": 0.4865298271179199, + "sampling/sampling_logp_difference/mean": 0.01380403246730566, + "step": 508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 483.0, + "completions/mean_terminated_length": 464.0000305175781, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.297705065459013, + "epoch": 0.27248394004282656, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007576956879347563, + "learning_rate": 1e-05, + "loss": -0.0033, + "num_tokens": 11226609.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4550213813781738, + "sampling/importance_sampling_ratio/mean": 1.0000966787338257, + "sampling/importance_sampling_ratio/min": 0.7399776577949524, + "sampling/sampling_logp_difference/max": 0.3750205636024475, + "sampling/sampling_logp_difference/mean": 0.01007781270891428, + "step": 509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 504.84375, + "completions/mean_terminated_length": 496.3548278808594, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3161064684391022, + "epoch": 0.27301927194860814, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0715, + "num_tokens": 11246188.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3199232816696167, + "sampling/importance_sampling_ratio/mean": 0.9998076558113098, + "sampling/importance_sampling_ratio/min": 0.6959797739982605, + "sampling/sampling_logp_difference/max": 0.36243462562561035, + "sampling/sampling_logp_difference/mean": 0.010502434335649014, + "step": 510 + }, + { + "clip_ratio/high_max": 5.5506217904621735e-05, + "clip_ratio/high_mean": 5.5506217904621735e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.5506217904621735e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 575.03125, + "completions/mean_terminated_length": 562.1666870117188, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 0.3339088074862957, + "epoch": 0.27355460385438973, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01582440920174122, + "learning_rate": 1e-05, + "loss": 0.0028, + "num_tokens": 11268429.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.8644335269927979, + "sampling/importance_sampling_ratio/mean": 1.000091552734375, + "sampling/importance_sampling_ratio/min": 0.45649251341819763, + "sampling/sampling_logp_difference/max": 0.7841830253601074, + "sampling/sampling_logp_difference/mean": 0.010884090326726437, + "step": 511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.778089977568015e-05, + "clip_ratio/low_min": 8.778089977568015e-05, + "clip_ratio/region_mean": 8.778089977568015e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 725.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 377.34375, + "completions/mean_terminated_length": 377.34375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.34434812515974045, + "epoch": 0.2740899357601713, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024170197546482086, + "learning_rate": 1e-05, + "loss": 0.0183, + "num_tokens": 11284296.0, + "reward": 0.625, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7866148948669434, + "sampling/importance_sampling_ratio/mean": 1.0001447200775146, + "sampling/importance_sampling_ratio/min": 0.7076085209846497, + "sampling/sampling_logp_difference/max": 0.5803227424621582, + "sampling/sampling_logp_difference/mean": 0.011088111437857151, + "step": 512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 715.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 476.15625, + "completions/mean_terminated_length": 476.15625, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.31146080791950226, + "epoch": 0.2746252676659529, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006982438266277313, + "learning_rate": 1e-05, + "loss": -0.0112, + "num_tokens": 11303469.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4836581945419312, + "sampling/importance_sampling_ratio/mean": 0.9999073147773743, + "sampling/importance_sampling_ratio/min": 0.675791323184967, + "sampling/sampling_logp_difference/max": 0.39451074600219727, + "sampling/sampling_logp_difference/mean": 0.01101579237729311, + "step": 513 + }, + { + "clip_ratio/high_max": 4.740235090139322e-05, + "clip_ratio/high_mean": 4.740235090139322e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.740235090139322e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 583.0, + "completions/mean_terminated_length": 540.3077392578125, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.32112976163625717, + "epoch": 0.2751605995717345, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002383845392614603, + "learning_rate": 1e-05, + "loss": -0.0029, + "num_tokens": 11326253.0, + "reward": 0.46875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.461322546005249, + "sampling/importance_sampling_ratio/mean": 0.9995347857475281, + "sampling/importance_sampling_ratio/min": 0.6077783703804016, + "sampling/sampling_logp_difference/max": 0.49794501066207886, + "sampling/sampling_logp_difference/mean": 0.010457371361553669, + "step": 514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 614.3125, + "completions/mean_terminated_length": 554.1739501953125, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.2877458203583956, + "epoch": 0.2756959314775161, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002158162649720907, + "learning_rate": 1e-05, + "loss": 0.0301, + "num_tokens": 11349871.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3167599439620972, + "sampling/importance_sampling_ratio/mean": 0.9999128580093384, + "sampling/importance_sampling_ratio/min": 0.7256494164466858, + "sampling/sampling_logp_difference/max": 0.32068824768066406, + "sampling/sampling_logp_difference/mean": 0.009066893719136715, + "step": 515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 640.28125, + "completions/mean_terminated_length": 597.7083740234375, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.36829761415719986, + "epoch": 0.2762312633832976, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012869792059063911, + "learning_rate": 1e-05, + "loss": 0.023, + "num_tokens": 11374280.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4000791311264038, + "sampling/importance_sampling_ratio/mean": 0.9999721646308899, + "sampling/importance_sampling_ratio/min": 0.6369316577911377, + "sampling/sampling_logp_difference/max": 0.4510929584503174, + "sampling/sampling_logp_difference/mean": 0.011919982731342316, + "step": 516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012242898810654879, + "clip_ratio/low_min": 0.00012242898810654879, + "clip_ratio/region_mean": 0.00012242898810654879, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 549.34375, + "completions/mean_terminated_length": 508.85186767578125, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.4445907063782215, + "epoch": 0.2767665952890792, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0458, + "num_tokens": 11396179.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.44916832447052, + "sampling/importance_sampling_ratio/mean": 1.0002517700195312, + "sampling/importance_sampling_ratio/min": 0.7010741233825684, + "sampling/sampling_logp_difference/max": 0.3709897994995117, + "sampling/sampling_logp_difference/mean": 0.013188843615353107, + "step": 517 + }, + { + "clip_ratio/high_max": 5.303351645125076e-05, + "clip_ratio/high_mean": 5.303351645125076e-05, + "clip_ratio/low_mean": 0.00010527325866860338, + "clip_ratio/low_min": 0.00010527325866860338, + "clip_ratio/region_mean": 0.00015830677511985414, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 564.625, + "completions/mean_terminated_length": 496.8333435058594, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.3561297971755266, + "epoch": 0.2773019271948608, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013775930739939213, + "learning_rate": 1e-05, + "loss": 0.0579, + "num_tokens": 11418127.0, + "reward": 0.46875, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.437573790550232, + "sampling/importance_sampling_ratio/mean": 0.9998301863670349, + "sampling/importance_sampling_ratio/min": 0.5677351951599121, + "sampling/sampling_logp_difference/max": 0.5661001205444336, + "sampling/sampling_logp_difference/mean": 0.011913307942450047, + "step": 518 + }, + { + "clip_ratio/high_max": 5.1062092097708955e-05, + "clip_ratio/high_mean": 5.1062092097708955e-05, + "clip_ratio/low_mean": 0.00016235495422733948, + "clip_ratio/low_min": 0.00016235495422733948, + "clip_ratio/region_mean": 0.00021341704632504843, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 571.96875, + "completions/mean_terminated_length": 535.6666870117188, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.41886864602565765, + "epoch": 0.2778372591006424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009069961495697498, + "learning_rate": 1e-05, + "loss": 0.0859, + "num_tokens": 11440222.0, + "reward": 0.53125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4745533466339111, + "sampling/importance_sampling_ratio/mean": 1.0001682043075562, + "sampling/importance_sampling_ratio/min": 0.5070000886917114, + "sampling/sampling_logp_difference/max": 0.6792440414428711, + "sampling/sampling_logp_difference/mean": 0.013481111265718937, + "step": 519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020833496819250286, + "clip_ratio/low_min": 0.00020833496819250286, + "clip_ratio/region_mean": 0.00020833496819250286, + "completions/clipped_ratio": 0.0, + "completions/max_length": 591.0, + "completions/max_terminated_length": 591.0, + "completions/mean_length": 410.4375, + "completions/mean_terminated_length": 410.4375, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.3321453146636486, + "epoch": 0.278372591006424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02371736243367195, + "learning_rate": 1e-05, + "loss": 0.0636, + "num_tokens": 11456932.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3492980003356934, + "sampling/importance_sampling_ratio/mean": 0.9997901320457458, + "sampling/importance_sampling_ratio/min": 0.756468653678894, + "sampling/sampling_logp_difference/max": 0.29958438873291016, + "sampling/sampling_logp_difference/mean": 0.010396013036370277, + "step": 520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.492091440828517e-05, + "clip_ratio/low_min": 5.492091440828517e-05, + "clip_ratio/region_mean": 5.492091440828517e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 608.90625, + "completions/mean_terminated_length": 536.5909423828125, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.3451676666736603, + "epoch": 0.27890792291220556, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00869609136134386, + "learning_rate": 1e-05, + "loss": 0.0798, + "num_tokens": 11480577.0, + "reward": 0.53125, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6381021738052368, + "sampling/importance_sampling_ratio/mean": 0.9999784827232361, + "sampling/importance_sampling_ratio/min": 0.7408223152160645, + "sampling/sampling_logp_difference/max": 0.49353837966918945, + "sampling/sampling_logp_difference/mean": 0.01023965235799551, + "step": 521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00025105642998823896, + "clip_ratio/low_min": 0.00025105642998823896, + "clip_ratio/region_mean": 0.00025105642998823896, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 638.90625, + "completions/mean_terminated_length": 561.4500122070312, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.44716671109199524, + "epoch": 0.27944325481798715, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022217893972992897, + "learning_rate": 1e-05, + "loss": 0.0313, + "num_tokens": 11504902.0, + "reward": 0.46875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.9299200773239136, + "sampling/importance_sampling_ratio/mean": 1.0000885725021362, + "sampling/importance_sampling_ratio/min": 0.5071318745613098, + "sampling/sampling_logp_difference/max": 0.6789841651916504, + "sampling/sampling_logp_difference/mean": 0.013543638400733471, + "step": 522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 486.8125, + "completions/mean_terminated_length": 457.72412109375, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "entropy": 0.34625769034028053, + "epoch": 0.27997858672376874, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015269968658685684, + "learning_rate": 1e-05, + "loss": 0.0073, + "num_tokens": 11523664.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.474115252494812, + "sampling/importance_sampling_ratio/mean": 0.9997300505638123, + "sampling/importance_sampling_ratio/min": 0.7066593766212463, + "sampling/sampling_logp_difference/max": 0.3880579471588135, + "sampling/sampling_logp_difference/mean": 0.010783224366605282, + "step": 523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002079651203530375, + "clip_ratio/low_min": 0.0002079651203530375, + "clip_ratio/region_mean": 0.0002079651203530375, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 571.21875, + "completions/mean_terminated_length": 516.1199951171875, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.4020249657332897, + "epoch": 0.28051391862955033, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012099537998437881, + "learning_rate": 1e-05, + "loss": 0.0118, + "num_tokens": 11545791.0, + "reward": 0.53125, + "reward_std": 0.521792471408844, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4420913457870483, + "sampling/importance_sampling_ratio/mean": 1.0004171133041382, + "sampling/importance_sampling_ratio/min": 0.7396310567855835, + "sampling/sampling_logp_difference/max": 0.36609435081481934, + "sampling/sampling_logp_difference/mean": 0.012760459445416927, + "step": 524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 441.28125, + "completions/mean_terminated_length": 419.5000305175781, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.31816796585917473, + "epoch": 0.2810492505353319, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01995597779750824, + "learning_rate": 1e-05, + "loss": 0.0577, + "num_tokens": 11563208.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.328745722770691, + "sampling/importance_sampling_ratio/mean": 0.9999685287475586, + "sampling/importance_sampling_ratio/min": 0.700685441493988, + "sampling/sampling_logp_difference/max": 0.3556962013244629, + "sampling/sampling_logp_difference/mean": 0.01083736028522253, + "step": 525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.703365503810346e-05, + "clip_ratio/low_min": 9.703365503810346e-05, + "clip_ratio/region_mean": 9.703365503810346e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 625.875, + "completions/mean_terminated_length": 561.2727661132812, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.5319276303052902, + "epoch": 0.2815845824411135, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013550633564591408, + "learning_rate": 1e-05, + "loss": 0.1078, + "num_tokens": 11586780.0, + "reward": 0.40625, + "reward_std": 0.521792471408844, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4476656913757324, + "sampling/importance_sampling_ratio/mean": 1.0002061128616333, + "sampling/importance_sampling_ratio/min": 0.7269487380981445, + "sampling/sampling_logp_difference/max": 0.3699524402618408, + "sampling/sampling_logp_difference/mean": 0.0151499193161726, + "step": 526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 492.96875, + "completions/mean_terminated_length": 484.0967712402344, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "entropy": 0.3193536028265953, + "epoch": 0.2821199143468951, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004949445836246014, + "learning_rate": 1e-05, + "loss": -0.0045, + "num_tokens": 11606163.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.6154122352600098, + "sampling/importance_sampling_ratio/mean": 0.9998866319656372, + "sampling/importance_sampling_ratio/min": 0.6933082342147827, + "sampling/sampling_logp_difference/max": 0.47959017753601074, + "sampling/sampling_logp_difference/mean": 0.010449059307575226, + "step": 527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 453.375, + "completions/mean_terminated_length": 443.2257995605469, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.28020163998007774, + "epoch": 0.2826552462526767, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0057, + "num_tokens": 11624119.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3215128183364868, + "sampling/importance_sampling_ratio/mean": 1.0000460147857666, + "sampling/importance_sampling_ratio/min": 0.7348396182060242, + "sampling/sampling_logp_difference/max": 0.308103084564209, + "sampling/sampling_logp_difference/mean": 0.009280155412852764, + "step": 528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.166748789837584e-05, + "clip_ratio/low_min": 6.166748789837584e-05, + "clip_ratio/region_mean": 6.166748789837584e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 500.96875, + "completions/mean_terminated_length": 451.5185241699219, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "entropy": 0.3511334862560034, + "epoch": 0.2831905781584582, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014866421930491924, + "learning_rate": 1e-05, + "loss": 0.0598, + "num_tokens": 11643870.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4438130855560303, + "sampling/importance_sampling_ratio/mean": 1.0003409385681152, + "sampling/importance_sampling_ratio/min": 0.7352181077003479, + "sampling/sampling_logp_difference/max": 0.36728763580322266, + "sampling/sampling_logp_difference/mean": 0.011323046870529652, + "step": 529 + }, + { + "clip_ratio/high_max": 6.332320481305942e-05, + "clip_ratio/high_mean": 6.332320481305942e-05, + "clip_ratio/low_mean": 0.0001737024140311405, + "clip_ratio/low_min": 0.0001737024140311405, + "clip_ratio/region_mean": 0.00023702561884419993, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 535.71875, + "completions/mean_terminated_length": 482.11541748046875, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.34200095385313034, + "epoch": 0.2837259100642398, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004239071160554886, + "learning_rate": 1e-05, + "loss": 0.0319, + "num_tokens": 11664997.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.9145855903625488, + "sampling/importance_sampling_ratio/mean": 1.0002284049987793, + "sampling/importance_sampling_ratio/min": 0.6010749936103821, + "sampling/sampling_logp_difference/max": 0.6495012044906616, + "sampling/sampling_logp_difference/mean": 0.011249025352299213, + "step": 530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000150988700625021, + "clip_ratio/low_min": 0.000150988700625021, + "clip_ratio/region_mean": 0.000150988700625021, + "completions/clipped_ratio": 0.0, + "completions/max_length": 680.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 419.21875, + "completions/mean_terminated_length": 419.21875, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.32063016667962074, + "epoch": 0.2842612419700214, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0190244410187006, + "learning_rate": 1e-05, + "loss": 0.0072, + "num_tokens": 11682012.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5228393077850342, + "sampling/importance_sampling_ratio/mean": 1.0001142024993896, + "sampling/importance_sampling_ratio/min": 0.6502863764762878, + "sampling/sampling_logp_difference/max": 0.430342435836792, + "sampling/sampling_logp_difference/mean": 0.011216445825994015, + "step": 531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 551.46875, + "completions/mean_terminated_length": 511.370361328125, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.4353875555098057, + "epoch": 0.284796573875803, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0205855555832386, + "learning_rate": 1e-05, + "loss": 0.0758, + "num_tokens": 11703315.0, + "reward": 0.65625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4349011182785034, + "sampling/importance_sampling_ratio/mean": 1.0000529289245605, + "sampling/importance_sampling_ratio/min": 0.6882743239402771, + "sampling/sampling_logp_difference/max": 0.3735678195953369, + "sampling/sampling_logp_difference/mean": 0.01333760004490614, + "step": 532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.05019774613902e-05, + "clip_ratio/low_min": 7.05019774613902e-05, + "clip_ratio/region_mean": 7.05019774613902e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 528.09375, + "completions/mean_terminated_length": 512.1000366210938, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.32369837909936905, + "epoch": 0.28533190578158457, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012548979371786118, + "learning_rate": 1e-05, + "loss": 0.0006, + "num_tokens": 11724038.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4544031620025635, + "sampling/importance_sampling_ratio/mean": 0.9998462796211243, + "sampling/importance_sampling_ratio/min": 0.6570523977279663, + "sampling/sampling_logp_difference/max": 0.41999149322509766, + "sampling/sampling_logp_difference/mean": 0.010763757862150669, + "step": 533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.18242122780066e-05, + "clip_ratio/low_min": 5.18242122780066e-05, + "clip_ratio/region_mean": 5.18242122780066e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 479.875, + "completions/mean_terminated_length": 438.71429443359375, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.3744928799569607, + "epoch": 0.28586723768736616, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.029886197298765182, + "learning_rate": 1e-05, + "loss": 0.1306, + "num_tokens": 11743138.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4820278882980347, + "sampling/importance_sampling_ratio/mean": 0.999498724937439, + "sampling/importance_sampling_ratio/min": 0.6708641648292542, + "sampling/sampling_logp_difference/max": 0.3991886377334595, + "sampling/sampling_logp_difference/mean": 0.012439063750207424, + "step": 534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 490.1875, + "completions/mean_terminated_length": 481.2257995605469, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.34023406356573105, + "epoch": 0.28640256959314775, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0046, + "num_tokens": 11762680.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5184926986694336, + "sampling/importance_sampling_ratio/mean": 0.9999358654022217, + "sampling/importance_sampling_ratio/min": 0.7156133651733398, + "sampling/sampling_logp_difference/max": 0.41771817207336426, + "sampling/sampling_logp_difference/mean": 0.010797393508255482, + "step": 535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 702.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 470.0625, + "completions/mean_terminated_length": 470.0625, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.32570483535528183, + "epoch": 0.28693790149892934, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0123, + "num_tokens": 11781426.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.465185523033142, + "sampling/importance_sampling_ratio/mean": 0.9999537467956543, + "sampling/importance_sampling_ratio/min": 0.6084758639335632, + "sampling/sampling_logp_difference/max": 0.496798038482666, + "sampling/sampling_logp_difference/mean": 0.011000039055943489, + "step": 536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019886280279024504, + "clip_ratio/low_min": 0.00019886280279024504, + "clip_ratio/region_mean": 0.00019886280279024504, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 599.40625, + "completions/mean_terminated_length": 581.9655151367188, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.2990373261272907, + "epoch": 0.2874732334047109, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010818886570632458, + "learning_rate": 1e-05, + "loss": 0.0459, + "num_tokens": 11804671.0, + "reward": 0.5625, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4307703971862793, + "sampling/importance_sampling_ratio/mean": 1.0002832412719727, + "sampling/importance_sampling_ratio/min": 0.64666748046875, + "sampling/sampling_logp_difference/max": 0.43592309951782227, + "sampling/sampling_logp_difference/mean": 0.01015778910368681, + "step": 537 + }, + { + "clip_ratio/high_max": 5.7313158322358504e-05, + "clip_ratio/high_mean": 5.7313158322358504e-05, + "clip_ratio/low_mean": 0.00017786582247936167, + "clip_ratio/low_min": 0.00017786582247936167, + "clip_ratio/region_mean": 0.00023517898080172017, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 593.46875, + "completions/mean_terminated_length": 514.1363525390625, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.5299359522759914, + "epoch": 0.2880085653104925, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006210138089954853, + "learning_rate": 1e-05, + "loss": -0.024, + "num_tokens": 11828142.0, + "reward": 0.4375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3749855756759644, + "sampling/importance_sampling_ratio/mean": 1.0002870559692383, + "sampling/importance_sampling_ratio/min": 0.6991753578186035, + "sampling/sampling_logp_difference/max": 0.35785365104675293, + "sampling/sampling_logp_difference/mean": 0.014098601415753365, + "step": 538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019079378762398846, + "clip_ratio/low_min": 0.00019079378762398846, + "clip_ratio/region_mean": 0.00019079378762398846, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 449.6875, + "completions/mean_terminated_length": 428.4666748046875, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.44281355291604996, + "epoch": 0.2885438972162741, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021223677322268486, + "learning_rate": 1e-05, + "loss": -0.0587, + "num_tokens": 11846244.0, + "reward": 0.625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.628201961517334, + "sampling/importance_sampling_ratio/mean": 1.000152826309204, + "sampling/importance_sampling_ratio/min": 0.5882433652877808, + "sampling/sampling_logp_difference/max": 0.5306146144866943, + "sampling/sampling_logp_difference/mean": 0.01380736194550991, + "step": 539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012186438470962457, + "clip_ratio/low_min": 0.00012186438470962457, + "clip_ratio/region_mean": 0.00012186438470962457, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 584.3125, + "completions/mean_terminated_length": 500.8182067871094, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.4238555245101452, + "epoch": 0.2890792291220557, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008518840186297894, + "learning_rate": 1e-05, + "loss": 0.0765, + "num_tokens": 11868750.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4809027910232544, + "sampling/importance_sampling_ratio/mean": 0.9999627470970154, + "sampling/importance_sampling_ratio/min": 0.7298859357833862, + "sampling/sampling_logp_difference/max": 0.39265191555023193, + "sampling/sampling_logp_difference/mean": 0.013026274740695953, + "step": 540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.415401053847745e-05, + "clip_ratio/low_min": 4.415401053847745e-05, + "clip_ratio/region_mean": 4.415401053847745e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 559.9375, + "completions/mean_terminated_length": 530.2142944335938, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.3965966682881117, + "epoch": 0.2896145610278373, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0122, + "num_tokens": 11890508.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4583379030227661, + "sampling/importance_sampling_ratio/mean": 1.000077247619629, + "sampling/importance_sampling_ratio/min": 0.7309221625328064, + "sampling/sampling_logp_difference/max": 0.37729740142822266, + "sampling/sampling_logp_difference/mean": 0.011999874375760555, + "step": 541 + }, + { + "clip_ratio/high_max": 4.954419273417443e-05, + "clip_ratio/high_mean": 4.954419273417443e-05, + "clip_ratio/low_mean": 0.00013027618115302175, + "clip_ratio/low_min": 0.00013027618115302175, + "clip_ratio/region_mean": 0.00017982037388719618, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 515.375, + "completions/mean_terminated_length": 431.16668701171875, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.43001551553606987, + "epoch": 0.29014989293361887, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021864552050828934, + "learning_rate": 1e-05, + "loss": -0.0041, + "num_tokens": 11911192.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4208108186721802, + "sampling/importance_sampling_ratio/mean": 1.0005956888198853, + "sampling/importance_sampling_ratio/min": 0.7243146896362305, + "sampling/sampling_logp_difference/max": 0.3512277603149414, + "sampling/sampling_logp_difference/mean": 0.013221126049757004, + "step": 542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 550.375, + "completions/mean_terminated_length": 465.2174072265625, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.3935386463999748, + "epoch": 0.2906852248394004, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012153339572250843, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 11932948.0, + "reward": 0.5, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.5028960704803467, + "sampling/importance_sampling_ratio/mean": 1.0005394220352173, + "sampling/importance_sampling_ratio/min": 0.684719443321228, + "sampling/sampling_logp_difference/max": 0.4073939323425293, + "sampling/sampling_logp_difference/mean": 0.012549460865557194, + "step": 543 + }, + { + "clip_ratio/high_max": 6.811989442212507e-05, + "clip_ratio/high_mean": 6.811989442212507e-05, + "clip_ratio/low_mean": 5.6921675422927365e-05, + "clip_ratio/low_min": 5.6921675422927365e-05, + "clip_ratio/region_mean": 0.00012504156984505244, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 435.40625, + "completions/mean_terminated_length": 424.6773986816406, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.3174975700676441, + "epoch": 0.291220556745182, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0212, + "num_tokens": 11950273.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4301387071609497, + "sampling/importance_sampling_ratio/mean": 0.9999814033508301, + "sampling/importance_sampling_ratio/min": 0.7221091389656067, + "sampling/sampling_logp_difference/max": 0.3577713966369629, + "sampling/sampling_logp_difference/mean": 0.011437466368079185, + "step": 544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.720823901356198e-05, + "clip_ratio/low_min": 5.720823901356198e-05, + "clip_ratio/region_mean": 5.720823901356198e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 585.5625, + "completions/mean_terminated_length": 534.47998046875, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "entropy": 0.40324000269174576, + "epoch": 0.2917558886509636, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004175969399511814, + "learning_rate": 1e-05, + "loss": -0.002, + "num_tokens": 11972979.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4461041688919067, + "sampling/importance_sampling_ratio/mean": 0.9998451471328735, + "sampling/importance_sampling_ratio/min": 0.618733823299408, + "sampling/sampling_logp_difference/max": 0.48008012771606445, + "sampling/sampling_logp_difference/mean": 0.012317562475800514, + "step": 545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.638218888430856e-05, + "clip_ratio/low_min": 4.638218888430856e-05, + "clip_ratio/region_mean": 4.638218888430856e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 557.90625, + "completions/mean_terminated_length": 509.423095703125, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.3849346190690994, + "epoch": 0.29229122055674517, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010279565118253231, + "learning_rate": 1e-05, + "loss": 0.0736, + "num_tokens": 11994784.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4003819227218628, + "sampling/importance_sampling_ratio/mean": 0.9998975992202759, + "sampling/importance_sampling_ratio/min": 0.10590694099664688, + "sampling/sampling_logp_difference/max": 2.245194435119629, + "sampling/sampling_logp_difference/mean": 0.012231167405843735, + "step": 546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 543.65625, + "completions/mean_terminated_length": 502.1111145019531, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.30879425071179867, + "epoch": 0.29282655246252676, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02512621320784092, + "learning_rate": 1e-05, + "loss": 0.0556, + "num_tokens": 12015829.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4133927822113037, + "sampling/importance_sampling_ratio/mean": 0.9995831251144409, + "sampling/importance_sampling_ratio/min": 0.7208872437477112, + "sampling/sampling_logp_difference/max": 0.3459930419921875, + "sampling/sampling_logp_difference/mean": 0.010667083784937859, + "step": 547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 557.625, + "completions/mean_terminated_length": 543.6000366210938, + "completions/min_length": 367.0, + "completions/min_terminated_length": 367.0, + "entropy": 0.3389309160411358, + "epoch": 0.29336188436830835, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0692, + "num_tokens": 12038129.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4502341747283936, + "sampling/importance_sampling_ratio/mean": 1.0001298189163208, + "sampling/importance_sampling_ratio/min": 0.6379529237747192, + "sampling/sampling_logp_difference/max": 0.4494907855987549, + "sampling/sampling_logp_difference/mean": 0.011395558714866638, + "step": 548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.753106572432443e-05, + "clip_ratio/low_min": 6.753106572432443e-05, + "clip_ratio/region_mean": 6.753106572432443e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 420.28125, + "completions/mean_terminated_length": 420.28125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "entropy": 0.32912104576826096, + "epoch": 0.29389721627408993, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011712674051523209, + "learning_rate": 1e-05, + "loss": -0.0219, + "num_tokens": 12055554.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5154039859771729, + "sampling/importance_sampling_ratio/mean": 0.9998295903205872, + "sampling/importance_sampling_ratio/min": 0.7116454243659973, + "sampling/sampling_logp_difference/max": 0.4156820774078369, + "sampling/sampling_logp_difference/mean": 0.011668115854263306, + "step": 549 + }, + { + "clip_ratio/high_max": 6.0299083997961134e-05, + "clip_ratio/high_mean": 6.0299083997961134e-05, + "clip_ratio/low_mean": 5.397236600401811e-05, + "clip_ratio/low_min": 5.397236600401811e-05, + "clip_ratio/region_mean": 0.00011427145000197925, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 483.0, + "completions/mean_terminated_length": 453.5172424316406, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.34380580112338066, + "epoch": 0.2944325481798715, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003323654644191265, + "learning_rate": 1e-05, + "loss": 0.057, + "num_tokens": 12074282.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.466320514678955, + "sampling/importance_sampling_ratio/mean": 1.0004823207855225, + "sampling/importance_sampling_ratio/min": 0.6793035268783569, + "sampling/sampling_logp_difference/max": 0.3866872787475586, + "sampling/sampling_logp_difference/mean": 0.011829284951090813, + "step": 550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.412650251761079e-05, + "clip_ratio/low_min": 9.412650251761079e-05, + "clip_ratio/region_mean": 9.412650251761079e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 452.21875, + "completions/mean_terminated_length": 442.0322570800781, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "entropy": 0.3522225581109524, + "epoch": 0.2949678800856531, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017959894612431526, + "learning_rate": 1e-05, + "loss": -0.0623, + "num_tokens": 12092393.0, + "reward": 0.65625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5850427150726318, + "sampling/importance_sampling_ratio/mean": 0.999859631061554, + "sampling/importance_sampling_ratio/min": 0.5930550694465637, + "sampling/sampling_logp_difference/max": 0.522468090057373, + "sampling/sampling_logp_difference/mean": 0.01135376188904047, + "step": 551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001395868166582659, + "clip_ratio/low_min": 0.0001395868166582659, + "clip_ratio/region_mean": 0.0001395868166582659, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 476.15625, + "completions/mean_terminated_length": 476.15625, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.3376327082514763, + "epoch": 0.2955032119914347, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009012979455292225, + "learning_rate": 1e-05, + "loss": 0.0022, + "num_tokens": 12111486.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3721379041671753, + "sampling/importance_sampling_ratio/mean": 0.9999564290046692, + "sampling/importance_sampling_ratio/min": 0.21763287484645844, + "sampling/sampling_logp_difference/max": 1.5249457359313965, + "sampling/sampling_logp_difference/mean": 0.011011257767677307, + "step": 552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 711.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 455.09375, + "completions/mean_terminated_length": 455.09375, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.3443738520145416, + "epoch": 0.2960385438972163, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.013324462808668613, + "learning_rate": 1e-05, + "loss": -0.0503, + "num_tokens": 12129569.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3456470966339111, + "sampling/importance_sampling_ratio/mean": 0.9997557997703552, + "sampling/importance_sampling_ratio/min": 0.7060806751251221, + "sampling/sampling_logp_difference/max": 0.3480257987976074, + "sampling/sampling_logp_difference/mean": 0.01204129308462143, + "step": 553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 455.0625, + "completions/mean_terminated_length": 444.9677429199219, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.35447244346141815, + "epoch": 0.2965738758029979, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005047038197517395, + "learning_rate": 1e-05, + "loss": 0.0414, + "num_tokens": 12147731.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4405754804611206, + "sampling/importance_sampling_ratio/mean": 0.9999611973762512, + "sampling/importance_sampling_ratio/min": 0.6882584095001221, + "sampling/sampling_logp_difference/max": 0.37359094619750977, + "sampling/sampling_logp_difference/mean": 0.011543944478034973, + "step": 554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002068972826236859, + "clip_ratio/low_min": 0.0002068972826236859, + "clip_ratio/region_mean": 0.0002068972826236859, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 448.03125, + "completions/mean_terminated_length": 437.70965576171875, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.41757863387465477, + "epoch": 0.29710920770877947, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02503243274986744, + "learning_rate": 1e-05, + "loss": -0.0516, + "num_tokens": 12166124.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4345734119415283, + "sampling/importance_sampling_ratio/mean": 1.0002120733261108, + "sampling/importance_sampling_ratio/min": 0.6146836280822754, + "sampling/sampling_logp_difference/max": 0.4866476058959961, + "sampling/sampling_logp_difference/mean": 0.01272653415799141, + "step": 555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.715592124033719e-05, + "clip_ratio/low_min": 5.715592124033719e-05, + "clip_ratio/region_mean": 5.715592124033719e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 518.125, + "completions/mean_terminated_length": 460.4615478515625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.3800526633858681, + "epoch": 0.29764453961456105, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008308999240398407, + "learning_rate": 1e-05, + "loss": 0.039, + "num_tokens": 12186216.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5458557605743408, + "sampling/importance_sampling_ratio/mean": 0.9998978972434998, + "sampling/importance_sampling_ratio/min": 0.5737590193748474, + "sampling/sampling_logp_difference/max": 0.5555458068847656, + "sampling/sampling_logp_difference/mean": 0.012542897835373878, + "step": 556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.745633850689046e-05, + "clip_ratio/low_min": 4.745633850689046e-05, + "clip_ratio/region_mean": 4.745633850689046e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 548.15625, + "completions/mean_terminated_length": 486.5999755859375, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.38176218047738075, + "epoch": 0.2981798715203426, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024729037657380104, + "learning_rate": 1e-05, + "loss": 0.0284, + "num_tokens": 12207485.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4402064085006714, + "sampling/importance_sampling_ratio/mean": 0.9997570514678955, + "sampling/importance_sampling_ratio/min": 0.6999471187591553, + "sampling/sampling_logp_difference/max": 0.36478638648986816, + "sampling/sampling_logp_difference/mean": 0.012147819623351097, + "step": 557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.106497312430292e-05, + "clip_ratio/low_min": 6.106497312430292e-05, + "clip_ratio/region_mean": 6.106497312430292e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 580.65625, + "completions/mean_terminated_length": 507.34783935546875, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.35670357197523117, + "epoch": 0.2987152034261242, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01215875893831253, + "learning_rate": 1e-05, + "loss": 0.0056, + "num_tokens": 12229690.0, + "reward": 0.4375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3789316415786743, + "sampling/importance_sampling_ratio/mean": 0.9998000860214233, + "sampling/importance_sampling_ratio/min": 0.6110749840736389, + "sampling/sampling_logp_difference/max": 0.4925355911254883, + "sampling/sampling_logp_difference/mean": 0.01118595339357853, + "step": 558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014841670781606808, + "clip_ratio/low_min": 0.00014841670781606808, + "clip_ratio/region_mean": 0.00014841670781606808, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 510.875, + "completions/mean_terminated_length": 502.58062744140625, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.3493451811373234, + "epoch": 0.29925053533190576, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006535404361784458, + "learning_rate": 1e-05, + "loss": -0.0476, + "num_tokens": 12249870.0, + "reward": 0.84375, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3719756603240967, + "sampling/importance_sampling_ratio/mean": 1.0001106262207031, + "sampling/importance_sampling_ratio/min": 0.4449857175350189, + "sampling/sampling_logp_difference/max": 0.8097131252288818, + "sampling/sampling_logp_difference/mean": 0.011852250434458256, + "step": 559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 437.46875, + "completions/mean_terminated_length": 426.8064270019531, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.32386602833867073, + "epoch": 0.29978586723768735, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006508872844278812, + "learning_rate": 1e-05, + "loss": 0.0394, + "num_tokens": 12267429.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.7509204149246216, + "sampling/importance_sampling_ratio/mean": 1.0000189542770386, + "sampling/importance_sampling_ratio/min": 0.6039376258850098, + "sampling/sampling_logp_difference/max": 0.5601415634155273, + "sampling/sampling_logp_difference/mean": 0.01119659747928381, + "step": 560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 601.15625, + "completions/mean_terminated_length": 535.8695678710938, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "entropy": 0.36822259798645973, + "epoch": 0.30032119914346894, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015724875032901764, + "learning_rate": 1e-05, + "loss": -0.0018, + "num_tokens": 12289858.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3015179634094238, + "sampling/importance_sampling_ratio/mean": 0.9997707605361938, + "sampling/importance_sampling_ratio/min": 0.6908484101295471, + "sampling/sampling_logp_difference/max": 0.36983489990234375, + "sampling/sampling_logp_difference/mean": 0.011337664909660816, + "step": 561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.808108370634727e-05, + "clip_ratio/low_min": 9.808108370634727e-05, + "clip_ratio/region_mean": 9.808108370634727e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 540.5, + "completions/mean_terminated_length": 516.9655151367188, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.31132769770920277, + "epoch": 0.30085653104925053, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011534156277775764, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 12311138.0, + "reward": 0.5, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3694326877593994, + "sampling/importance_sampling_ratio/mean": 1.0003710985183716, + "sampling/importance_sampling_ratio/min": 0.7437523007392883, + "sampling/sampling_logp_difference/max": 0.31439661979675293, + "sampling/sampling_logp_difference/mean": 0.010466148145496845, + "step": 562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.724504004931077e-05, + "clip_ratio/low_min": 8.724504004931077e-05, + "clip_ratio/region_mean": 8.724504004931077e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 568.84375, + "completions/mean_terminated_length": 502.4583435058594, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.38482216745615005, + "epoch": 0.3013918629550321, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007253366056829691, + "learning_rate": 1e-05, + "loss": 0.0984, + "num_tokens": 12332877.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3173551559448242, + "sampling/importance_sampling_ratio/mean": 0.9999594688415527, + "sampling/importance_sampling_ratio/min": 0.6961379647254944, + "sampling/sampling_logp_difference/max": 0.36220741271972656, + "sampling/sampling_logp_difference/mean": 0.011871026828885078, + "step": 563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 585.65625, + "completions/mean_terminated_length": 490.1428527832031, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.5359249711036682, + "epoch": 0.3019271948608137, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012351706624031067, + "learning_rate": 1e-05, + "loss": 0.0201, + "num_tokens": 12355762.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4052972793579102, + "sampling/importance_sampling_ratio/mean": 1.000327229499817, + "sampling/importance_sampling_ratio/min": 0.7129547595977783, + "sampling/sampling_logp_difference/max": 0.34024882316589355, + "sampling/sampling_logp_difference/mean": 0.016371745616197586, + "step": 564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020264643535483629, + "clip_ratio/low_min": 0.00020264643535483629, + "clip_ratio/region_mean": 0.00020264643535483629, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 432.6875, + "completions/mean_terminated_length": 421.8709411621094, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.3549719266593456, + "epoch": 0.3024625267665953, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014779582619667053, + "learning_rate": 1e-05, + "loss": -0.0115, + "num_tokens": 12373136.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5077543258666992, + "sampling/importance_sampling_ratio/mean": 0.999976396560669, + "sampling/importance_sampling_ratio/min": 0.6832563877105713, + "sampling/sampling_logp_difference/max": 0.41062140464782715, + "sampling/sampling_logp_difference/mean": 0.011491380631923676, + "step": 565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 466.21875, + "completions/mean_terminated_length": 456.4838562011719, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.28736421652138233, + "epoch": 0.3029978586723769, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0026263478212058544, + "learning_rate": 1e-05, + "loss": 0.0274, + "num_tokens": 12391703.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6552960872650146, + "sampling/importance_sampling_ratio/mean": 0.9999865889549255, + "sampling/importance_sampling_ratio/min": 0.7178400158882141, + "sampling/sampling_logp_difference/max": 0.5039799213409424, + "sampling/sampling_logp_difference/mean": 0.00997740589082241, + "step": 566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.221259147627279e-05, + "clip_ratio/low_min": 7.221259147627279e-05, + "clip_ratio/region_mean": 7.221259147627279e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 422.875, + "completions/mean_terminated_length": 422.875, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.26030854880809784, + "epoch": 0.3035331905781585, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.022980762645602226, + "learning_rate": 1e-05, + "loss": -0.0205, + "num_tokens": 12408923.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4185572862625122, + "sampling/importance_sampling_ratio/mean": 1.000119924545288, + "sampling/importance_sampling_ratio/min": 0.7247578501701355, + "sampling/sampling_logp_difference/max": 0.3496403694152832, + "sampling/sampling_logp_difference/mean": 0.009362556971609592, + "step": 567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 404.40625, + "completions/mean_terminated_length": 392.6773986816406, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.32211763970553875, + "epoch": 0.30406852248394006, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006427176296710968, + "learning_rate": 1e-05, + "loss": 0.0127, + "num_tokens": 12425112.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3863672018051147, + "sampling/importance_sampling_ratio/mean": 0.9999868273735046, + "sampling/importance_sampling_ratio/min": 0.6178496479988098, + "sampling/sampling_logp_difference/max": 0.4815101623535156, + "sampling/sampling_logp_difference/mean": 0.011179612949490547, + "step": 568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 490.96875, + "completions/mean_terminated_length": 462.3103332519531, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.3328098524361849, + "epoch": 0.30460385438972165, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008083218708634377, + "learning_rate": 1e-05, + "loss": 0.0181, + "num_tokens": 12444359.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3812774419784546, + "sampling/importance_sampling_ratio/mean": 1.0000039339065552, + "sampling/importance_sampling_ratio/min": 0.6159681677818298, + "sampling/sampling_logp_difference/max": 0.4845600128173828, + "sampling/sampling_logp_difference/mean": 0.01154243666678667, + "step": 569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.977025961736217e-05, + "clip_ratio/low_min": 7.977025961736217e-05, + "clip_ratio/region_mean": 7.977025961736217e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 456.0, + "completions/mean_terminated_length": 435.20001220703125, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.4078424833714962, + "epoch": 0.30513918629550324, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008885356597602367, + "learning_rate": 1e-05, + "loss": -0.0218, + "num_tokens": 12463175.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.70856773853302, + "sampling/importance_sampling_ratio/mean": 0.9998947978019714, + "sampling/importance_sampling_ratio/min": 0.5030553340911865, + "sampling/sampling_logp_difference/max": 0.6870551109313965, + "sampling/sampling_logp_difference/mean": 0.013486012816429138, + "step": 570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 477.34375, + "completions/mean_terminated_length": 457.9667053222656, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.3029682785272598, + "epoch": 0.3056745182012848, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014453751966357231, + "learning_rate": 1e-05, + "loss": 0.067, + "num_tokens": 12482450.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4614412784576416, + "sampling/importance_sampling_ratio/mean": 0.9999015927314758, + "sampling/importance_sampling_ratio/min": 0.7071356177330017, + "sampling/sampling_logp_difference/max": 0.3794231414794922, + "sampling/sampling_logp_difference/mean": 0.010116026736795902, + "step": 571 + }, + { + "clip_ratio/high_max": 0.0002342275038245134, + "clip_ratio/high_mean": 0.0002342275038245134, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0002342275038245134, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 504.78125, + "completions/mean_terminated_length": 467.1785888671875, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.38962068036198616, + "epoch": 0.30620985010706636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024353552609682083, + "learning_rate": 1e-05, + "loss": 0.0685, + "num_tokens": 12502091.0, + "reward": 0.625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4311225414276123, + "sampling/importance_sampling_ratio/mean": 1.0002267360687256, + "sampling/importance_sampling_ratio/min": 0.4781179428100586, + "sampling/sampling_logp_difference/max": 0.7378978729248047, + "sampling/sampling_logp_difference/mean": 0.012861599214375019, + "step": 572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 432.90625, + "completions/mean_terminated_length": 422.0967712402344, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3488805815577507, + "epoch": 0.30674518201284795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002496341010555625, + "learning_rate": 1e-05, + "loss": 0.0121, + "num_tokens": 12519144.0, + "reward": 0.59375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5523734092712402, + "sampling/importance_sampling_ratio/mean": 1.0000110864639282, + "sampling/importance_sampling_ratio/min": 0.6951249241828918, + "sampling/sampling_logp_difference/max": 0.4397850036621094, + "sampling/sampling_logp_difference/mean": 0.01103675551712513, + "step": 573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002104796549247112, + "clip_ratio/low_min": 0.0002104796549247112, + "clip_ratio/region_mean": 0.0002104796549247112, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 504.90625, + "completions/mean_terminated_length": 456.1851806640625, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.36549049615859985, + "epoch": 0.30728051391862954, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021736497059464455, + "learning_rate": 1e-05, + "loss": -0.0014, + "num_tokens": 12539293.0, + "reward": 0.71875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6297128200531006, + "sampling/importance_sampling_ratio/mean": 0.9993993639945984, + "sampling/importance_sampling_ratio/min": 0.676750659942627, + "sampling/sampling_logp_difference/max": 0.4884037971496582, + "sampling/sampling_logp_difference/mean": 0.012100109830498695, + "step": 574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.701015495811589e-05, + "clip_ratio/low_min": 4.701015495811589e-05, + "clip_ratio/region_mean": 4.701015495811589e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 504.96875, + "completions/mean_terminated_length": 467.39288330078125, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "entropy": 0.4503924436867237, + "epoch": 0.3078158458244111, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007269065361469984, + "learning_rate": 1e-05, + "loss": 0.0752, + "num_tokens": 12559260.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5005733966827393, + "sampling/importance_sampling_ratio/mean": 1.0002752542495728, + "sampling/importance_sampling_ratio/min": 0.6220771670341492, + "sampling/sampling_logp_difference/max": 0.47469115257263184, + "sampling/sampling_logp_difference/mean": 0.01278842706233263, + "step": 575 + }, + { + "clip_ratio/high_max": 7.237985118990764e-05, + "clip_ratio/high_mean": 7.237985118990764e-05, + "clip_ratio/low_mean": 0.00014186622865963727, + "clip_ratio/low_min": 0.00014186622865963727, + "clip_ratio/region_mean": 0.0002142460798495449, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 425.34375, + "completions/mean_terminated_length": 425.34375, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.412844717502594, + "epoch": 0.3083511777301927, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0110018914565444, + "learning_rate": 1e-05, + "loss": -0.003, + "num_tokens": 12576791.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.409265160560608, + "sampling/importance_sampling_ratio/mean": 1.0001308917999268, + "sampling/importance_sampling_ratio/min": 0.6833316683769226, + "sampling/sampling_logp_difference/max": 0.38077497482299805, + "sampling/sampling_logp_difference/mean": 0.013048562221229076, + "step": 576 + }, + { + "clip_ratio/high_max": 5.8548008382786065e-05, + "clip_ratio/high_mean": 5.8548008382786065e-05, + "clip_ratio/low_mean": 4.8809059080667794e-05, + "clip_ratio/low_min": 4.8809059080667794e-05, + "clip_ratio/region_mean": 0.00010735706746345386, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 506.9375, + "completions/mean_terminated_length": 489.5333557128906, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.42174794897437096, + "epoch": 0.3088865096359743, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03381386399269104, + "learning_rate": 1e-05, + "loss": 0.045, + "num_tokens": 12596885.0, + "reward": 0.5625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4981865882873535, + "sampling/importance_sampling_ratio/mean": 1.000162959098816, + "sampling/importance_sampling_ratio/min": 0.6529225707054138, + "sampling/sampling_logp_difference/max": 0.4262967109680176, + "sampling/sampling_logp_difference/mean": 0.012921232730150223, + "step": 577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014186279440764338, + "clip_ratio/low_min": 0.00014186279440764338, + "clip_ratio/region_mean": 0.00014186279440764338, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 461.3125, + "completions/mean_terminated_length": 440.86669921875, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.3352612443268299, + "epoch": 0.3094218415417559, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009724777191877365, + "learning_rate": 1e-05, + "loss": 0.0122, + "num_tokens": 12614999.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4091874361038208, + "sampling/importance_sampling_ratio/mean": 1.0004322528839111, + "sampling/importance_sampling_ratio/min": 0.3782390058040619, + "sampling/sampling_logp_difference/max": 0.97222900390625, + "sampling/sampling_logp_difference/mean": 0.010968740098178387, + "step": 578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 485.65625, + "completions/mean_terminated_length": 476.5483703613281, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.30073725804686546, + "epoch": 0.3099571734475375, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019250115379691124, + "learning_rate": 1e-05, + "loss": -0.0194, + "num_tokens": 12634788.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3734782934188843, + "sampling/importance_sampling_ratio/mean": 0.9999491572380066, + "sampling/importance_sampling_ratio/min": 0.7112151980400085, + "sampling/sampling_logp_difference/max": 0.34078025817871094, + "sampling/sampling_logp_difference/mean": 0.010382591746747494, + "step": 579 + }, + { + "clip_ratio/high_max": 7.494004967156798e-05, + "clip_ratio/high_mean": 7.494004967156798e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.494004967156798e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 473.09375, + "completions/mean_terminated_length": 473.09375, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.39355795457959175, + "epoch": 0.31049250535331907, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0271613709628582, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 12654271.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3442903757095337, + "sampling/importance_sampling_ratio/mean": 1.000354528427124, + "sampling/importance_sampling_ratio/min": 0.6165125966072083, + "sampling/sampling_logp_difference/max": 0.483676552772522, + "sampling/sampling_logp_difference/mean": 0.012498216703534126, + "step": 580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 457.03125, + "completions/mean_terminated_length": 457.03125, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.30188281275331974, + "epoch": 0.31102783725910066, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005763284396380186, + "learning_rate": 1e-05, + "loss": 0.0036, + "num_tokens": 12672720.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3802168369293213, + "sampling/importance_sampling_ratio/mean": 0.9999557137489319, + "sampling/importance_sampling_ratio/min": 0.6441707611083984, + "sampling/sampling_logp_difference/max": 0.4397914409637451, + "sampling/sampling_logp_difference/mean": 0.009894377551972866, + "step": 581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.163155583431944e-05, + "clip_ratio/low_min": 5.163155583431944e-05, + "clip_ratio/region_mean": 5.163155583431944e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 548.875, + "completions/mean_terminated_length": 526.2069091796875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.2989288941025734, + "epoch": 0.31156316916488225, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005480234511196613, + "learning_rate": 1e-05, + "loss": 0.0605, + "num_tokens": 12693604.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4595129489898682, + "sampling/importance_sampling_ratio/mean": 1.0002418756484985, + "sampling/importance_sampling_ratio/min": 0.6605982780456543, + "sampling/sampling_logp_difference/max": 0.414609432220459, + "sampling/sampling_logp_difference/mean": 0.01005987636744976, + "step": 582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001239581215486396, + "clip_ratio/low_min": 0.0001239581215486396, + "clip_ratio/region_mean": 0.0001239581215486396, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 473.5, + "completions/mean_terminated_length": 431.4285888671875, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.39790476486086845, + "epoch": 0.31209850107066384, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004399289842694998, + "learning_rate": 1e-05, + "loss": 0.0551, + "num_tokens": 12712172.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4628119468688965, + "sampling/importance_sampling_ratio/mean": 0.9997876882553101, + "sampling/importance_sampling_ratio/min": 0.6796148419380188, + "sampling/sampling_logp_difference/max": 0.3862290382385254, + "sampling/sampling_logp_difference/mean": 0.012877034954726696, + "step": 583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 423.0, + "completions/mean_terminated_length": 400.0000305175781, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.35159216448664665, + "epoch": 0.31263383297644537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.062, + "num_tokens": 12729068.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4646342992782593, + "sampling/importance_sampling_ratio/mean": 1.0000327825546265, + "sampling/importance_sampling_ratio/min": 0.7577028870582581, + "sampling/sampling_logp_difference/max": 0.3816056251525879, + "sampling/sampling_logp_difference/mean": 0.011340050958096981, + "step": 584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 473.375, + "completions/mean_terminated_length": 463.8709411621094, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.3590838871896267, + "epoch": 0.31316916488222696, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0332, + "num_tokens": 12747752.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5954334735870361, + "sampling/importance_sampling_ratio/mean": 0.9996042847633362, + "sampling/importance_sampling_ratio/min": 0.6934288740158081, + "sampling/sampling_logp_difference/max": 0.4671454429626465, + "sampling/sampling_logp_difference/mean": 0.012334608472883701, + "step": 585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010694679440348409, + "clip_ratio/low_min": 0.00010694679440348409, + "clip_ratio/region_mean": 0.00010694679440348409, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 568.90625, + "completions/mean_terminated_length": 502.54168701171875, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "entropy": 0.35023289918899536, + "epoch": 0.31370449678800855, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034463456831872463, + "learning_rate": 1e-05, + "loss": 0.0571, + "num_tokens": 12769461.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4596868753433228, + "sampling/importance_sampling_ratio/mean": 1.0000150203704834, + "sampling/importance_sampling_ratio/min": 0.6247786283493042, + "sampling/sampling_logp_difference/max": 0.47035789489746094, + "sampling/sampling_logp_difference/mean": 0.011078916490077972, + "step": 586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.214851989876479e-05, + "clip_ratio/low_min": 5.214851989876479e-05, + "clip_ratio/region_mean": 5.214851989876479e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 576.09375, + "completions/mean_terminated_length": 512.125, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3751702047884464, + "epoch": 0.31423982869379014, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009753352031111717, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 12791336.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.380725383758545, + "sampling/importance_sampling_ratio/mean": 0.9998529553413391, + "sampling/importance_sampling_ratio/min": 0.7274522185325623, + "sampling/sampling_logp_difference/max": 0.32260894775390625, + "sampling/sampling_logp_difference/mean": 0.011273259297013283, + "step": 587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 434.28125, + "completions/mean_terminated_length": 434.28125, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "entropy": 0.3039466142654419, + "epoch": 0.3147751605995717, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0028925358783453703, + "learning_rate": 1e-05, + "loss": 0.0296, + "num_tokens": 12808297.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4272725582122803, + "sampling/importance_sampling_ratio/mean": 1.0001274347305298, + "sampling/importance_sampling_ratio/min": 0.6404282450675964, + "sampling/sampling_logp_difference/max": 0.4456181526184082, + "sampling/sampling_logp_difference/mean": 0.009803217835724354, + "step": 588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.305647497763857e-05, + "clip_ratio/low_min": 8.305647497763857e-05, + "clip_ratio/region_mean": 8.305647497763857e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 558.0, + "completions/max_terminated_length": 558.0, + "completions/mean_length": 391.28125, + "completions/mean_terminated_length": 391.28125, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.316328976303339, + "epoch": 0.3153104925053533, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01038505882024765, + "learning_rate": 1e-05, + "loss": -0.0428, + "num_tokens": 12824410.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3519842624664307, + "sampling/importance_sampling_ratio/mean": 0.9998332262039185, + "sampling/importance_sampling_ratio/min": 0.7554951906204224, + "sampling/sampling_logp_difference/max": 0.3015732765197754, + "sampling/sampling_logp_difference/mean": 0.010774289257824421, + "step": 589 + }, + { + "clip_ratio/high_max": 0.00013501863577403128, + "clip_ratio/high_mean": 0.00013501863577403128, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00013501863577403128, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 523.03125, + "completions/mean_terminated_length": 497.6896667480469, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.31476516649127007, + "epoch": 0.3158458244111349, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027100680395960808, + "learning_rate": 1e-05, + "loss": 0.0568, + "num_tokens": 12844723.0, + "reward": 0.625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3829365968704224, + "sampling/importance_sampling_ratio/mean": 1.0001263618469238, + "sampling/importance_sampling_ratio/min": 0.781120777130127, + "sampling/sampling_logp_difference/max": 0.32420921325683594, + "sampling/sampling_logp_difference/mean": 0.010345133021473885, + "step": 590 + }, + { + "clip_ratio/high_max": 5.369415885070339e-05, + "clip_ratio/high_mean": 5.369415885070339e-05, + "clip_ratio/low_mean": 0.00013852096162736416, + "clip_ratio/low_min": 0.00013852096162736416, + "clip_ratio/region_mean": 0.00019221512047806755, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 502.28125, + "completions/mean_terminated_length": 464.3214416503906, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.3764626607298851, + "epoch": 0.3163811563169165, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011633516289293766, + "learning_rate": 1e-05, + "loss": 0.106, + "num_tokens": 12864684.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5665745735168457, + "sampling/importance_sampling_ratio/mean": 1.0000176429748535, + "sampling/importance_sampling_ratio/min": 0.6921218037605286, + "sampling/sampling_logp_difference/max": 0.44889140129089355, + "sampling/sampling_logp_difference/mean": 0.011256412602961063, + "step": 591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 663.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 439.375, + "completions/mean_terminated_length": 439.375, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.3273787572979927, + "epoch": 0.3169164882226981, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003047094913199544, + "learning_rate": 1e-05, + "loss": 0.0143, + "num_tokens": 12882304.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.386520266532898, + "sampling/importance_sampling_ratio/mean": 1.0001561641693115, + "sampling/importance_sampling_ratio/min": 0.7159668207168579, + "sampling/sampling_logp_difference/max": 0.3341214656829834, + "sampling/sampling_logp_difference/mean": 0.010446848347783089, + "step": 592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021715218099416234, + "clip_ratio/low_min": 0.00021715218099416234, + "clip_ratio/region_mean": 0.00021715218099416234, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 501.5625, + "completions/mean_terminated_length": 426.9599914550781, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.5103813372552395, + "epoch": 0.31745182012847967, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011841914616525173, + "learning_rate": 1e-05, + "loss": 0.0229, + "num_tokens": 12902210.0, + "reward": 0.5, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6238415241241455, + "sampling/importance_sampling_ratio/mean": 1.000093698501587, + "sampling/importance_sampling_ratio/min": 0.7020272016525269, + "sampling/sampling_logp_difference/max": 0.48479461669921875, + "sampling/sampling_logp_difference/mean": 0.014888807199895382, + "step": 593 + }, + { + "clip_ratio/high_max": 6.760410906281322e-05, + "clip_ratio/high_mean": 6.760410906281322e-05, + "clip_ratio/low_mean": 6.760410906281322e-05, + "clip_ratio/low_min": 6.760410906281322e-05, + "clip_ratio/region_mean": 0.00013520821812562644, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 612.875, + "completions/mean_terminated_length": 552.1739501953125, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.46455319225788116, + "epoch": 0.31798715203426126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009063370525836945, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 12926246.0, + "reward": 0.625, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4822250604629517, + "sampling/importance_sampling_ratio/mean": 0.9995887279510498, + "sampling/importance_sampling_ratio/min": 0.14249230921268463, + "sampling/sampling_logp_difference/max": 1.9484672546386719, + "sampling/sampling_logp_difference/mean": 0.013255693949759007, + "step": 594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002372380367887672, + "clip_ratio/low_min": 0.0002372380367887672, + "clip_ratio/region_mean": 0.0002372380367887672, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 546.875, + "completions/mean_terminated_length": 505.9259338378906, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.3897904269397259, + "epoch": 0.31852248394004284, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010369587689638138, + "learning_rate": 1e-05, + "loss": 0.0675, + "num_tokens": 12947642.0, + "reward": 0.65625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9995553493499756, + "sampling/importance_sampling_ratio/min": 0.6306949257850647, + "sampling/sampling_logp_difference/max": 0.8285665512084961, + "sampling/sampling_logp_difference/mean": 0.012024225667119026, + "step": 595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015292173338821158, + "clip_ratio/low_min": 0.00015292173338821158, + "clip_ratio/region_mean": 0.00015292173338821158, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 491.84375, + "completions/mean_terminated_length": 452.39288330078125, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.3545280061662197, + "epoch": 0.31905781584582443, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012320420704782009, + "learning_rate": 1e-05, + "loss": 0.064, + "num_tokens": 12967469.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2975786924362183, + "sampling/importance_sampling_ratio/mean": 1.0001546144485474, + "sampling/importance_sampling_ratio/min": 0.7114093899726868, + "sampling/sampling_logp_difference/max": 0.34050726890563965, + "sampling/sampling_logp_difference/mean": 0.01130654476583004, + "step": 596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014027773431735113, + "clip_ratio/low_min": 0.00014027773431735113, + "clip_ratio/region_mean": 0.00014027773431735113, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 533.34375, + "completions/mean_terminated_length": 479.19232177734375, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.3938218764960766, + "epoch": 0.319593147751606, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019227659329771996, + "learning_rate": 1e-05, + "loss": 0.0173, + "num_tokens": 12988344.0, + "reward": 0.46875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4319961071014404, + "sampling/importance_sampling_ratio/mean": 0.999968409538269, + "sampling/importance_sampling_ratio/min": 0.6174544095993042, + "sampling/sampling_logp_difference/max": 0.4821500778198242, + "sampling/sampling_logp_difference/mean": 0.0121418172493577, + "step": 597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012024539319099858, + "clip_ratio/low_min": 0.00012024539319099858, + "clip_ratio/region_mean": 0.00012024539319099858, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 532.59375, + "completions/mean_terminated_length": 489.0, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.39213399589061737, + "epoch": 0.32012847965738755, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02263588458299637, + "learning_rate": 1e-05, + "loss": -0.0016, + "num_tokens": 13009667.0, + "reward": 0.5625, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.452756404876709, + "sampling/importance_sampling_ratio/mean": 1.0003527402877808, + "sampling/importance_sampling_ratio/min": 0.6124302744865417, + "sampling/sampling_logp_difference/max": 0.49032020568847656, + "sampling/sampling_logp_difference/mean": 0.012032617814838886, + "step": 598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 509.9375, + "completions/mean_terminated_length": 483.2413635253906, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "entropy": 0.417438130825758, + "epoch": 0.32066381156316914, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0060572451911866665, + "learning_rate": 1e-05, + "loss": 0.0458, + "num_tokens": 13030321.0, + "reward": 0.65625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3527554273605347, + "sampling/importance_sampling_ratio/mean": 1.000324010848999, + "sampling/importance_sampling_ratio/min": 0.6157258749008179, + "sampling/sampling_logp_difference/max": 0.4849534034729004, + "sampling/sampling_logp_difference/mean": 0.012862362898886204, + "step": 599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 420.5, + "completions/mean_terminated_length": 409.2903137207031, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.3093317486345768, + "epoch": 0.32119914346895073, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.019723497331142426, + "learning_rate": 1e-05, + "loss": -0.0183, + "num_tokens": 13047321.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3038541078567505, + "sampling/importance_sampling_ratio/mean": 0.9999650716781616, + "sampling/importance_sampling_ratio/min": 0.7139387726783752, + "sampling/sampling_logp_difference/max": 0.3369581699371338, + "sampling/sampling_logp_difference/mean": 0.010076196864247322, + "step": 600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.907734546577558e-05, + "clip_ratio/low_min": 4.907734546577558e-05, + "clip_ratio/region_mean": 4.907734546577558e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 570.65625, + "completions/mean_terminated_length": 504.875, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.36661629751324654, + "epoch": 0.3217344753747323, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011421416886150837, + "learning_rate": 1e-05, + "loss": 0.0478, + "num_tokens": 13069318.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6252936124801636, + "sampling/importance_sampling_ratio/mean": 0.999879777431488, + "sampling/importance_sampling_ratio/min": 0.5668175220489502, + "sampling/sampling_logp_difference/max": 0.5677177906036377, + "sampling/sampling_logp_difference/mean": 0.011020981706678867, + "step": 601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 489.625, + "completions/mean_terminated_length": 460.82757568359375, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.3008575960993767, + "epoch": 0.3222698072805139, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013026995584368706, + "learning_rate": 1e-05, + "loss": 0.0337, + "num_tokens": 13088866.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4073952436447144, + "sampling/importance_sampling_ratio/mean": 0.9999853372573853, + "sampling/importance_sampling_ratio/min": 0.6923598051071167, + "sampling/sampling_logp_difference/max": 0.36764955520629883, + "sampling/sampling_logp_difference/mean": 0.010159152559936047, + "step": 602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 492.4375, + "completions/mean_terminated_length": 474.0666809082031, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 0.3233669772744179, + "epoch": 0.3228051391862955, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004429081454873085, + "learning_rate": 1e-05, + "loss": 0.0324, + "num_tokens": 13108544.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4388188123703003, + "sampling/importance_sampling_ratio/mean": 1.0002284049987793, + "sampling/importance_sampling_ratio/min": 0.7001235485076904, + "sampling/sampling_logp_difference/max": 0.36382246017456055, + "sampling/sampling_logp_difference/mean": 0.010552718304097652, + "step": 603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001708725503704045, + "clip_ratio/low_min": 0.0001708725503704045, + "clip_ratio/region_mean": 0.0001708725503704045, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 545.34375, + "completions/mean_terminated_length": 483.0, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.47946010157465935, + "epoch": 0.3233404710920771, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.031759437173604965, + "learning_rate": 1e-05, + "loss": -0.0158, + "num_tokens": 13130379.0, + "reward": 0.4375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2905904054641724, + "sampling/importance_sampling_ratio/mean": 0.9998774528503418, + "sampling/importance_sampling_ratio/min": 0.7416612505912781, + "sampling/sampling_logp_difference/max": 0.2988626956939697, + "sampling/sampling_logp_difference/mean": 0.013906342908740044, + "step": 604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023261904789251275, + "clip_ratio/low_min": 0.00023261904789251275, + "clip_ratio/region_mean": 0.00023261904789251275, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 575.65625, + "completions/mean_terminated_length": 511.54168701171875, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.42365530505776405, + "epoch": 0.3238758029978587, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006852906662970781, + "learning_rate": 1e-05, + "loss": 0.0172, + "num_tokens": 13152448.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2941696643829346, + "sampling/importance_sampling_ratio/mean": 1.0000784397125244, + "sampling/importance_sampling_ratio/min": 0.7816289663314819, + "sampling/sampling_logp_difference/max": 0.25786924362182617, + "sampling/sampling_logp_difference/mean": 0.012645269744098186, + "step": 605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 478.6875, + "completions/mean_terminated_length": 469.3548278808594, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.35248585045337677, + "epoch": 0.32441113490364026, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0067518651485443115, + "learning_rate": 1e-05, + "loss": 0.0042, + "num_tokens": 13171142.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3850748538970947, + "sampling/importance_sampling_ratio/mean": 0.999954104423523, + "sampling/importance_sampling_ratio/min": 0.729388415813446, + "sampling/sampling_logp_difference/max": 0.32575416564941406, + "sampling/sampling_logp_difference/mean": 0.010965636000037193, + "step": 606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 494.78125, + "completions/mean_terminated_length": 466.5172424316406, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.4147927723824978, + "epoch": 0.32494646680942185, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00472274562343955, + "learning_rate": 1e-05, + "loss": 0.0092, + "num_tokens": 13190343.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4420902729034424, + "sampling/importance_sampling_ratio/mean": 1.0001254081726074, + "sampling/importance_sampling_ratio/min": 0.698153555393219, + "sampling/sampling_logp_difference/max": 0.36609363555908203, + "sampling/sampling_logp_difference/mean": 0.012433970347046852, + "step": 607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015350939429481514, + "clip_ratio/low_min": 0.00015350939429481514, + "clip_ratio/region_mean": 0.00015350939429481514, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 491.25, + "completions/mean_terminated_length": 399.0, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.353031437844038, + "epoch": 0.32548179871520344, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011456039734184742, + "learning_rate": 1e-05, + "loss": 0.0453, + "num_tokens": 13209543.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9996740818023682, + "sampling/importance_sampling_ratio/min": 0.4374963045120239, + "sampling/sampling_logp_difference/max": 1.7040212154388428, + "sampling/sampling_logp_difference/mean": 0.010685842484235764, + "step": 608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001578001247253269, + "clip_ratio/low_min": 0.0001578001247253269, + "clip_ratio/region_mean": 0.0001578001247253269, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 584.0, + "completions/mean_terminated_length": 522.6666870117188, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.41221435368061066, + "epoch": 0.32601713062098503, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007403896190226078, + "learning_rate": 1e-05, + "loss": 0.0575, + "num_tokens": 13231847.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4222649335861206, + "sampling/importance_sampling_ratio/mean": 1.0001360177993774, + "sampling/importance_sampling_ratio/min": 0.7329180240631104, + "sampling/sampling_logp_difference/max": 0.3522505760192871, + "sampling/sampling_logp_difference/mean": 0.012082134373486042, + "step": 609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 443.5625, + "completions/mean_terminated_length": 421.933349609375, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.3271928485482931, + "epoch": 0.3265524625267666, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0491, + "num_tokens": 13249457.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0004411935806274, + "sampling/importance_sampling_ratio/min": 0.7110956907272339, + "sampling/sampling_logp_difference/max": 0.8487842082977295, + "sampling/sampling_logp_difference/mean": 0.010361955501139164, + "step": 610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 505.6875, + "completions/mean_terminated_length": 488.20001220703125, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.2920081876218319, + "epoch": 0.3270877944325482, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0692, + "num_tokens": 13269135.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4318369626998901, + "sampling/importance_sampling_ratio/mean": 1.0002622604370117, + "sampling/importance_sampling_ratio/min": 0.7309180498123169, + "sampling/sampling_logp_difference/max": 0.35895824432373047, + "sampling/sampling_logp_difference/mean": 0.009328608401119709, + "step": 611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017318427126156166, + "clip_ratio/low_min": 0.00017318427126156166, + "clip_ratio/region_mean": 0.00017318427126156166, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 538.90625, + "completions/mean_terminated_length": 496.4814758300781, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.32793927378952503, + "epoch": 0.32762312633832974, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013294613920152187, + "learning_rate": 1e-05, + "loss": 0.0699, + "num_tokens": 13290108.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4561046361923218, + "sampling/importance_sampling_ratio/mean": 1.0001935958862305, + "sampling/importance_sampling_ratio/min": 0.6996236443519592, + "sampling/sampling_logp_difference/max": 0.3757648468017578, + "sampling/sampling_logp_difference/mean": 0.010137120261788368, + "step": 612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 410.5625, + "completions/mean_terminated_length": 399.0322570800781, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.37601885199546814, + "epoch": 0.32815845824411133, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0261, + "num_tokens": 13306742.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3682092428207397, + "sampling/importance_sampling_ratio/mean": 0.9998409748077393, + "sampling/importance_sampling_ratio/min": 0.7513484358787537, + "sampling/sampling_logp_difference/max": 0.31350278854370117, + "sampling/sampling_logp_difference/mean": 0.011797838844358921, + "step": 613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.13346419413574e-05, + "clip_ratio/low_min": 6.13346419413574e-05, + "clip_ratio/region_mean": 6.13346419413574e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 518.09375, + "completions/mean_terminated_length": 482.39288330078125, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3716732859611511, + "epoch": 0.3286937901498929, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0584, + "num_tokens": 13326969.0, + "reward": 0.5, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3558627367019653, + "sampling/importance_sampling_ratio/mean": 1.0003249645233154, + "sampling/importance_sampling_ratio/min": 0.629155158996582, + "sampling/sampling_logp_difference/max": 0.46337735652923584, + "sampling/sampling_logp_difference/mean": 0.011459769681096077, + "step": 614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013365456834435463, + "clip_ratio/low_min": 0.00013365456834435463, + "clip_ratio/region_mean": 0.00013365456834435463, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 429.375, + "completions/mean_terminated_length": 429.375, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.34134896472096443, + "epoch": 0.3292291220556745, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009187165647745132, + "learning_rate": 1e-05, + "loss": 0.0177, + "num_tokens": 13344245.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.372597575187683, + "sampling/importance_sampling_ratio/mean": 1.0002752542495728, + "sampling/importance_sampling_ratio/min": 0.6704867482185364, + "sampling/sampling_logp_difference/max": 0.3997514247894287, + "sampling/sampling_logp_difference/mean": 0.011531875468790531, + "step": 615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010604775889078155, + "clip_ratio/low_min": 0.00010604775889078155, + "clip_ratio/region_mean": 0.00010604775889078155, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 571.84375, + "completions/mean_terminated_length": 551.5516967773438, + "completions/min_length": 390.0, + "completions/min_terminated_length": 390.0, + "entropy": 0.25688663870096207, + "epoch": 0.3297644539614561, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005713587626814842, + "learning_rate": 1e-05, + "loss": 0.0339, + "num_tokens": 13366408.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3288211822509766, + "sampling/importance_sampling_ratio/mean": 0.9999892711639404, + "sampling/importance_sampling_ratio/min": 0.7728637456893921, + "sampling/sampling_logp_difference/max": 0.28429222106933594, + "sampling/sampling_logp_difference/mean": 0.008276664651930332, + "step": 616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 514.59375, + "completions/mean_terminated_length": 506.4193420410156, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.3054012767970562, + "epoch": 0.3302997858672377, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13386995.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.3915033340454102, + "sampling/importance_sampling_ratio/mean": 1.0001293420791626, + "sampling/importance_sampling_ratio/min": 0.7308658957481384, + "sampling/sampling_logp_difference/max": 0.3303847312927246, + "sampling/sampling_logp_difference/mean": 0.010608382523059845, + "step": 617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 680.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 423.84375, + "completions/mean_terminated_length": 423.84375, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.2652575895190239, + "epoch": 0.33083511777301927, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13404278.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.5266615152359009, + "sampling/importance_sampling_ratio/mean": 0.9996988773345947, + "sampling/importance_sampling_ratio/min": 0.7503836750984192, + "sampling/sampling_logp_difference/max": 0.4230833053588867, + "sampling/sampling_logp_difference/mean": 0.008782808668911457, + "step": 618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 479.375, + "completions/mean_terminated_length": 460.13336181640625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.2553304620087147, + "epoch": 0.33137044967880086, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0158, + "num_tokens": 13422898.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3723410367965698, + "sampling/importance_sampling_ratio/mean": 1.0000486373901367, + "sampling/importance_sampling_ratio/min": 0.7063539624214172, + "sampling/sampling_logp_difference/max": 0.3476388454437256, + "sampling/sampling_logp_difference/mean": 0.00843969825655222, + "step": 619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 564.71875, + "completions/mean_terminated_length": 527.0740966796875, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.3039424940943718, + "epoch": 0.33190578158458245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014260738156735897, + "learning_rate": 1e-05, + "loss": 0.0973, + "num_tokens": 13444929.0, + "reward": 0.5, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.298768401145935, + "sampling/importance_sampling_ratio/mean": 0.9998161792755127, + "sampling/importance_sampling_ratio/min": 0.7113383412361145, + "sampling/sampling_logp_difference/max": 0.3406071662902832, + "sampling/sampling_logp_difference/mean": 0.009649362415075302, + "step": 620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 493.53125, + "completions/mean_terminated_length": 475.2333679199219, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.37491171061992645, + "epoch": 0.33244111349036404, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0036310877185314894, + "learning_rate": 1e-05, + "loss": 0.0184, + "num_tokens": 13464506.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4245522022247314, + "sampling/importance_sampling_ratio/mean": 0.9994794130325317, + "sampling/importance_sampling_ratio/min": 0.7612655758857727, + "sampling/sampling_logp_difference/max": 0.35385751724243164, + "sampling/sampling_logp_difference/mean": 0.011728239245712757, + "step": 621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 445.15625, + "completions/mean_terminated_length": 434.7419128417969, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3791900798678398, + "epoch": 0.3329764453961456, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011801331304013729, + "learning_rate": 1e-05, + "loss": -0.0266, + "num_tokens": 13482463.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.329850673675537, + "sampling/importance_sampling_ratio/mean": 1.0001842975616455, + "sampling/importance_sampling_ratio/min": 0.5571074485778809, + "sampling/sampling_logp_difference/max": 0.5849971771240234, + "sampling/sampling_logp_difference/mean": 0.011895262636244297, + "step": 622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 518.0, + "completions/max_terminated_length": 518.0, + "completions/mean_length": 343.84375, + "completions/mean_terminated_length": 343.84375, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "entropy": 0.28707359731197357, + "epoch": 0.3335117773019272, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010176939889788628, + "learning_rate": 1e-05, + "loss": -0.0191, + "num_tokens": 13496850.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3418649435043335, + "sampling/importance_sampling_ratio/mean": 0.9998665452003479, + "sampling/importance_sampling_ratio/min": 0.6146644353866577, + "sampling/sampling_logp_difference/max": 0.4866788387298584, + "sampling/sampling_logp_difference/mean": 0.010198241099715233, + "step": 623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 485.59375, + "completions/mean_terminated_length": 456.3793029785156, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.3566009998321533, + "epoch": 0.3340471092077088, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004952109418809414, + "learning_rate": 1e-05, + "loss": -0.0494, + "num_tokens": 13516069.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4125614166259766, + "sampling/importance_sampling_ratio/mean": 0.9998021125793457, + "sampling/importance_sampling_ratio/min": 0.7557869553565979, + "sampling/sampling_logp_difference/max": 0.34540462493896484, + "sampling/sampling_logp_difference/mean": 0.011198865249752998, + "step": 624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 569.75, + "completions/mean_terminated_length": 524.0, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.3010255694389343, + "epoch": 0.33458244111349034, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002087995409965515, + "learning_rate": 1e-05, + "loss": -0.0104, + "num_tokens": 13538709.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.391015648841858, + "sampling/importance_sampling_ratio/mean": 0.9999024868011475, + "sampling/importance_sampling_ratio/min": 0.6822434067726135, + "sampling/sampling_logp_difference/max": 0.382368803024292, + "sampling/sampling_logp_difference/mean": 0.010051458142697811, + "step": 625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 765.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 460.3125, + "completions/mean_terminated_length": 460.3125, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.36738942563533783, + "epoch": 0.3351177730192719, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018555957823991776, + "learning_rate": 1e-05, + "loss": 0.026, + "num_tokens": 13556767.0, + "reward": 0.6875, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.55997896194458, + "sampling/importance_sampling_ratio/mean": 1.00010085105896, + "sampling/importance_sampling_ratio/min": 0.6896438002586365, + "sampling/sampling_logp_difference/max": 0.4446723461151123, + "sampling/sampling_logp_difference/mean": 0.012395805679261684, + "step": 626 + }, + { + "clip_ratio/high_max": 7.916402682894841e-05, + "clip_ratio/high_mean": 7.916402682894841e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.916402682894841e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 528.84375, + "completions/mean_terminated_length": 449.125, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.47731801494956017, + "epoch": 0.3356531049250535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0235456395894289, + "learning_rate": 1e-05, + "loss": 0.0211, + "num_tokens": 13577474.0, + "reward": 0.65625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.8571079969406128, + "sampling/importance_sampling_ratio/mean": 1.0001784563064575, + "sampling/importance_sampling_ratio/min": 0.2742895483970642, + "sampling/sampling_logp_difference/max": 1.2935709953308105, + "sampling/sampling_logp_difference/mean": 0.013619859702885151, + "step": 627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 540.59375, + "completions/mean_terminated_length": 525.433349609375, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.2912454418838024, + "epoch": 0.3361884368308351, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02371685579419136, + "learning_rate": 1e-05, + "loss": 0.0311, + "num_tokens": 13598141.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3892697095870972, + "sampling/importance_sampling_ratio/mean": 0.9999818205833435, + "sampling/importance_sampling_ratio/min": 0.5333466529846191, + "sampling/sampling_logp_difference/max": 0.6285836696624756, + "sampling/sampling_logp_difference/mean": 0.009452036581933498, + "step": 628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 596.15625, + "completions/mean_terminated_length": 528.9130249023438, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.264277670532465, + "epoch": 0.3367237687366167, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029641496017575264, + "learning_rate": 1e-05, + "loss": 0.0604, + "num_tokens": 13620954.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.2616605758666992, + "sampling/importance_sampling_ratio/mean": 1.0001510381698608, + "sampling/importance_sampling_ratio/min": 0.6960204243659973, + "sampling/sampling_logp_difference/max": 0.36237621307373047, + "sampling/sampling_logp_difference/mean": 0.008369022980332375, + "step": 629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.439979188144207e-05, + "clip_ratio/low_min": 6.439979188144207e-05, + "clip_ratio/region_mean": 6.439979188144207e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 472.0625, + "completions/mean_terminated_length": 441.4482727050781, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.26084725745022297, + "epoch": 0.3372591006423983, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02300383523106575, + "learning_rate": 1e-05, + "loss": 0.034, + "num_tokens": 13639708.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2997791767120361, + "sampling/importance_sampling_ratio/mean": 0.9998540282249451, + "sampling/importance_sampling_ratio/min": 0.7616881728172302, + "sampling/sampling_logp_difference/max": 0.2722179889678955, + "sampling/sampling_logp_difference/mean": 0.00920176226645708, + "step": 630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 691.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 426.5, + "completions/mean_terminated_length": 426.5, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.311075109988451, + "epoch": 0.33779443254817987, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006628968752920628, + "learning_rate": 1e-05, + "loss": -0.0028, + "num_tokens": 13657076.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4387227296829224, + "sampling/importance_sampling_ratio/mean": 0.9996896982192993, + "sampling/importance_sampling_ratio/min": 0.6817339658737183, + "sampling/sampling_logp_difference/max": 0.3831157684326172, + "sampling/sampling_logp_difference/mean": 0.010619109496474266, + "step": 631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 509.125, + "completions/mean_terminated_length": 500.774169921875, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "entropy": 0.2965533994138241, + "epoch": 0.33832976445396146, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13676968.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.3712502717971802, + "sampling/importance_sampling_ratio/mean": 1.0000545978546143, + "sampling/importance_sampling_ratio/min": 0.7730183005332947, + "sampling/sampling_logp_difference/max": 0.3157229423522949, + "sampling/sampling_logp_difference/mean": 0.009084545075893402, + "step": 632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012959235755261034, + "clip_ratio/low_min": 0.00012959235755261034, + "clip_ratio/region_mean": 0.00012959235755261034, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 524.375, + "completions/mean_terminated_length": 499.17242431640625, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.41893286257982254, + "epoch": 0.33886509635974305, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00824392307549715, + "learning_rate": 1e-05, + "loss": 0.0185, + "num_tokens": 13698004.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4280670881271362, + "sampling/importance_sampling_ratio/mean": 1.0004184246063232, + "sampling/importance_sampling_ratio/min": 0.5557708740234375, + "sampling/sampling_logp_difference/max": 0.5873992443084717, + "sampling/sampling_logp_difference/mean": 0.012900300323963165, + "step": 633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 395.78125, + "completions/mean_terminated_length": 383.774169921875, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.32786916568875313, + "epoch": 0.33940042826552463, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.002, + "num_tokens": 13714381.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3311372995376587, + "sampling/importance_sampling_ratio/mean": 1.000272274017334, + "sampling/importance_sampling_ratio/min": 0.7063815593719482, + "sampling/sampling_logp_difference/max": 0.34759974479675293, + "sampling/sampling_logp_difference/mean": 0.011236365884542465, + "step": 634 + }, + { + "clip_ratio/high_max": 6.55479816487059e-05, + "clip_ratio/high_mean": 6.55479816487059e-05, + "clip_ratio/low_mean": 0.00023448842330253683, + "clip_ratio/low_min": 0.00023448842330253683, + "clip_ratio/region_mean": 0.00030003640495124273, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 530.1875, + "completions/mean_terminated_length": 450.91668701171875, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.354931078851223, + "epoch": 0.3399357601713062, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021596211940050125, + "learning_rate": 1e-05, + "loss": -0.0105, + "num_tokens": 13735019.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5256561040878296, + "sampling/importance_sampling_ratio/mean": 1.0000821352005005, + "sampling/importance_sampling_ratio/min": 0.6678833961486816, + "sampling/sampling_logp_difference/max": 0.4224245548248291, + "sampling/sampling_logp_difference/mean": 0.01115508284419775, + "step": 635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 675.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 429.15625, + "completions/mean_terminated_length": 429.15625, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.29260409623384476, + "epoch": 0.3404710920770878, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13752848.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4287768602371216, + "sampling/importance_sampling_ratio/mean": 0.9997872114181519, + "sampling/importance_sampling_ratio/min": 0.7665042281150818, + "sampling/sampling_logp_difference/max": 0.35681867599487305, + "sampling/sampling_logp_difference/mean": 0.009826496243476868, + "step": 636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015769056699355133, + "clip_ratio/low_min": 0.00015769056699355133, + "clip_ratio/region_mean": 0.00015769056699355133, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 554.96875, + "completions/mean_terminated_length": 540.7667236328125, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.3323060441762209, + "epoch": 0.3410064239828694, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014778217300772667, + "learning_rate": 1e-05, + "loss": 0.0251, + "num_tokens": 13775199.0, + "reward": 0.59375, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4481210708618164, + "sampling/importance_sampling_ratio/mean": 1.000173568725586, + "sampling/importance_sampling_ratio/min": 0.7709746956825256, + "sampling/sampling_logp_difference/max": 0.3702669143676758, + "sampling/sampling_logp_difference/mean": 0.011516406200826168, + "step": 637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 554.5625, + "completions/mean_terminated_length": 505.3077087402344, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.45366695895791054, + "epoch": 0.341541755888651, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015367541462182999, + "learning_rate": 1e-05, + "loss": -0.0083, + "num_tokens": 13797153.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.461901307106018, + "sampling/importance_sampling_ratio/mean": 1.0005213022232056, + "sampling/importance_sampling_ratio/min": 0.6465102434158325, + "sampling/sampling_logp_difference/max": 0.43616628646850586, + "sampling/sampling_logp_difference/mean": 0.013607689179480076, + "step": 638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.610259151784703e-05, + "clip_ratio/low_min": 6.610259151784703e-05, + "clip_ratio/region_mean": 6.610259151784703e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 682.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 475.34375, + "completions/mean_terminated_length": 475.34375, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.26533373817801476, + "epoch": 0.3420770877944325, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026921937242150307, + "learning_rate": 1e-05, + "loss": 0.0135, + "num_tokens": 13815828.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4592902660369873, + "sampling/importance_sampling_ratio/mean": 0.9997817873954773, + "sampling/importance_sampling_ratio/min": 0.7529975771903992, + "sampling/sampling_logp_difference/max": 0.37795019149780273, + "sampling/sampling_logp_difference/mean": 0.009174923412501812, + "step": 639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 735.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 410.84375, + "completions/mean_terminated_length": 410.84375, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.3154573068022728, + "epoch": 0.3426124197002141, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01859649457037449, + "learning_rate": 1e-05, + "loss": -0.0104, + "num_tokens": 13832559.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.2813645601272583, + "sampling/importance_sampling_ratio/mean": 1.0000611543655396, + "sampling/importance_sampling_ratio/min": 0.700233519077301, + "sampling/sampling_logp_difference/max": 0.3563413619995117, + "sampling/sampling_logp_difference/mean": 0.010139619931578636, + "step": 640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 477.09375, + "completions/mean_terminated_length": 467.70965576171875, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.2767404969781637, + "epoch": 0.3431477516059957, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0419, + "num_tokens": 13851330.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3960591554641724, + "sampling/importance_sampling_ratio/mean": 1.0000600814819336, + "sampling/importance_sampling_ratio/min": 0.7296417951583862, + "sampling/sampling_logp_difference/max": 0.33365345001220703, + "sampling/sampling_logp_difference/mean": 0.00965786911547184, + "step": 641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.972288636257872e-05, + "clip_ratio/low_min": 5.972288636257872e-05, + "clip_ratio/region_mean": 5.972288636257872e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 456.15625, + "completions/mean_terminated_length": 423.89654541015625, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.3610168881714344, + "epoch": 0.3436830835117773, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011776655912399292, + "learning_rate": 1e-05, + "loss": 0.0774, + "num_tokens": 13869991.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6153571605682373, + "sampling/importance_sampling_ratio/mean": 1.0004236698150635, + "sampling/importance_sampling_ratio/min": 0.7631751298904419, + "sampling/sampling_logp_difference/max": 0.4795560836791992, + "sampling/sampling_logp_difference/mean": 0.012264749966561794, + "step": 642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 570.5, + "completions/mean_terminated_length": 480.727294921875, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.4524570107460022, + "epoch": 0.3442184154175589, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005275725852698088, + "learning_rate": 1e-05, + "loss": 0.0277, + "num_tokens": 13892391.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6950676441192627, + "sampling/importance_sampling_ratio/mean": 0.9998581409454346, + "sampling/importance_sampling_ratio/min": 0.6842971444129944, + "sampling/sampling_logp_difference/max": 0.5277225971221924, + "sampling/sampling_logp_difference/mean": 0.014087657444179058, + "step": 643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 432.8125, + "completions/mean_terminated_length": 422.0, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "entropy": 0.31366887129843235, + "epoch": 0.34475374732334046, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012850224040448666, + "learning_rate": 1e-05, + "loss": 0.0203, + "num_tokens": 13909697.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.425286054611206, + "sampling/importance_sampling_ratio/mean": 1.0000022649765015, + "sampling/importance_sampling_ratio/min": 0.6593077182769775, + "sampling/sampling_logp_difference/max": 0.41656494140625, + "sampling/sampling_logp_difference/mean": 0.009947948157787323, + "step": 644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001031969441100955, + "clip_ratio/low_min": 0.0001031969441100955, + "clip_ratio/region_mean": 0.0001031969441100955, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 544.3125, + "completions/mean_terminated_length": 456.7826232910156, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.37259775772690773, + "epoch": 0.34528907922912205, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010465526022017002, + "learning_rate": 1e-05, + "loss": 0.1164, + "num_tokens": 13930667.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3667930364608765, + "sampling/importance_sampling_ratio/mean": 1.0000056028366089, + "sampling/importance_sampling_ratio/min": 0.6974546909332275, + "sampling/sampling_logp_difference/max": 0.3603177070617676, + "sampling/sampling_logp_difference/mean": 0.011153936386108398, + "step": 645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011498959065647796, + "clip_ratio/low_min": 0.00011498959065647796, + "clip_ratio/region_mean": 0.00011498959065647796, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 492.8125, + "completions/mean_terminated_length": 453.5000305175781, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3283007889986038, + "epoch": 0.34582441113490364, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008446267805993557, + "learning_rate": 1e-05, + "loss": 0.0214, + "num_tokens": 13950317.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4042552709579468, + "sampling/importance_sampling_ratio/mean": 1.0003057718276978, + "sampling/importance_sampling_ratio/min": 0.7099096775054932, + "sampling/sampling_logp_difference/max": 0.3426175117492676, + "sampling/sampling_logp_difference/mean": 0.011360456235706806, + "step": 646 + }, + { + "clip_ratio/high_max": 6.0444875998655334e-05, + "clip_ratio/high_mean": 6.0444875998655334e-05, + "clip_ratio/low_mean": 0.00010931084398180246, + "clip_ratio/low_min": 0.00010931084398180246, + "clip_ratio/region_mean": 0.0001697557199804578, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 546.28125, + "completions/mean_terminated_length": 472.375, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.42438481748104095, + "epoch": 0.34635974304068523, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0141336964443326, + "learning_rate": 1e-05, + "loss": -0.0179, + "num_tokens": 13972102.0, + "reward": 0.40625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6954357624053955, + "sampling/importance_sampling_ratio/mean": 1.000226616859436, + "sampling/importance_sampling_ratio/min": 0.6467899084091187, + "sampling/sampling_logp_difference/max": 0.5279397964477539, + "sampling/sampling_logp_difference/mean": 0.013065065257251263, + "step": 647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013569513976108283, + "clip_ratio/low_min": 0.00013569513976108283, + "clip_ratio/region_mean": 0.00013569513976108283, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 524.4375, + "completions/mean_terminated_length": 489.64288330078125, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.30432552844285965, + "epoch": 0.3468950749464668, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004665049258619547, + "learning_rate": 1e-05, + "loss": 0.0898, + "num_tokens": 13992652.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4343668222427368, + "sampling/importance_sampling_ratio/mean": 1.0000685453414917, + "sampling/importance_sampling_ratio/min": 0.6709504127502441, + "sampling/sampling_logp_difference/max": 0.3990600109100342, + "sampling/sampling_logp_difference/mean": 0.009907660074532032, + "step": 648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 475.53125, + "completions/mean_terminated_length": 466.0967712402344, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.23276705294847488, + "epoch": 0.3474304068522484, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009352444671094418, + "learning_rate": 1e-05, + "loss": 0.0069, + "num_tokens": 14011093.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4106377363204956, + "sampling/importance_sampling_ratio/mean": 1.0000784397125244, + "sampling/importance_sampling_ratio/min": 0.7408989667892456, + "sampling/sampling_logp_difference/max": 0.3440418243408203, + "sampling/sampling_logp_difference/mean": 0.0077589633874595165, + "step": 649 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023136611707741395, + "clip_ratio/low_min": 0.00023136611707741395, + "clip_ratio/region_mean": 0.00023136611707741395, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 426.28125, + "completions/mean_terminated_length": 415.258056640625, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.37946831807494164, + "epoch": 0.34796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015928439795970917, + "learning_rate": 1e-05, + "loss": 0.0351, + "num_tokens": 14028910.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3176966905593872, + "sampling/importance_sampling_ratio/mean": 0.9998493790626526, + "sampling/importance_sampling_ratio/min": 0.7060791850090027, + "sampling/sampling_logp_difference/max": 0.34802794456481934, + "sampling/sampling_logp_difference/mean": 0.012711400166153908, + "step": 650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010587761789793149, + "clip_ratio/low_min": 0.00010587761789793149, + "clip_ratio/region_mean": 0.00010587761789793149, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 561.4375, + "completions/mean_terminated_length": 492.5833435058594, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.40715794265270233, + "epoch": 0.3485010706638116, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009008923545479774, + "learning_rate": 1e-05, + "loss": 0.047, + "num_tokens": 14051060.0, + "reward": 0.40625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.556815266609192, + "sampling/importance_sampling_ratio/mean": 0.9994319081306458, + "sampling/importance_sampling_ratio/min": 0.6722291707992554, + "sampling/sampling_logp_difference/max": 0.4426422119140625, + "sampling/sampling_logp_difference/mean": 0.01241252664476633, + "step": 651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.698677993379533e-05, + "clip_ratio/low_min": 8.698677993379533e-05, + "clip_ratio/region_mean": 8.698677993379533e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 733.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 408.71875, + "completions/mean_terminated_length": 408.71875, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.350768007338047, + "epoch": 0.3490364025695932, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03018689714372158, + "learning_rate": 1e-05, + "loss": 0.0085, + "num_tokens": 14067859.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3940702676773071, + "sampling/importance_sampling_ratio/mean": 0.9998666048049927, + "sampling/importance_sampling_ratio/min": 0.7021839022636414, + "sampling/sampling_logp_difference/max": 0.3535599708557129, + "sampling/sampling_logp_difference/mean": 0.011442244052886963, + "step": 652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 525.9375, + "completions/mean_terminated_length": 525.9375, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "entropy": 0.26179109886288643, + "epoch": 0.3495717344753747, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014005845412611961, + "learning_rate": 1e-05, + "loss": -0.0034, + "num_tokens": 14088801.0, + "reward": 0.6875, + "reward_std": 0.3924052119255066, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3349133729934692, + "sampling/importance_sampling_ratio/mean": 0.9999707937240601, + "sampling/importance_sampling_ratio/min": 0.7767595648765564, + "sampling/sampling_logp_difference/max": 0.28886640071868896, + "sampling/sampling_logp_difference/mean": 0.008738524280488491, + "step": 653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011466631258372217, + "clip_ratio/low_min": 0.00011466631258372217, + "clip_ratio/region_mean": 0.00011466631258372217, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 486.71875, + "completions/mean_terminated_length": 407.9599914550781, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.5003787465393543, + "epoch": 0.3501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.028450779616832733, + "learning_rate": 1e-05, + "loss": 0.0732, + "num_tokens": 14107896.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.380290150642395, + "sampling/importance_sampling_ratio/mean": 0.9999485611915588, + "sampling/importance_sampling_ratio/min": 0.5255972743034363, + "sampling/sampling_logp_difference/max": 0.6432199478149414, + "sampling/sampling_logp_difference/mean": 0.015035094693303108, + "step": 654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.720823901356198e-05, + "clip_ratio/low_min": 5.720823901356198e-05, + "clip_ratio/region_mean": 5.720823901356198e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 469.28125, + "completions/mean_terminated_length": 413.96295166015625, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3441755622625351, + "epoch": 0.3506423982869379, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0384136363863945, + "learning_rate": 1e-05, + "loss": 0.1092, + "num_tokens": 14126193.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4771353006362915, + "sampling/importance_sampling_ratio/mean": 1.0000910758972168, + "sampling/importance_sampling_ratio/min": 0.5364977121353149, + "sampling/sampling_logp_difference/max": 0.6226930618286133, + "sampling/sampling_logp_difference/mean": 0.011351614259183407, + "step": 655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.860290548298508e-05, + "clip_ratio/low_min": 5.860290548298508e-05, + "clip_ratio/region_mean": 5.860290548298508e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 570.46875, + "completions/mean_terminated_length": 550.0344848632812, + "completions/min_length": 365.0, + "completions/min_terminated_length": 365.0, + "entropy": 0.3153909929096699, + "epoch": 0.3511777301927195, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0221896693110466, + "learning_rate": 1e-05, + "loss": 0.028, + "num_tokens": 14147664.0, + "reward": 0.6875, + "reward_std": 0.4808131456375122, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.607094645500183, + "sampling/importance_sampling_ratio/mean": 1.0001755952835083, + "sampling/importance_sampling_ratio/min": 0.7048884630203247, + "sampling/sampling_logp_difference/max": 0.4744279384613037, + "sampling/sampling_logp_difference/mean": 0.009652480483055115, + "step": 656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 392.90625, + "completions/mean_terminated_length": 392.90625, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.34484558179974556, + "epoch": 0.35171306209850106, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012628809548914433, + "learning_rate": 1e-05, + "loss": 0.0514, + "num_tokens": 14164893.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3562942743301392, + "sampling/importance_sampling_ratio/mean": 0.999950647354126, + "sampling/importance_sampling_ratio/min": 0.7395182251930237, + "sampling/sampling_logp_difference/max": 0.30475616455078125, + "sampling/sampling_logp_difference/mean": 0.011204061098396778, + "step": 657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 451.875, + "completions/mean_terminated_length": 441.6773986816406, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.30138526670634747, + "epoch": 0.35224839400428265, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014551598578691483, + "learning_rate": 1e-05, + "loss": 0.0002, + "num_tokens": 14183577.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.8423817157745361, + "sampling/importance_sampling_ratio/mean": 1.0001647472381592, + "sampling/importance_sampling_ratio/min": 0.6992892622947693, + "sampling/sampling_logp_difference/max": 0.6110591888427734, + "sampling/sampling_logp_difference/mean": 0.010228762403130531, + "step": 658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010833868509507738, + "clip_ratio/low_min": 0.00010833868509507738, + "clip_ratio/region_mean": 0.00010833868509507738, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 517.625, + "completions/mean_terminated_length": 419.6521911621094, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.38055815920233727, + "epoch": 0.35278372591006424, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008826745674014091, + "learning_rate": 1e-05, + "loss": 0.0248, + "num_tokens": 14204133.0, + "reward": 0.46875, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4185012578964233, + "sampling/importance_sampling_ratio/mean": 1.0001429319381714, + "sampling/importance_sampling_ratio/min": 0.7214440703392029, + "sampling/sampling_logp_difference/max": 0.34960079193115234, + "sampling/sampling_logp_difference/mean": 0.011931713670492172, + "step": 659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012206177780171856, + "clip_ratio/low_min": 0.00012206177780171856, + "clip_ratio/region_mean": 0.00012206177780171856, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 514.0625, + "completions/mean_terminated_length": 497.13336181640625, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.2995508201420307, + "epoch": 0.3533190578158458, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010350965894758701, + "learning_rate": 1e-05, + "loss": 0.0044, + "num_tokens": 14224871.0, + "reward": 0.65625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.434417486190796, + "sampling/importance_sampling_ratio/mean": 0.9998875260353088, + "sampling/importance_sampling_ratio/min": 0.6783097386360168, + "sampling/sampling_logp_difference/max": 0.3881511688232422, + "sampling/sampling_logp_difference/mean": 0.009691186249256134, + "step": 660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 433.25, + "completions/mean_terminated_length": 433.25, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.32164764404296875, + "epoch": 0.3538543897216274, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010363774374127388, + "learning_rate": 1e-05, + "loss": -0.0382, + "num_tokens": 14242391.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.413983941078186, + "sampling/importance_sampling_ratio/mean": 1.0000758171081543, + "sampling/importance_sampling_ratio/min": 0.6953238248825073, + "sampling/sampling_logp_difference/max": 0.36337757110595703, + "sampling/sampling_logp_difference/mean": 0.010926454328000546, + "step": 661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 713.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 436.09375, + "completions/mean_terminated_length": 436.09375, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.25281756557524204, + "epoch": 0.354389721627409, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02384403720498085, + "learning_rate": 1e-05, + "loss": -0.0308, + "num_tokens": 14259514.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.2802929878234863, + "sampling/importance_sampling_ratio/mean": 0.999589741230011, + "sampling/importance_sampling_ratio/min": 0.6354244351387024, + "sampling/sampling_logp_difference/max": 0.4534621238708496, + "sampling/sampling_logp_difference/mean": 0.008979644626379013, + "step": 662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021215934248175472, + "clip_ratio/low_min": 0.00021215934248175472, + "clip_ratio/region_mean": 0.00021215934248175472, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 560.6875, + "completions/mean_terminated_length": 466.4545593261719, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.43268299102783203, + "epoch": 0.3549250535331906, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020790722221136093, + "learning_rate": 1e-05, + "loss": 0.0202, + "num_tokens": 14281560.0, + "reward": 0.46875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4320964813232422, + "sampling/importance_sampling_ratio/mean": 0.9997743964195251, + "sampling/importance_sampling_ratio/min": 0.5930963158607483, + "sampling/sampling_logp_difference/max": 0.5223984718322754, + "sampling/sampling_logp_difference/mean": 0.013722680509090424, + "step": 663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 464.51611328125, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.4267589747905731, + "epoch": 0.3554603854389722, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009368442930281162, + "learning_rate": 1e-05, + "loss": 0.0372, + "num_tokens": 14301176.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.8331667184829712, + "sampling/importance_sampling_ratio/mean": 0.9994924664497375, + "sampling/importance_sampling_ratio/min": 0.5951104164123535, + "sampling/sampling_logp_difference/max": 0.6060450077056885, + "sampling/sampling_logp_difference/mean": 0.01349746622145176, + "step": 664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.79301772126928e-05, + "clip_ratio/low_min": 7.79301772126928e-05, + "clip_ratio/region_mean": 7.79301772126928e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 418.25, + "completions/mean_terminated_length": 394.933349609375, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "entropy": 0.2833059933036566, + "epoch": 0.35599571734475377, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008563783019781113, + "learning_rate": 1e-05, + "loss": 0.0533, + "num_tokens": 14317992.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.484289288520813, + "sampling/importance_sampling_ratio/mean": 1.0001963376998901, + "sampling/importance_sampling_ratio/min": 0.6188217401504517, + "sampling/sampling_logp_difference/max": 0.4799380302429199, + "sampling/sampling_logp_difference/mean": 0.01006927527487278, + "step": 665 + }, + { + "clip_ratio/high_max": 5.397236600401811e-05, + "clip_ratio/high_mean": 5.397236600401811e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.397236600401811e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 561.25, + "completions/mean_terminated_length": 531.7142944335938, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.29960734583437443, + "epoch": 0.35653104925053536, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007625725120306015, + "learning_rate": 1e-05, + "loss": 0.0084, + "num_tokens": 14339680.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.432206392288208, + "sampling/importance_sampling_ratio/mean": 1.0004332065582275, + "sampling/importance_sampling_ratio/min": 0.7404946088790894, + "sampling/sampling_logp_difference/max": 0.35921621322631836, + "sampling/sampling_logp_difference/mean": 0.010498258285224438, + "step": 666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000206250864721369, + "clip_ratio/low_min": 0.000206250864721369, + "clip_ratio/region_mean": 0.000206250864721369, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 427.5625, + "completions/mean_terminated_length": 404.86669921875, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.3941080830991268, + "epoch": 0.3570663811563169, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020041802898049355, + "learning_rate": 1e-05, + "loss": 0.0143, + "num_tokens": 14356858.0, + "reward": 0.65625, + "reward_std": 0.38816186785697937, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.396597146987915, + "sampling/importance_sampling_ratio/mean": 0.9999650120735168, + "sampling/importance_sampling_ratio/min": 0.6440496444702148, + "sampling/sampling_logp_difference/max": 0.4399794340133667, + "sampling/sampling_logp_difference/mean": 0.012557687237858772, + "step": 667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.006727562635206e-05, + "clip_ratio/low_min": 6.006727562635206e-05, + "clip_ratio/region_mean": 6.006727562635206e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 449.6875, + "completions/mean_terminated_length": 439.4193420410156, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.2734356876462698, + "epoch": 0.3576017130620985, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006712873931974173, + "learning_rate": 1e-05, + "loss": 0.0749, + "num_tokens": 14374464.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.2982434034347534, + "sampling/importance_sampling_ratio/mean": 1.000036597251892, + "sampling/importance_sampling_ratio/min": 0.7399297952651978, + "sampling/sampling_logp_difference/max": 0.30119991302490234, + "sampling/sampling_logp_difference/mean": 0.009010636247694492, + "step": 668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 399.25, + "completions/mean_terminated_length": 399.25, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.2868831194937229, + "epoch": 0.35813704496788007, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004024685360491276, + "learning_rate": 1e-05, + "loss": -0.0013, + "num_tokens": 14391328.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.7804118394851685, + "sampling/importance_sampling_ratio/mean": 0.9999974966049194, + "sampling/importance_sampling_ratio/min": 0.749528706073761, + "sampling/sampling_logp_difference/max": 0.5768446922302246, + "sampling/sampling_logp_difference/mean": 0.009922800585627556, + "step": 669 + }, + { + "clip_ratio/high_max": 5.55308761249762e-05, + "clip_ratio/high_mean": 5.55308761249762e-05, + "clip_ratio/low_mean": 5.582849553320557e-05, + "clip_ratio/low_min": 5.582849553320557e-05, + "clip_ratio/region_mean": 0.00011135937165818177, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 503.875, + "completions/mean_terminated_length": 415.8333435058594, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.4187277927994728, + "epoch": 0.35867237687366166, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00617664260789752, + "learning_rate": 1e-05, + "loss": 0.0103, + "num_tokens": 14411612.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4836184978485107, + "sampling/importance_sampling_ratio/mean": 1.0000054836273193, + "sampling/importance_sampling_ratio/min": 0.610197126865387, + "sampling/sampling_logp_difference/max": 0.4939732551574707, + "sampling/sampling_logp_difference/mean": 0.012816681526601315, + "step": 670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.668934318237007e-05, + "clip_ratio/low_min": 5.668934318237007e-05, + "clip_ratio/region_mean": 5.668934318237007e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 472.78125, + "completions/mean_terminated_length": 442.2413635253906, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.28476603515446186, + "epoch": 0.35920770877944325, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01838054321706295, + "learning_rate": 1e-05, + "loss": 0.0899, + "num_tokens": 14430253.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3775122165679932, + "sampling/importance_sampling_ratio/mean": 0.9997398257255554, + "sampling/importance_sampling_ratio/min": 0.6943086981773376, + "sampling/sampling_logp_difference/max": 0.3648386001586914, + "sampling/sampling_logp_difference/mean": 0.009384599514305592, + "step": 671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 456.875, + "completions/mean_terminated_length": 446.83868408203125, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3012051973491907, + "epoch": 0.35974304068522484, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01845335215330124, + "learning_rate": 1e-05, + "loss": 0.0221, + "num_tokens": 14448897.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.390568494796753, + "sampling/importance_sampling_ratio/mean": 1.000192642211914, + "sampling/importance_sampling_ratio/min": 0.5708194375038147, + "sampling/sampling_logp_difference/max": 0.5606822967529297, + "sampling/sampling_logp_difference/mean": 0.010296937078237534, + "step": 672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 393.03125, + "completions/mean_terminated_length": 393.03125, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.27892127074301243, + "epoch": 0.3602783725910064, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024538559839129448, + "learning_rate": 1e-05, + "loss": -0.0102, + "num_tokens": 14465018.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.41757071018219, + "sampling/importance_sampling_ratio/mean": 0.9998502731323242, + "sampling/importance_sampling_ratio/min": 0.700309693813324, + "sampling/sampling_logp_difference/max": 0.3562326431274414, + "sampling/sampling_logp_difference/mean": 0.010319615714251995, + "step": 673 + }, + { + "clip_ratio/high_max": 6.558236782439053e-05, + "clip_ratio/high_mean": 6.558236782439053e-05, + "clip_ratio/low_mean": 5.653550397255458e-05, + "clip_ratio/low_min": 5.653550397255458e-05, + "clip_ratio/region_mean": 0.0001221178717969451, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 543.15625, + "completions/mean_terminated_length": 501.5185241699219, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.4119257219135761, + "epoch": 0.360813704496788, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02290952019393444, + "learning_rate": 1e-05, + "loss": 0.0779, + "num_tokens": 14486527.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5678411722183228, + "sampling/importance_sampling_ratio/mean": 0.9999833703041077, + "sampling/importance_sampling_ratio/min": 0.7061614990234375, + "sampling/sampling_logp_difference/max": 0.44969964027404785, + "sampling/sampling_logp_difference/mean": 0.01298696082085371, + "step": 674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.417508070124313e-05, + "clip_ratio/low_min": 8.417508070124313e-05, + "clip_ratio/region_mean": 8.417508070124313e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 418.03125, + "completions/mean_terminated_length": 406.7419128417969, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.3174128942191601, + "epoch": 0.3613490364025696, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0409, + "num_tokens": 14503728.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3247625827789307, + "sampling/importance_sampling_ratio/mean": 1.0001379251480103, + "sampling/importance_sampling_ratio/min": 0.6964145302772522, + "sampling/sampling_logp_difference/max": 0.36181020736694336, + "sampling/sampling_logp_difference/mean": 0.01076096948236227, + "step": 675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001639887923374772, + "clip_ratio/low_min": 0.0001639887923374772, + "clip_ratio/region_mean": 0.0001639887923374772, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 398.21875, + "completions/mean_terminated_length": 386.2903137207031, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.42826948687434196, + "epoch": 0.3618843683083512, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018404463306069374, + "learning_rate": 1e-05, + "loss": 0.0451, + "num_tokens": 14520439.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4523186683654785, + "sampling/importance_sampling_ratio/mean": 1.0004849433898926, + "sampling/importance_sampling_ratio/min": 0.5930233597755432, + "sampling/sampling_logp_difference/max": 0.5225214958190918, + "sampling/sampling_logp_difference/mean": 0.01220714021474123, + "step": 676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 527.5625, + "completions/mean_terminated_length": 472.0769348144531, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.398250425234437, + "epoch": 0.3624197002141328, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 14541169.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.8422622680664062, + "sampling/importance_sampling_ratio/mean": 1.0002211332321167, + "sampling/importance_sampling_ratio/min": 0.6787064075469971, + "sampling/sampling_logp_difference/max": 0.6109943389892578, + "sampling/sampling_logp_difference/mean": 0.012101640924811363, + "step": 677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 454.25, + "completions/mean_terminated_length": 444.1290283203125, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.2895977236330509, + "epoch": 0.36295503211991437, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018193962052464485, + "learning_rate": 1e-05, + "loss": 0.0717, + "num_tokens": 14558889.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.408778429031372, + "sampling/importance_sampling_ratio/mean": 0.9998857975006104, + "sampling/importance_sampling_ratio/min": 0.715449333190918, + "sampling/sampling_logp_difference/max": 0.34272289276123047, + "sampling/sampling_logp_difference/mean": 0.009631810709834099, + "step": 678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 617.0, + "completions/mean_length": 448.21875, + "completions/mean_terminated_length": 437.9031982421875, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.2796888202428818, + "epoch": 0.36349036402569596, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0073, + "num_tokens": 14576816.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6304254531860352, + "sampling/importance_sampling_ratio/mean": 1.0001057386398315, + "sampling/importance_sampling_ratio/min": 0.7328755855560303, + "sampling/sampling_logp_difference/max": 0.48884105682373047, + "sampling/sampling_logp_difference/mean": 0.010036729276180267, + "step": 679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.365940109593794e-05, + "clip_ratio/low_min": 7.365940109593794e-05, + "clip_ratio/region_mean": 7.365940109593794e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 435.03125, + "completions/mean_terminated_length": 435.03125, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.2705319318920374, + "epoch": 0.3640256959314775, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005537100601941347, + "learning_rate": 1e-05, + "loss": -0.0327, + "num_tokens": 14594233.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.7540359497070312, + "sampling/importance_sampling_ratio/mean": 0.9998466968536377, + "sampling/importance_sampling_ratio/min": 0.6739554405212402, + "sampling/sampling_logp_difference/max": 0.5619193315505981, + "sampling/sampling_logp_difference/mean": 0.009636824019253254, + "step": 680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.959911115700379e-05, + "clip_ratio/low_min": 6.959911115700379e-05, + "clip_ratio/region_mean": 6.959911115700379e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 499.6875, + "completions/mean_terminated_length": 461.357177734375, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.403135072439909, + "epoch": 0.3645610278372591, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004972268361598253, + "learning_rate": 1e-05, + "loss": 0.012, + "num_tokens": 14613935.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5715802907943726, + "sampling/importance_sampling_ratio/mean": 0.9998313784599304, + "sampling/importance_sampling_ratio/min": 0.7040213346481323, + "sampling/sampling_logp_difference/max": 0.45208168029785156, + "sampling/sampling_logp_difference/mean": 0.012608659453690052, + "step": 681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.022471982054412e-05, + "clip_ratio/low_min": 7.022471982054412e-05, + "clip_ratio/region_mean": 7.022471982054412e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 498.59375, + "completions/mean_terminated_length": 470.72412109375, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.3240054100751877, + "epoch": 0.36509635974304067, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01842893287539482, + "learning_rate": 1e-05, + "loss": 0.0445, + "num_tokens": 14634002.0, + "reward": 0.6875, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.2981414794921875, + "sampling/importance_sampling_ratio/mean": 0.9996616840362549, + "sampling/importance_sampling_ratio/min": 0.731063961982727, + "sampling/sampling_logp_difference/max": 0.31325435638427734, + "sampling/sampling_logp_difference/mean": 0.010245262645184994, + "step": 682 + }, + { + "clip_ratio/high_max": 8.79662184161134e-05, + "clip_ratio/high_mean": 8.79662184161134e-05, + "clip_ratio/low_mean": 0.00017959770048037171, + "clip_ratio/low_min": 0.00017959770048037171, + "clip_ratio/region_mean": 0.0002675639188964851, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 373.71875, + "completions/mean_terminated_length": 373.71875, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.32563577592372894, + "epoch": 0.36563169164882225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015845371410250664, + "learning_rate": 1e-05, + "loss": 0.0122, + "num_tokens": 14649825.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3533612489700317, + "sampling/importance_sampling_ratio/mean": 0.9999992251396179, + "sampling/importance_sampling_ratio/min": 0.7251482605934143, + "sampling/sampling_logp_difference/max": 0.3213791847229004, + "sampling/sampling_logp_difference/mean": 0.011381244286894798, + "step": 683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 436.875, + "completions/mean_terminated_length": 426.19354248046875, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.39201460406184196, + "epoch": 0.36616702355460384, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01667793281376362, + "learning_rate": 1e-05, + "loss": 0.0147, + "num_tokens": 14667477.0, + "reward": 0.71875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6575206518173218, + "sampling/importance_sampling_ratio/mean": 0.9999275803565979, + "sampling/importance_sampling_ratio/min": 0.739303469657898, + "sampling/sampling_logp_difference/max": 0.5053229331970215, + "sampling/sampling_logp_difference/mean": 0.012439138256013393, + "step": 684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.534659380326048e-05, + "clip_ratio/low_min": 7.534659380326048e-05, + "clip_ratio/region_mean": 7.534659380326048e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 537.28125, + "completions/mean_terminated_length": 472.67999267578125, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.42449041455984116, + "epoch": 0.36670235546038543, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006100288592278957, + "learning_rate": 1e-05, + "loss": 0.0174, + "num_tokens": 14688654.0, + "reward": 0.46875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.693973422050476, + "sampling/importance_sampling_ratio/mean": 1.0002092123031616, + "sampling/importance_sampling_ratio/min": 0.7011035680770874, + "sampling/sampling_logp_difference/max": 0.5270768404006958, + "sampling/sampling_logp_difference/mean": 0.012892551720142365, + "step": 685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001289334468310699, + "clip_ratio/low_min": 0.0001289334468310699, + "clip_ratio/region_mean": 0.0001289334468310699, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 454.875, + "completions/mean_terminated_length": 434.0000305175781, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3620806783437729, + "epoch": 0.367237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010676519945263863, + "learning_rate": 1e-05, + "loss": 0.0076, + "num_tokens": 14706810.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3780028820037842, + "sampling/importance_sampling_ratio/mean": 0.9999836087226868, + "sampling/importance_sampling_ratio/min": 0.7050690054893494, + "sampling/sampling_logp_difference/max": 0.3494596481323242, + "sampling/sampling_logp_difference/mean": 0.01154541689902544, + "step": 686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.456611299654469e-05, + "clip_ratio/low_min": 6.456611299654469e-05, + "clip_ratio/region_mean": 6.456611299654469e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 617.0, + "completions/mean_length": 425.8125, + "completions/mean_terminated_length": 414.774169921875, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.31526566483080387, + "epoch": 0.3677730192719486, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.014808454550802708, + "learning_rate": 1e-05, + "loss": -0.0027, + "num_tokens": 14723460.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5825399160385132, + "sampling/importance_sampling_ratio/mean": 1.0002360343933105, + "sampling/importance_sampling_ratio/min": 0.6621766686439514, + "sampling/sampling_logp_difference/max": 0.4590311050415039, + "sampling/sampling_logp_difference/mean": 0.010774160735309124, + "step": 687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001803751802071929, + "clip_ratio/low_min": 0.0001803751802071929, + "clip_ratio/region_mean": 0.0001803751802071929, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 465.59375, + "completions/mean_terminated_length": 395.8077087402344, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.36562727577984333, + "epoch": 0.3683083511777302, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009232750162482262, + "learning_rate": 1e-05, + "loss": 0.0656, + "num_tokens": 14741935.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.477591633796692, + "sampling/importance_sampling_ratio/mean": 1.0000759363174438, + "sampling/importance_sampling_ratio/min": 0.5472881197929382, + "sampling/sampling_logp_difference/max": 0.6027798652648926, + "sampling/sampling_logp_difference/mean": 0.012475117109715939, + "step": 688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 549.25, + "completions/mean_terminated_length": 449.8182067871094, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.4224976561963558, + "epoch": 0.3688436830835118, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00646795891225338, + "learning_rate": 1e-05, + "loss": 0.0225, + "num_tokens": 14764047.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.6040548086166382, + "sampling/importance_sampling_ratio/mean": 0.9998417496681213, + "sampling/importance_sampling_ratio/min": 0.721508264541626, + "sampling/sampling_logp_difference/max": 0.4725346565246582, + "sampling/sampling_logp_difference/mean": 0.012980914674699306, + "step": 689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 451.125, + "completions/mean_terminated_length": 430.0000305175781, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.3737363927066326, + "epoch": 0.3693790149892934, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012766394764184952, + "learning_rate": 1e-05, + "loss": -0.0639, + "num_tokens": 14782235.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4845302104949951, + "sampling/importance_sampling_ratio/mean": 1.0001002550125122, + "sampling/importance_sampling_ratio/min": 0.669922947883606, + "sampling/sampling_logp_difference/max": 0.400592565536499, + "sampling/sampling_logp_difference/mean": 0.012106068432331085, + "step": 690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000156429159687832, + "clip_ratio/low_min": 0.000156429159687832, + "clip_ratio/region_mean": 0.000156429159687832, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 551.46875, + "completions/mean_terminated_length": 466.7391357421875, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.5542093142867088, + "epoch": 0.36991434689507496, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.028639044612646103, + "learning_rate": 1e-05, + "loss": 0.045, + "num_tokens": 14803578.0, + "reward": 0.375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4161304235458374, + "sampling/importance_sampling_ratio/mean": 1.0001494884490967, + "sampling/importance_sampling_ratio/min": 0.6985909938812256, + "sampling/sampling_logp_difference/max": 0.3586897850036621, + "sampling/sampling_logp_difference/mean": 0.01604504883289337, + "step": 691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 537.0, + "completions/mean_terminated_length": 513.1034545898438, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.2902977168560028, + "epoch": 0.37044967880085655, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004602069966495037, + "learning_rate": 1e-05, + "loss": 0.0993, + "num_tokens": 14824722.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3806614875793457, + "sampling/importance_sampling_ratio/mean": 0.9998685121536255, + "sampling/importance_sampling_ratio/min": 0.6865155100822449, + "sampling/sampling_logp_difference/max": 0.37612640857696533, + "sampling/sampling_logp_difference/mean": 0.009488875977694988, + "step": 692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.275902316905558e-05, + "clip_ratio/low_min": 7.275902316905558e-05, + "clip_ratio/region_mean": 7.275902316905558e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 421.90625, + "completions/mean_terminated_length": 410.7419128417969, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "entropy": 0.2983164694160223, + "epoch": 0.37098501070663814, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00871491339057684, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 14841839.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3694186210632324, + "sampling/importance_sampling_ratio/mean": 1.000069260597229, + "sampling/importance_sampling_ratio/min": 0.7014452815055847, + "sampling/sampling_logp_difference/max": 0.3546123504638672, + "sampling/sampling_logp_difference/mean": 0.0099724680185318, + "step": 693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 555.375, + "completions/mean_terminated_length": 495.8399963378906, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.3679488059133291, + "epoch": 0.3715203426124197, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.03551452234387398, + "learning_rate": 1e-05, + "loss": -0.0027, + "num_tokens": 14863243.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3860419988632202, + "sampling/importance_sampling_ratio/mean": 0.9996388554573059, + "sampling/importance_sampling_ratio/min": 0.5520268082618713, + "sampling/sampling_logp_difference/max": 0.5941586494445801, + "sampling/sampling_logp_difference/mean": 0.011161968111991882, + "step": 694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 482.84375, + "completions/mean_terminated_length": 482.84375, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.30257387086749077, + "epoch": 0.37205567451820126, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007895922288298607, + "learning_rate": 1e-05, + "loss": -0.0119, + "num_tokens": 14882014.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2971529960632324, + "sampling/importance_sampling_ratio/mean": 0.9995283484458923, + "sampling/importance_sampling_ratio/min": 0.718858003616333, + "sampling/sampling_logp_difference/max": 0.3300914764404297, + "sampling/sampling_logp_difference/mean": 0.01028190553188324, + "step": 695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.085686618462205e-05, + "clip_ratio/low_min": 6.085686618462205e-05, + "clip_ratio/region_mean": 6.085686618462205e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 514.09375, + "completions/mean_terminated_length": 497.16668701171875, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 0.27033248357474804, + "epoch": 0.37259100642398285, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00923229567706585, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 14902369.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4416664838790894, + "sampling/importance_sampling_ratio/mean": 1.0003470182418823, + "sampling/importance_sampling_ratio/min": 0.6697207689285278, + "sampling/sampling_logp_difference/max": 0.4008944034576416, + "sampling/sampling_logp_difference/mean": 0.009560334496200085, + "step": 696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014302176714409143, + "clip_ratio/low_min": 0.00014302176714409143, + "clip_ratio/region_mean": 0.00014302176714409143, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 416.0, + "completions/mean_terminated_length": 404.6451416015625, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.3604154847562313, + "epoch": 0.37312633832976444, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02464476227760315, + "learning_rate": 1e-05, + "loss": 0.0041, + "num_tokens": 14919249.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4501818418502808, + "sampling/importance_sampling_ratio/mean": 0.9995229244232178, + "sampling/importance_sampling_ratio/min": 0.700311005115509, + "sampling/sampling_logp_difference/max": 0.37168896198272705, + "sampling/sampling_logp_difference/mean": 0.011596843600273132, + "step": 697 + }, + { + "clip_ratio/high_max": 7.42280317354016e-05, + "clip_ratio/high_mean": 7.42280317354016e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.42280317354016e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 498.5, + "completions/mean_terminated_length": 489.8064270019531, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.3239190801978111, + "epoch": 0.37366167023554603, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010498818010091782, + "learning_rate": 1e-05, + "loss": 0.03, + "num_tokens": 14939201.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4165321588516235, + "sampling/importance_sampling_ratio/mean": 0.9996457695960999, + "sampling/importance_sampling_ratio/min": 0.39121150970458984, + "sampling/sampling_logp_difference/max": 0.9385068416595459, + "sampling/sampling_logp_difference/mean": 0.010730743408203125, + "step": 698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 414.5, + "completions/mean_terminated_length": 414.5, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.2429221346974373, + "epoch": 0.3741970021413276, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 14955817.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.348206639289856, + "sampling/importance_sampling_ratio/mean": 1.0001676082611084, + "sampling/importance_sampling_ratio/min": 0.6665967106819153, + "sampling/sampling_logp_difference/max": 0.40557003021240234, + "sampling/sampling_logp_difference/mean": 0.008237956091761589, + "step": 699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 496.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 309.5, + "completions/mean_terminated_length": 309.5, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.32242966815829277, + "epoch": 0.3747323340471092, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007265747524797916, + "learning_rate": 1e-05, + "loss": -0.0232, + "num_tokens": 14968929.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2998929023742676, + "sampling/importance_sampling_ratio/mean": 0.9999431371688843, + "sampling/importance_sampling_ratio/min": 0.7561819553375244, + "sampling/sampling_logp_difference/max": 0.2794732451438904, + "sampling/sampling_logp_difference/mean": 0.010783866979181767, + "step": 700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 454.53125, + "completions/mean_terminated_length": 444.4193420410156, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "entropy": 0.2625401485711336, + "epoch": 0.3752676659528908, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00635444326326251, + "learning_rate": 1e-05, + "loss": 0.0379, + "num_tokens": 14987066.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4335325956344604, + "sampling/importance_sampling_ratio/mean": 1.000280737876892, + "sampling/importance_sampling_ratio/min": 0.6617458462715149, + "sampling/sampling_logp_difference/max": 0.4128737449645996, + "sampling/sampling_logp_difference/mean": 0.009299966506659985, + "step": 701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.1546390750445426e-05, + "clip_ratio/low_min": 5.1546390750445426e-05, + "clip_ratio/region_mean": 5.1546390750445426e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 479.375, + "completions/mean_terminated_length": 398.55999755859375, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.5339374206960201, + "epoch": 0.3758029978586724, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0073, + "num_tokens": 15006518.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.397071123123169, + "sampling/importance_sampling_ratio/mean": 0.999973714351654, + "sampling/importance_sampling_ratio/min": 0.6090170741081238, + "sampling/sampling_logp_difference/max": 0.4959089756011963, + "sampling/sampling_logp_difference/mean": 0.014633250422775745, + "step": 702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.88146280683577e-05, + "clip_ratio/low_min": 7.88146280683577e-05, + "clip_ratio/region_mean": 7.88146280683577e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 372.96875, + "completions/mean_terminated_length": 372.96875, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.3855822868645191, + "epoch": 0.37633832976445397, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019846171140670776, + "learning_rate": 1e-05, + "loss": 0.0753, + "num_tokens": 15021965.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.2919931411743164, + "sampling/importance_sampling_ratio/mean": 0.9998896718025208, + "sampling/importance_sampling_ratio/min": 0.7070300579071045, + "sampling/sampling_logp_difference/max": 0.346682071685791, + "sampling/sampling_logp_difference/mean": 0.012079265899956226, + "step": 703 + }, + { + "clip_ratio/high_max": 5.087505269329995e-05, + "clip_ratio/high_mean": 5.087505269329995e-05, + "clip_ratio/low_mean": 0.00010730029680416919, + "clip_ratio/low_min": 0.00010730029680416919, + "clip_ratio/region_mean": 0.00015817534949746914, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 601.5, + "completions/mean_terminated_length": 501.6000061035156, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.5670550353825092, + "epoch": 0.37687366167023556, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01242276281118393, + "learning_rate": 1e-05, + "loss": 0.0049, + "num_tokens": 15045205.0, + "reward": 0.34375, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5675126314163208, + "sampling/importance_sampling_ratio/mean": 1.000095248222351, + "sampling/importance_sampling_ratio/min": 0.7066628932952881, + "sampling/sampling_logp_difference/max": 0.4494900703430176, + "sampling/sampling_logp_difference/mean": 0.015393679030239582, + "step": 704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 700.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 417.71875, + "completions/mean_terminated_length": 417.71875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.35111863166093826, + "epoch": 0.37740899357601715, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01782170869410038, + "learning_rate": 1e-05, + "loss": 0.0099, + "num_tokens": 15062500.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3412227630615234, + "sampling/importance_sampling_ratio/mean": 0.9997622966766357, + "sampling/importance_sampling_ratio/min": 0.7273291945457458, + "sampling/sampling_logp_difference/max": 0.31837618350982666, + "sampling/sampling_logp_difference/mean": 0.011413737200200558, + "step": 705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 668.0, + "completions/mean_length": 383.84375, + "completions/mean_terminated_length": 371.45159912109375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.37662991508841515, + "epoch": 0.37794432548179874, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007424172479659319, + "learning_rate": 1e-05, + "loss": 0.0188, + "num_tokens": 15078527.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3684529066085815, + "sampling/importance_sampling_ratio/mean": 1.000205636024475, + "sampling/importance_sampling_ratio/min": 0.44091466069221497, + "sampling/sampling_logp_difference/max": 0.818903923034668, + "sampling/sampling_logp_difference/mean": 0.01036691665649414, + "step": 706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 490.40625, + "completions/mean_terminated_length": 471.9000244140625, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.30377865210175514, + "epoch": 0.3784796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004978889599442482, + "learning_rate": 1e-05, + "loss": 0.0435, + "num_tokens": 15097820.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.451399564743042, + "sampling/importance_sampling_ratio/mean": 0.9998151063919067, + "sampling/importance_sampling_ratio/min": 0.6608973741531372, + "sampling/sampling_logp_difference/max": 0.4141566753387451, + "sampling/sampling_logp_difference/mean": 0.009623161517083645, + "step": 707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 422.375, + "completions/mean_terminated_length": 422.375, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.28235390968620777, + "epoch": 0.37901498929336186, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00819606613367796, + "learning_rate": 1e-05, + "loss": 0.0011, + "num_tokens": 15114736.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3884201049804688, + "sampling/importance_sampling_ratio/mean": 1.000165343284607, + "sampling/importance_sampling_ratio/min": 0.33030667901039124, + "sampling/sampling_logp_difference/max": 1.1077337265014648, + "sampling/sampling_logp_difference/mean": 0.0095993522554636, + "step": 708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.069722389336675e-05, + "clip_ratio/low_min": 8.069722389336675e-05, + "clip_ratio/region_mean": 8.069722389336675e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 390.78125, + "completions/mean_terminated_length": 390.78125, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.3380414769053459, + "epoch": 0.37955032119914345, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01816956326365471, + "learning_rate": 1e-05, + "loss": -0.0078, + "num_tokens": 15131313.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.9131790399551392, + "sampling/importance_sampling_ratio/mean": 0.9997246265411377, + "sampling/importance_sampling_ratio/min": 0.7011801600456238, + "sampling/sampling_logp_difference/max": 0.648766279220581, + "sampling/sampling_logp_difference/mean": 0.011466284282505512, + "step": 709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011698735397658311, + "clip_ratio/low_min": 0.00011698735397658311, + "clip_ratio/region_mean": 0.00011698735397658311, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 499.96875, + "completions/mean_terminated_length": 461.6785888671875, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.35549603775143623, + "epoch": 0.38008565310492504, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02748221345245838, + "learning_rate": 1e-05, + "loss": -0.022, + "num_tokens": 15151992.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4221206903457642, + "sampling/importance_sampling_ratio/mean": 0.9998782277107239, + "sampling/importance_sampling_ratio/min": 0.6979137659072876, + "sampling/sampling_logp_difference/max": 0.35965967178344727, + "sampling/sampling_logp_difference/mean": 0.011623858474195004, + "step": 710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.294625149574131e-05, + "clip_ratio/low_min": 8.294625149574131e-05, + "clip_ratio/region_mean": 8.294625149574131e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 709.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 427.46875, + "completions/mean_terminated_length": 427.46875, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.30363865941762924, + "epoch": 0.3806209850107066, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00964261032640934, + "learning_rate": 1e-05, + "loss": 0.0051, + "num_tokens": 15169079.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4173977375030518, + "sampling/importance_sampling_ratio/mean": 0.999652624130249, + "sampling/importance_sampling_ratio/min": 0.698860228061676, + "sampling/sampling_logp_difference/max": 0.358304500579834, + "sampling/sampling_logp_difference/mean": 0.010039136745035648, + "step": 711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 545.0, + "completions/mean_terminated_length": 482.55999755859375, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.44356758520007133, + "epoch": 0.3811563169164882, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021267326548695564, + "learning_rate": 1e-05, + "loss": 0.0082, + "num_tokens": 15190719.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4318623542785645, + "sampling/importance_sampling_ratio/mean": 1.000069499015808, + "sampling/importance_sampling_ratio/min": 0.7342823147773743, + "sampling/sampling_logp_difference/max": 0.358975887298584, + "sampling/sampling_logp_difference/mean": 0.013153758831322193, + "step": 712 + }, + { + "clip_ratio/high_max": 8.468834857922047e-05, + "clip_ratio/high_mean": 8.468834857922047e-05, + "clip_ratio/low_mean": 8.085381705313921e-05, + "clip_ratio/low_min": 8.085381705313921e-05, + "clip_ratio/region_mean": 0.00016554216563235968, + "completions/clipped_ratio": 0.0, + "completions/max_length": 643.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 425.84375, + "completions/mean_terminated_length": 425.84375, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.2659711726009846, + "epoch": 0.3816916488222698, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00843278132379055, + "learning_rate": 1e-05, + "loss": -0.002, + "num_tokens": 15207610.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.30759859085083, + "sampling/importance_sampling_ratio/mean": 1.0002485513687134, + "sampling/importance_sampling_ratio/min": 0.7000202536582947, + "sampling/sampling_logp_difference/max": 0.3566460609436035, + "sampling/sampling_logp_difference/mean": 0.009244011715054512, + "step": 713 + }, + { + "clip_ratio/high_max": 5.310110282152891e-05, + "clip_ratio/high_mean": 5.310110282152891e-05, + "clip_ratio/low_mean": 0.00016015375149436295, + "clip_ratio/low_min": 0.00016015375149436295, + "clip_ratio/region_mean": 0.00021325485431589186, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 507.75, + "completions/mean_terminated_length": 490.4000244140625, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.28116846084594727, + "epoch": 0.3822269807280514, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0116853853687644, + "learning_rate": 1e-05, + "loss": -0.0388, + "num_tokens": 15227658.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4448853731155396, + "sampling/importance_sampling_ratio/mean": 1.000178337097168, + "sampling/importance_sampling_ratio/min": 0.69992995262146, + "sampling/sampling_logp_difference/max": 0.3680300712585449, + "sampling/sampling_logp_difference/mean": 0.009712045080959797, + "step": 714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001412651254213415, + "clip_ratio/low_min": 0.0001412651254213415, + "clip_ratio/region_mean": 0.0001412651254213415, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 578.90625, + "completions/mean_terminated_length": 449.52630615234375, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.4918227419257164, + "epoch": 0.382762312633833, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03207610547542572, + "learning_rate": 1e-05, + "loss": 0.0564, + "num_tokens": 15250231.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3958628177642822, + "sampling/importance_sampling_ratio/mean": 1.0001263618469238, + "sampling/importance_sampling_ratio/min": 0.6963813304901123, + "sampling/sampling_logp_difference/max": 0.36185789108276367, + "sampling/sampling_logp_difference/mean": 0.014611990191042423, + "step": 715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 470.625, + "completions/mean_terminated_length": 439.862060546875, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.3172050081193447, + "epoch": 0.38329764453961457, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021208766847848892, + "learning_rate": 1e-05, + "loss": 0.0068, + "num_tokens": 15268931.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4504036903381348, + "sampling/importance_sampling_ratio/mean": 1.000354528427124, + "sampling/importance_sampling_ratio/min": 0.7036083340644836, + "sampling/sampling_logp_difference/max": 0.3718419075012207, + "sampling/sampling_logp_difference/mean": 0.010729103349149227, + "step": 716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 471.25, + "completions/mean_terminated_length": 451.4667053222656, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.2803878542035818, + "epoch": 0.38383297644539616, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005157788749784231, + "learning_rate": 1e-05, + "loss": 0.0383, + "num_tokens": 15288067.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3918088674545288, + "sampling/importance_sampling_ratio/mean": 0.9998689293861389, + "sampling/importance_sampling_ratio/min": 0.7035257816314697, + "sampling/sampling_logp_difference/max": 0.3516507148742676, + "sampling/sampling_logp_difference/mean": 0.009469738230109215, + "step": 717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 591.125, + "completions/mean_terminated_length": 541.5999755859375, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.28335285373032093, + "epoch": 0.38436830835117775, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00884544849395752, + "learning_rate": 1e-05, + "loss": 0.0188, + "num_tokens": 15310607.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.476885437965393, + "sampling/importance_sampling_ratio/mean": 0.9997802972793579, + "sampling/importance_sampling_ratio/min": 0.5552170872688293, + "sampling/sampling_logp_difference/max": 0.5883960723876953, + "sampling/sampling_logp_difference/mean": 0.009259765967726707, + "step": 718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 470.5625, + "completions/mean_terminated_length": 460.9677429199219, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.34930726885795593, + "epoch": 0.38490364025695933, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.022168725728988647, + "learning_rate": 1e-05, + "loss": 0.0259, + "num_tokens": 15329457.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4237807989120483, + "sampling/importance_sampling_ratio/mean": 0.9998915791511536, + "sampling/importance_sampling_ratio/min": 0.6903370022773743, + "sampling/sampling_logp_difference/max": 0.3705754280090332, + "sampling/sampling_logp_difference/mean": 0.010541636496782303, + "step": 719 + }, + { + "clip_ratio/high_max": 8.272667037090287e-05, + "clip_ratio/high_mean": 8.272667037090287e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 8.272667037090287e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 422.15625, + "completions/mean_terminated_length": 411.0, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.2984021846204996, + "epoch": 0.3854389721627409, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018680274486541748, + "learning_rate": 1e-05, + "loss": 0.1035, + "num_tokens": 15346166.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4660590887069702, + "sampling/importance_sampling_ratio/mean": 0.9999105334281921, + "sampling/importance_sampling_ratio/min": 0.5593701004981995, + "sampling/sampling_logp_difference/max": 0.5809439420700073, + "sampling/sampling_logp_difference/mean": 0.009945410303771496, + "step": 720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 689.0, + "completions/max_terminated_length": 689.0, + "completions/mean_length": 430.53125, + "completions/mean_terminated_length": 430.53125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.3552410937845707, + "epoch": 0.3859743040685225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008144088089466095, + "learning_rate": 1e-05, + "loss": -0.023, + "num_tokens": 15363543.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2996821403503418, + "sampling/importance_sampling_ratio/mean": 1.000144124031067, + "sampling/importance_sampling_ratio/min": 0.7797349691390991, + "sampling/sampling_logp_difference/max": 0.26211977005004883, + "sampling/sampling_logp_difference/mean": 0.010755490511655807, + "step": 721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 698.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 449.15625, + "completions/mean_terminated_length": 449.15625, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.3446524739265442, + "epoch": 0.38650963597430404, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003281314391642809, + "learning_rate": 1e-05, + "loss": 0.0254, + "num_tokens": 15381876.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4003485441207886, + "sampling/importance_sampling_ratio/mean": 0.999922513961792, + "sampling/importance_sampling_ratio/min": 0.6165217757225037, + "sampling/sampling_logp_difference/max": 0.4836616516113281, + "sampling/sampling_logp_difference/mean": 0.011187918484210968, + "step": 722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 440.75, + "completions/mean_terminated_length": 430.19354248046875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.35840288922190666, + "epoch": 0.38704496788008563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013158319517970085, + "learning_rate": 1e-05, + "loss": -0.029, + "num_tokens": 15400068.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.431912899017334, + "sampling/importance_sampling_ratio/mean": 0.9998740553855896, + "sampling/importance_sampling_ratio/min": 0.5467763543128967, + "sampling/sampling_logp_difference/max": 0.6037154197692871, + "sampling/sampling_logp_difference/mean": 0.011617033742368221, + "step": 723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013079975178698078, + "clip_ratio/low_min": 0.00013079975178698078, + "clip_ratio/region_mean": 0.00013079975178698078, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 461.59375, + "completions/mean_terminated_length": 451.70965576171875, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.41740207001566887, + "epoch": 0.3875802997858672, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008753396570682526, + "learning_rate": 1e-05, + "loss": 0.028, + "num_tokens": 15418471.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.7728537321090698, + "sampling/importance_sampling_ratio/mean": 0.9997082948684692, + "sampling/importance_sampling_ratio/min": 0.468240350484848, + "sampling/sampling_logp_difference/max": 0.7587735652923584, + "sampling/sampling_logp_difference/mean": 0.013727336190640926, + "step": 724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 698.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 425.1875, + "completions/mean_terminated_length": 425.1875, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.3067735768854618, + "epoch": 0.3881156316916488, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016379397362470627, + "learning_rate": 1e-05, + "loss": 0.0134, + "num_tokens": 15435469.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.302474021911621, + "sampling/importance_sampling_ratio/mean": 1.000181794166565, + "sampling/importance_sampling_ratio/min": 0.7234917879104614, + "sampling/sampling_logp_difference/max": 0.3236660957336426, + "sampling/sampling_logp_difference/mean": 0.010212586261332035, + "step": 725 + }, + { + "clip_ratio/high_max": 7.188039307948202e-05, + "clip_ratio/high_mean": 7.188039307948202e-05, + "clip_ratio/low_mean": 6.234413740457967e-05, + "clip_ratio/low_min": 6.234413740457967e-05, + "clip_ratio/region_mean": 0.0001342245304840617, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 617.0, + "completions/mean_length": 469.875, + "completions/mean_terminated_length": 427.2857360839844, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.33584076538681984, + "epoch": 0.3886509635974304, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011967268772423267, + "learning_rate": 1e-05, + "loss": 0.1041, + "num_tokens": 15453993.0, + "reward": 0.71875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5494422912597656, + "sampling/importance_sampling_ratio/mean": 0.9996713399887085, + "sampling/importance_sampling_ratio/min": 0.5857813954353333, + "sampling/sampling_logp_difference/max": 0.5348086357116699, + "sampling/sampling_logp_difference/mean": 0.0111548425629735, + "step": 726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 429.0, + "completions/mean_terminated_length": 418.06451416015625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.322954460978508, + "epoch": 0.389186295503212, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013946806080639362, + "learning_rate": 1e-05, + "loss": 0.0573, + "num_tokens": 15471481.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3577029705047607, + "sampling/importance_sampling_ratio/mean": 1.000420331954956, + "sampling/importance_sampling_ratio/min": 0.7600410580635071, + "sampling/sampling_logp_difference/max": 0.30579423904418945, + "sampling/sampling_logp_difference/mean": 0.011020390316843987, + "step": 727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010679096158128232, + "clip_ratio/low_min": 0.00010679096158128232, + "clip_ratio/region_mean": 0.00010679096158128232, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 539.78125, + "completions/mean_terminated_length": 497.5185241699219, + "completions/min_length": 359.0, + "completions/min_terminated_length": 359.0, + "entropy": 0.2796822488307953, + "epoch": 0.3897216274089936, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0059602754190564156, + "learning_rate": 1e-05, + "loss": 0.0727, + "num_tokens": 15492594.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3901206254959106, + "sampling/importance_sampling_ratio/mean": 0.9996997117996216, + "sampling/importance_sampling_ratio/min": 0.7212743759155273, + "sampling/sampling_logp_difference/max": 0.3293905258178711, + "sampling/sampling_logp_difference/mean": 0.009473959915339947, + "step": 728 + }, + { + "clip_ratio/high_max": 5.56297272851225e-05, + "clip_ratio/high_mean": 5.56297272851225e-05, + "clip_ratio/low_mean": 0.000152136773976963, + "clip_ratio/low_min": 0.000152136773976963, + "clip_ratio/region_mean": 0.0002077665012620855, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 563.03125, + "completions/mean_terminated_length": 494.7083435058594, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4573715813457966, + "epoch": 0.39025695931477516, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0070287566632032394, + "learning_rate": 1e-05, + "loss": 0.0796, + "num_tokens": 15514627.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6117656230926514, + "sampling/importance_sampling_ratio/mean": 1.0001170635223389, + "sampling/importance_sampling_ratio/min": 0.5874090194702148, + "sampling/sampling_logp_difference/max": 0.5320339202880859, + "sampling/sampling_logp_difference/mean": 0.014223533682525158, + "step": 729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 420.53125, + "completions/mean_terminated_length": 409.32257080078125, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.31619079411029816, + "epoch": 0.39079229122055675, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005202142987400293, + "learning_rate": 1e-05, + "loss": 0.058, + "num_tokens": 15531860.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4242578744888306, + "sampling/importance_sampling_ratio/mean": 1.000077247619629, + "sampling/importance_sampling_ratio/min": 0.6973584294319153, + "sampling/sampling_logp_difference/max": 0.3604557514190674, + "sampling/sampling_logp_difference/mean": 0.010554124601185322, + "step": 730 + }, + { + "clip_ratio/high_max": 8.080155384959653e-05, + "clip_ratio/high_mean": 8.080155384959653e-05, + "clip_ratio/low_mean": 5.87406029808335e-05, + "clip_ratio/low_min": 5.87406029808335e-05, + "clip_ratio/region_mean": 0.00013954215683043003, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 427.125, + "completions/mean_terminated_length": 416.1290283203125, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.3117987625300884, + "epoch": 0.39132762312633834, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007393387146294117, + "learning_rate": 1e-05, + "loss": 0.0688, + "num_tokens": 15549240.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.433593511581421, + "sampling/importance_sampling_ratio/mean": 1.000236988067627, + "sampling/importance_sampling_ratio/min": 0.7485927939414978, + "sampling/sampling_logp_difference/max": 0.3601841926574707, + "sampling/sampling_logp_difference/mean": 0.0106127904728055, + "step": 731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013024853251408786, + "clip_ratio/low_min": 0.00013024853251408786, + "clip_ratio/region_mean": 0.00013024853251408786, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 482.625, + "completions/mean_terminated_length": 441.857177734375, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.4086911641061306, + "epoch": 0.39186295503211993, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028034420683979988, + "learning_rate": 1e-05, + "loss": 0.0302, + "num_tokens": 15568316.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4345265626907349, + "sampling/importance_sampling_ratio/mean": 0.9996673464775085, + "sampling/importance_sampling_ratio/min": 0.7197425961494446, + "sampling/sampling_logp_difference/max": 0.36083483695983887, + "sampling/sampling_logp_difference/mean": 0.012979852966964245, + "step": 732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018101435853168368, + "clip_ratio/low_min": 0.00018101435853168368, + "clip_ratio/region_mean": 0.00018101435853168368, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 465.03125, + "completions/mean_terminated_length": 433.6896667480469, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.39621807262301445, + "epoch": 0.3923982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013066532090306282, + "learning_rate": 1e-05, + "loss": 0.0734, + "num_tokens": 15587013.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4360840320587158, + "sampling/importance_sampling_ratio/mean": 0.9997668862342834, + "sampling/importance_sampling_ratio/min": 0.7086175084114075, + "sampling/sampling_logp_difference/max": 0.3619198799133301, + "sampling/sampling_logp_difference/mean": 0.012843320146203041, + "step": 733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 667.0, + "completions/max_terminated_length": 667.0, + "completions/mean_length": 457.84375, + "completions/mean_terminated_length": 457.84375, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.34751833975315094, + "epoch": 0.3929336188436831, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0152, + "num_tokens": 15604888.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.579491138458252, + "sampling/importance_sampling_ratio/mean": 1.0000112056732178, + "sampling/importance_sampling_ratio/min": 0.743756115436554, + "sampling/sampling_logp_difference/max": 0.45710277557373047, + "sampling/sampling_logp_difference/mean": 0.010801946744322777, + "step": 734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010974539327435195, + "clip_ratio/low_min": 0.00010974539327435195, + "clip_ratio/region_mean": 0.00010974539327435195, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 539.78125, + "completions/mean_terminated_length": 436.04547119140625, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.3937859311699867, + "epoch": 0.39346895074946464, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015017692930996418, + "learning_rate": 1e-05, + "loss": 0.0916, + "num_tokens": 15626313.0, + "reward": 0.5625, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3079745769500732, + "sampling/importance_sampling_ratio/mean": 1.0001164674758911, + "sampling/importance_sampling_ratio/min": 0.6615281701087952, + "sampling/sampling_logp_difference/max": 0.41320276260375977, + "sampling/sampling_logp_difference/mean": 0.01178530603647232, + "step": 735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 617.0, + "completions/mean_length": 464.96875, + "completions/mean_terminated_length": 444.7666931152344, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.3442009389400482, + "epoch": 0.39400428265524623, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017365379258990288, + "learning_rate": 1e-05, + "loss": -0.0806, + "num_tokens": 15644872.0, + "reward": 0.5625, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4252296686172485, + "sampling/importance_sampling_ratio/mean": 0.9997314214706421, + "sampling/importance_sampling_ratio/min": 0.6978505849838257, + "sampling/sampling_logp_difference/max": 0.35975027084350586, + "sampling/sampling_logp_difference/mean": 0.011152451857924461, + "step": 736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 695.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 411.125, + "completions/mean_terminated_length": 411.125, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.3204801678657532, + "epoch": 0.3945396145610278, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02090236358344555, + "learning_rate": 1e-05, + "loss": -0.0372, + "num_tokens": 15661492.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4939148426055908, + "sampling/importance_sampling_ratio/mean": 1.000043511390686, + "sampling/importance_sampling_ratio/min": 0.7046853303909302, + "sampling/sampling_logp_difference/max": 0.401400089263916, + "sampling/sampling_logp_difference/mean": 0.010819543153047562, + "step": 737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 708.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 460.34375, + "completions/mean_terminated_length": 460.34375, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.2755957208573818, + "epoch": 0.3950749464668094, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005674439016729593, + "learning_rate": 1e-05, + "loss": 0.019, + "num_tokens": 15679663.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4159061908721924, + "sampling/importance_sampling_ratio/mean": 0.9999711513519287, + "sampling/importance_sampling_ratio/min": 0.7319160103797913, + "sampling/sampling_logp_difference/max": 0.34776973724365234, + "sampling/sampling_logp_difference/mean": 0.009373167529702187, + "step": 738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 399.625, + "completions/mean_terminated_length": 387.7419128417969, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.32343524135649204, + "epoch": 0.395610278372591, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.015889469534158707, + "learning_rate": 1e-05, + "loss": 0.0031, + "num_tokens": 15696403.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5283796787261963, + "sampling/importance_sampling_ratio/mean": 0.9995576739311218, + "sampling/importance_sampling_ratio/min": 0.702329695224762, + "sampling/sampling_logp_difference/max": 0.4242081642150879, + "sampling/sampling_logp_difference/mean": 0.011235849931836128, + "step": 739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.8809059080667794e-05, + "clip_ratio/low_min": 4.8809059080667794e-05, + "clip_ratio/region_mean": 4.8809059080667794e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 526.40625, + "completions/mean_terminated_length": 458.7599792480469, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.31673742085695267, + "epoch": 0.3961456102783726, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028274651616811752, + "learning_rate": 1e-05, + "loss": 0.0256, + "num_tokens": 15717128.0, + "reward": 0.65625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3521307706832886, + "sampling/importance_sampling_ratio/mean": 1.0007063150405884, + "sampling/importance_sampling_ratio/min": 0.6952041387557983, + "sampling/sampling_logp_difference/max": 0.36354970932006836, + "sampling/sampling_logp_difference/mean": 0.01056370697915554, + "step": 740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 474.9375, + "completions/mean_terminated_length": 433.0714416503906, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.35712743923068047, + "epoch": 0.3966809421841542, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0037, + "num_tokens": 15736694.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4580076932907104, + "sampling/importance_sampling_ratio/mean": 0.9994637966156006, + "sampling/importance_sampling_ratio/min": 0.6935352683067322, + "sampling/sampling_logp_difference/max": 0.37707090377807617, + "sampling/sampling_logp_difference/mean": 0.012560808099806309, + "step": 741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 709.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 423.5, + "completions/mean_terminated_length": 423.5, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "entropy": 0.2690687384456396, + "epoch": 0.39721627408993576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007789425551891327, + "learning_rate": 1e-05, + "loss": 0.0069, + "num_tokens": 15753486.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.2923842668533325, + "sampling/importance_sampling_ratio/mean": 0.9999104738235474, + "sampling/importance_sampling_ratio/min": 0.5045144557952881, + "sampling/sampling_logp_difference/max": 0.6841588020324707, + "sampling/sampling_logp_difference/mean": 0.009072489105165005, + "step": 742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 756.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 407.625, + "completions/mean_terminated_length": 407.625, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.23075343668460846, + "epoch": 0.39775160599571735, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011375130154192448, + "learning_rate": 1e-05, + "loss": -0.0061, + "num_tokens": 15770138.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.285789966583252, + "sampling/importance_sampling_ratio/mean": 0.9999356269836426, + "sampling/importance_sampling_ratio/min": 0.6539907455444336, + "sampling/sampling_logp_difference/max": 0.42466211318969727, + "sampling/sampling_logp_difference/mean": 0.008017098531126976, + "step": 743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 743.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 421.5, + "completions/mean_terminated_length": 421.5, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.2794561889022589, + "epoch": 0.39828693790149894, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0023, + "num_tokens": 15786994.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.405788540840149, + "sampling/importance_sampling_ratio/mean": 0.999731183052063, + "sampling/importance_sampling_ratio/min": 0.729838490486145, + "sampling/sampling_logp_difference/max": 0.34059834480285645, + "sampling/sampling_logp_difference/mean": 0.010126659646630287, + "step": 744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012459380013751797, + "clip_ratio/low_min": 0.00012459380013751797, + "clip_ratio/region_mean": 0.00012459380013751797, + "completions/clipped_ratio": 0.0, + "completions/max_length": 660.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 430.78125, + "completions/mean_terminated_length": 430.78125, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.2542971894145012, + "epoch": 0.3988222698072805, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009447346441447735, + "learning_rate": 1e-05, + "loss": 0.0376, + "num_tokens": 15804395.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2802484035491943, + "sampling/importance_sampling_ratio/mean": 0.9995415210723877, + "sampling/importance_sampling_ratio/min": 0.5784639716148376, + "sampling/sampling_logp_difference/max": 0.5473790168762207, + "sampling/sampling_logp_difference/mean": 0.009644705802202225, + "step": 745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 381.875, + "completions/mean_terminated_length": 381.875, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.3233192600309849, + "epoch": 0.3993576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008273539133369923, + "learning_rate": 1e-05, + "loss": 0.0202, + "num_tokens": 15820103.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4061501026153564, + "sampling/importance_sampling_ratio/mean": 0.9998979568481445, + "sampling/importance_sampling_ratio/min": 0.7168700695037842, + "sampling/sampling_logp_difference/max": 0.34085559844970703, + "sampling/sampling_logp_difference/mean": 0.009892796166241169, + "step": 746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013287238834891468, + "clip_ratio/low_min": 0.00013287238834891468, + "clip_ratio/region_mean": 0.00013287238834891468, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 395.03125, + "completions/mean_terminated_length": 383.0, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.3803809992969036, + "epoch": 0.3998929336188437, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.058856669813394547, + "learning_rate": 1e-05, + "loss": 0.0639, + "num_tokens": 15836504.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3565399646759033, + "sampling/importance_sampling_ratio/mean": 0.9999346733093262, + "sampling/importance_sampling_ratio/min": 0.7007579207420349, + "sampling/sampling_logp_difference/max": 0.3555927276611328, + "sampling/sampling_logp_difference/mean": 0.011545914225280285, + "step": 747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 410.8125, + "completions/mean_terminated_length": 399.2903137207031, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.3029617629945278, + "epoch": 0.4004282655246253, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014787835069000721, + "learning_rate": 1e-05, + "loss": 0.0361, + "num_tokens": 15853162.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4724562168121338, + "sampling/importance_sampling_ratio/mean": 0.9996786117553711, + "sampling/importance_sampling_ratio/min": 0.6643556356430054, + "sampling/sampling_logp_difference/max": 0.4089376926422119, + "sampling/sampling_logp_difference/mean": 0.010183299891650677, + "step": 748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017294470671913587, + "clip_ratio/low_min": 0.00017294470671913587, + "clip_ratio/region_mean": 0.00017294470671913587, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 538.6875, + "completions/mean_terminated_length": 474.47998046875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3989364579319954, + "epoch": 0.4009635974304068, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008750291541218758, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 15874688.0, + "reward": 0.4375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.420766830444336, + "sampling/importance_sampling_ratio/mean": 0.9997806549072266, + "sampling/importance_sampling_ratio/min": 0.7073161602020264, + "sampling/sampling_logp_difference/max": 0.3511967658996582, + "sampling/sampling_logp_difference/mean": 0.012256912887096405, + "step": 749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.765682726632804e-05, + "clip_ratio/low_min": 5.765682726632804e-05, + "clip_ratio/region_mean": 5.765682726632804e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 493.03125, + "completions/mean_terminated_length": 453.7500305175781, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.26635151728987694, + "epoch": 0.4014989293361884, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013368546962738037, + "learning_rate": 1e-05, + "loss": 0.064, + "num_tokens": 15894089.0, + "reward": 0.5, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.5822577476501465, + "sampling/importance_sampling_ratio/mean": 1.000051736831665, + "sampling/importance_sampling_ratio/min": 0.2953316867351532, + "sampling/sampling_logp_difference/max": 1.219656229019165, + "sampling/sampling_logp_difference/mean": 0.00905274972319603, + "step": 750 + }, + { + "clip_ratio/high_max": 7.802746404195204e-05, + "clip_ratio/high_mean": 7.802746404195204e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.802746404195204e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 481.25, + "completions/mean_terminated_length": 462.13336181640625, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.3769327774643898, + "epoch": 0.40203426124197, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0483182817697525, + "learning_rate": 1e-05, + "loss": 0.0122, + "num_tokens": 15912921.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4769721031188965, + "sampling/importance_sampling_ratio/mean": 0.9998060464859009, + "sampling/importance_sampling_ratio/min": 0.6666865944862366, + "sampling/sampling_logp_difference/max": 0.4054352045059204, + "sampling/sampling_logp_difference/mean": 0.012469490990042686, + "step": 751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016942126967478544, + "clip_ratio/low_min": 0.00016942126967478544, + "clip_ratio/region_mean": 0.00016942126967478544, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 485.03125, + "completions/mean_terminated_length": 432.629638671875, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.38218012638390064, + "epoch": 0.4025695931477516, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012462539598345757, + "learning_rate": 1e-05, + "loss": 0.0415, + "num_tokens": 15932250.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002398490905762, + "sampling/importance_sampling_ratio/min": 0.48620763421058655, + "sampling/sampling_logp_difference/max": 0.7410974502563477, + "sampling/sampling_logp_difference/mean": 0.011902324855327606, + "step": 752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.6484408560208976e-05, + "clip_ratio/low_min": 5.6484408560208976e-05, + "clip_ratio/region_mean": 5.6484408560208976e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 645.0, + "completions/mean_length": 455.4375, + "completions/mean_terminated_length": 434.60003662109375, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.40006024576723576, + "epoch": 0.4031049250535332, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010189701803028584, + "learning_rate": 1e-05, + "loss": 0.0842, + "num_tokens": 15950536.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5445659160614014, + "sampling/importance_sampling_ratio/mean": 1.0000519752502441, + "sampling/importance_sampling_ratio/min": 0.7623915076255798, + "sampling/sampling_logp_difference/max": 0.43474292755126953, + "sampling/sampling_logp_difference/mean": 0.011419408023357391, + "step": 753 + }, + { + "clip_ratio/high_max": 5.7444853155175224e-05, + "clip_ratio/high_mean": 5.7444853155175224e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.7444853155175224e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 559.5, + "completions/mean_terminated_length": 490.0, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.4570811428129673, + "epoch": 0.40364025695931477, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014431962743401527, + "learning_rate": 1e-05, + "loss": -0.0126, + "num_tokens": 15972584.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5669465065002441, + "sampling/importance_sampling_ratio/mean": 1.0001393556594849, + "sampling/importance_sampling_ratio/min": 0.686478316783905, + "sampling/sampling_logp_difference/max": 0.4491288661956787, + "sampling/sampling_logp_difference/mean": 0.015018288977444172, + "step": 754 + }, + { + "clip_ratio/high_max": 6.316321378108114e-05, + "clip_ratio/high_mean": 6.316321378108114e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.316321378108114e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 432.0625, + "completions/mean_terminated_length": 354.5384826660156, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.39246325194835663, + "epoch": 0.40417558886509636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014133045449852943, + "learning_rate": 1e-05, + "loss": -0.0695, + "num_tokens": 15989626.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4653111696243286, + "sampling/importance_sampling_ratio/mean": 1.0005357265472412, + "sampling/importance_sampling_ratio/min": 0.7505085468292236, + "sampling/sampling_logp_difference/max": 0.3820676803588867, + "sampling/sampling_logp_difference/mean": 0.011345033533871174, + "step": 755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 484.5625, + "completions/mean_terminated_length": 475.4193420410156, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.35822298005223274, + "epoch": 0.40471092077087795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.028463730588555336, + "learning_rate": 1e-05, + "loss": 0.0381, + "num_tokens": 16009412.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6963238716125488, + "sampling/importance_sampling_ratio/mean": 0.9997175931930542, + "sampling/importance_sampling_ratio/min": 0.5386660695075989, + "sampling/sampling_logp_difference/max": 0.618659496307373, + "sampling/sampling_logp_difference/mean": 0.011470532976090908, + "step": 756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 556.0, + "completions/max_terminated_length": 556.0, + "completions/mean_length": 408.375, + "completions/mean_terminated_length": 408.375, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.27229262702167034, + "epoch": 0.40524625267665954, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02580346167087555, + "learning_rate": 1e-05, + "loss": 0.0344, + "num_tokens": 16026240.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4563825130462646, + "sampling/importance_sampling_ratio/mean": 0.9996753931045532, + "sampling/importance_sampling_ratio/min": 0.7031433582305908, + "sampling/sampling_logp_difference/max": 0.37595558166503906, + "sampling/sampling_logp_difference/mean": 0.009219473227858543, + "step": 757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 414.59375, + "completions/mean_terminated_length": 414.59375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.29809033684432507, + "epoch": 0.4057815845824411, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005617144517600536, + "learning_rate": 1e-05, + "loss": -0.0031, + "num_tokens": 16042971.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2700101137161255, + "sampling/importance_sampling_ratio/mean": 1.000062346458435, + "sampling/importance_sampling_ratio/min": 0.706326961517334, + "sampling/sampling_logp_difference/max": 0.34767699241638184, + "sampling/sampling_logp_difference/mean": 0.009595252573490143, + "step": 758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.901960710412823e-05, + "clip_ratio/low_min": 4.901960710412823e-05, + "clip_ratio/region_mean": 4.901960710412823e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 620.0, + "completions/mean_length": 532.28125, + "completions/mean_terminated_length": 466.2799987792969, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3346907217055559, + "epoch": 0.4063169164882227, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015848277136683464, + "learning_rate": 1e-05, + "loss": 0.0345, + "num_tokens": 16064428.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3548301458358765, + "sampling/importance_sampling_ratio/mean": 0.9993706941604614, + "sampling/importance_sampling_ratio/min": 0.6472514271736145, + "sampling/sampling_logp_difference/max": 0.43502044677734375, + "sampling/sampling_logp_difference/mean": 0.010415096767246723, + "step": 759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.706008571200073e-05, + "clip_ratio/low_min": 6.706008571200073e-05, + "clip_ratio/region_mean": 6.706008571200073e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 523.0625, + "completions/mean_terminated_length": 497.72412109375, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.2949142698198557, + "epoch": 0.4068522483940043, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008824774995446205, + "learning_rate": 1e-05, + "loss": -0.0136, + "num_tokens": 16084710.0, + "reward": 0.375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.624711275100708, + "sampling/importance_sampling_ratio/mean": 1.0001001358032227, + "sampling/importance_sampling_ratio/min": 0.7189587950706482, + "sampling/sampling_logp_difference/max": 0.48533010482788086, + "sampling/sampling_logp_difference/mean": 0.009899305179715157, + "step": 760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.441880784928799e-05, + "clip_ratio/low_min": 5.441880784928799e-05, + "clip_ratio/region_mean": 5.441880784928799e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 632.0, + "completions/max_terminated_length": 632.0, + "completions/mean_length": 480.40625, + "completions/mean_terminated_length": 480.40625, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.27928940765559673, + "epoch": 0.4073875802997859, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012388971634209156, + "learning_rate": 1e-05, + "loss": -0.0378, + "num_tokens": 16103971.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.424300193786621, + "sampling/importance_sampling_ratio/mean": 0.9999654293060303, + "sampling/importance_sampling_ratio/min": 0.5367613434791565, + "sampling/sampling_logp_difference/max": 0.622201681137085, + "sampling/sampling_logp_difference/mean": 0.009574158117175102, + "step": 761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 509.625, + "completions/mean_terminated_length": 482.89654541015625, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.2994512524455786, + "epoch": 0.4079229122055675, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008833318948745728, + "learning_rate": 1e-05, + "loss": 0.0644, + "num_tokens": 16123567.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5718441009521484, + "sampling/importance_sampling_ratio/mean": 1.0001444816589355, + "sampling/importance_sampling_ratio/min": 0.5895460844039917, + "sampling/sampling_logp_difference/max": 0.5284023284912109, + "sampling/sampling_logp_difference/mean": 0.009947758167982101, + "step": 762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 444.34375, + "completions/mean_terminated_length": 433.9031982421875, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.27685064263641834, + "epoch": 0.408458244111349, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003286470426246524, + "learning_rate": 1e-05, + "loss": 0.065, + "num_tokens": 16141594.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2818659543991089, + "sampling/importance_sampling_ratio/mean": 1.0000402927398682, + "sampling/importance_sampling_ratio/min": 0.7070391774177551, + "sampling/sampling_logp_difference/max": 0.34666919708251953, + "sampling/sampling_logp_difference/mean": 0.009597057476639748, + "step": 763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.339598283171654e-05, + "clip_ratio/low_min": 5.339598283171654e-05, + "clip_ratio/region_mean": 5.339598283171654e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 501.71875, + "completions/mean_terminated_length": 474.17242431640625, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.4013344496488571, + "epoch": 0.4089935760171306, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023167669773101807, + "learning_rate": 1e-05, + "loss": -0.0163, + "num_tokens": 16161313.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3985191583633423, + "sampling/importance_sampling_ratio/mean": 1.000298261642456, + "sampling/importance_sampling_ratio/min": 0.7825467586517334, + "sampling/sampling_logp_difference/max": 0.33541393280029297, + "sampling/sampling_logp_difference/mean": 0.012038334272801876, + "step": 764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 747.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 438.0625, + "completions/mean_terminated_length": 438.0625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.30539108626544476, + "epoch": 0.4095289079229122, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016800880432128906, + "learning_rate": 1e-05, + "loss": -0.0384, + "num_tokens": 16179387.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.8013707399368286, + "sampling/importance_sampling_ratio/mean": 0.999898374080658, + "sampling/importance_sampling_ratio/min": 0.6412967443466187, + "sampling/sampling_logp_difference/max": 0.588547945022583, + "sampling/sampling_logp_difference/mean": 0.009695417247712612, + "step": 765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001732737509883009, + "clip_ratio/low_min": 0.0001732737509883009, + "clip_ratio/region_mean": 0.0001732737509883009, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 562.71875, + "completions/mean_terminated_length": 524.7037353515625, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.29583428986370564, + "epoch": 0.4100642398286938, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009501498192548752, + "learning_rate": 1e-05, + "loss": 0.0282, + "num_tokens": 16202186.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4362612962722778, + "sampling/importance_sampling_ratio/mean": 0.999686062335968, + "sampling/importance_sampling_ratio/min": 0.48731547594070435, + "sampling/sampling_logp_difference/max": 0.7188435792922974, + "sampling/sampling_logp_difference/mean": 0.010522641241550446, + "step": 766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 507.53125, + "completions/mean_terminated_length": 480.5862121582031, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.3411188870668411, + "epoch": 0.41059957173447537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018128888681530952, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 16222331.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4630672931671143, + "sampling/importance_sampling_ratio/mean": 1.0001051425933838, + "sampling/importance_sampling_ratio/min": 0.6867654323577881, + "sampling/sampling_logp_difference/max": 0.3805351257324219, + "sampling/sampling_logp_difference/mean": 0.011262279003858566, + "step": 767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 398.9375, + "completions/mean_terminated_length": 398.9375, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.2790275290608406, + "epoch": 0.41113490364025695, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005060677882283926, + "learning_rate": 1e-05, + "loss": -0.0046, + "num_tokens": 16238849.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.435973048210144, + "sampling/importance_sampling_ratio/mean": 0.9995302557945251, + "sampling/importance_sampling_ratio/min": 0.7087464928627014, + "sampling/sampling_logp_difference/max": 0.36184263229370117, + "sampling/sampling_logp_difference/mean": 0.009966474957764149, + "step": 768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 457.4375, + "completions/mean_terminated_length": 436.7333679199219, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.294871186837554, + "epoch": 0.41167023554603854, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009670751169323921, + "learning_rate": 1e-05, + "loss": 0.0044, + "num_tokens": 16256663.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3708109855651855, + "sampling/importance_sampling_ratio/mean": 0.9998509287834167, + "sampling/importance_sampling_ratio/min": 0.7468981146812439, + "sampling/sampling_logp_difference/max": 0.3154025077819824, + "sampling/sampling_logp_difference/mean": 0.009731477126479149, + "step": 769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013690898776985705, + "clip_ratio/low_min": 0.00013690898776985705, + "clip_ratio/region_mean": 0.00013690898776985705, + "completions/clipped_ratio": 0.0, + "completions/max_length": 741.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 455.5, + "completions/mean_terminated_length": 455.5, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.34996984153985977, + "epoch": 0.41220556745182013, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005920007824897766, + "learning_rate": 1e-05, + "loss": -0.0093, + "num_tokens": 16275295.0, + "reward": 0.59375, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4538800716400146, + "sampling/importance_sampling_ratio/mean": 0.9999443292617798, + "sampling/importance_sampling_ratio/min": 0.7225494980812073, + "sampling/sampling_logp_difference/max": 0.3742358684539795, + "sampling/sampling_logp_difference/mean": 0.011001553386449814, + "step": 770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 469.46875, + "completions/mean_terminated_length": 449.5666809082031, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.33411771804094315, + "epoch": 0.4127408993576017, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015054845251142979, + "learning_rate": 1e-05, + "loss": -0.0378, + "num_tokens": 16294782.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.363189697265625, + "sampling/importance_sampling_ratio/mean": 1.0000303983688354, + "sampling/importance_sampling_ratio/min": 0.6979761719703674, + "sampling/sampling_logp_difference/max": 0.35957032442092896, + "sampling/sampling_logp_difference/mean": 0.011455297470092773, + "step": 771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 607.0, + "completions/max_terminated_length": 607.0, + "completions/mean_length": 435.90625, + "completions/mean_terminated_length": 435.90625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.27202615328133106, + "epoch": 0.4132762312633833, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0109, + "num_tokens": 16313067.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.378258228302002, + "sampling/importance_sampling_ratio/mean": 0.9994197487831116, + "sampling/importance_sampling_ratio/min": 0.6492050290107727, + "sampling/sampling_logp_difference/max": 0.4320066571235657, + "sampling/sampling_logp_difference/mean": 0.009395753964781761, + "step": 772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 373.5625, + "completions/mean_terminated_length": 373.5625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.29544753581285477, + "epoch": 0.4138115631691649, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003570796921849251, + "learning_rate": 1e-05, + "loss": 0.0375, + "num_tokens": 16328933.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3228645324707031, + "sampling/importance_sampling_ratio/mean": 1.0000734329223633, + "sampling/importance_sampling_ratio/min": 0.7217989563941956, + "sampling/sampling_logp_difference/max": 0.326008677482605, + "sampling/sampling_logp_difference/mean": 0.00953470729291439, + "step": 773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.433962259208784e-05, + "clip_ratio/low_min": 9.433962259208784e-05, + "clip_ratio/region_mean": 9.433962259208784e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 664.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 371.3125, + "completions/mean_terminated_length": 371.3125, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.3113366477191448, + "epoch": 0.4143468950749465, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013471437618136406, + "learning_rate": 1e-05, + "loss": -0.0216, + "num_tokens": 16344423.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3202178478240967, + "sampling/importance_sampling_ratio/mean": 0.9994405508041382, + "sampling/importance_sampling_ratio/min": 0.7053040266036987, + "sampling/sampling_logp_difference/max": 0.34912633895874023, + "sampling/sampling_logp_difference/mean": 0.01060332078486681, + "step": 774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.802746404195204e-05, + "clip_ratio/low_min": 7.802746404195204e-05, + "clip_ratio/region_mean": 7.802746404195204e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 437.0625, + "completions/mean_terminated_length": 415.0000305175781, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.43285075947642326, + "epoch": 0.4148822269807281, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02249554730951786, + "learning_rate": 1e-05, + "loss": -0.0058, + "num_tokens": 16362817.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5441429615020752, + "sampling/importance_sampling_ratio/mean": 0.999977171421051, + "sampling/importance_sampling_ratio/min": 0.6840009093284607, + "sampling/sampling_logp_difference/max": 0.43446898460388184, + "sampling/sampling_logp_difference/mean": 0.013381147757172585, + "step": 775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015147724479902536, + "clip_ratio/low_min": 0.00015147724479902536, + "clip_ratio/region_mean": 0.00015147724479902536, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 542.15625, + "completions/mean_terminated_length": 490.0384826660156, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.3197234198451042, + "epoch": 0.41541755888650966, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018198829144239426, + "learning_rate": 1e-05, + "loss": -0.0302, + "num_tokens": 16384078.0, + "reward": 0.5625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.501399040222168, + "sampling/importance_sampling_ratio/mean": 0.9999122023582458, + "sampling/importance_sampling_ratio/min": 0.7645636200904846, + "sampling/sampling_logp_difference/max": 0.40639734268188477, + "sampling/sampling_logp_difference/mean": 0.010852006264030933, + "step": 776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.164222865365446e-05, + "clip_ratio/low_min": 9.164222865365446e-05, + "clip_ratio/region_mean": 9.164222865365446e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 723.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 434.5625, + "completions/mean_terminated_length": 434.5625, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.3417657408863306, + "epoch": 0.4159528907922912, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025946030393242836, + "learning_rate": 1e-05, + "loss": 0.0231, + "num_tokens": 16401288.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5710872411727905, + "sampling/importance_sampling_ratio/mean": 0.9997629523277283, + "sampling/importance_sampling_ratio/min": 0.6923314332962036, + "sampling/sampling_logp_difference/max": 0.4517679214477539, + "sampling/sampling_logp_difference/mean": 0.011955196969211102, + "step": 777 + }, + { + "clip_ratio/high_max": 6.443299207603559e-05, + "clip_ratio/high_mean": 6.443299207603559e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.443299207603559e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 553.21875, + "completions/mean_terminated_length": 455.5909118652344, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "entropy": 0.4989879857748747, + "epoch": 0.4164882226980728, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02220899611711502, + "learning_rate": 1e-05, + "loss": 0.0271, + "num_tokens": 16422823.0, + "reward": 0.34375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.999857485294342, + "sampling/importance_sampling_ratio/min": 0.7533624768257141, + "sampling/sampling_logp_difference/max": 0.7256240844726562, + "sampling/sampling_logp_difference/mean": 0.013844046741724014, + "step": 778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 362.0625, + "completions/mean_terminated_length": 348.9677429199219, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.34428347274661064, + "epoch": 0.4170235546038544, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027528373524546623, + "learning_rate": 1e-05, + "loss": -0.0426, + "num_tokens": 16438033.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.345342993736267, + "sampling/importance_sampling_ratio/mean": 0.9997936487197876, + "sampling/importance_sampling_ratio/min": 0.7098269462585449, + "sampling/sampling_logp_difference/max": 0.34273409843444824, + "sampling/sampling_logp_difference/mean": 0.011760414578020573, + "step": 779 + }, + { + "clip_ratio/high_max": 9.735202183946967e-05, + "clip_ratio/high_mean": 9.735202183946967e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.735202183946967e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 412.21875, + "completions/mean_terminated_length": 412.21875, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.29027098417282104, + "epoch": 0.41755888650963596, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016562609001994133, + "learning_rate": 1e-05, + "loss": 0.025, + "num_tokens": 16454424.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4183530807495117, + "sampling/importance_sampling_ratio/mean": 0.9999801516532898, + "sampling/importance_sampling_ratio/min": 0.724010169506073, + "sampling/sampling_logp_difference/max": 0.34949636459350586, + "sampling/sampling_logp_difference/mean": 0.010326000861823559, + "step": 780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 494.25, + "completions/mean_terminated_length": 455.14288330078125, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.34014772810041904, + "epoch": 0.41809421841541755, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005914956796914339, + "learning_rate": 1e-05, + "loss": 0.0079, + "num_tokens": 16474184.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3385014533996582, + "sampling/importance_sampling_ratio/mean": 1.0002925395965576, + "sampling/importance_sampling_ratio/min": 0.7563097476959229, + "sampling/sampling_logp_difference/max": 0.2915506362915039, + "sampling/sampling_logp_difference/mean": 0.011034255847334862, + "step": 781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 493.46875, + "completions/mean_terminated_length": 475.16668701171875, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.3546532765030861, + "epoch": 0.41862955032119914, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014526438899338245, + "learning_rate": 1e-05, + "loss": 0.0861, + "num_tokens": 16493543.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4465943574905396, + "sampling/importance_sampling_ratio/mean": 0.9996225237846375, + "sampling/importance_sampling_ratio/min": 0.7496525049209595, + "sampling/sampling_logp_difference/max": 0.3692120313644409, + "sampling/sampling_logp_difference/mean": 0.01135631650686264, + "step": 782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 487.0, + "completions/mean_terminated_length": 468.2666931152344, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.287403654307127, + "epoch": 0.41916488222698073, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0461, + "num_tokens": 16513167.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3896218538284302, + "sampling/importance_sampling_ratio/mean": 1.0000495910644531, + "sampling/importance_sampling_ratio/min": 0.6388620734214783, + "sampling/sampling_logp_difference/max": 0.44806671142578125, + "sampling/sampling_logp_difference/mean": 0.01001773588359356, + "step": 783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010822511103469878, + "clip_ratio/low_min": 0.00010822511103469878, + "clip_ratio/region_mean": 0.00010822511103469878, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 460.625, + "completions/mean_terminated_length": 403.7037048339844, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.4263018108904362, + "epoch": 0.4197002141327623, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028048962354660034, + "learning_rate": 1e-05, + "loss": 0.015, + "num_tokens": 16532059.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5398837327957153, + "sampling/importance_sampling_ratio/mean": 1.0002316236495972, + "sampling/importance_sampling_ratio/min": 0.7910860180854797, + "sampling/sampling_logp_difference/max": 0.43170690536499023, + "sampling/sampling_logp_difference/mean": 0.01344273705035448, + "step": 784 + }, + { + "clip_ratio/high_max": 7.846829976188019e-05, + "clip_ratio/high_mean": 7.846829976188019e-05, + "clip_ratio/low_mean": 0.000147733255289495, + "clip_ratio/low_min": 0.000147733255289495, + "clip_ratio/region_mean": 0.00022620155505137518, + "completions/clipped_ratio": 0.0, + "completions/max_length": 550.0, + "completions/max_terminated_length": 550.0, + "completions/mean_length": 394.84375, + "completions/mean_terminated_length": 394.84375, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.30854377523064613, + "epoch": 0.4202355460385439, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03182852640748024, + "learning_rate": 1e-05, + "loss": 0.0189, + "num_tokens": 16548814.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4465000629425049, + "sampling/importance_sampling_ratio/mean": 1.000160813331604, + "sampling/importance_sampling_ratio/min": 0.599141001701355, + "sampling/sampling_logp_difference/max": 0.5122582912445068, + "sampling/sampling_logp_difference/mean": 0.010697192512452602, + "step": 785 + }, + { + "clip_ratio/high_max": 8.934953802963719e-05, + "clip_ratio/high_mean": 8.934953802963719e-05, + "clip_ratio/low_mean": 5.6357079301960766e-05, + "clip_ratio/low_min": 5.6357079301960766e-05, + "clip_ratio/region_mean": 0.00014570661733159795, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 540.0, + "completions/mean_length": 397.125, + "completions/mean_terminated_length": 385.1612854003906, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.29431120678782463, + "epoch": 0.4207708779443255, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006559995003044605, + "learning_rate": 1e-05, + "loss": 0.0352, + "num_tokens": 16565282.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.673102855682373, + "sampling/importance_sampling_ratio/mean": 1.0000230073928833, + "sampling/importance_sampling_ratio/min": 0.5996344089508057, + "sampling/sampling_logp_difference/max": 0.5146799087524414, + "sampling/sampling_logp_difference/mean": 0.010234890505671501, + "step": 786 + }, + { + "clip_ratio/high_max": 7.318500865949318e-05, + "clip_ratio/high_mean": 7.318500865949318e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.318500865949318e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 700.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 358.0, + "completions/mean_terminated_length": 358.0, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.31584992073476315, + "epoch": 0.4213062098501071, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0464, + "num_tokens": 16580154.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5974339246749878, + "sampling/importance_sampling_ratio/mean": 0.9998888969421387, + "sampling/importance_sampling_ratio/min": 0.6470592021942139, + "sampling/sampling_logp_difference/max": 0.4683985710144043, + "sampling/sampling_logp_difference/mean": 0.01112749520689249, + "step": 787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 408.875, + "completions/mean_terminated_length": 397.2903137207031, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.2817312330007553, + "epoch": 0.42184154175588867, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.04354003816843033, + "learning_rate": 1e-05, + "loss": 0.0319, + "num_tokens": 16596990.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.485916018486023, + "sampling/importance_sampling_ratio/mean": 0.9996706247329712, + "sampling/importance_sampling_ratio/min": 0.7777830362319946, + "sampling/sampling_logp_difference/max": 0.39603137969970703, + "sampling/sampling_logp_difference/mean": 0.009876232594251633, + "step": 788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 466.59375, + "completions/mean_terminated_length": 456.8709411621094, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.38442598655819893, + "epoch": 0.42237687366167026, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008006955496966839, + "learning_rate": 1e-05, + "loss": -0.0064, + "num_tokens": 16616201.0, + "reward": 0.40625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6210137605667114, + "sampling/importance_sampling_ratio/mean": 1.000440239906311, + "sampling/importance_sampling_ratio/min": 0.7053622007369995, + "sampling/sampling_logp_difference/max": 0.48305177688598633, + "sampling/sampling_logp_difference/mean": 0.01286507397890091, + "step": 789 + }, + { + "clip_ratio/high_max": 6.503642362076789e-05, + "clip_ratio/high_mean": 6.503642362076789e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.503642362076789e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 440.125, + "completions/mean_terminated_length": 379.40740966796875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.3840542919933796, + "epoch": 0.4229122055674518, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021103430539369583, + "learning_rate": 1e-05, + "loss": -0.0078, + "num_tokens": 16634061.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3124192953109741, + "sampling/importance_sampling_ratio/mean": 0.9999834895133972, + "sampling/importance_sampling_ratio/min": 0.5593166947364807, + "sampling/sampling_logp_difference/max": 0.5810394287109375, + "sampling/sampling_logp_difference/mean": 0.012441441416740417, + "step": 790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.036261063651182e-05, + "clip_ratio/low_min": 5.036261063651182e-05, + "clip_ratio/region_mean": 5.036261063651182e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 549.8125, + "completions/mean_terminated_length": 509.40740966796875, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.4020552970468998, + "epoch": 0.4234475374732334, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023112032562494278, + "learning_rate": 1e-05, + "loss": -0.0065, + "num_tokens": 16655855.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3809304237365723, + "sampling/importance_sampling_ratio/mean": 0.999682605266571, + "sampling/importance_sampling_ratio/min": 0.7346566915512085, + "sampling/sampling_logp_difference/max": 0.3227574825286865, + "sampling/sampling_logp_difference/mean": 0.012609808705747128, + "step": 791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011947887469432317, + "clip_ratio/low_min": 0.00011947887469432317, + "clip_ratio/region_mean": 0.00011947887469432317, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 484.46875, + "completions/mean_terminated_length": 465.5666809082031, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.4358667880296707, + "epoch": 0.42398286937901497, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0032, + "num_tokens": 16674934.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3661179542541504, + "sampling/importance_sampling_ratio/mean": 0.9998723864555359, + "sampling/importance_sampling_ratio/min": 0.782680332660675, + "sampling/sampling_logp_difference/max": 0.31197309494018555, + "sampling/sampling_logp_difference/mean": 0.013119467534124851, + "step": 792 + }, + { + "clip_ratio/high_max": 8.400537626584992e-05, + "clip_ratio/high_mean": 8.400537626584992e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 8.400537626584992e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 358.9375, + "completions/mean_terminated_length": 358.9375, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.25597029365599155, + "epoch": 0.42451820128479656, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0038700823206454515, + "learning_rate": 1e-05, + "loss": 0.0041, + "num_tokens": 16689908.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.420012354850769, + "sampling/importance_sampling_ratio/mean": 0.9999418258666992, + "sampling/importance_sampling_ratio/min": 0.7710124254226685, + "sampling/sampling_logp_difference/max": 0.3506655693054199, + "sampling/sampling_logp_difference/mean": 0.008947829715907574, + "step": 793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 533.0, + "completions/max_terminated_length": 533.0, + "completions/mean_length": 339.84375, + "completions/mean_terminated_length": 339.84375, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "entropy": 0.26448273472487926, + "epoch": 0.42505353319057815, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0002, + "num_tokens": 16704335.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.359431266784668, + "sampling/importance_sampling_ratio/mean": 0.9999831318855286, + "sampling/importance_sampling_ratio/min": 0.7856153845787048, + "sampling/sampling_logp_difference/max": 0.3070664405822754, + "sampling/sampling_logp_difference/mean": 0.009440883062779903, + "step": 794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 669.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 433.25, + "completions/mean_terminated_length": 433.25, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.3539182171225548, + "epoch": 0.42558886509635974, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015658603981137276, + "learning_rate": 1e-05, + "loss": -0.0086, + "num_tokens": 16721991.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6626696586608887, + "sampling/importance_sampling_ratio/mean": 0.9999551773071289, + "sampling/importance_sampling_ratio/min": 0.6779660582542419, + "sampling/sampling_logp_difference/max": 0.5084245800971985, + "sampling/sampling_logp_difference/mean": 0.01210436224937439, + "step": 795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.886435196269304e-05, + "clip_ratio/low_min": 7.886435196269304e-05, + "clip_ratio/region_mean": 7.886435196269304e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 424.0, + "completions/mean_terminated_length": 412.9031982421875, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.3245571702718735, + "epoch": 0.4261241970021413, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005203715991228819, + "learning_rate": 1e-05, + "loss": 0.0258, + "num_tokens": 16739199.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3860565423965454, + "sampling/importance_sampling_ratio/mean": 1.0002363920211792, + "sampling/importance_sampling_ratio/min": 0.7304226756095886, + "sampling/sampling_logp_difference/max": 0.3264627456665039, + "sampling/sampling_logp_difference/mean": 0.01050733681768179, + "step": 796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 370.96875, + "completions/mean_terminated_length": 358.1612854003906, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "entropy": 0.3662017323076725, + "epoch": 0.4266595289079229, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0327, + "num_tokens": 16754558.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4157353639602661, + "sampling/importance_sampling_ratio/mean": 1.0003262758255005, + "sampling/importance_sampling_ratio/min": 0.7653398513793945, + "sampling/sampling_logp_difference/max": 0.34764909744262695, + "sampling/sampling_logp_difference/mean": 0.011724627576768398, + "step": 797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.468066592584364e-05, + "clip_ratio/low_min": 5.468066592584364e-05, + "clip_ratio/region_mean": 5.468066592584364e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 455.65625, + "completions/mean_terminated_length": 445.58062744140625, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.3370545245707035, + "epoch": 0.4271948608137045, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0043823388405144215, + "learning_rate": 1e-05, + "loss": -0.0209, + "num_tokens": 16772859.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4160512685775757, + "sampling/importance_sampling_ratio/mean": 1.0002046823501587, + "sampling/importance_sampling_ratio/min": 0.6508194804191589, + "sampling/sampling_logp_difference/max": 0.4295229911804199, + "sampling/sampling_logp_difference/mean": 0.010617252439260483, + "step": 798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.989458441035822e-05, + "clip_ratio/low_min": 5.989458441035822e-05, + "clip_ratio/region_mean": 5.989458441035822e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 543.09375, + "completions/mean_terminated_length": 519.8275756835938, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.3294404186308384, + "epoch": 0.4277301927194861, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007699902169406414, + "learning_rate": 1e-05, + "loss": 0.0544, + "num_tokens": 16794214.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4315295219421387, + "sampling/importance_sampling_ratio/mean": 1.0000755786895752, + "sampling/importance_sampling_ratio/min": 0.6101194024085999, + "sampling/sampling_logp_difference/max": 0.49410057067871094, + "sampling/sampling_logp_difference/mean": 0.011136038228869438, + "step": 799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 587.0, + "completions/max_terminated_length": 587.0, + "completions/mean_length": 407.21875, + "completions/mean_terminated_length": 407.21875, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.29633622989058495, + "epoch": 0.4282655246252677, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02443593367934227, + "learning_rate": 1e-05, + "loss": 0.0021, + "num_tokens": 16811309.0, + "reward": 0.84375, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.593308448791504, + "sampling/importance_sampling_ratio/mean": 1.000170350074768, + "sampling/importance_sampling_ratio/min": 0.7495781779289246, + "sampling/sampling_logp_difference/max": 0.46581268310546875, + "sampling/sampling_logp_difference/mean": 0.010436818934977055, + "step": 800 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.132726361509413e-05, + "clip_ratio/low_min": 8.132726361509413e-05, + "clip_ratio/region_mean": 8.132726361509413e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 452.96875, + "completions/mean_terminated_length": 452.96875, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.2979501448571682, + "epoch": 0.42880085653104927, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028952719643712044, + "learning_rate": 1e-05, + "loss": -0.0137, + "num_tokens": 16829436.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3987958431243896, + "sampling/importance_sampling_ratio/mean": 1.0001310110092163, + "sampling/importance_sampling_ratio/min": 0.6112988591194153, + "sampling/sampling_logp_difference/max": 0.4921693801879883, + "sampling/sampling_logp_difference/mean": 0.010467601008713245, + "step": 801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.757715189247392e-05, + "clip_ratio/low_min": 5.757715189247392e-05, + "clip_ratio/region_mean": 5.757715189247392e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 482.3125, + "completions/mean_terminated_length": 473.0967712402344, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.32549588568508625, + "epoch": 0.42933618843683086, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007285809610038996, + "learning_rate": 1e-05, + "loss": 0.0304, + "num_tokens": 16849150.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3400046825408936, + "sampling/importance_sampling_ratio/mean": 0.9999621510505676, + "sampling/importance_sampling_ratio/min": 0.6875501871109009, + "sampling/sampling_logp_difference/max": 0.3746204376220703, + "sampling/sampling_logp_difference/mean": 0.01079532690346241, + "step": 802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 434.59375, + "completions/mean_terminated_length": 423.83868408203125, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.30391672998666763, + "epoch": 0.42987152034261245, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.022445475682616234, + "learning_rate": 1e-05, + "loss": 0.0314, + "num_tokens": 16866753.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5056238174438477, + "sampling/importance_sampling_ratio/mean": 0.9998089075088501, + "sampling/importance_sampling_ratio/min": 0.6181923747062683, + "sampling/sampling_logp_difference/max": 0.4809556007385254, + "sampling/sampling_logp_difference/mean": 0.009911777451634407, + "step": 803 + }, + { + "clip_ratio/high_max": 7.405213546007872e-05, + "clip_ratio/high_mean": 7.405213546007872e-05, + "clip_ratio/low_mean": 0.00016379839144065045, + "clip_ratio/low_min": 0.00016379839144065045, + "clip_ratio/region_mean": 0.00023785052690072916, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 438.78125, + "completions/mean_terminated_length": 377.8148193359375, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.3829710967838764, + "epoch": 0.430406852248394, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019850274547934532, + "learning_rate": 1e-05, + "loss": 0.0196, + "num_tokens": 16884810.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5259016752243042, + "sampling/importance_sampling_ratio/mean": 1.0003832578659058, + "sampling/importance_sampling_ratio/min": 0.732100248336792, + "sampling/sampling_logp_difference/max": 0.42258548736572266, + "sampling/sampling_logp_difference/mean": 0.012629854492843151, + "step": 804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 419.46875, + "completions/mean_terminated_length": 408.2257995605469, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.2942862119525671, + "epoch": 0.43094218415417557, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.020406479015946388, + "learning_rate": 1e-05, + "loss": 0.0575, + "num_tokens": 16901569.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4498920440673828, + "sampling/importance_sampling_ratio/mean": 0.9996421933174133, + "sampling/importance_sampling_ratio/min": 0.697530210018158, + "sampling/sampling_logp_difference/max": 0.3714890480041504, + "sampling/sampling_logp_difference/mean": 0.009771422483026981, + "step": 805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 550.0, + "completions/mean_length": 350.125, + "completions/mean_terminated_length": 336.6451416015625, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.3032227195799351, + "epoch": 0.43147751605995716, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007902158424258232, + "learning_rate": 1e-05, + "loss": -0.0176, + "num_tokens": 16916125.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4670381546020508, + "sampling/importance_sampling_ratio/mean": 0.9999794363975525, + "sampling/importance_sampling_ratio/min": 0.7704863548278809, + "sampling/sampling_logp_difference/max": 0.38324546813964844, + "sampling/sampling_logp_difference/mean": 0.010330552235245705, + "step": 806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 439.5, + "completions/mean_terminated_length": 439.5, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.3029349409043789, + "epoch": 0.43201284796573874, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 16933789.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4466451406478882, + "sampling/importance_sampling_ratio/mean": 1.0001295804977417, + "sampling/importance_sampling_ratio/min": 0.6920611262321472, + "sampling/sampling_logp_difference/max": 0.3692471981048584, + "sampling/sampling_logp_difference/mean": 0.010409644804894924, + "step": 807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 407.03125, + "completions/mean_terminated_length": 407.03125, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.39045247435569763, + "epoch": 0.43254817987152033, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0006, + "num_tokens": 16950534.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4458879232406616, + "sampling/importance_sampling_ratio/mean": 1.0001180171966553, + "sampling/importance_sampling_ratio/min": 0.6843310594558716, + "sampling/sampling_logp_difference/max": 0.37931346893310547, + "sampling/sampling_logp_difference/mean": 0.012879184447228909, + "step": 808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 530.875, + "completions/mean_terminated_length": 497.0000305175781, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.3060468230396509, + "epoch": 0.4330835117773019, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008273949846625328, + "learning_rate": 1e-05, + "loss": 0.0921, + "num_tokens": 16971354.0, + "reward": 0.78125, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3393288850784302, + "sampling/importance_sampling_ratio/mean": 0.999944806098938, + "sampling/importance_sampling_ratio/min": 0.6818230748176575, + "sampling/sampling_logp_difference/max": 0.38298511505126953, + "sampling/sampling_logp_difference/mean": 0.010227049700915813, + "step": 809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.231306178960949e-05, + "clip_ratio/low_min": 6.231306178960949e-05, + "clip_ratio/region_mean": 6.231306178960949e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 455.9375, + "completions/mean_terminated_length": 445.8709411621094, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.2802313603460789, + "epoch": 0.4336188436830835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005037680268287659, + "learning_rate": 1e-05, + "loss": 0.0307, + "num_tokens": 16989528.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3644779920578003, + "sampling/importance_sampling_ratio/mean": 0.9999107122421265, + "sampling/importance_sampling_ratio/min": 0.7234307527542114, + "sampling/sampling_logp_difference/max": 0.32375049591064453, + "sampling/sampling_logp_difference/mean": 0.009951348416507244, + "step": 810 + }, + { + "clip_ratio/high_max": 5.5163283832371235e-05, + "clip_ratio/high_mean": 5.5163283832371235e-05, + "clip_ratio/low_mean": 0.00012614567822311074, + "clip_ratio/low_min": 0.00012614567822311074, + "clip_ratio/region_mean": 0.00018130896205548197, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 474.0, + "completions/mean_terminated_length": 432.0000305175781, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.3423337507992983, + "epoch": 0.4341541755888651, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012094578705728054, + "learning_rate": 1e-05, + "loss": 0.0034, + "num_tokens": 17008328.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4612789154052734, + "sampling/importance_sampling_ratio/mean": 1.000400424003601, + "sampling/importance_sampling_ratio/min": 0.6412340998649597, + "sampling/sampling_logp_difference/max": 0.44436073303222656, + "sampling/sampling_logp_difference/mean": 0.011007783003151417, + "step": 811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9465768825029954e-05, + "clip_ratio/low_min": 4.9465768825029954e-05, + "clip_ratio/region_mean": 4.9465768825029954e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 513.875, + "completions/mean_terminated_length": 487.5862121582031, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.27780028618872166, + "epoch": 0.4346895074946467, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019300971180200577, + "learning_rate": 1e-05, + "loss": 0.0486, + "num_tokens": 17028564.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4398139715194702, + "sampling/importance_sampling_ratio/mean": 1.0000369548797607, + "sampling/importance_sampling_ratio/min": 0.6995759010314941, + "sampling/sampling_logp_difference/max": 0.3645138740539551, + "sampling/sampling_logp_difference/mean": 0.009919270873069763, + "step": 812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001410715631209314, + "clip_ratio/low_min": 0.0001410715631209314, + "clip_ratio/region_mean": 0.0001410715631209314, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 492.4375, + "completions/mean_terminated_length": 441.40740966796875, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.3248841241002083, + "epoch": 0.4352248394004283, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017952993512153625, + "learning_rate": 1e-05, + "loss": 0.0108, + "num_tokens": 17047946.0, + "reward": 0.34375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3786996603012085, + "sampling/importance_sampling_ratio/mean": 0.9999850988388062, + "sampling/importance_sampling_ratio/min": 0.6992431879043579, + "sampling/sampling_logp_difference/max": 0.3577566146850586, + "sampling/sampling_logp_difference/mean": 0.010284186340868473, + "step": 813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015852885553613305, + "clip_ratio/low_min": 0.00015852885553613305, + "clip_ratio/region_mean": 0.00015852885553613305, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 371.15625, + "completions/mean_terminated_length": 358.3548278808594, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "entropy": 0.3041163217276335, + "epoch": 0.43576017130620986, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01717095822095871, + "learning_rate": 1e-05, + "loss": -0.0204, + "num_tokens": 17063431.0, + "reward": 0.6875, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3836407661437988, + "sampling/importance_sampling_ratio/mean": 1.0002788305282593, + "sampling/importance_sampling_ratio/min": 0.5404121279716492, + "sampling/sampling_logp_difference/max": 0.6154232025146484, + "sampling/sampling_logp_difference/mean": 0.011140824295580387, + "step": 814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 756.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 437.84375, + "completions/mean_terminated_length": 437.84375, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.3168862573802471, + "epoch": 0.43629550321199145, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01972295716404915, + "learning_rate": 1e-05, + "loss": -0.0744, + "num_tokens": 17081010.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4323437213897705, + "sampling/importance_sampling_ratio/mean": 0.9998100399971008, + "sampling/importance_sampling_ratio/min": 0.7168599963188171, + "sampling/sampling_logp_difference/max": 0.3593120574951172, + "sampling/sampling_logp_difference/mean": 0.010707537643611431, + "step": 815 + }, + { + "clip_ratio/high_max": 7.51653642510064e-05, + "clip_ratio/high_mean": 7.51653642510064e-05, + "clip_ratio/low_mean": 8.941345004132017e-05, + "clip_ratio/low_min": 8.941345004132017e-05, + "clip_ratio/region_mean": 0.00016457881429232657, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 376.03125, + "completions/mean_terminated_length": 376.03125, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3296228162944317, + "epoch": 0.43683083511777304, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02471880428493023, + "learning_rate": 1e-05, + "loss": -0.0007, + "num_tokens": 17096931.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3028932809829712, + "sampling/importance_sampling_ratio/mean": 0.9998487234115601, + "sampling/importance_sampling_ratio/min": 0.6383896470069885, + "sampling/sampling_logp_difference/max": 0.4488065242767334, + "sampling/sampling_logp_difference/mean": 0.012072399258613586, + "step": 816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 757.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 365.21875, + "completions/mean_terminated_length": 365.21875, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.3616967312991619, + "epoch": 0.43736616702355463, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009743794798851013, + "learning_rate": 1e-05, + "loss": 0.0026, + "num_tokens": 17112514.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.7478456497192383, + "sampling/importance_sampling_ratio/mean": 0.9996093511581421, + "sampling/importance_sampling_ratio/min": 0.6976439356803894, + "sampling/sampling_logp_difference/max": 0.5583839416503906, + "sampling/sampling_logp_difference/mean": 0.012133362703025341, + "step": 817 + }, + { + "clip_ratio/high_max": 6.237524939933792e-05, + "clip_ratio/high_mean": 6.237524939933792e-05, + "clip_ratio/low_mean": 0.00017773100989870727, + "clip_ratio/low_min": 0.00017773100989870727, + "clip_ratio/region_mean": 0.0002401062592980452, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 513.21875, + "completions/mean_terminated_length": 505.0, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.32191819325089455, + "epoch": 0.43790149892933616, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03233141452074051, + "learning_rate": 1e-05, + "loss": 0.003, + "num_tokens": 17133425.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5630940198898315, + "sampling/importance_sampling_ratio/mean": 0.9998918771743774, + "sampling/importance_sampling_ratio/min": 0.662834107875824, + "sampling/sampling_logp_difference/max": 0.4466671943664551, + "sampling/sampling_logp_difference/mean": 0.011077848263084888, + "step": 818 + }, + { + "clip_ratio/high_max": 5.6357079301960766e-05, + "clip_ratio/high_mean": 5.6357079301960766e-05, + "clip_ratio/low_mean": 0.00011214663391001523, + "clip_ratio/low_min": 0.00011214663391001523, + "clip_ratio/region_mean": 0.000168503713211976, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 500.0625, + "completions/mean_terminated_length": 472.3448181152344, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.2890834603458643, + "epoch": 0.43843683083511775, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015631988644599915, + "learning_rate": 1e-05, + "loss": -0.0032, + "num_tokens": 17153331.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3648066520690918, + "sampling/importance_sampling_ratio/mean": 1.0000524520874023, + "sampling/importance_sampling_ratio/min": 0.5533369779586792, + "sampling/sampling_logp_difference/max": 0.5917880535125732, + "sampling/sampling_logp_difference/mean": 0.01031559333205223, + "step": 819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 449.59375, + "completions/mean_terminated_length": 449.59375, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.3009734135121107, + "epoch": 0.43897216274089934, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008712420240044594, + "learning_rate": 1e-05, + "loss": -0.0699, + "num_tokens": 17171550.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.999958872795105, + "sampling/importance_sampling_ratio/min": 0.6119124293327332, + "sampling/sampling_logp_difference/max": 0.7268886566162109, + "sampling/sampling_logp_difference/mean": 0.010710617527365685, + "step": 820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019959078053943813, + "clip_ratio/low_min": 0.00019959078053943813, + "clip_ratio/region_mean": 0.00019959078053943813, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 605.75, + "completions/mean_terminated_length": 551.6666870117188, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.5200091004371643, + "epoch": 0.43950749464668093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012178665027022362, + "learning_rate": 1e-05, + "loss": 0.0472, + "num_tokens": 17196494.0, + "reward": 0.3125, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5874097347259521, + "sampling/importance_sampling_ratio/mean": 0.9998118877410889, + "sampling/importance_sampling_ratio/min": 0.17467598617076874, + "sampling/sampling_logp_difference/max": 1.7448225021362305, + "sampling/sampling_logp_difference/mean": 0.015267375856637955, + "step": 821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 439.96875, + "completions/mean_terminated_length": 418.10003662109375, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.29386604204773903, + "epoch": 0.4400428265524625, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.022035328671336174, + "learning_rate": 1e-05, + "loss": 0.029, + "num_tokens": 17214245.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.2908525466918945, + "sampling/importance_sampling_ratio/mean": 0.9993357062339783, + "sampling/importance_sampling_ratio/min": 0.6960011720657349, + "sampling/sampling_logp_difference/max": 0.36240386962890625, + "sampling/sampling_logp_difference/mean": 0.010311836376786232, + "step": 822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 534.46875, + "completions/mean_terminated_length": 456.625, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.42147471755743027, + "epoch": 0.4405781584582441, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0033640230540186167, + "learning_rate": 1e-05, + "loss": 0.0025, + "num_tokens": 17235044.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5563052892684937, + "sampling/importance_sampling_ratio/mean": 1.0000025033950806, + "sampling/importance_sampling_ratio/min": 0.7584670782089233, + "sampling/sampling_logp_difference/max": 0.44231462478637695, + "sampling/sampling_logp_difference/mean": 0.013455349020659924, + "step": 823 + }, + { + "clip_ratio/high_max": 6.786102312617004e-05, + "clip_ratio/high_mean": 6.786102312617004e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.786102312617004e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 451.75, + "completions/mean_terminated_length": 451.75, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.26561981998384, + "epoch": 0.4411134903640257, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007244923152029514, + "learning_rate": 1e-05, + "loss": 0.0117, + "num_tokens": 17252796.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.662221908569336, + "sampling/importance_sampling_ratio/mean": 0.9998593926429749, + "sampling/importance_sampling_ratio/min": 0.7401193380355835, + "sampling/sampling_logp_difference/max": 0.5081552267074585, + "sampling/sampling_logp_difference/mean": 0.008950168266892433, + "step": 824 + }, + { + "clip_ratio/high_max": 4.950494985678233e-05, + "clip_ratio/high_mean": 4.950494985678233e-05, + "clip_ratio/low_mean": 5.169561700313352e-05, + "clip_ratio/low_min": 5.169561700313352e-05, + "clip_ratio/region_mean": 0.00010120056685991585, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 567.09375, + "completions/mean_terminated_length": 510.8399963378906, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.39459048956632614, + "epoch": 0.4416488222698073, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009630724787712097, + "learning_rate": 1e-05, + "loss": 0.0559, + "num_tokens": 17274927.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6130503416061401, + "sampling/importance_sampling_ratio/mean": 1.0000581741333008, + "sampling/importance_sampling_ratio/min": 0.6157658100128174, + "sampling/sampling_logp_difference/max": 0.48488855361938477, + "sampling/sampling_logp_difference/mean": 0.012862004339694977, + "step": 825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 405.375, + "completions/mean_terminated_length": 393.6773986816406, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.34616445004940033, + "epoch": 0.4421841541755889, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016571659594774246, + "learning_rate": 1e-05, + "loss": 0.0699, + "num_tokens": 17291267.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.00043523311615, + "sampling/importance_sampling_ratio/min": 0.6178522706031799, + "sampling/sampling_logp_difference/max": 0.9045839309692383, + "sampling/sampling_logp_difference/mean": 0.011966786347329617, + "step": 826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001231828791787848, + "clip_ratio/low_min": 0.0001231828791787848, + "clip_ratio/region_mean": 0.0001231828791787848, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 441.28125, + "completions/mean_terminated_length": 430.7419128417969, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.49824395403265953, + "epoch": 0.44271948608137046, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018703874200582504, + "learning_rate": 1e-05, + "loss": -0.0075, + "num_tokens": 17309068.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.9725242853164673, + "sampling/importance_sampling_ratio/mean": 1.0002188682556152, + "sampling/importance_sampling_ratio/min": 0.7081458568572998, + "sampling/sampling_logp_difference/max": 0.679314136505127, + "sampling/sampling_logp_difference/mean": 0.014864131808280945, + "step": 827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.579272252973169e-05, + "clip_ratio/low_min": 8.579272252973169e-05, + "clip_ratio/region_mean": 8.579272252973169e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 373.3125, + "completions/mean_terminated_length": 373.3125, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "entropy": 0.37105197459459305, + "epoch": 0.44325481798715205, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.04320777580142021, + "learning_rate": 1e-05, + "loss": -0.0142, + "num_tokens": 17324638.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4403419494628906, + "sampling/importance_sampling_ratio/mean": 1.0000241994857788, + "sampling/importance_sampling_ratio/min": 0.4905491769313812, + "sampling/sampling_logp_difference/max": 0.7122297286987305, + "sampling/sampling_logp_difference/mean": 0.012443293817341328, + "step": 828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 482.625, + "completions/mean_terminated_length": 473.4193420410156, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.23537281714379787, + "epoch": 0.44379014989293364, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17343866.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5877076387405396, + "sampling/importance_sampling_ratio/mean": 1.0002354383468628, + "sampling/importance_sampling_ratio/min": 0.7195164561271667, + "sampling/sampling_logp_difference/max": 0.46229124069213867, + "sampling/sampling_logp_difference/mean": 0.008366691879928112, + "step": 829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 470.03125, + "completions/mean_terminated_length": 414.85186767578125, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.4646605346351862, + "epoch": 0.4443254817987152, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02096404694020748, + "learning_rate": 1e-05, + "loss": 0.0515, + "num_tokens": 17363595.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5607069730758667, + "sampling/importance_sampling_ratio/mean": 0.9995356798171997, + "sampling/importance_sampling_ratio/min": 0.5868687629699707, + "sampling/sampling_logp_difference/max": 0.5329539775848389, + "sampling/sampling_logp_difference/mean": 0.014165048487484455, + "step": 830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.4656757129123434e-05, + "clip_ratio/low_min": 5.4656757129123434e-05, + "clip_ratio/region_mean": 5.4656757129123434e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 514.15625, + "completions/mean_terminated_length": 467.1481628417969, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "entropy": 0.29630786553025246, + "epoch": 0.44486081370449676, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022731341421604156, + "learning_rate": 1e-05, + "loss": 0.0357, + "num_tokens": 17384136.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4652239084243774, + "sampling/importance_sampling_ratio/mean": 1.0000675916671753, + "sampling/importance_sampling_ratio/min": 0.6110819578170776, + "sampling/sampling_logp_difference/max": 0.4925241470336914, + "sampling/sampling_logp_difference/mean": 0.009815549477934837, + "step": 831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 664.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 418.3125, + "completions/mean_terminated_length": 418.3125, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "entropy": 0.30645952746272087, + "epoch": 0.44539614561027835, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006999002303928137, + "learning_rate": 1e-05, + "loss": 0.0061, + "num_tokens": 17400818.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4104315042495728, + "sampling/importance_sampling_ratio/mean": 1.0002522468566895, + "sampling/importance_sampling_ratio/min": 0.7386618256568909, + "sampling/sampling_logp_difference/max": 0.34389567375183105, + "sampling/sampling_logp_difference/mean": 0.011039087548851967, + "step": 832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 755.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 473.6875, + "completions/mean_terminated_length": 473.6875, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.24417481571435928, + "epoch": 0.44593147751605994, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004943369887769222, + "learning_rate": 1e-05, + "loss": 0.0137, + "num_tokens": 17419960.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3704149723052979, + "sampling/importance_sampling_ratio/mean": 1.000150203704834, + "sampling/importance_sampling_ratio/min": 0.6169997453689575, + "sampling/sampling_logp_difference/max": 0.4828866720199585, + "sampling/sampling_logp_difference/mean": 0.009111866354942322, + "step": 833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 643.0, + "completions/mean_length": 480.25, + "completions/mean_terminated_length": 450.4827575683594, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.46574817411601543, + "epoch": 0.4464668094218415, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004699389915913343, + "learning_rate": 1e-05, + "loss": -0.0046, + "num_tokens": 17439064.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4031789302825928, + "sampling/importance_sampling_ratio/mean": 1.0000865459442139, + "sampling/importance_sampling_ratio/min": 0.6470046043395996, + "sampling/sampling_logp_difference/max": 0.43540191650390625, + "sampling/sampling_logp_difference/mean": 0.014360631816089153, + "step": 834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010442862912896089, + "clip_ratio/low_min": 0.00010442862912896089, + "clip_ratio/region_mean": 0.00010442862912896089, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 525.25, + "completions/mean_terminated_length": 457.2799987792969, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.2754701804369688, + "epoch": 0.4470021413276231, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01315237581729889, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 17459720.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4207916259765625, + "sampling/importance_sampling_ratio/mean": 0.9995573163032532, + "sampling/importance_sampling_ratio/min": 0.5681164860725403, + "sampling/sampling_logp_difference/max": 0.5654287338256836, + "sampling/sampling_logp_difference/mean": 0.01060456968843937, + "step": 835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 529.0, + "completions/max_terminated_length": 529.0, + "completions/mean_length": 383.125, + "completions/mean_terminated_length": 383.125, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.2697168178856373, + "epoch": 0.4475374732334047, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.048002421855926514, + "learning_rate": 1e-05, + "loss": -0.025, + "num_tokens": 17475268.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.454479455947876, + "sampling/importance_sampling_ratio/mean": 1.00013267993927, + "sampling/importance_sampling_ratio/min": 0.7839727401733398, + "sampling/sampling_logp_difference/max": 0.3746480941772461, + "sampling/sampling_logp_difference/mean": 0.009478505700826645, + "step": 836 + }, + { + "clip_ratio/high_max": 5.439512460725382e-05, + "clip_ratio/high_mean": 5.439512460725382e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.439512460725382e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 518.375, + "completions/mean_terminated_length": 472.1481628417969, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.40216017700731754, + "epoch": 0.4480728051391863, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013944534584879875, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 17495768.0, + "reward": 0.6875, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.513625979423523, + "sampling/importance_sampling_ratio/mean": 1.0001559257507324, + "sampling/importance_sampling_ratio/min": 0.24801397323608398, + "sampling/sampling_logp_difference/max": 1.3942701816558838, + "sampling/sampling_logp_difference/mean": 0.013445986434817314, + "step": 837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 695.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 502.8125, + "completions/mean_terminated_length": 502.8125, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.2928779497742653, + "epoch": 0.4486081370449679, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0052935900166630745, + "learning_rate": 1e-05, + "loss": 0.0092, + "num_tokens": 17515722.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4502661228179932, + "sampling/importance_sampling_ratio/mean": 0.9999672770500183, + "sampling/importance_sampling_ratio/min": 0.7170757055282593, + "sampling/sampling_logp_difference/max": 0.3717470169067383, + "sampling/sampling_logp_difference/mean": 0.010355195961892605, + "step": 838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.115459837019444e-05, + "clip_ratio/low_min": 6.115459837019444e-05, + "clip_ratio/region_mean": 6.115459837019444e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 420.59375, + "completions/mean_terminated_length": 397.433349609375, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.3525313138961792, + "epoch": 0.44914346895074947, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.020515669137239456, + "learning_rate": 1e-05, + "loss": 0.024, + "num_tokens": 17533253.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.8683252334594727, + "sampling/importance_sampling_ratio/mean": 0.9998475909233093, + "sampling/importance_sampling_ratio/min": 0.5709669589996338, + "sampling/sampling_logp_difference/max": 0.6250424385070801, + "sampling/sampling_logp_difference/mean": 0.01299965288490057, + "step": 839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 493.65625, + "completions/mean_terminated_length": 484.8064270019531, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "entropy": 0.22328947111964226, + "epoch": 0.44967880085653106, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005702413152903318, + "learning_rate": 1e-05, + "loss": -0.038, + "num_tokens": 17553082.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.2993196249008179, + "sampling/importance_sampling_ratio/mean": 0.9999359846115112, + "sampling/importance_sampling_ratio/min": 0.6806808710098267, + "sampling/sampling_logp_difference/max": 0.3846616744995117, + "sampling/sampling_logp_difference/mean": 0.008328845724463463, + "step": 840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 546.34375, + "completions/mean_terminated_length": 505.2962951660156, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.3904264606535435, + "epoch": 0.45021413276231265, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17575165.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4561699628829956, + "sampling/importance_sampling_ratio/mean": 0.9996989965438843, + "sampling/importance_sampling_ratio/min": 0.6488072872161865, + "sampling/sampling_logp_difference/max": 0.432619571685791, + "sampling/sampling_logp_difference/mean": 0.013349946588277817, + "step": 841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001283141755266115, + "clip_ratio/low_min": 0.0001283141755266115, + "clip_ratio/region_mean": 0.0001283141755266115, + "completions/clipped_ratio": 0.0, + "completions/max_length": 719.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 456.59375, + "completions/mean_terminated_length": 456.59375, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.28330995328724384, + "epoch": 0.45074946466809424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.034676454961299896, + "learning_rate": 1e-05, + "loss": -0.0453, + "num_tokens": 17593376.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.404496431350708, + "sampling/importance_sampling_ratio/mean": 0.9998428225517273, + "sampling/importance_sampling_ratio/min": 0.6960118412971497, + "sampling/sampling_logp_difference/max": 0.36238861083984375, + "sampling/sampling_logp_difference/mean": 0.010486968792974949, + "step": 842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 535.0, + "completions/mean_length": 401.90625, + "completions/mean_terminated_length": 390.0967712402344, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.25354092195630074, + "epoch": 0.4512847965738758, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013399692252278328, + "learning_rate": 1e-05, + "loss": 0.074, + "num_tokens": 17609973.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4726992845535278, + "sampling/importance_sampling_ratio/mean": 0.999975860118866, + "sampling/importance_sampling_ratio/min": 0.6956537961959839, + "sampling/sampling_logp_difference/max": 0.3870968818664551, + "sampling/sampling_logp_difference/mean": 0.009599726647138596, + "step": 843 + }, + { + "clip_ratio/high_max": 6.551362457685173e-05, + "clip_ratio/high_mean": 6.551362457685173e-05, + "clip_ratio/low_mean": 0.0002793692401610315, + "clip_ratio/low_min": 0.0002793692401610315, + "clip_ratio/region_mean": 0.0003448828647378832, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 437.28125, + "completions/mean_terminated_length": 403.0689697265625, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "entropy": 0.4792197309434414, + "epoch": 0.4518201284796574, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.040485985577106476, + "learning_rate": 1e-05, + "loss": 0.0784, + "num_tokens": 17627462.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.9082411527633667, + "sampling/importance_sampling_ratio/mean": 1.0005013942718506, + "sampling/importance_sampling_ratio/min": 0.6889159083366394, + "sampling/sampling_logp_difference/max": 0.6461819410324097, + "sampling/sampling_logp_difference/mean": 0.016048213467001915, + "step": 844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.2126772061455995e-05, + "clip_ratio/low_min": 5.2126772061455995e-05, + "clip_ratio/region_mean": 5.2126772061455995e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 484.0, + "completions/mean_terminated_length": 465.0666809082031, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.3404336478561163, + "epoch": 0.45235546038543895, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.018093738704919815, + "learning_rate": 1e-05, + "loss": 0.0077, + "num_tokens": 17647278.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6443889141082764, + "sampling/importance_sampling_ratio/mean": 0.9996999502182007, + "sampling/importance_sampling_ratio/min": 0.6987136602401733, + "sampling/sampling_logp_difference/max": 0.49736881256103516, + "sampling/sampling_logp_difference/mean": 0.01183375995606184, + "step": 845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012124221029807813, + "clip_ratio/low_min": 0.00012124221029807813, + "clip_ratio/region_mean": 0.00012124221029807813, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 517.59375, + "completions/mean_terminated_length": 509.51611328125, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.28196615166962147, + "epoch": 0.45289079229122053, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00997235719114542, + "learning_rate": 1e-05, + "loss": -0.035, + "num_tokens": 17667409.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5072767734527588, + "sampling/importance_sampling_ratio/mean": 1.0000327825546265, + "sampling/importance_sampling_ratio/min": 0.7333748936653137, + "sampling/sampling_logp_difference/max": 0.41030454635620117, + "sampling/sampling_logp_difference/mean": 0.010541322641074657, + "step": 846 + }, + { + "clip_ratio/high_max": 6.698820652673021e-05, + "clip_ratio/high_mean": 6.698820652673021e-05, + "clip_ratio/low_mean": 0.00011798779814853333, + "clip_ratio/low_min": 0.00011798779814853333, + "clip_ratio/region_mean": 0.00018497600467526354, + "completions/clipped_ratio": 0.0, + "completions/max_length": 727.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 475.46875, + "completions/mean_terminated_length": 475.46875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.34865502640604973, + "epoch": 0.4534261241970021, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004660570994019508, + "learning_rate": 1e-05, + "loss": 0.0368, + "num_tokens": 17686608.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.470431923866272, + "sampling/importance_sampling_ratio/mean": 1.0003079175949097, + "sampling/importance_sampling_ratio/min": 0.7072607278823853, + "sampling/sampling_logp_difference/max": 0.3855562210083008, + "sampling/sampling_logp_difference/mean": 0.011790947988629341, + "step": 847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015960823293426074, + "clip_ratio/low_min": 0.00015960823293426074, + "clip_ratio/region_mean": 0.00015960823293426074, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 590.625, + "completions/mean_terminated_length": 565.2857666015625, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.33926576375961304, + "epoch": 0.4539614561027837, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01985170878469944, + "learning_rate": 1e-05, + "loss": -0.0024, + "num_tokens": 17710020.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.8691842555999756, + "sampling/importance_sampling_ratio/mean": 1.0000360012054443, + "sampling/importance_sampling_ratio/min": 0.6046698093414307, + "sampling/sampling_logp_difference/max": 0.6255021095275879, + "sampling/sampling_logp_difference/mean": 0.012216202914714813, + "step": 848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 552.0, + "completions/max_terminated_length": 552.0, + "completions/mean_length": 412.21875, + "completions/mean_terminated_length": 412.21875, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.2868776898831129, + "epoch": 0.4544967880085653, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013748974539339542, + "learning_rate": 1e-05, + "loss": -0.0423, + "num_tokens": 17726899.0, + "reward": 0.71875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3464021682739258, + "sampling/importance_sampling_ratio/mean": 1.0001211166381836, + "sampling/importance_sampling_ratio/min": 0.7243185043334961, + "sampling/sampling_logp_difference/max": 0.3225240707397461, + "sampling/sampling_logp_difference/mean": 0.010825544595718384, + "step": 849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.994963769102469e-05, + "clip_ratio/low_min": 6.994963769102469e-05, + "clip_ratio/region_mean": 6.994963769102469e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 752.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 442.65625, + "completions/mean_terminated_length": 442.65625, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "entropy": 0.21593933179974556, + "epoch": 0.4550321199143469, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006543995812535286, + "learning_rate": 1e-05, + "loss": -0.0158, + "num_tokens": 17744664.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5651553869247437, + "sampling/importance_sampling_ratio/mean": 0.9995262622833252, + "sampling/importance_sampling_ratio/min": 0.624371349811554, + "sampling/sampling_logp_difference/max": 0.4710099697113037, + "sampling/sampling_logp_difference/mean": 0.00816783681511879, + "step": 850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.952380888629705e-05, + "clip_ratio/low_min": 5.952380888629705e-05, + "clip_ratio/region_mean": 5.952380888629705e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 520.90625, + "completions/mean_terminated_length": 495.3448181152344, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.3044566884636879, + "epoch": 0.4555674518201285, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02042035572230816, + "learning_rate": 1e-05, + "loss": 0.0118, + "num_tokens": 17765173.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4871103763580322, + "sampling/importance_sampling_ratio/mean": 1.0001747608184814, + "sampling/importance_sampling_ratio/min": 0.6212068796157837, + "sampling/sampling_logp_difference/max": 0.4760911464691162, + "sampling/sampling_logp_difference/mean": 0.011123652569949627, + "step": 851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.124448555056006e-05, + "clip_ratio/low_min": 6.124448555056006e-05, + "clip_ratio/region_mean": 6.124448555056006e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 454.25, + "completions/mean_terminated_length": 444.1290283203125, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.2763819079846144, + "epoch": 0.45610278372591007, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013571832329034805, + "learning_rate": 1e-05, + "loss": 0.0155, + "num_tokens": 17783517.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4096670150756836, + "sampling/importance_sampling_ratio/mean": 1.0002992153167725, + "sampling/importance_sampling_ratio/min": 0.682148277759552, + "sampling/sampling_logp_difference/max": 0.3825082778930664, + "sampling/sampling_logp_difference/mean": 0.010273137129843235, + "step": 852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 418.5625, + "completions/mean_terminated_length": 395.2666931152344, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.32329537719488144, + "epoch": 0.45663811563169165, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17800271.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5444774627685547, + "sampling/importance_sampling_ratio/mean": 1.0000261068344116, + "sampling/importance_sampling_ratio/min": 0.6960228085517883, + "sampling/sampling_logp_difference/max": 0.43468570709228516, + "sampling/sampling_logp_difference/mean": 0.01200791634619236, + "step": 853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014849321451038122, + "clip_ratio/low_min": 0.00014849321451038122, + "clip_ratio/region_mean": 0.00014849321451038122, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 436.34375, + "completions/mean_terminated_length": 425.6451416015625, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3880876265466213, + "epoch": 0.45717344753747324, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009868266992270947, + "learning_rate": 1e-05, + "loss": 0.0124, + "num_tokens": 17818162.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4362332820892334, + "sampling/importance_sampling_ratio/mean": 0.9998043179512024, + "sampling/importance_sampling_ratio/min": 0.5585951805114746, + "sampling/sampling_logp_difference/max": 0.5823302268981934, + "sampling/sampling_logp_difference/mean": 0.01380274910479784, + "step": 854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013056294847046956, + "clip_ratio/low_min": 0.00013056294847046956, + "clip_ratio/region_mean": 0.00013056294847046956, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 474.1875, + "completions/mean_terminated_length": 454.60003662109375, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3601721115410328, + "epoch": 0.45770877944325483, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009773055091500282, + "learning_rate": 1e-05, + "loss": 0.0507, + "num_tokens": 17836960.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5590063333511353, + "sampling/importance_sampling_ratio/mean": 0.9999298453330994, + "sampling/importance_sampling_ratio/min": 0.6473665833473206, + "sampling/sampling_logp_difference/max": 0.4440486431121826, + "sampling/sampling_logp_difference/mean": 0.012558458372950554, + "step": 855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.846585554536432e-05, + "clip_ratio/low_min": 5.846585554536432e-05, + "clip_ratio/region_mean": 5.846585554536432e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 639.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 448.5, + "completions/mean_terminated_length": 448.5, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.2852116785943508, + "epoch": 0.4582441113490364, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0109, + "num_tokens": 17854680.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.6794205904006958, + "sampling/importance_sampling_ratio/mean": 1.0000758171081543, + "sampling/importance_sampling_ratio/min": 0.665300190448761, + "sampling/sampling_logp_difference/max": 0.5184488296508789, + "sampling/sampling_logp_difference/mean": 0.01116214320063591, + "step": 856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 421.6875, + "completions/mean_terminated_length": 421.6875, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.29658959805965424, + "epoch": 0.458779443254818, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021478310227394104, + "learning_rate": 1e-05, + "loss": -0.0052, + "num_tokens": 17871486.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.834847331047058, + "sampling/importance_sampling_ratio/mean": 1.0001215934753418, + "sampling/importance_sampling_ratio/min": 0.6851791143417358, + "sampling/sampling_logp_difference/max": 0.6069612503051758, + "sampling/sampling_logp_difference/mean": 0.01114323828369379, + "step": 857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.139802669873461e-05, + "clip_ratio/low_min": 5.139802669873461e-05, + "clip_ratio/region_mean": 5.139802669873461e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 456.9375, + "completions/mean_terminated_length": 456.9375, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.28951916098594666, + "epoch": 0.4593147751605996, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003334330627694726, + "learning_rate": 1e-05, + "loss": 0.0173, + "num_tokens": 17890260.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.7477837800979614, + "sampling/importance_sampling_ratio/mean": 0.999552845954895, + "sampling/importance_sampling_ratio/min": 0.6054671406745911, + "sampling/sampling_logp_difference/max": 0.558348536491394, + "sampling/sampling_logp_difference/mean": 0.011437330394983292, + "step": 858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 457.9375, + "completions/mean_terminated_length": 447.9354553222656, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "entropy": 0.34119654819369316, + "epoch": 0.45985010706638113, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019065650179982185, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 17908162.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4342588186264038, + "sampling/importance_sampling_ratio/mean": 1.0001591444015503, + "sampling/importance_sampling_ratio/min": 0.6432639956474304, + "sampling/sampling_logp_difference/max": 0.44120001792907715, + "sampling/sampling_logp_difference/mean": 0.011838923208415508, + "step": 859 + }, + { + "clip_ratio/high_max": 6.309944728855044e-05, + "clip_ratio/high_mean": 6.309944728855044e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.309944728855044e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 436.65625, + "completions/mean_terminated_length": 414.5666809082031, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.32465410977602005, + "epoch": 0.4603854389721627, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006189401727169752, + "learning_rate": 1e-05, + "loss": 0.0391, + "num_tokens": 17925495.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4466276168823242, + "sampling/importance_sampling_ratio/mean": 0.9999515414237976, + "sampling/importance_sampling_ratio/min": 0.6255346536636353, + "sampling/sampling_logp_difference/max": 0.4691486358642578, + "sampling/sampling_logp_difference/mean": 0.012850682251155376, + "step": 860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 520.90625, + "completions/mean_terminated_length": 475.1481628417969, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.29688988626003265, + "epoch": 0.4609207708779443, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0018, + "num_tokens": 17945460.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4301317930221558, + "sampling/importance_sampling_ratio/mean": 0.9998509883880615, + "sampling/importance_sampling_ratio/min": 0.6935265064239502, + "sampling/sampling_logp_difference/max": 0.3659658432006836, + "sampling/sampling_logp_difference/mean": 0.01100583653897047, + "step": 861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 527.34375, + "completions/mean_terminated_length": 502.4482727050781, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.25307290256023407, + "epoch": 0.4614561027837259, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005747991148382425, + "learning_rate": 1e-05, + "loss": 0.0376, + "num_tokens": 17965943.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5645287036895752, + "sampling/importance_sampling_ratio/mean": 0.9996405243873596, + "sampling/importance_sampling_ratio/min": 0.6969373822212219, + "sampling/sampling_logp_difference/max": 0.4475846290588379, + "sampling/sampling_logp_difference/mean": 0.010516190901398659, + "step": 862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 547.1875, + "completions/mean_terminated_length": 460.7826232910156, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.3906844034790993, + "epoch": 0.4619914346895075, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02722861059010029, + "learning_rate": 1e-05, + "loss": 0.0244, + "num_tokens": 17987221.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6273584365844727, + "sampling/importance_sampling_ratio/mean": 1.000257134437561, + "sampling/importance_sampling_ratio/min": 0.6665515303611755, + "sampling/sampling_logp_difference/max": 0.48695802688598633, + "sampling/sampling_logp_difference/mean": 0.013111263513565063, + "step": 863 + }, + { + "clip_ratio/high_max": 0.00010442773782415316, + "clip_ratio/high_mean": 0.00010442773782415316, + "clip_ratio/low_mean": 7.018529140623286e-05, + "clip_ratio/low_min": 7.018529140623286e-05, + "clip_ratio/region_mean": 0.00017461302923038602, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 353.9375, + "completions/mean_terminated_length": 353.9375, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.2766990512609482, + "epoch": 0.4625267665952891, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005675820168107748, + "learning_rate": 1e-05, + "loss": -0.0002, + "num_tokens": 18001803.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.403257966041565, + "sampling/importance_sampling_ratio/mean": 0.9998831152915955, + "sampling/importance_sampling_ratio/min": 0.6779201626777649, + "sampling/sampling_logp_difference/max": 0.38872575759887695, + "sampling/sampling_logp_difference/mean": 0.01106823980808258, + "step": 864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 468.375, + "completions/mean_terminated_length": 425.5714416503906, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.35452619194984436, + "epoch": 0.46306209850107066, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014592459425330162, + "learning_rate": 1e-05, + "loss": 0.0309, + "num_tokens": 18020807.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.8562095165252686, + "sampling/importance_sampling_ratio/mean": 0.9994197487831116, + "sampling/importance_sampling_ratio/min": 0.3569437563419342, + "sampling/sampling_logp_difference/max": 1.030177116394043, + "sampling/sampling_logp_difference/mean": 0.013007337227463722, + "step": 865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 453.625, + "completions/mean_terminated_length": 443.4838562011719, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.261502867564559, + "epoch": 0.46359743040685225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008076409809291363, + "learning_rate": 1e-05, + "loss": 0.0763, + "num_tokens": 18038851.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4596995115280151, + "sampling/importance_sampling_ratio/mean": 0.9996519088745117, + "sampling/importance_sampling_ratio/min": 0.6547124981880188, + "sampling/sampling_logp_difference/max": 0.4235590696334839, + "sampling/sampling_logp_difference/mean": 0.010198540985584259, + "step": 866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 510.65625, + "completions/mean_terminated_length": 502.3548278808594, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.35885627567768097, + "epoch": 0.46413276231263384, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009353122673928738, + "learning_rate": 1e-05, + "loss": 0.0001, + "num_tokens": 18058928.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6648247241973877, + "sampling/importance_sampling_ratio/mean": 0.999761700630188, + "sampling/importance_sampling_ratio/min": 0.4991922974586487, + "sampling/sampling_logp_difference/max": 0.6947638988494873, + "sampling/sampling_logp_difference/mean": 0.012562422081828117, + "step": 867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011859582446049899, + "clip_ratio/low_min": 0.00011859582446049899, + "clip_ratio/region_mean": 0.00011859582446049899, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 454.375, + "completions/mean_terminated_length": 444.258056640625, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.29925916343927383, + "epoch": 0.46466809421841543, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005252790171653032, + "learning_rate": 1e-05, + "loss": 0.0172, + "num_tokens": 18077300.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.480759620666504, + "sampling/importance_sampling_ratio/mean": 0.9997273683547974, + "sampling/importance_sampling_ratio/min": 0.6663631200790405, + "sampling/sampling_logp_difference/max": 0.40592050552368164, + "sampling/sampling_logp_difference/mean": 0.012263418175280094, + "step": 868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 450.65625, + "completions/mean_terminated_length": 429.5000305175781, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.2597888894379139, + "epoch": 0.465203426124197, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.018060805276036263, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 18095649.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.722191333770752, + "sampling/importance_sampling_ratio/mean": 1.0004228353500366, + "sampling/importance_sampling_ratio/min": 0.7221609354019165, + "sampling/sampling_logp_difference/max": 0.5435974597930908, + "sampling/sampling_logp_difference/mean": 0.010789213702082634, + "step": 869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 603.78125, + "completions/mean_terminated_length": 580.3214721679688, + "completions/min_length": 379.0, + "completions/min_terminated_length": 379.0, + "entropy": 0.26773758232593536, + "epoch": 0.4657387580299786, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016819944605231285, + "learning_rate": 1e-05, + "loss": 0.0553, + "num_tokens": 18119234.0, + "reward": 0.75, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7391873598098755, + "sampling/importance_sampling_ratio/mean": 0.999520480632782, + "sampling/importance_sampling_ratio/min": 0.6458948850631714, + "sampling/sampling_logp_difference/max": 0.5534179210662842, + "sampling/sampling_logp_difference/mean": 0.009992285631597042, + "step": 870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 463.5, + "completions/mean_terminated_length": 432.0, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.28490811958909035, + "epoch": 0.4662740899357602, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01612742803990841, + "learning_rate": 1e-05, + "loss": 0.0194, + "num_tokens": 18138290.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.589380145072937, + "sampling/importance_sampling_ratio/mean": 0.9997609853744507, + "sampling/importance_sampling_ratio/min": 0.6430565714836121, + "sampling/sampling_logp_difference/max": 0.46334409713745117, + "sampling/sampling_logp_difference/mean": 0.012222224846482277, + "step": 871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 516.40625, + "completions/mean_terminated_length": 516.40625, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.2322842124849558, + "epoch": 0.4668094218415418, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006723431404680014, + "learning_rate": 1e-05, + "loss": 0.0083, + "num_tokens": 18158759.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.440084457397461, + "sampling/importance_sampling_ratio/mean": 1.0000141859054565, + "sampling/importance_sampling_ratio/min": 0.685984194278717, + "sampling/sampling_logp_difference/max": 0.37690067291259766, + "sampling/sampling_logp_difference/mean": 0.009048039093613625, + "step": 872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.4324205848388374e-05, + "clip_ratio/low_min": 5.4324205848388374e-05, + "clip_ratio/region_mean": 5.4324205848388374e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 497.125, + "completions/mean_terminated_length": 458.4285888671875, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.4040381032973528, + "epoch": 0.4673447537473233, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018366919830441475, + "learning_rate": 1e-05, + "loss": 0.0542, + "num_tokens": 18178155.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.440218448638916, + "sampling/importance_sampling_ratio/mean": 0.9998831748962402, + "sampling/importance_sampling_ratio/min": 0.5919627547264099, + "sampling/sampling_logp_difference/max": 0.5243115425109863, + "sampling/sampling_logp_difference/mean": 0.013409332372248173, + "step": 873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 734.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 383.75, + "completions/mean_terminated_length": 383.75, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.22543956339359283, + "epoch": 0.4678800856531049, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 18193835.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002186298370361, + "sampling/importance_sampling_ratio/min": 0.5272040367126465, + "sampling/sampling_logp_difference/max": 0.8156676292419434, + "sampling/sampling_logp_difference/mean": 0.010029658675193787, + "step": 874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001817653828766197, + "clip_ratio/low_min": 0.0001817653828766197, + "clip_ratio/region_mean": 0.0001817653828766197, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 586.28125, + "completions/mean_terminated_length": 477.25, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3310044165700674, + "epoch": 0.4684154175588865, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00836801528930664, + "learning_rate": 1e-05, + "loss": 0.0389, + "num_tokens": 18216260.0, + "reward": 0.53125, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4283980131149292, + "sampling/importance_sampling_ratio/mean": 0.9998232126235962, + "sampling/importance_sampling_ratio/min": 0.6860008239746094, + "sampling/sampling_logp_difference/max": 0.3768763542175293, + "sampling/sampling_logp_difference/mean": 0.011945049278438091, + "step": 875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 447.59375, + "completions/mean_terminated_length": 437.258056640625, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.33078383654356003, + "epoch": 0.4689507494646681, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006502924952656031, + "learning_rate": 1e-05, + "loss": 0.0448, + "num_tokens": 18234087.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5555464029312134, + "sampling/importance_sampling_ratio/mean": 1.0001003742218018, + "sampling/importance_sampling_ratio/min": 0.5430275797843933, + "sampling/sampling_logp_difference/max": 0.6105952262878418, + "sampling/sampling_logp_difference/mean": 0.012804846279323101, + "step": 876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 557.6875, + "completions/mean_terminated_length": 487.5833435058594, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.23251344822347164, + "epoch": 0.46948608137044967, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003787594148889184, + "learning_rate": 1e-05, + "loss": 0.0259, + "num_tokens": 18255877.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.658536672592163, + "sampling/importance_sampling_ratio/mean": 0.9999313950538635, + "sampling/importance_sampling_ratio/min": 0.6560901999473572, + "sampling/sampling_logp_difference/max": 0.5059356689453125, + "sampling/sampling_logp_difference/mean": 0.009480309672653675, + "step": 877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 425.1875, + "completions/mean_terminated_length": 425.1875, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.2553399093449116, + "epoch": 0.47002141327623126, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0038415465969592333, + "learning_rate": 1e-05, + "loss": -0.0391, + "num_tokens": 18273027.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5385226011276245, + "sampling/importance_sampling_ratio/mean": 0.9999315142631531, + "sampling/importance_sampling_ratio/min": 0.6782143115997314, + "sampling/sampling_logp_difference/max": 0.43082261085510254, + "sampling/sampling_logp_difference/mean": 0.010707005858421326, + "step": 878 + }, + { + "clip_ratio/high_max": 5.118755143485032e-05, + "clip_ratio/high_mean": 5.118755143485032e-05, + "clip_ratio/low_mean": 0.00014566063691745512, + "clip_ratio/low_min": 0.00014566063691745512, + "clip_ratio/region_mean": 0.00019684818835230544, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 550.90625, + "completions/mean_terminated_length": 465.95654296875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.44744790345430374, + "epoch": 0.47055674518201285, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007869189605116844, + "learning_rate": 1e-05, + "loss": 0.0408, + "num_tokens": 18294632.0, + "reward": 0.59375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5502578020095825, + "sampling/importance_sampling_ratio/mean": 0.9996722936630249, + "sampling/importance_sampling_ratio/min": 0.69932621717453, + "sampling/sampling_logp_difference/max": 0.43842124938964844, + "sampling/sampling_logp_difference/mean": 0.015066186897456646, + "step": 879 + }, + { + "clip_ratio/high_max": 6.067961294320412e-05, + "clip_ratio/high_mean": 6.067961294320412e-05, + "clip_ratio/low_mean": 6.067961294320412e-05, + "clip_ratio/low_min": 6.067961294320412e-05, + "clip_ratio/region_mean": 0.00012135922588640824, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 528.96875, + "completions/mean_terminated_length": 473.8077087402344, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.40720242634415627, + "epoch": 0.47109207708779444, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019042128697037697, + "learning_rate": 1e-05, + "loss": 0.0633, + "num_tokens": 18315839.0, + "reward": 0.5625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3979129791259766, + "sampling/importance_sampling_ratio/mean": 0.9999843239784241, + "sampling/importance_sampling_ratio/min": 0.6468991041183472, + "sampling/sampling_logp_difference/max": 0.4355649948120117, + "sampling/sampling_logp_difference/mean": 0.013768176548182964, + "step": 880 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 652.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 434.59375, + "completions/mean_terminated_length": 434.59375, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.2813435569405556, + "epoch": 0.471627408993576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010080586187541485, + "learning_rate": 1e-05, + "loss": -0.0253, + "num_tokens": 18333386.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.568997859954834, + "sampling/importance_sampling_ratio/mean": 1.000688076019287, + "sampling/importance_sampling_ratio/min": 0.6839987635612488, + "sampling/sampling_logp_difference/max": 0.450437068939209, + "sampling/sampling_logp_difference/mean": 0.011479225009679794, + "step": 881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010245901648886502, + "clip_ratio/low_min": 0.00010245901648886502, + "clip_ratio/region_mean": 0.00010245901648886502, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 559.75, + "completions/mean_terminated_length": 511.69232177734375, + "completions/min_length": 346.0, + "completions/min_terminated_length": 346.0, + "entropy": 0.31052958965301514, + "epoch": 0.4721627408993576, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009617138653993607, + "learning_rate": 1e-05, + "loss": 0.011, + "num_tokens": 18355410.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.640962839126587, + "sampling/importance_sampling_ratio/mean": 0.9999303221702576, + "sampling/importance_sampling_ratio/min": 0.695469081401825, + "sampling/sampling_logp_difference/max": 0.4952831268310547, + "sampling/sampling_logp_difference/mean": 0.01131723914295435, + "step": 882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 543.0, + "completions/max_terminated_length": 543.0, + "completions/mean_length": 383.9375, + "completions/mean_terminated_length": 383.9375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.25118929147720337, + "epoch": 0.4726980728051392, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01852281764149666, + "learning_rate": 1e-05, + "loss": 0.0192, + "num_tokens": 18371328.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.9445576667785645, + "sampling/importance_sampling_ratio/mean": 1.000382900238037, + "sampling/importance_sampling_ratio/min": 0.6961176991462708, + "sampling/sampling_logp_difference/max": 0.6650345325469971, + "sampling/sampling_logp_difference/mean": 0.010267798788845539, + "step": 883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 344.65625, + "completions/mean_terminated_length": 344.65625, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.2916817367076874, + "epoch": 0.4732334047109208, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 18385613.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.7127189636230469, + "sampling/importance_sampling_ratio/mean": 0.9995903968811035, + "sampling/importance_sampling_ratio/min": 0.7017140984535217, + "sampling/sampling_logp_difference/max": 0.5380821228027344, + "sampling/sampling_logp_difference/mean": 0.011689784936606884, + "step": 884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 440.9375, + "completions/mean_terminated_length": 419.13336181640625, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.33362437784671783, + "epoch": 0.4737687366167024, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0481, + "num_tokens": 18403219.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.8563076257705688, + "sampling/importance_sampling_ratio/mean": 1.0002373456954956, + "sampling/importance_sampling_ratio/min": 0.6271288394927979, + "sampling/sampling_logp_difference/max": 0.6185894012451172, + "sampling/sampling_logp_difference/mean": 0.012039019726216793, + "step": 885 + }, + { + "clip_ratio/high_max": 5.5358723329845816e-05, + "clip_ratio/high_mean": 5.5358723329845816e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.5358723329845816e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 751.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 503.09375, + "completions/mean_terminated_length": 503.09375, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.32552289217710495, + "epoch": 0.4743040685224839, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01108392421156168, + "learning_rate": 1e-05, + "loss": 0.0072, + "num_tokens": 18423718.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4468801021575928, + "sampling/importance_sampling_ratio/mean": 0.9990369081497192, + "sampling/importance_sampling_ratio/min": 0.5851335525512695, + "sampling/sampling_logp_difference/max": 0.5359151363372803, + "sampling/sampling_logp_difference/mean": 0.012145783752202988, + "step": 886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 453.34375, + "completions/mean_terminated_length": 443.19354248046875, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.29032275825738907, + "epoch": 0.4748394004282655, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007234309334307909, + "learning_rate": 1e-05, + "loss": 0.0227, + "num_tokens": 18441785.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.438271403312683, + "sampling/importance_sampling_ratio/mean": 1.000157117843628, + "sampling/importance_sampling_ratio/min": 0.7075569033622742, + "sampling/sampling_logp_difference/max": 0.36344194412231445, + "sampling/sampling_logp_difference/mean": 0.010901025496423244, + "step": 887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 759.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 442.96875, + "completions/mean_terminated_length": 442.96875, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.23408159986138344, + "epoch": 0.4753747323340471, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005237105768173933, + "learning_rate": 1e-05, + "loss": 0.0134, + "num_tokens": 18459912.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5362108945846558, + "sampling/importance_sampling_ratio/mean": 1.0000702142715454, + "sampling/importance_sampling_ratio/min": 0.6879655718803406, + "sampling/sampling_logp_difference/max": 0.429318904876709, + "sampling/sampling_logp_difference/mean": 0.009599080309271812, + "step": 888 + }, + { + "clip_ratio/high_max": 0.00010968405695166439, + "clip_ratio/high_mean": 0.00010968405695166439, + "clip_ratio/low_mean": 5.887894440093078e-05, + "clip_ratio/low_min": 5.887894440093078e-05, + "clip_ratio/region_mean": 0.00016856300135259517, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 539.34375, + "completions/mean_terminated_length": 524.1000366210938, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.30345601215958595, + "epoch": 0.4759100642398287, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019864946603775024, + "learning_rate": 1e-05, + "loss": 0.0569, + "num_tokens": 18481267.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.72467839717865, + "sampling/importance_sampling_ratio/mean": 1.0002552270889282, + "sampling/importance_sampling_ratio/min": 0.4199666678905487, + "sampling/sampling_logp_difference/max": 0.8675799369812012, + "sampling/sampling_logp_difference/mean": 0.012069141492247581, + "step": 889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 542.40625, + "completions/mean_terminated_length": 527.36669921875, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.26934675872325897, + "epoch": 0.47644539614561027, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01202559657394886, + "learning_rate": 1e-05, + "loss": 0.0332, + "num_tokens": 18502776.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3873212337493896, + "sampling/importance_sampling_ratio/mean": 0.9999104738235474, + "sampling/importance_sampling_ratio/min": 0.6175732016563416, + "sampling/sampling_logp_difference/max": 0.48195767402648926, + "sampling/sampling_logp_difference/mean": 0.010256065987050533, + "step": 890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 384.53125, + "completions/mean_terminated_length": 384.53125, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.28182504139840603, + "epoch": 0.47698072805139186, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004495512694120407, + "learning_rate": 1e-05, + "loss": 0.0301, + "num_tokens": 18518745.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.8272864818572998, + "sampling/importance_sampling_ratio/mean": 1.0004711151123047, + "sampling/importance_sampling_ratio/min": 0.5638600587844849, + "sampling/sampling_logp_difference/max": 0.6028320789337158, + "sampling/sampling_logp_difference/mean": 0.011364540085196495, + "step": 891 + }, + { + "clip_ratio/high_max": 5.5654498282819986e-05, + "clip_ratio/high_mean": 5.5654498282819986e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.5654498282819986e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 521.09375, + "completions/mean_terminated_length": 504.63336181640625, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.2611116748303175, + "epoch": 0.47751605995717344, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010341753251850605, + "learning_rate": 1e-05, + "loss": 0.076, + "num_tokens": 18539180.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.443167805671692, + "sampling/importance_sampling_ratio/mean": 0.9995496273040771, + "sampling/importance_sampling_ratio/min": 0.43118199706077576, + "sampling/sampling_logp_difference/max": 0.8412249684333801, + "sampling/sampling_logp_difference/mean": 0.010036716237664223, + "step": 892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00022684068244416267, + "clip_ratio/low_min": 0.00022684068244416267, + "clip_ratio/region_mean": 0.00022684068244416267, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 497.125, + "completions/mean_terminated_length": 446.96295166015625, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.22813864424824715, + "epoch": 0.47805139186295503, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007934284396469593, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 18559168.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.7819143533706665, + "sampling/importance_sampling_ratio/mean": 1.0001704692840576, + "sampling/importance_sampling_ratio/min": 0.6121436953544617, + "sampling/sampling_logp_difference/max": 0.5776882171630859, + "sampling/sampling_logp_difference/mean": 0.009797493927180767, + "step": 893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 718.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 439.46875, + "completions/mean_terminated_length": 439.46875, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.3192239012569189, + "epoch": 0.4785867237687366, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020979948341846466, + "learning_rate": 1e-05, + "loss": -0.0382, + "num_tokens": 18576847.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.503618597984314, + "sampling/importance_sampling_ratio/mean": 0.9997166395187378, + "sampling/importance_sampling_ratio/min": 0.6501674056053162, + "sampling/sampling_logp_difference/max": 0.43052542209625244, + "sampling/sampling_logp_difference/mean": 0.011881766840815544, + "step": 894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014281420590123162, + "clip_ratio/low_min": 0.00014281420590123162, + "clip_ratio/region_mean": 0.00014281420590123162, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 607.0, + "completions/mean_length": 406.78125, + "completions/mean_terminated_length": 369.4137878417969, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.35287977010011673, + "epoch": 0.4791220556745182, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010944359004497528, + "learning_rate": 1e-05, + "loss": -0.0141, + "num_tokens": 18593472.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6390905380249023, + "sampling/importance_sampling_ratio/mean": 1.0002306699752808, + "sampling/importance_sampling_ratio/min": 0.7132964134216309, + "sampling/sampling_logp_difference/max": 0.4941415786743164, + "sampling/sampling_logp_difference/mean": 0.013677633367478848, + "step": 895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 437.5, + "completions/mean_terminated_length": 437.5, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.2677403874695301, + "epoch": 0.4796573875802998, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008695017546415329, + "learning_rate": 1e-05, + "loss": 0.0143, + "num_tokens": 18611384.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4183658361434937, + "sampling/importance_sampling_ratio/mean": 0.9993807673454285, + "sampling/importance_sampling_ratio/min": 0.6329377293586731, + "sampling/sampling_logp_difference/max": 0.45738327503204346, + "sampling/sampling_logp_difference/mean": 0.010964793153107166, + "step": 896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 433.28125, + "completions/mean_terminated_length": 433.28125, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.299940075725317, + "epoch": 0.4801927194860814, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0108, + "num_tokens": 18628953.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4305717945098877, + "sampling/importance_sampling_ratio/mean": 1.0000569820404053, + "sampling/importance_sampling_ratio/min": 0.7020472884178162, + "sampling/sampling_logp_difference/max": 0.3580741882324219, + "sampling/sampling_logp_difference/mean": 0.0108436094596982, + "step": 897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 391.71875, + "completions/mean_terminated_length": 379.58062744140625, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.34211036562919617, + "epoch": 0.480728051391863, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007606659084558487, + "learning_rate": 1e-05, + "loss": 0.0051, + "num_tokens": 18645168.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6583902835845947, + "sampling/importance_sampling_ratio/mean": 1.0000258684158325, + "sampling/importance_sampling_ratio/min": 0.6280116438865662, + "sampling/sampling_logp_difference/max": 0.5058474540710449, + "sampling/sampling_logp_difference/mean": 0.013064631260931492, + "step": 898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 671.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 518.28125, + "completions/mean_terminated_length": 518.28125, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.2641910742968321, + "epoch": 0.48126338329764456, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003466410096734762, + "learning_rate": 1e-05, + "loss": -0.0089, + "num_tokens": 18665681.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4305813312530518, + "sampling/importance_sampling_ratio/mean": 1.000329613685608, + "sampling/importance_sampling_ratio/min": 0.6659950017929077, + "sampling/sampling_logp_difference/max": 0.40647315979003906, + "sampling/sampling_logp_difference/mean": 0.010509542189538479, + "step": 899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 682.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 446.15625, + "completions/mean_terminated_length": 446.15625, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.29691479727625847, + "epoch": 0.4817987152034261, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007280933205038309, + "learning_rate": 1e-05, + "loss": -0.0126, + "num_tokens": 18683910.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001438856124878, + "sampling/importance_sampling_ratio/min": 0.6874964237213135, + "sampling/sampling_logp_difference/max": 0.8669991493225098, + "sampling/sampling_logp_difference/mean": 0.011581506580114365, + "step": 900 + }, + { + "clip_ratio/high_max": 7.720815483480692e-05, + "clip_ratio/high_mean": 7.720815483480692e-05, + "clip_ratio/low_mean": 0.00012594458530656993, + "clip_ratio/low_min": 0.00012594458530656993, + "clip_ratio/region_mean": 0.00020315274014137685, + "completions/clipped_ratio": 0.0, + "completions/max_length": 685.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 452.9375, + "completions/mean_terminated_length": 452.9375, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.2824072800576687, + "epoch": 0.4823340471092077, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006068243179470301, + "learning_rate": 1e-05, + "loss": -0.0023, + "num_tokens": 18702316.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.819222331047058, + "sampling/importance_sampling_ratio/mean": 1.000252604484558, + "sampling/importance_sampling_ratio/min": 0.4884468913078308, + "sampling/sampling_logp_difference/max": 0.716524600982666, + "sampling/sampling_logp_difference/mean": 0.01148994266986847, + "step": 901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 546.6875, + "completions/mean_terminated_length": 515.0714721679688, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.3716841824352741, + "epoch": 0.4828693790149893, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009000168181955814, + "learning_rate": 1e-05, + "loss": 0.0521, + "num_tokens": 18723578.0, + "reward": 0.40625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998276233673096, + "sampling/importance_sampling_ratio/min": 0.6062214970588684, + "sampling/sampling_logp_difference/max": 0.7087397575378418, + "sampling/sampling_logp_difference/mean": 0.01339913159608841, + "step": 902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016592920292168856, + "clip_ratio/low_min": 0.00016592920292168856, + "clip_ratio/region_mean": 0.00016592920292168856, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 533.375, + "completions/mean_terminated_length": 479.23077392578125, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.23833789117634296, + "epoch": 0.48340471092077086, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005179865751415491, + "learning_rate": 1e-05, + "loss": 0.0308, + "num_tokens": 18745302.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.599319577217102, + "sampling/importance_sampling_ratio/mean": 1.0000009536743164, + "sampling/importance_sampling_ratio/min": 0.702337920665741, + "sampling/sampling_logp_difference/max": 0.46957826614379883, + "sampling/sampling_logp_difference/mean": 0.009353325702250004, + "step": 903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 440.5, + "completions/mean_terminated_length": 440.5, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.21238282695412636, + "epoch": 0.48394004282655245, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 18762782.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.434478998184204, + "sampling/importance_sampling_ratio/mean": 0.999982476234436, + "sampling/importance_sampling_ratio/min": 0.6461242437362671, + "sampling/sampling_logp_difference/max": 0.4367635250091553, + "sampling/sampling_logp_difference/mean": 0.008587619289755821, + "step": 904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002008641604334116, + "clip_ratio/low_min": 0.0002008641604334116, + "clip_ratio/region_mean": 0.0002008641604334116, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 462.03125, + "completions/mean_terminated_length": 462.03125, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.3878296688199043, + "epoch": 0.48447537473233404, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005296432413160801, + "learning_rate": 1e-05, + "loss": 0.013, + "num_tokens": 18781703.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001065731048584, + "sampling/importance_sampling_ratio/min": 0.7359341979026794, + "sampling/sampling_logp_difference/max": 0.7660646438598633, + "sampling/sampling_logp_difference/mean": 0.013224497437477112, + "step": 905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017650244990363717, + "clip_ratio/low_min": 0.00017650244990363717, + "clip_ratio/region_mean": 0.00017650244990363717, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 478.0, + "completions/mean_terminated_length": 424.2962951660156, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.3274830374866724, + "epoch": 0.48501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009160587564110756, + "learning_rate": 1e-05, + "loss": 0.0395, + "num_tokens": 18800815.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.8400568962097168, + "sampling/importance_sampling_ratio/mean": 1.0003446340560913, + "sampling/importance_sampling_ratio/min": 0.6211739778518677, + "sampling/sampling_logp_difference/max": 0.6097965240478516, + "sampling/sampling_logp_difference/mean": 0.011642465367913246, + "step": 906 + }, + { + "clip_ratio/high_max": 6.041565939085558e-05, + "clip_ratio/high_mean": 6.041565939085558e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.041565939085558e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 475.34375, + "completions/mean_terminated_length": 475.34375, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.202494403347373, + "epoch": 0.4855460385438972, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012039056979119778, + "learning_rate": 1e-05, + "loss": 0.025, + "num_tokens": 18819354.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6875321865081787, + "sampling/importance_sampling_ratio/mean": 0.9998688697814941, + "sampling/importance_sampling_ratio/min": 0.43752893805503845, + "sampling/sampling_logp_difference/max": 0.8266124725341797, + "sampling/sampling_logp_difference/mean": 0.00876142829656601, + "step": 907 + }, + { + "clip_ratio/high_max": 5.6999542721314356e-05, + "clip_ratio/high_mean": 5.6999542721314356e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.6999542721314356e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 501.03125, + "completions/mean_terminated_length": 492.4193420410156, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.31029269844293594, + "epoch": 0.4860813704496788, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014903610572218895, + "learning_rate": 1e-05, + "loss": -0.0225, + "num_tokens": 18839035.0, + "reward": 0.46875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0003976821899414, + "sampling/importance_sampling_ratio/min": 0.6708811521530151, + "sampling/sampling_logp_difference/max": 0.7926762104034424, + "sampling/sampling_logp_difference/mean": 0.011670663021504879, + "step": 908 + }, + { + "clip_ratio/high_max": 6.572029087692499e-05, + "clip_ratio/high_mean": 6.572029087692499e-05, + "clip_ratio/low_mean": 0.0001427755632903427, + "clip_ratio/low_min": 0.0001427755632903427, + "clip_ratio/region_mean": 0.00020849585416726768, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 479.4375, + "completions/mean_terminated_length": 460.20001220703125, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.35903792828321457, + "epoch": 0.4866167023554604, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008260451257228851, + "learning_rate": 1e-05, + "loss": 0.0072, + "num_tokens": 18859217.0, + "reward": 0.53125, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6009653806686401, + "sampling/importance_sampling_ratio/mean": 0.9997239112854004, + "sampling/importance_sampling_ratio/min": 0.661887526512146, + "sampling/sampling_logp_difference/max": 0.47060680389404297, + "sampling/sampling_logp_difference/mean": 0.013056580908596516, + "step": 909 + }, + { + "clip_ratio/high_max": 6.35485484963283e-05, + "clip_ratio/high_mean": 6.35485484963283e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.35485484963283e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 750.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 483.1875, + "completions/mean_terminated_length": 483.1875, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.2662975788116455, + "epoch": 0.487152034261242, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003052448621019721, + "learning_rate": 1e-05, + "loss": 0.0272, + "num_tokens": 18878551.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4324358701705933, + "sampling/importance_sampling_ratio/mean": 1.000091552734375, + "sampling/importance_sampling_ratio/min": 0.7687998414039612, + "sampling/sampling_logp_difference/max": 0.3593764305114746, + "sampling/sampling_logp_difference/mean": 0.010334242135286331, + "step": 910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 514.90625, + "completions/mean_terminated_length": 488.72412109375, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.2492699082940817, + "epoch": 0.4876873661670236, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014037317596375942, + "learning_rate": 1e-05, + "loss": 0.0038, + "num_tokens": 18898660.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.437874436378479, + "sampling/importance_sampling_ratio/mean": 0.9996024370193481, + "sampling/importance_sampling_ratio/min": 0.5474321842193604, + "sampling/sampling_logp_difference/max": 0.6025166511535645, + "sampling/sampling_logp_difference/mean": 0.009555388242006302, + "step": 911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 750.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 510.5625, + "completions/mean_terminated_length": 510.5625, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.30975694209337234, + "epoch": 0.48822269807280516, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013880766928195953, + "learning_rate": 1e-05, + "loss": -0.0137, + "num_tokens": 18919062.0, + "reward": 0.40625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5324641466140747, + "sampling/importance_sampling_ratio/mean": 1.0001721382141113, + "sampling/importance_sampling_ratio/min": 0.6259236931800842, + "sampling/sampling_logp_difference/max": 0.46852684020996094, + "sampling/sampling_logp_difference/mean": 0.012039105407893658, + "step": 912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.206164132687263e-05, + "clip_ratio/low_min": 5.206164132687263e-05, + "clip_ratio/region_mean": 5.206164132687263e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 529.375, + "completions/mean_terminated_length": 504.6896667480469, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.333847489207983, + "epoch": 0.48875802997858675, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03074287250638008, + "learning_rate": 1e-05, + "loss": 0.0924, + "num_tokens": 18940234.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.6131587028503418, + "sampling/importance_sampling_ratio/mean": 1.0000418424606323, + "sampling/importance_sampling_ratio/min": 0.6844300031661987, + "sampling/sampling_logp_difference/max": 0.4781942367553711, + "sampling/sampling_logp_difference/mean": 0.012957409024238586, + "step": 913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013158086221665144, + "clip_ratio/low_min": 0.00013158086221665144, + "clip_ratio/region_mean": 0.00013158086221665144, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 464.09375, + "completions/mean_terminated_length": 454.2903137207031, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.30686079524457455, + "epoch": 0.4892933618843683, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01318507269024849, + "learning_rate": 1e-05, + "loss": -0.0319, + "num_tokens": 18958749.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.36928129196167, + "sampling/importance_sampling_ratio/mean": 0.9999698400497437, + "sampling/importance_sampling_ratio/min": 0.5420946478843689, + "sampling/sampling_logp_difference/max": 0.6123147010803223, + "sampling/sampling_logp_difference/mean": 0.011734270490705967, + "step": 914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.259389192564413e-05, + "clip_ratio/low_min": 6.259389192564413e-05, + "clip_ratio/region_mean": 6.259389192564413e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 476.53125, + "completions/mean_terminated_length": 457.10003662109375, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.31297939643263817, + "epoch": 0.48982869379014987, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0388018861413002, + "learning_rate": 1e-05, + "loss": 0.066, + "num_tokens": 18978182.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7275769710540771, + "sampling/importance_sampling_ratio/mean": 0.9997548460960388, + "sampling/importance_sampling_ratio/min": 0.6455977559089661, + "sampling/sampling_logp_difference/max": 0.5467197895050049, + "sampling/sampling_logp_difference/mean": 0.011493703350424767, + "step": 915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 435.0, + "completions/mean_terminated_length": 424.258056640625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.28156609646975994, + "epoch": 0.49036402569593146, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0062019904144108295, + "learning_rate": 1e-05, + "loss": 0.0003, + "num_tokens": 18995990.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5014770030975342, + "sampling/importance_sampling_ratio/mean": 0.9995688199996948, + "sampling/importance_sampling_ratio/min": 0.6919264197349548, + "sampling/sampling_logp_difference/max": 0.4064493179321289, + "sampling/sampling_logp_difference/mean": 0.010881432332098484, + "step": 916 + }, + { + "clip_ratio/high_max": 4.7063251258805394e-05, + "clip_ratio/high_mean": 4.7063251258805394e-05, + "clip_ratio/low_mean": 0.00015507238276768476, + "clip_ratio/low_min": 0.00015507238276768476, + "clip_ratio/region_mean": 0.00020213563402649015, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 575.9375, + "completions/mean_terminated_length": 522.1599731445312, + "completions/min_length": 365.0, + "completions/min_terminated_length": 365.0, + "entropy": 0.24633987620472908, + "epoch": 0.49089935760171305, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007041865028440952, + "learning_rate": 1e-05, + "loss": 0.0363, + "num_tokens": 19018220.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.662911057472229, + "sampling/importance_sampling_ratio/mean": 1.000190258026123, + "sampling/importance_sampling_ratio/min": 0.7511054873466492, + "sampling/sampling_logp_difference/max": 0.5085697174072266, + "sampling/sampling_logp_difference/mean": 0.009644015692174435, + "step": 917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 423.3125, + "completions/mean_terminated_length": 423.3125, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.2914689928293228, + "epoch": 0.49143468950749464, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006776359397917986, + "learning_rate": 1e-05, + "loss": 0.0137, + "num_tokens": 19035510.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.555033564567566, + "sampling/importance_sampling_ratio/mean": 0.9998643398284912, + "sampling/importance_sampling_ratio/min": 0.6702762842178345, + "sampling/sampling_logp_difference/max": 0.4414970874786377, + "sampling/sampling_logp_difference/mean": 0.011066603474318981, + "step": 918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 491.1875, + "completions/mean_terminated_length": 482.258056640625, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.28279813192784786, + "epoch": 0.4919700214132762, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00830013770610094, + "learning_rate": 1e-05, + "loss": 0.0235, + "num_tokens": 19055164.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.8878000974655151, + "sampling/importance_sampling_ratio/mean": 1.0000550746917725, + "sampling/importance_sampling_ratio/min": 0.7158579230308533, + "sampling/sampling_logp_difference/max": 0.6354122161865234, + "sampling/sampling_logp_difference/mean": 0.00993894599378109, + "step": 919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 618.90625, + "completions/mean_terminated_length": 529.4500122070312, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "entropy": 0.5270113684237003, + "epoch": 0.4925053533190578, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010443360544741154, + "learning_rate": 1e-05, + "loss": 0.0142, + "num_tokens": 19079321.0, + "reward": 0.25, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5007963180541992, + "sampling/importance_sampling_ratio/mean": 1.000497817993164, + "sampling/importance_sampling_ratio/min": 0.6487916111946106, + "sampling/sampling_logp_difference/max": 0.4326436519622803, + "sampling/sampling_logp_difference/mean": 0.01748787797987461, + "step": 920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 489.4375, + "completions/mean_terminated_length": 489.4375, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.23764795064926147, + "epoch": 0.4930406852248394, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0094, + "num_tokens": 19099007.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5893279314041138, + "sampling/importance_sampling_ratio/mean": 0.9997685551643372, + "sampling/importance_sampling_ratio/min": 0.667430579662323, + "sampling/sampling_logp_difference/max": 0.46331119537353516, + "sampling/sampling_logp_difference/mean": 0.009766027331352234, + "step": 921 + }, + { + "clip_ratio/high_max": 5.765682726632804e-05, + "clip_ratio/high_mean": 5.765682726632804e-05, + "clip_ratio/low_mean": 0.00015271549636963755, + "clip_ratio/low_min": 0.00015271549636963755, + "clip_ratio/region_mean": 0.00021037232363596559, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 569.6875, + "completions/mean_terminated_length": 532.9629516601562, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.3499591462314129, + "epoch": 0.493576017130621, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019014639779925346, + "learning_rate": 1e-05, + "loss": 0.0438, + "num_tokens": 19121261.0, + "reward": 0.5625, + "reward_std": 0.48503684997558594, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.464830994606018, + "sampling/importance_sampling_ratio/mean": 1.0005414485931396, + "sampling/importance_sampling_ratio/min": 0.650802731513977, + "sampling/sampling_logp_difference/max": 0.4295487403869629, + "sampling/sampling_logp_difference/mean": 0.012619705870747566, + "step": 922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001937806373462081, + "clip_ratio/low_min": 0.0001937806373462081, + "clip_ratio/region_mean": 0.0001937806373462081, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 538.0625, + "completions/mean_terminated_length": 473.67999267578125, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.39270614087581635, + "epoch": 0.4941113490364026, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013273780234158039, + "learning_rate": 1e-05, + "loss": 0.0109, + "num_tokens": 19142279.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4174294471740723, + "sampling/importance_sampling_ratio/mean": 0.9999698400497437, + "sampling/importance_sampling_ratio/min": 0.6977755427360535, + "sampling/sampling_logp_difference/max": 0.35985779762268066, + "sampling/sampling_logp_difference/mean": 0.013609446585178375, + "step": 923 + }, + { + "clip_ratio/high_max": 5.776340185548179e-05, + "clip_ratio/high_mean": 5.776340185548179e-05, + "clip_ratio/low_mean": 0.00017970944463741034, + "clip_ratio/low_min": 0.00017970944463741034, + "clip_ratio/region_mean": 0.00023747284649289213, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 539.53125, + "completions/mean_terminated_length": 497.22222900390625, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.2898131124675274, + "epoch": 0.49464668094218417, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029068931937217712, + "learning_rate": 1e-05, + "loss": 0.0778, + "num_tokens": 19163376.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4534404277801514, + "sampling/importance_sampling_ratio/mean": 1.000167727470398, + "sampling/importance_sampling_ratio/min": 0.6783241629600525, + "sampling/sampling_logp_difference/max": 0.38812994956970215, + "sampling/sampling_logp_difference/mean": 0.010268832556903362, + "step": 924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00022056644229451194, + "clip_ratio/low_min": 0.00022056644229451194, + "clip_ratio/region_mean": 0.00022056644229451194, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 522.84375, + "completions/mean_terminated_length": 477.4444580078125, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.3398930039256811, + "epoch": 0.49518201284796576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0146, + "num_tokens": 19184131.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4498913288116455, + "sampling/importance_sampling_ratio/mean": 1.000004768371582, + "sampling/importance_sampling_ratio/min": 0.6497147679328918, + "sampling/sampling_logp_difference/max": 0.43122178316116333, + "sampling/sampling_logp_difference/mean": 0.012509508058428764, + "step": 925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 534.0, + "completions/mean_terminated_length": 509.7930908203125, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 0.32699492760002613, + "epoch": 0.49571734475374735, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013125520199537277, + "learning_rate": 1e-05, + "loss": 0.0375, + "num_tokens": 19205331.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4404257535934448, + "sampling/importance_sampling_ratio/mean": 0.9994783401489258, + "sampling/importance_sampling_ratio/min": 0.6050943732261658, + "sampling/sampling_logp_difference/max": 0.5023708343505859, + "sampling/sampling_logp_difference/mean": 0.012279679998755455, + "step": 926 + }, + { + "clip_ratio/high_max": 6.348400347633287e-05, + "clip_ratio/high_mean": 6.348400347633287e-05, + "clip_ratio/low_mean": 0.00016947766562225297, + "clip_ratio/low_min": 0.00016947766562225297, + "clip_ratio/region_mean": 0.00023296166909858584, + "completions/clipped_ratio": 0.0, + "completions/max_length": 757.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 435.3125, + "completions/mean_terminated_length": 435.3125, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.340041421353817, + "epoch": 0.49625267665952894, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011166700161993504, + "learning_rate": 1e-05, + "loss": 0.0299, + "num_tokens": 19222317.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.612455129623413, + "sampling/importance_sampling_ratio/mean": 0.9999621510505676, + "sampling/importance_sampling_ratio/min": 0.739499568939209, + "sampling/sampling_logp_difference/max": 0.47775793075561523, + "sampling/sampling_logp_difference/mean": 0.011311161331832409, + "step": 927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012195505041745491, + "clip_ratio/low_min": 0.00012195505041745491, + "clip_ratio/region_mean": 0.00012195505041745491, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 547.53125, + "completions/mean_terminated_length": 447.3182067871094, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.5082920771092176, + "epoch": 0.49678800856531047, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0019, + "num_tokens": 19244222.0, + "reward": 0.53125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6648000478744507, + "sampling/importance_sampling_ratio/mean": 0.9998617768287659, + "sampling/importance_sampling_ratio/min": 0.6047614812850952, + "sampling/sampling_logp_difference/max": 0.5097050666809082, + "sampling/sampling_logp_difference/mean": 0.017021482810378075, + "step": 928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.752851600642316e-05, + "clip_ratio/low_min": 4.752851600642316e-05, + "clip_ratio/region_mean": 4.752851600642316e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 518.25, + "completions/mean_terminated_length": 472.0, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 0.3265595231205225, + "epoch": 0.49732334047109206, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007692792918533087, + "learning_rate": 1e-05, + "loss": 0.0579, + "num_tokens": 19264606.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.456028938293457, + "sampling/importance_sampling_ratio/mean": 0.9998459815979004, + "sampling/importance_sampling_ratio/min": 0.7026187777519226, + "sampling/sampling_logp_difference/max": 0.37571287155151367, + "sampling/sampling_logp_difference/mean": 0.011296630837023258, + "step": 929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 692.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 437.59375, + "completions/mean_terminated_length": 437.59375, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.2326170839369297, + "epoch": 0.49785867237687365, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 19281905.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4700236320495605, + "sampling/importance_sampling_ratio/mean": 0.999994158744812, + "sampling/importance_sampling_ratio/min": 0.6835604310035706, + "sampling/sampling_logp_difference/max": 0.38527846336364746, + "sampling/sampling_logp_difference/mean": 0.009263720363378525, + "step": 930 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013016723823966458, + "clip_ratio/low_min": 0.00013016723823966458, + "clip_ratio/region_mean": 0.00013016723823966458, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 499.78125, + "completions/mean_terminated_length": 461.46429443359375, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.32206207886338234, + "epoch": 0.49839400428265523, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014004467986524105, + "learning_rate": 1e-05, + "loss": -0.0124, + "num_tokens": 19301706.0, + "reward": 0.40625, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7464927434921265, + "sampling/importance_sampling_ratio/mean": 1.0003650188446045, + "sampling/importance_sampling_ratio/min": 0.5425254702568054, + "sampling/sampling_logp_difference/max": 0.6115202903747559, + "sampling/sampling_logp_difference/mean": 0.011845179833471775, + "step": 931 + }, + { + "clip_ratio/high_max": 6.118453165981919e-05, + "clip_ratio/high_mean": 6.118453165981919e-05, + "clip_ratio/low_mean": 0.00010584674964775331, + "clip_ratio/low_min": 0.00010584674964775331, + "clip_ratio/region_mean": 0.0001670312813075725, + "completions/clipped_ratio": 0.0, + "completions/max_length": 727.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 468.125, + "completions/mean_terminated_length": 468.125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.45232093334198, + "epoch": 0.4989293361884368, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011814016848802567, + "learning_rate": 1e-05, + "loss": 0.0338, + "num_tokens": 19320934.0, + "reward": 0.5, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6969050168991089, + "sampling/importance_sampling_ratio/mean": 0.9999136328697205, + "sampling/importance_sampling_ratio/min": 0.3159007728099823, + "sampling/sampling_logp_difference/max": 1.152327060699463, + "sampling/sampling_logp_difference/mean": 0.014239568263292313, + "step": 932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 491.125, + "completions/mean_terminated_length": 472.66668701171875, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.2690283786505461, + "epoch": 0.4994646680942184, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02261507138609886, + "learning_rate": 1e-05, + "loss": 0.015, + "num_tokens": 19340258.0, + "reward": 0.8125, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5850526094436646, + "sampling/importance_sampling_ratio/mean": 0.9998255372047424, + "sampling/importance_sampling_ratio/min": 0.6380040049552917, + "sampling/sampling_logp_difference/max": 0.4606175422668457, + "sampling/sampling_logp_difference/mean": 0.01035875640809536, + "step": 933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.02008042531088e-05, + "clip_ratio/low_min": 5.02008042531088e-05, + "clip_ratio/region_mean": 5.02008042531088e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 501.8125, + "completions/mean_terminated_length": 474.2758483886719, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.26222812198102474, + "epoch": 0.5, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013308855704963207, + "learning_rate": 1e-05, + "loss": 0.0239, + "num_tokens": 19359980.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5375230312347412, + "sampling/importance_sampling_ratio/mean": 1.0001217126846313, + "sampling/importance_sampling_ratio/min": 0.7298194766044617, + "sampling/sampling_logp_difference/max": 0.4301726818084717, + "sampling/sampling_logp_difference/mean": 0.01018164400011301, + "step": 934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.446622137445956e-05, + "clip_ratio/low_min": 6.446622137445956e-05, + "clip_ratio/region_mean": 6.446622137445956e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 494.28125, + "completions/mean_terminated_length": 417.6399841308594, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.2620402257889509, + "epoch": 0.5005353319057816, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.010162093676626682, + "learning_rate": 1e-05, + "loss": -0.0005, + "num_tokens": 19379717.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4264450073242188, + "sampling/importance_sampling_ratio/mean": 1.0002689361572266, + "sampling/importance_sampling_ratio/min": 0.6831101179122925, + "sampling/sampling_logp_difference/max": 0.3810991942882538, + "sampling/sampling_logp_difference/mean": 0.010737966746091843, + "step": 935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 439.28125, + "completions/mean_terminated_length": 428.6773986816406, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.2794332280755043, + "epoch": 0.5010706638115632, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010731112211942673, + "learning_rate": 1e-05, + "loss": 0.0324, + "num_tokens": 19397078.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6577776670455933, + "sampling/importance_sampling_ratio/mean": 1.0000596046447754, + "sampling/importance_sampling_ratio/min": 0.6590961217880249, + "sampling/sampling_logp_difference/max": 0.5054779052734375, + "sampling/sampling_logp_difference/mean": 0.010796277783811092, + "step": 936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.713211769238114e-05, + "clip_ratio/low_min": 6.713211769238114e-05, + "clip_ratio/region_mean": 6.713211769238114e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 662.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 485.65625, + "completions/mean_terminated_length": 485.65625, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.2834157757461071, + "epoch": 0.5016059957173448, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0138821080327034, + "learning_rate": 1e-05, + "loss": 0.0267, + "num_tokens": 19416451.0, + "reward": 0.625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4637072086334229, + "sampling/importance_sampling_ratio/mean": 0.9997812509536743, + "sampling/importance_sampling_ratio/min": 0.680621862411499, + "sampling/sampling_logp_difference/max": 0.3847484588623047, + "sampling/sampling_logp_difference/mean": 0.011037930846214294, + "step": 937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 583.0, + "completions/max_terminated_length": 583.0, + "completions/mean_length": 398.28125, + "completions/mean_terminated_length": 398.28125, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.29254633747041225, + "epoch": 0.5021413276231264, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.010301598347723484, + "learning_rate": 1e-05, + "loss": 0.0259, + "num_tokens": 19433308.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.8035612106323242, + "sampling/importance_sampling_ratio/mean": 1.0004361867904663, + "sampling/importance_sampling_ratio/min": 0.6567971110343933, + "sampling/sampling_logp_difference/max": 0.5897631645202637, + "sampling/sampling_logp_difference/mean": 0.012045230716466904, + "step": 938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.246376480907202e-05, + "clip_ratio/low_min": 7.246376480907202e-05, + "clip_ratio/region_mean": 7.246376480907202e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 406.9375, + "completions/mean_terminated_length": 406.9375, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.2541348449885845, + "epoch": 0.5026766595289079, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0059, + "num_tokens": 19449730.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.466042399406433, + "sampling/importance_sampling_ratio/mean": 0.9999431371688843, + "sampling/importance_sampling_ratio/min": 0.757443368434906, + "sampling/sampling_logp_difference/max": 0.3825664520263672, + "sampling/sampling_logp_difference/mean": 0.01026766374707222, + "step": 939 + }, + { + "clip_ratio/high_max": 6.883259629830718e-05, + "clip_ratio/high_mean": 6.883259629830718e-05, + "clip_ratio/low_mean": 0.00010640990876709111, + "clip_ratio/low_min": 0.00010640990876709111, + "clip_ratio/region_mean": 0.0001752425050653983, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 527.90625, + "completions/mean_terminated_length": 493.607177734375, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.3679649531841278, + "epoch": 0.5032119914346895, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004698208533227444, + "learning_rate": 1e-05, + "loss": 0.0624, + "num_tokens": 19470351.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4841936826705933, + "sampling/importance_sampling_ratio/mean": 0.9998572468757629, + "sampling/importance_sampling_ratio/min": 0.7141680121421814, + "sampling/sampling_logp_difference/max": 0.39487171173095703, + "sampling/sampling_logp_difference/mean": 0.012638048268854618, + "step": 940 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.868132186355069e-05, + "clip_ratio/low_min": 6.868132186355069e-05, + "clip_ratio/region_mean": 6.868132186355069e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 463.40625, + "completions/mean_terminated_length": 463.40625, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.2769796848297119, + "epoch": 0.5037473233404711, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.030556363984942436, + "learning_rate": 1e-05, + "loss": -0.0122, + "num_tokens": 19489316.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4240692853927612, + "sampling/importance_sampling_ratio/mean": 1.0002126693725586, + "sampling/importance_sampling_ratio/min": 0.6985477209091187, + "sampling/sampling_logp_difference/max": 0.3587517738342285, + "sampling/sampling_logp_difference/mean": 0.010646170936524868, + "step": 941 + }, + { + "clip_ratio/high_max": 6.041565939085558e-05, + "clip_ratio/high_mean": 6.041565939085558e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.041565939085558e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 457.21875, + "completions/mean_terminated_length": 425.0689697265625, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.28365115635097027, + "epoch": 0.5042826552462527, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011648779734969139, + "learning_rate": 1e-05, + "loss": -0.0076, + "num_tokens": 19507731.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7110072374343872, + "sampling/importance_sampling_ratio/mean": 1.0001929998397827, + "sampling/importance_sampling_ratio/min": 0.6931325793266296, + "sampling/sampling_logp_difference/max": 0.5370821952819824, + "sampling/sampling_logp_difference/mean": 0.011281576938927174, + "step": 942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 436.84375, + "completions/mean_terminated_length": 436.84375, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.2924124225974083, + "epoch": 0.5048179871520343, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0055651580914855, + "learning_rate": 1e-05, + "loss": 0.0339, + "num_tokens": 19525430.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.46877121925354, + "sampling/importance_sampling_ratio/mean": 0.9997760057449341, + "sampling/importance_sampling_ratio/min": 0.5539229512214661, + "sampling/sampling_logp_difference/max": 0.5907297134399414, + "sampling/sampling_logp_difference/mean": 0.011159236542880535, + "step": 943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.297229265328497e-05, + "clip_ratio/low_min": 6.297229265328497e-05, + "clip_ratio/region_mean": 6.297229265328497e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 608.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 429.96875, + "completions/mean_terminated_length": 429.96875, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.24778622947633266, + "epoch": 0.5053533190578159, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012465167790651321, + "learning_rate": 1e-05, + "loss": 0.019, + "num_tokens": 19542381.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5475302934646606, + "sampling/importance_sampling_ratio/mean": 1.0002715587615967, + "sampling/importance_sampling_ratio/min": 0.6991207003593445, + "sampling/sampling_logp_difference/max": 0.4366602897644043, + "sampling/sampling_logp_difference/mean": 0.010228806175291538, + "step": 944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 467.03125, + "completions/mean_terminated_length": 446.9666748046875, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.2548218183219433, + "epoch": 0.5058886509635975, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.027409628033638, + "learning_rate": 1e-05, + "loss": 0.0826, + "num_tokens": 19560566.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4531505107879639, + "sampling/importance_sampling_ratio/mean": 1.0004417896270752, + "sampling/importance_sampling_ratio/min": 0.7483683228492737, + "sampling/sampling_logp_difference/max": 0.3737339973449707, + "sampling/sampling_logp_difference/mean": 0.009670830331742764, + "step": 945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010070791176985949, + "clip_ratio/low_min": 0.00010070791176985949, + "clip_ratio/region_mean": 0.00010070791176985949, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 554.53125, + "completions/mean_terminated_length": 515.0, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.41932306811213493, + "epoch": 0.5064239828693791, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019317878410220146, + "learning_rate": 1e-05, + "loss": 0.0388, + "num_tokens": 19581975.0, + "reward": 0.53125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4860639572143555, + "sampling/importance_sampling_ratio/mean": 1.0001702308654785, + "sampling/importance_sampling_ratio/min": 0.6995219588279724, + "sampling/sampling_logp_difference/max": 0.39613091945648193, + "sampling/sampling_logp_difference/mean": 0.013263840228319168, + "step": 946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 411.28125, + "completions/mean_terminated_length": 387.5000305175781, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.322050666436553, + "epoch": 0.5069593147751607, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.036013711243867874, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 19598336.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5235679149627686, + "sampling/importance_sampling_ratio/mean": 1.0000337362289429, + "sampling/importance_sampling_ratio/min": 0.7071281671524048, + "sampling/sampling_logp_difference/max": 0.4210548400878906, + "sampling/sampling_logp_difference/mean": 0.012662426568567753, + "step": 947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 483.09375, + "completions/mean_terminated_length": 473.9031982421875, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "entropy": 0.2943005654960871, + "epoch": 0.5074946466809421, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011393197812139988, + "learning_rate": 1e-05, + "loss": 0.0245, + "num_tokens": 19617427.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4300520420074463, + "sampling/importance_sampling_ratio/mean": 1.0001416206359863, + "sampling/importance_sampling_ratio/min": 0.7061121463775635, + "sampling/sampling_logp_difference/max": 0.3577108383178711, + "sampling/sampling_logp_difference/mean": 0.011147226206958294, + "step": 948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010816605572472326, + "clip_ratio/low_min": 0.00010816605572472326, + "clip_ratio/region_mean": 0.00010816605572472326, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 535.28125, + "completions/mean_terminated_length": 502.0357360839844, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.34803836792707443, + "epoch": 0.5080299785867237, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011546298861503601, + "learning_rate": 1e-05, + "loss": 0.032, + "num_tokens": 19638516.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7856738567352295, + "sampling/importance_sampling_ratio/mean": 0.9996559619903564, + "sampling/importance_sampling_ratio/min": 0.6398385763168335, + "sampling/sampling_logp_difference/max": 0.5797958374023438, + "sampling/sampling_logp_difference/mean": 0.01192948967218399, + "step": 949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 694.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 500.0, + "completions/mean_terminated_length": 500.0, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.32023950666189194, + "epoch": 0.5085653104925053, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003911130130290985, + "learning_rate": 1e-05, + "loss": -0.0045, + "num_tokens": 19658284.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4485037326812744, + "sampling/importance_sampling_ratio/mean": 0.9998983144760132, + "sampling/importance_sampling_ratio/min": 0.7037187218666077, + "sampling/sampling_logp_difference/max": 0.3705310821533203, + "sampling/sampling_logp_difference/mean": 0.011814751662313938, + "step": 950 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013728720659855753, + "clip_ratio/low_min": 0.00013728720659855753, + "clip_ratio/region_mean": 0.00013728720659855753, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 412.875, + "completions/mean_terminated_length": 401.4193420410156, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.3241432998329401, + "epoch": 0.5091006423982869, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.037214525043964386, + "learning_rate": 1e-05, + "loss": 0.0511, + "num_tokens": 19675016.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4139505624771118, + "sampling/importance_sampling_ratio/mean": 1.0000523328781128, + "sampling/importance_sampling_ratio/min": 0.7438786029815674, + "sampling/sampling_logp_difference/max": 0.3463876247406006, + "sampling/sampling_logp_difference/mean": 0.011365550570189953, + "step": 951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 480.53125, + "completions/mean_terminated_length": 461.36669921875, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.254257895052433, + "epoch": 0.5096359743040685, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01980079524219036, + "learning_rate": 1e-05, + "loss": 0.0157, + "num_tokens": 19693849.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4840246438980103, + "sampling/importance_sampling_ratio/mean": 1.0001052618026733, + "sampling/importance_sampling_ratio/min": 0.7634907364845276, + "sampling/sampling_logp_difference/max": 0.3947577476501465, + "sampling/sampling_logp_difference/mean": 0.010038191452622414, + "step": 952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 457.96875, + "completions/mean_terminated_length": 425.89654541015625, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.41520047187805176, + "epoch": 0.5101713062098501, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0030097414273768663, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 19712288.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3846614360809326, + "sampling/importance_sampling_ratio/mean": 1.0002185106277466, + "sampling/importance_sampling_ratio/min": 0.6740763783454895, + "sampling/sampling_logp_difference/max": 0.3944118618965149, + "sampling/sampling_logp_difference/mean": 0.012530251406133175, + "step": 953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000273627694696188, + "clip_ratio/low_min": 0.000273627694696188, + "clip_ratio/region_mean": 0.000273627694696188, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 518.5625, + "completions/mean_terminated_length": 482.9285888671875, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.324110297486186, + "epoch": 0.5107066381156317, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005960370879620314, + "learning_rate": 1e-05, + "loss": 0.0614, + "num_tokens": 19733346.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5565340518951416, + "sampling/importance_sampling_ratio/mean": 1.0002589225769043, + "sampling/importance_sampling_ratio/min": 0.7417091727256775, + "sampling/sampling_logp_difference/max": 0.4424614906311035, + "sampling/sampling_logp_difference/mean": 0.011185092851519585, + "step": 954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012192071153549477, + "clip_ratio/low_min": 0.00012192071153549477, + "clip_ratio/region_mean": 0.00012192071153549477, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 465.0, + "completions/mean_terminated_length": 455.2257995605469, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.28435055166482925, + "epoch": 0.5112419700214133, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014217003248631954, + "learning_rate": 1e-05, + "loss": -0.0178, + "num_tokens": 19752074.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.476675033569336, + "sampling/importance_sampling_ratio/mean": 0.9998407363891602, + "sampling/importance_sampling_ratio/min": 0.6523876190185547, + "sampling/sampling_logp_difference/max": 0.42711639404296875, + "sampling/sampling_logp_difference/mean": 0.010454796254634857, + "step": 955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 416.21875, + "completions/mean_terminated_length": 416.21875, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.2670032139867544, + "epoch": 0.5117773019271948, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013444775715470314, + "learning_rate": 1e-05, + "loss": 0.0405, + "num_tokens": 19768905.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.599617838859558, + "sampling/importance_sampling_ratio/mean": 0.9995540380477905, + "sampling/importance_sampling_ratio/min": 0.6160816550254822, + "sampling/sampling_logp_difference/max": 0.4843757152557373, + "sampling/sampling_logp_difference/mean": 0.010016650892794132, + "step": 956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 617.0, + "completions/max_terminated_length": 617.0, + "completions/mean_length": 470.03125, + "completions/mean_terminated_length": 470.03125, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.2323832232505083, + "epoch": 0.5123126338329764, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016881059855222702, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 19787314.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5071568489074707, + "sampling/importance_sampling_ratio/mean": 1.0001609325408936, + "sampling/importance_sampling_ratio/min": 0.5303636193275452, + "sampling/sampling_logp_difference/max": 0.6341924667358398, + "sampling/sampling_logp_difference/mean": 0.009288148023188114, + "step": 957 + }, + { + "clip_ratio/high_max": 0.00012261857045814395, + "clip_ratio/high_mean": 0.00012261857045814395, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012261857045814395, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 490.78125, + "completions/mean_terminated_length": 472.3000183105469, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.26723513193428516, + "epoch": 0.512847965738758, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00785796344280243, + "learning_rate": 1e-05, + "loss": -0.0331, + "num_tokens": 19806971.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.8052798509597778, + "sampling/importance_sampling_ratio/mean": 1.0002069473266602, + "sampling/importance_sampling_ratio/min": 0.30172789096832275, + "sampling/sampling_logp_difference/max": 1.1982296705245972, + "sampling/sampling_logp_difference/mean": 0.010774476453661919, + "step": 958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 473.15625, + "completions/mean_terminated_length": 463.6451416015625, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.2213722411543131, + "epoch": 0.5133832976445396, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.015048615634441376, + "learning_rate": 1e-05, + "loss": -0.0126, + "num_tokens": 19825768.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4348511695861816, + "sampling/importance_sampling_ratio/mean": 0.9997027516365051, + "sampling/importance_sampling_ratio/min": 0.6895332932472229, + "sampling/sampling_logp_difference/max": 0.37174034118652344, + "sampling/sampling_logp_difference/mean": 0.00852168258279562, + "step": 959 + }, + { + "clip_ratio/high_max": 5.602868623100221e-05, + "clip_ratio/high_mean": 5.602868623100221e-05, + "clip_ratio/low_mean": 0.0001959605870069936, + "clip_ratio/low_min": 0.0001959605870069936, + "clip_ratio/region_mean": 0.0002519892732379958, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 603.8125, + "completions/mean_terminated_length": 565.923095703125, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.38207705318927765, + "epoch": 0.5139186295503212, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022195030003786087, + "learning_rate": 1e-05, + "loss": 0.0283, + "num_tokens": 19849026.0, + "reward": 0.34375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4360729455947876, + "sampling/importance_sampling_ratio/mean": 0.9999850392341614, + "sampling/importance_sampling_ratio/min": 0.6669304966926575, + "sampling/sampling_logp_difference/max": 0.4050694704055786, + "sampling/sampling_logp_difference/mean": 0.013118598610162735, + "step": 960 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012831228377763182, + "clip_ratio/low_min": 0.00012831228377763182, + "clip_ratio/region_mean": 0.00012831228377763182, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 466.34375, + "completions/mean_terminated_length": 456.6128845214844, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.24108667485415936, + "epoch": 0.5144539614561028, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004858960397541523, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 19867597.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5404936075210571, + "sampling/importance_sampling_ratio/mean": 0.999920129776001, + "sampling/importance_sampling_ratio/min": 0.7229648232460022, + "sampling/sampling_logp_difference/max": 0.43210291862487793, + "sampling/sampling_logp_difference/mean": 0.009461749345064163, + "step": 961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011436413478804752, + "clip_ratio/low_min": 0.00011436413478804752, + "clip_ratio/region_mean": 0.00011436413478804752, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 535.6875, + "completions/mean_terminated_length": 511.6551818847656, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.3625764474272728, + "epoch": 0.5149892933618844, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010485700331628323, + "learning_rate": 1e-05, + "loss": -0.0269, + "num_tokens": 19888667.0, + "reward": 0.59375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6535862684249878, + "sampling/importance_sampling_ratio/mean": 0.9999732971191406, + "sampling/importance_sampling_ratio/min": 0.6269338726997375, + "sampling/sampling_logp_difference/max": 0.5029463768005371, + "sampling/sampling_logp_difference/mean": 0.012667607516050339, + "step": 962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 434.15625, + "completions/mean_terminated_length": 434.15625, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.2798925694078207, + "epoch": 0.515524625267666, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004387347027659416, + "learning_rate": 1e-05, + "loss": 0.0161, + "num_tokens": 19906608.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5710426568984985, + "sampling/importance_sampling_ratio/mean": 1.0001790523529053, + "sampling/importance_sampling_ratio/min": 0.7475011348724365, + "sampling/sampling_logp_difference/max": 0.4517395496368408, + "sampling/sampling_logp_difference/mean": 0.01011952105909586, + "step": 963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 481.1875, + "completions/mean_terminated_length": 471.9354553222656, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.2522803172469139, + "epoch": 0.5160599571734475, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005507873836904764, + "learning_rate": 1e-05, + "loss": 0.0394, + "num_tokens": 19925614.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3990647792816162, + "sampling/importance_sampling_ratio/mean": 0.9996039271354675, + "sampling/importance_sampling_ratio/min": 0.7229023575782776, + "sampling/sampling_logp_difference/max": 0.3358039855957031, + "sampling/sampling_logp_difference/mean": 0.009419592097401619, + "step": 964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023852916638134047, + "clip_ratio/low_min": 0.00023852916638134047, + "clip_ratio/region_mean": 0.00023852916638134047, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 531.875, + "completions/mean_terminated_length": 498.14288330078125, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.29346702992916107, + "epoch": 0.5165952890792291, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017893539741635323, + "learning_rate": 1e-05, + "loss": 0.0876, + "num_tokens": 19946362.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4782878160476685, + "sampling/importance_sampling_ratio/mean": 1.000130295753479, + "sampling/importance_sampling_ratio/min": 0.6992011666297913, + "sampling/sampling_logp_difference/max": 0.39088451862335205, + "sampling/sampling_logp_difference/mean": 0.01042831502854824, + "step": 965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.256518124951981e-05, + "clip_ratio/low_min": 5.256518124951981e-05, + "clip_ratio/region_mean": 5.256518124951981e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 462.3125, + "completions/mean_terminated_length": 430.6896667480469, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.23513570055365562, + "epoch": 0.5171306209850107, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013849371112883091, + "learning_rate": 1e-05, + "loss": 0.0202, + "num_tokens": 19965212.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5709261894226074, + "sampling/importance_sampling_ratio/mean": 1.0000474452972412, + "sampling/importance_sampling_ratio/min": 0.6098437309265137, + "sampling/sampling_logp_difference/max": 0.4945526123046875, + "sampling/sampling_logp_difference/mean": 0.009246601723134518, + "step": 966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 725.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 479.25, + "completions/mean_terminated_length": 479.25, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.22078409604728222, + "epoch": 0.5176659528907923, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.018964843824505806, + "learning_rate": 1e-05, + "loss": -0.0053, + "num_tokens": 19984596.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3845479488372803, + "sampling/importance_sampling_ratio/mean": 0.9999772310256958, + "sampling/importance_sampling_ratio/min": 0.7026873230934143, + "sampling/sampling_logp_difference/max": 0.3528432846069336, + "sampling/sampling_logp_difference/mean": 0.008462992496788502, + "step": 967 + }, + { + "clip_ratio/high_max": 5.623032120638527e-05, + "clip_ratio/high_mean": 5.623032120638527e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.623032120638527e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 576.4375, + "completions/mean_terminated_length": 532.2307739257812, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.3101378045976162, + "epoch": 0.5182012847965739, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027649912983179092, + "learning_rate": 1e-05, + "loss": -0.017, + "num_tokens": 20007330.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3878732919692993, + "sampling/importance_sampling_ratio/mean": 0.9999299645423889, + "sampling/importance_sampling_ratio/min": 0.6943771839141846, + "sampling/sampling_logp_difference/max": 0.36473989486694336, + "sampling/sampling_logp_difference/mean": 0.01125099416822195, + "step": 968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 598.0, + "completions/mean_terminated_length": 520.727294921875, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 0.3771156594157219, + "epoch": 0.5187366167023555, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007927303202450275, + "learning_rate": 1e-05, + "loss": 0.0808, + "num_tokens": 20030898.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4083565473556519, + "sampling/importance_sampling_ratio/mean": 0.9997958540916443, + "sampling/importance_sampling_ratio/min": 0.7026858329772949, + "sampling/sampling_logp_difference/max": 0.3528454303741455, + "sampling/sampling_logp_difference/mean": 0.012313827872276306, + "step": 969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 348.3125, + "completions/mean_terminated_length": 348.3125, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.24957561120390892, + "epoch": 0.5192719486081371, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20045676.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4911633729934692, + "sampling/importance_sampling_ratio/mean": 0.9995911717414856, + "sampling/importance_sampling_ratio/min": 0.7303797006607056, + "sampling/sampling_logp_difference/max": 0.39955663681030273, + "sampling/sampling_logp_difference/mean": 0.009428753517568111, + "step": 970 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010276515240548179, + "clip_ratio/low_min": 0.00010276515240548179, + "clip_ratio/region_mean": 0.00010276515240548179, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 555.59375, + "completions/mean_terminated_length": 533.6206665039062, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.2982930038124323, + "epoch": 0.5198072805139187, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.032353516668081284, + "learning_rate": 1e-05, + "loss": 0.0612, + "num_tokens": 20067407.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4002083539962769, + "sampling/importance_sampling_ratio/mean": 0.9995220899581909, + "sampling/importance_sampling_ratio/min": 0.6951965689659119, + "sampling/sampling_logp_difference/max": 0.36356067657470703, + "sampling/sampling_logp_difference/mean": 0.01075439341366291, + "step": 971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 468.8125, + "completions/mean_terminated_length": 369.0833435058594, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.3024728912860155, + "epoch": 0.5203426124197003, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0036959454882889986, + "learning_rate": 1e-05, + "loss": 0.0278, + "num_tokens": 20085825.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998359680175781, + "sampling/importance_sampling_ratio/min": 0.6323160529136658, + "sampling/sampling_logp_difference/max": 0.7227559089660645, + "sampling/sampling_logp_difference/mean": 0.010977193713188171, + "step": 972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014327518510981463, + "clip_ratio/low_min": 0.00014327518510981463, + "clip_ratio/region_mean": 0.00014327518510981463, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 505.0, + "completions/mean_terminated_length": 444.3077087402344, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.336171830072999, + "epoch": 0.5208779443254818, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0036234515719115734, + "learning_rate": 1e-05, + "loss": 0.0217, + "num_tokens": 20105985.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6552815437316895, + "sampling/importance_sampling_ratio/mean": 1.000277042388916, + "sampling/importance_sampling_ratio/min": 0.6642733812332153, + "sampling/sampling_logp_difference/max": 0.5039710998535156, + "sampling/sampling_logp_difference/mean": 0.012055602855980396, + "step": 973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 560.0, + "completions/max_terminated_length": 560.0, + "completions/mean_length": 381.59375, + "completions/mean_terminated_length": 381.59375, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.246996833011508, + "epoch": 0.5214132762312634, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0135, + "num_tokens": 20121892.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.428590178489685, + "sampling/importance_sampling_ratio/mean": 1.0002020597457886, + "sampling/importance_sampling_ratio/min": 0.6473625302314758, + "sampling/sampling_logp_difference/max": 0.4348487854003906, + "sampling/sampling_logp_difference/mean": 0.00973924994468689, + "step": 974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00029816240930813365, + "clip_ratio/low_min": 0.00029816240930813365, + "clip_ratio/region_mean": 0.00029816240930813365, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 506.1875, + "completions/mean_terminated_length": 488.7333679199219, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.26473431661725044, + "epoch": 0.521948608137045, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014563528820872307, + "learning_rate": 1e-05, + "loss": -0.0015, + "num_tokens": 20142122.0, + "reward": 0.46875, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.560551404953003, + "sampling/importance_sampling_ratio/mean": 0.9999781250953674, + "sampling/importance_sampling_ratio/min": 0.7204043865203857, + "sampling/sampling_logp_difference/max": 0.4450392723083496, + "sampling/sampling_logp_difference/mean": 0.009984633885324001, + "step": 975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 747.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 459.96875, + "completions/mean_terminated_length": 459.96875, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.29980884678661823, + "epoch": 0.5224839400428265, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.02271120250225067, + "learning_rate": 1e-05, + "loss": 0.0217, + "num_tokens": 20160417.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4021810293197632, + "sampling/importance_sampling_ratio/mean": 0.9993802309036255, + "sampling/importance_sampling_ratio/min": 0.7222098708152771, + "sampling/sampling_logp_difference/max": 0.3380289077758789, + "sampling/sampling_logp_difference/mean": 0.010800668969750404, + "step": 976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002829060031217523, + "clip_ratio/low_min": 0.0002829060031217523, + "clip_ratio/region_mean": 0.0002829060031217523, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 670.375, + "completions/mean_terminated_length": 632.1739501953125, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.28090241737663746, + "epoch": 0.5230192719486081, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014829407446086407, + "learning_rate": 1e-05, + "loss": 0.0415, + "num_tokens": 20186549.0, + "reward": 0.4375, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5468014478683472, + "sampling/importance_sampling_ratio/mean": 1.0000035762786865, + "sampling/importance_sampling_ratio/min": 0.6892077922821045, + "sampling/sampling_logp_difference/max": 0.4361891746520996, + "sampling/sampling_logp_difference/mean": 0.00957837700843811, + "step": 977 + }, + { + "clip_ratio/high_max": 7.97193861217238e-05, + "clip_ratio/high_mean": 7.97193861217238e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.97193861217238e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 454.5, + "completions/mean_terminated_length": 454.5, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.254776893183589, + "epoch": 0.5235546038543897, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0076627591624855995, + "learning_rate": 1e-05, + "loss": 0.0114, + "num_tokens": 20204333.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5157309770584106, + "sampling/importance_sampling_ratio/mean": 1.0001399517059326, + "sampling/importance_sampling_ratio/min": 0.6700972318649292, + "sampling/sampling_logp_difference/max": 0.41589784622192383, + "sampling/sampling_logp_difference/mean": 0.009625574573874474, + "step": 978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 571.0625, + "completions/mean_terminated_length": 525.6154174804688, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.37119146436452866, + "epoch": 0.5240899357601713, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010141514241695404, + "learning_rate": 1e-05, + "loss": 0.0273, + "num_tokens": 20226567.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.7598707675933838, + "sampling/importance_sampling_ratio/mean": 1.0000534057617188, + "sampling/importance_sampling_ratio/min": 0.6091075539588928, + "sampling/sampling_logp_difference/max": 0.5652403831481934, + "sampling/sampling_logp_difference/mean": 0.012942535802721977, + "step": 979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 505.125, + "completions/mean_terminated_length": 496.6451416015625, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.3061270248144865, + "epoch": 0.5246252676659529, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021014241501688957, + "learning_rate": 1e-05, + "loss": 0.0419, + "num_tokens": 20246699.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4478200674057007, + "sampling/importance_sampling_ratio/mean": 0.999735414981842, + "sampling/importance_sampling_ratio/min": 0.6959509253501892, + "sampling/sampling_logp_difference/max": 0.3700590133666992, + "sampling/sampling_logp_difference/mean": 0.01126969326287508, + "step": 980 + }, + { + "clip_ratio/high_max": 5.9326055634301156e-05, + "clip_ratio/high_mean": 5.9326055634301156e-05, + "clip_ratio/low_mean": 0.00011280367834842764, + "clip_ratio/low_min": 0.00011280367834842764, + "clip_ratio/region_mean": 0.0001721297339827288, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 507.53125, + "completions/mean_terminated_length": 490.16668701171875, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.3268727343529463, + "epoch": 0.5251605995717344, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03915729746222496, + "learning_rate": 1e-05, + "loss": 0.0258, + "num_tokens": 20266476.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5074266195297241, + "sampling/importance_sampling_ratio/mean": 1.0002350807189941, + "sampling/importance_sampling_ratio/min": 0.6859396696090698, + "sampling/sampling_logp_difference/max": 0.4104039669036865, + "sampling/sampling_logp_difference/mean": 0.012186874635517597, + "step": 981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 372.65625, + "completions/mean_terminated_length": 372.65625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.28994522243738174, + "epoch": 0.525695931477516, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.025411561131477356, + "learning_rate": 1e-05, + "loss": 0.0112, + "num_tokens": 20281681.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4376773834228516, + "sampling/importance_sampling_ratio/mean": 0.9996423125267029, + "sampling/importance_sampling_ratio/min": 0.65763258934021, + "sampling/sampling_logp_difference/max": 0.41910886764526367, + "sampling/sampling_logp_difference/mean": 0.010259519331157207, + "step": 982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.777290713624097e-05, + "clip_ratio/low_min": 8.777290713624097e-05, + "clip_ratio/region_mean": 8.777290713624097e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 573.15625, + "completions/mean_terminated_length": 508.2083435058594, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.36684974282979965, + "epoch": 0.5262312633832976, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016475502401590347, + "learning_rate": 1e-05, + "loss": 0.0193, + "num_tokens": 20304206.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6101723909378052, + "sampling/importance_sampling_ratio/mean": 1.0001775026321411, + "sampling/importance_sampling_ratio/min": 0.7203891277313232, + "sampling/sampling_logp_difference/max": 0.47634124755859375, + "sampling/sampling_logp_difference/mean": 0.012516574002802372, + "step": 983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 553.25, + "completions/mean_terminated_length": 531.0344848632812, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.3875699080526829, + "epoch": 0.5267665952890792, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015934111550450325, + "learning_rate": 1e-05, + "loss": -0.0034, + "num_tokens": 20325894.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4636443853378296, + "sampling/importance_sampling_ratio/mean": 1.000114917755127, + "sampling/importance_sampling_ratio/min": 0.7714143991470337, + "sampling/sampling_logp_difference/max": 0.38092947006225586, + "sampling/sampling_logp_difference/mean": 0.012341463938355446, + "step": 984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011558142432477325, + "clip_ratio/low_min": 0.00011558142432477325, + "clip_ratio/region_mean": 0.00011558142432477325, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 411.25927734375, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.3518456779420376, + "epoch": 0.5273019271948608, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004745997954159975, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 20344414.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.723758339881897, + "sampling/importance_sampling_ratio/mean": 1.0002174377441406, + "sampling/importance_sampling_ratio/min": 0.6284816265106201, + "sampling/sampling_logp_difference/max": 0.5445070266723633, + "sampling/sampling_logp_difference/mean": 0.013397024013102055, + "step": 985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 737.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 409.0, + "completions/mean_terminated_length": 409.0, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.259233757853508, + "epoch": 0.5278372591006424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03837405517697334, + "learning_rate": 1e-05, + "loss": 0.0469, + "num_tokens": 20361838.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.679805040359497, + "sampling/importance_sampling_ratio/mean": 1.0000888109207153, + "sampling/importance_sampling_ratio/min": 0.6981421113014221, + "sampling/sampling_logp_difference/max": 0.5186777114868164, + "sampling/sampling_logp_difference/mean": 0.010746989399194717, + "step": 986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 545.28125, + "completions/mean_terminated_length": 513.4642944335938, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.33587929606437683, + "epoch": 0.528372591006424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02470543049275875, + "learning_rate": 1e-05, + "loss": 0.0127, + "num_tokens": 20383519.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6979705095291138, + "sampling/importance_sampling_ratio/mean": 0.9997697472572327, + "sampling/importance_sampling_ratio/min": 0.6274191737174988, + "sampling/sampling_logp_difference/max": 0.5294337272644043, + "sampling/sampling_logp_difference/mean": 0.012272883206605911, + "step": 987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 486.5, + "completions/mean_terminated_length": 467.7333679199219, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.2786070182919502, + "epoch": 0.5289079229122056, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006380059290677309, + "learning_rate": 1e-05, + "loss": 0.0128, + "num_tokens": 20402767.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5402277708053589, + "sampling/importance_sampling_ratio/mean": 0.99989914894104, + "sampling/importance_sampling_ratio/min": 0.7686281800270081, + "sampling/sampling_logp_difference/max": 0.4319303035736084, + "sampling/sampling_logp_difference/mean": 0.010861505754292011, + "step": 988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 396.0, + "completions/mean_terminated_length": 396.0, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.23258525133132935, + "epoch": 0.5294432548179872, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20419511.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4397772550582886, + "sampling/importance_sampling_ratio/mean": 1.000404715538025, + "sampling/importance_sampling_ratio/min": 0.691949188709259, + "sampling/sampling_logp_difference/max": 0.3682427406311035, + "sampling/sampling_logp_difference/mean": 0.010228743776679039, + "step": 989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 620.0, + "completions/max_terminated_length": 620.0, + "completions/mean_length": 438.3125, + "completions/mean_terminated_length": 438.3125, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.33510708808898926, + "epoch": 0.5299785867237687, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008542737923562527, + "learning_rate": 1e-05, + "loss": -0.0146, + "num_tokens": 20437337.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.405011534690857, + "sampling/importance_sampling_ratio/mean": 0.9998782277107239, + "sampling/importance_sampling_ratio/min": 0.7477208971977234, + "sampling/sampling_logp_difference/max": 0.3400454521179199, + "sampling/sampling_logp_difference/mean": 0.012023129500448704, + "step": 990 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 529.34375, + "completions/mean_terminated_length": 521.6451416015625, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.28788173757493496, + "epoch": 0.5305139186295503, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00872648973017931, + "learning_rate": 1e-05, + "loss": 0.026, + "num_tokens": 20457724.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4477896690368652, + "sampling/importance_sampling_ratio/mean": 1.000186800956726, + "sampling/importance_sampling_ratio/min": 0.7323204278945923, + "sampling/sampling_logp_difference/max": 0.3700380325317383, + "sampling/sampling_logp_difference/mean": 0.010936199687421322, + "step": 991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011848341091535985, + "clip_ratio/low_min": 0.00011848341091535985, + "clip_ratio/region_mean": 0.00011848341091535985, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 389.09375, + "completions/mean_terminated_length": 389.09375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.3102747555822134, + "epoch": 0.5310492505353319, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008674847893416882, + "learning_rate": 1e-05, + "loss": 0.0388, + "num_tokens": 20473623.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4332579374313354, + "sampling/importance_sampling_ratio/mean": 1.000070571899414, + "sampling/importance_sampling_ratio/min": 0.6004974842071533, + "sampling/sampling_logp_difference/max": 0.5099968910217285, + "sampling/sampling_logp_difference/mean": 0.011831749230623245, + "step": 992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 677.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 425.71875, + "completions/mean_terminated_length": 425.71875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.22281677462160587, + "epoch": 0.5315845824411135, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.015275092795491219, + "learning_rate": 1e-05, + "loss": 0.0105, + "num_tokens": 20490782.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.9839650392532349, + "sampling/importance_sampling_ratio/mean": 1.0004388093948364, + "sampling/importance_sampling_ratio/min": 0.668385922908783, + "sampling/sampling_logp_difference/max": 0.6850974559783936, + "sampling/sampling_logp_difference/mean": 0.008745191618800163, + "step": 993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 414.5, + "completions/mean_terminated_length": 390.933349609375, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.2425446305423975, + "epoch": 0.5321199143468951, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.015331036411225796, + "learning_rate": 1e-05, + "loss": 0.0027, + "num_tokens": 20507494.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4391626119613647, + "sampling/importance_sampling_ratio/mean": 1.0001529455184937, + "sampling/importance_sampling_ratio/min": 0.7409034967422485, + "sampling/sampling_logp_difference/max": 0.3640613555908203, + "sampling/sampling_logp_difference/mean": 0.010235991328954697, + "step": 994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 489.0, + "completions/mean_terminated_length": 460.137939453125, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.3341273684054613, + "epoch": 0.5326552462526767, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02347775362432003, + "learning_rate": 1e-05, + "loss": 0.0056, + "num_tokens": 20526894.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4396318197250366, + "sampling/importance_sampling_ratio/mean": 1.000096082687378, + "sampling/importance_sampling_ratio/min": 0.6704294681549072, + "sampling/sampling_logp_difference/max": 0.39983677864074707, + "sampling/sampling_logp_difference/mean": 0.012613601982593536, + "step": 995 + }, + { + "clip_ratio/high_max": 7.631257903994992e-05, + "clip_ratio/high_mean": 7.631257903994992e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.631257903994992e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 448.28125, + "completions/mean_terminated_length": 402.6071472167969, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "entropy": 0.4117588307708502, + "epoch": 0.5331905781584583, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024844985455274582, + "learning_rate": 1e-05, + "loss": 0.0555, + "num_tokens": 20545055.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.382009744644165, + "sampling/importance_sampling_ratio/mean": 1.0003635883331299, + "sampling/importance_sampling_ratio/min": 0.48914584517478943, + "sampling/sampling_logp_difference/max": 0.7150945663452148, + "sampling/sampling_logp_difference/mean": 0.013542810454964638, + "step": 996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 584.0625, + "completions/mean_terminated_length": 565.0344848632812, + "completions/min_length": 355.0, + "completions/min_terminated_length": 355.0, + "entropy": 0.24145429581403732, + "epoch": 0.5337259100642399, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00853966549038887, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 20567793.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3396481275558472, + "sampling/importance_sampling_ratio/mean": 1.000004768371582, + "sampling/importance_sampling_ratio/min": 0.43636608123779297, + "sampling/sampling_logp_difference/max": 0.8292738199234009, + "sampling/sampling_logp_difference/mean": 0.008973004296422005, + "step": 997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 491.15625, + "completions/mean_terminated_length": 491.15625, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.20358617417514324, + "epoch": 0.5342612419700214, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006970930378884077, + "learning_rate": 1e-05, + "loss": -0.0082, + "num_tokens": 20587310.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4240021705627441, + "sampling/importance_sampling_ratio/mean": 0.9996641278266907, + "sampling/importance_sampling_ratio/min": 0.6888296604156494, + "sampling/sampling_logp_difference/max": 0.37276124954223633, + "sampling/sampling_logp_difference/mean": 0.008362756110727787, + "step": 998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017769212718121707, + "clip_ratio/low_min": 0.00017769212718121707, + "clip_ratio/region_mean": 0.00017769212718121707, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 374.375, + "completions/mean_terminated_length": 374.375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.30256051383912563, + "epoch": 0.534796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006298878695815802, + "learning_rate": 1e-05, + "loss": 0.0083, + "num_tokens": 20602714.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4680598974227905, + "sampling/importance_sampling_ratio/mean": 1.000386357307434, + "sampling/importance_sampling_ratio/min": 0.7151246070861816, + "sampling/sampling_logp_difference/max": 0.383941650390625, + "sampling/sampling_logp_difference/mean": 0.01191803626716137, + "step": 999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 566.75, + "completions/mean_terminated_length": 545.9310302734375, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.34574132412672043, + "epoch": 0.5353319057815846, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02217051014304161, + "learning_rate": 1e-05, + "loss": 0.0523, + "num_tokens": 20624474.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3802006244659424, + "sampling/importance_sampling_ratio/mean": 0.9995157718658447, + "sampling/importance_sampling_ratio/min": 0.70558100938797, + "sampling/sampling_logp_difference/max": 0.34873366355895996, + "sampling/sampling_logp_difference/mean": 0.01196676678955555, + "step": 1000 + }, + { + "clip_ratio/high_max": 6.742178811691701e-05, + "clip_ratio/high_mean": 6.742178811691701e-05, + "clip_ratio/low_mean": 6.271951860981062e-05, + "clip_ratio/low_min": 6.271951860981062e-05, + "clip_ratio/region_mean": 0.00013014130672672763, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 663.0, + "completions/mean_length": 461.09375, + "completions/mean_terminated_length": 440.63336181640625, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.29909978061914444, + "epoch": 0.5358672376873662, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006004105322062969, + "learning_rate": 1e-05, + "loss": 0.0916, + "num_tokens": 20643509.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5818359851837158, + "sampling/importance_sampling_ratio/mean": 0.999842643737793, + "sampling/importance_sampling_ratio/min": 0.6266263723373413, + "sampling/sampling_logp_difference/max": 0.467404842376709, + "sampling/sampling_logp_difference/mean": 0.011380976997315884, + "step": 1001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.003842281643301e-05, + "clip_ratio/low_min": 6.003842281643301e-05, + "clip_ratio/region_mean": 6.003842281643301e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 513.1875, + "completions/mean_terminated_length": 476.7857360839844, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.31066639348864555, + "epoch": 0.5364025695931478, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02892880327999592, + "learning_rate": 1e-05, + "loss": 0.0193, + "num_tokens": 20663891.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4119313955307007, + "sampling/importance_sampling_ratio/mean": 0.9999995231628418, + "sampling/importance_sampling_ratio/min": 0.7418161630630493, + "sampling/sampling_logp_difference/max": 0.3449585437774658, + "sampling/sampling_logp_difference/mean": 0.011235960759222507, + "step": 1002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014886206918163225, + "clip_ratio/low_min": 0.00014886206918163225, + "clip_ratio/region_mean": 0.00014886206918163225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 639.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 406.625, + "completions/mean_terminated_length": 406.625, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.32252445816993713, + "epoch": 0.5369379014989293, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008560850284993649, + "learning_rate": 1e-05, + "loss": 0.007, + "num_tokens": 20680823.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3867586851119995, + "sampling/importance_sampling_ratio/mean": 1.0000890493392944, + "sampling/importance_sampling_ratio/min": 0.6831401586532593, + "sampling/sampling_logp_difference/max": 0.38105523586273193, + "sampling/sampling_logp_difference/mean": 0.012363986112177372, + "step": 1003 + }, + { + "clip_ratio/high_max": 5.380973016144708e-05, + "clip_ratio/high_mean": 5.380973016144708e-05, + "clip_ratio/low_mean": 5.087505269329995e-05, + "clip_ratio/low_min": 5.087505269329995e-05, + "clip_ratio/region_mean": 0.00010468478285474703, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 553.0625, + "completions/mean_terminated_length": 530.8275756835938, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.41602638363838196, + "epoch": 0.5374732334047109, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01616629585623741, + "learning_rate": 1e-05, + "loss": -0.0111, + "num_tokens": 20702265.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.521829605102539, + "sampling/importance_sampling_ratio/mean": 0.9995875954627991, + "sampling/importance_sampling_ratio/min": 0.6779919266700745, + "sampling/sampling_logp_difference/max": 0.41991329193115234, + "sampling/sampling_logp_difference/mean": 0.013063188642263412, + "step": 1004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 375.8125, + "completions/mean_terminated_length": 363.1612854003906, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "entropy": 0.2924263346940279, + "epoch": 0.5380085653104925, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009580021724104881, + "learning_rate": 1e-05, + "loss": 0.0663, + "num_tokens": 20717299.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5672733783721924, + "sampling/importance_sampling_ratio/mean": 0.9999663233757019, + "sampling/importance_sampling_ratio/min": 0.6852931380271912, + "sampling/sampling_logp_difference/max": 0.4493374824523926, + "sampling/sampling_logp_difference/mean": 0.010947070084512234, + "step": 1005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 448.40625, + "completions/mean_terminated_length": 438.0967712402344, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.31122429855167866, + "epoch": 0.538543897216274, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024525612592697144, + "learning_rate": 1e-05, + "loss": -0.025, + "num_tokens": 20735240.0, + "reward": 0.625, + "reward_std": 0.3924052119255066, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7791000604629517, + "sampling/importance_sampling_ratio/mean": 0.9997047781944275, + "sampling/importance_sampling_ratio/min": 0.4711102843284607, + "sampling/sampling_logp_difference/max": 0.7526631355285645, + "sampling/sampling_logp_difference/mean": 0.011387272737920284, + "step": 1006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013269779447000474, + "clip_ratio/low_min": 0.00013269779447000474, + "clip_ratio/region_mean": 0.00013269779447000474, + "completions/clipped_ratio": 0.0, + "completions/max_length": 708.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 486.5625, + "completions/mean_terminated_length": 486.5625, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.2862442471086979, + "epoch": 0.5390792291220556, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006266626995056868, + "learning_rate": 1e-05, + "loss": 0.0265, + "num_tokens": 20754994.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5755714178085327, + "sampling/importance_sampling_ratio/mean": 0.9999701976776123, + "sampling/importance_sampling_ratio/min": 0.7431015968322754, + "sampling/sampling_logp_difference/max": 0.454617977142334, + "sampling/sampling_logp_difference/mean": 0.010682228952646255, + "step": 1007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 476.125, + "completions/mean_terminated_length": 476.125, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.31628429144620895, + "epoch": 0.5396145610278372, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0032, + "num_tokens": 20773798.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6980595588684082, + "sampling/importance_sampling_ratio/mean": 0.9999076724052429, + "sampling/importance_sampling_ratio/min": 0.5939722657203674, + "sampling/sampling_logp_difference/max": 0.5294861793518066, + "sampling/sampling_logp_difference/mean": 0.011187360621988773, + "step": 1008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.319514795904979e-05, + "clip_ratio/low_min": 6.319514795904979e-05, + "clip_ratio/region_mean": 6.319514795904979e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 485.59375, + "completions/mean_terminated_length": 466.7666931152344, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.33392519876360893, + "epoch": 0.5401498929336188, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013179289177060127, + "learning_rate": 1e-05, + "loss": 0.0438, + "num_tokens": 20793513.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4700065851211548, + "sampling/importance_sampling_ratio/mean": 0.9999333620071411, + "sampling/importance_sampling_ratio/min": 0.5124729871749878, + "sampling/sampling_logp_difference/max": 0.6685073375701904, + "sampling/sampling_logp_difference/mean": 0.011264905333518982, + "step": 1009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.950494985678233e-05, + "clip_ratio/low_min": 4.950494985678233e-05, + "clip_ratio/region_mean": 4.950494985678233e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 504.0625, + "completions/mean_terminated_length": 466.357177734375, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.380746778100729, + "epoch": 0.5406852248394004, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01088571734726429, + "learning_rate": 1e-05, + "loss": 0.0466, + "num_tokens": 20813331.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.389113426208496, + "sampling/importance_sampling_ratio/mean": 1.000091314315796, + "sampling/importance_sampling_ratio/min": 0.7165794372558594, + "sampling/sampling_logp_difference/max": 0.3332662582397461, + "sampling/sampling_logp_difference/mean": 0.012350971810519695, + "step": 1010 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 393.9375, + "completions/mean_terminated_length": 393.9375, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "entropy": 0.2761106453835964, + "epoch": 0.541220556745182, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20829409.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4311715364456177, + "sampling/importance_sampling_ratio/mean": 0.9997698068618774, + "sampling/importance_sampling_ratio/min": 0.6138607859611511, + "sampling/sampling_logp_difference/max": 0.4879871606826782, + "sampling/sampling_logp_difference/mean": 0.010651922784745693, + "step": 1011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 563.9375, + "completions/mean_terminated_length": 441.5, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.4022360257804394, + "epoch": 0.5417558886509636, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0235, + "num_tokens": 20851847.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5724354982376099, + "sampling/importance_sampling_ratio/mean": 1.0001391172409058, + "sampling/importance_sampling_ratio/min": 0.6602086424827576, + "sampling/sampling_logp_difference/max": 0.45262575149536133, + "sampling/sampling_logp_difference/mean": 0.012675242498517036, + "step": 1012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 548.0, + "completions/max_terminated_length": 548.0, + "completions/mean_length": 394.4375, + "completions/mean_terminated_length": 394.4375, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.2700550053268671, + "epoch": 0.5422912205567452, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006720743142068386, + "learning_rate": 1e-05, + "loss": 0.011, + "num_tokens": 20868293.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4353808164596558, + "sampling/importance_sampling_ratio/mean": 0.9999120831489563, + "sampling/importance_sampling_ratio/min": 0.7047362327575684, + "sampling/sampling_logp_difference/max": 0.36143016815185547, + "sampling/sampling_logp_difference/mean": 0.010281146503984928, + "step": 1013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 421.78125, + "completions/mean_terminated_length": 421.78125, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.24118858203291893, + "epoch": 0.5428265524625268, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20885838.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.427647352218628, + "sampling/importance_sampling_ratio/mean": 1.0003013610839844, + "sampling/importance_sampling_ratio/min": 0.7005481123924255, + "sampling/sampling_logp_difference/max": 0.35602784156799316, + "sampling/sampling_logp_difference/mean": 0.010140156373381615, + "step": 1014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002263921342091635, + "clip_ratio/low_min": 0.0002263921342091635, + "clip_ratio/region_mean": 0.0002263921342091635, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 410.65625, + "completions/mean_terminated_length": 410.65625, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.2819908335804939, + "epoch": 0.5433618843683083, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011591862887144089, + "learning_rate": 1e-05, + "loss": 0.0156, + "num_tokens": 20902467.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.436547875404358, + "sampling/importance_sampling_ratio/mean": 1.0001517534255981, + "sampling/importance_sampling_ratio/min": 0.7365150451660156, + "sampling/sampling_logp_difference/max": 0.3622429370880127, + "sampling/sampling_logp_difference/mean": 0.010819976218044758, + "step": 1015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 474.8125, + "completions/mean_terminated_length": 432.9285888671875, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.32844111137092113, + "epoch": 0.5438972162740899, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20921165.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4033128023147583, + "sampling/importance_sampling_ratio/mean": 1.0000876188278198, + "sampling/importance_sampling_ratio/min": 0.7019321322441101, + "sampling/sampling_logp_difference/max": 0.35391855239868164, + "sampling/sampling_logp_difference/mean": 0.011300837621092796, + "step": 1016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 438.25, + "completions/mean_terminated_length": 438.25, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.2606982924044132, + "epoch": 0.5444325481798715, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.013174247927963734, + "learning_rate": 1e-05, + "loss": -0.0082, + "num_tokens": 20938957.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.454656958580017, + "sampling/importance_sampling_ratio/mean": 1.0001221895217896, + "sampling/importance_sampling_ratio/min": 0.7321741580963135, + "sampling/sampling_logp_difference/max": 0.3747701644897461, + "sampling/sampling_logp_difference/mean": 0.010059934109449387, + "step": 1017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010052271682070568, + "clip_ratio/low_min": 0.00010052271682070568, + "clip_ratio/region_mean": 0.00010052271682070568, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 538.96875, + "completions/mean_terminated_length": 506.2500305175781, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.32434203661978245, + "epoch": 0.5449678800856531, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01634792611002922, + "learning_rate": 1e-05, + "loss": 0.014, + "num_tokens": 20960220.0, + "reward": 0.375, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3839634656906128, + "sampling/importance_sampling_ratio/mean": 0.9999974966049194, + "sampling/importance_sampling_ratio/min": 0.6750747561454773, + "sampling/sampling_logp_difference/max": 0.3929319381713867, + "sampling/sampling_logp_difference/mean": 0.011419091373682022, + "step": 1018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.715592124033719e-05, + "clip_ratio/low_min": 5.715592124033719e-05, + "clip_ratio/region_mean": 5.715592124033719e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 509.8125, + "completions/mean_terminated_length": 450.23077392578125, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.3778581041842699, + "epoch": 0.5455032119914347, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024147875607013702, + "learning_rate": 1e-05, + "loss": 0.0124, + "num_tokens": 20980374.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4404075145721436, + "sampling/importance_sampling_ratio/mean": 1.0000821352005005, + "sampling/importance_sampling_ratio/min": 0.712488055229187, + "sampling/sampling_logp_difference/max": 0.3649260997772217, + "sampling/sampling_logp_difference/mean": 0.011622502468526363, + "step": 1019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 448.09375, + "completions/mean_terminated_length": 426.7666931152344, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.35178061947226524, + "epoch": 0.5460385438972163, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03291179984807968, + "learning_rate": 1e-05, + "loss": 0.0386, + "num_tokens": 20998073.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4602396488189697, + "sampling/importance_sampling_ratio/mean": 0.9998698234558105, + "sampling/importance_sampling_ratio/min": 0.6329456567764282, + "sampling/sampling_logp_difference/max": 0.4573707580566406, + "sampling/sampling_logp_difference/mean": 0.011893614195287228, + "step": 1020 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010640751861501485, + "clip_ratio/low_min": 0.00010640751861501485, + "clip_ratio/region_mean": 0.00010640751861501485, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 559.125, + "completions/mean_terminated_length": 520.4444580078125, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.40700463578104973, + "epoch": 0.5465738758029979, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01419112365692854, + "learning_rate": 1e-05, + "loss": 0.0215, + "num_tokens": 21019837.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7007182836532593, + "sampling/importance_sampling_ratio/mean": 0.9997385740280151, + "sampling/importance_sampling_ratio/min": 0.695031464099884, + "sampling/sampling_logp_difference/max": 0.5310506820678711, + "sampling/sampling_logp_difference/mean": 0.01345874834805727, + "step": 1021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.739938155282289e-05, + "clip_ratio/low_min": 7.739938155282289e-05, + "clip_ratio/region_mean": 7.739938155282289e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 715.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 426.03125, + "completions/mean_terminated_length": 426.03125, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.30674983933568, + "epoch": 0.5471092077087795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02698620967566967, + "learning_rate": 1e-05, + "loss": -0.0005, + "num_tokens": 21037142.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5722301006317139, + "sampling/importance_sampling_ratio/mean": 0.9997084736824036, + "sampling/importance_sampling_ratio/min": 0.7135961651802063, + "sampling/sampling_logp_difference/max": 0.45249509811401367, + "sampling/sampling_logp_difference/mean": 0.01148021500557661, + "step": 1022 + }, + { + "clip_ratio/high_max": 6.300403038039804e-05, + "clip_ratio/high_mean": 6.300403038039804e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.300403038039804e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 425.5625, + "completions/mean_terminated_length": 425.5625, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.27811561338603497, + "epoch": 0.547644539614561, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.019728660583496094, + "learning_rate": 1e-05, + "loss": 0.0063, + "num_tokens": 21055040.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.6341397762298584, + "sampling/importance_sampling_ratio/mean": 0.9996365904808044, + "sampling/importance_sampling_ratio/min": 0.6480754017829895, + "sampling/sampling_logp_difference/max": 0.4911165237426758, + "sampling/sampling_logp_difference/mean": 0.010462039150297642, + "step": 1023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 491.125, + "completions/mean_terminated_length": 482.19354248046875, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.26878377981483936, + "epoch": 0.5481798715203426, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020172934979200363, + "learning_rate": 1e-05, + "loss": 0.0082, + "num_tokens": 21074644.0, + "reward": 0.40625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2830291986465454, + "sampling/importance_sampling_ratio/mean": 1.0000466108322144, + "sampling/importance_sampling_ratio/min": 0.643420934677124, + "sampling/sampling_logp_difference/max": 0.44095611572265625, + "sampling/sampling_logp_difference/mean": 0.009556285105645657, + "step": 1024 + } + ], + "logging_steps": 1, + "max_steps": 1024, + "num_input_tokens_seen": 21074644, + "num_train_epochs": 1, + "save_steps": 64, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}